Patient_wise_cross_validation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Patientwise_validation_fold_custom_weight.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "_eNqwrN19Ql-",
        "colab_type": "code",
        "outputId": "3ecca97d-d962-42b9-c054-0de66b3518a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VDAwqCP5kYby",
        "colab_type": "code",
        "outputId": "d9a20c34-c81c-499f-8e4a-364f4835f15b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "from keras.models import load_model\n",
        "from __future__ import print_function\n",
        "from keras.utils import np_utils\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras import applications\n",
        "import cv2 \n",
        "from os import listdir\n",
        "import numpy as np\n",
        "import matplotlib. pyplot as plt\n",
        "from keras import layers\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Input\n",
        "from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten, AveragePooling2D,GlobalAveragePooling2D,Activation,BatchNormalization\n",
        "from keras.models import Model\n",
        "from keras.applications import DenseNet121\n",
        "from keras.applications.inception_v3 import InceptionV3\n",
        "from keras.optimizers import Adam, Adamax\n",
        "from keras.callbacks import LearningRateScheduler\n",
        "from keras import optimizers\n",
        "#Augmentation\n",
        "from numpy import expand_dims\n",
        "from keras.preprocessing.image import load_img\n",
        "from keras.preprocessing.image import img_to_array\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from matplotlib import pyplot\n",
        "from sklearn.utils import shuffle\n",
        "from sklearn.utils import class_weight\n",
        "from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, ReduceLROnPlateau, TensorBoard, EarlyStopping\n",
        "from sklearn.model_selection import KFold,StratifiedKFold\n",
        "from sklearn.model_selection import train_test_split\n",
        "import urllib,math"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TQHSEqGFkecc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "kaggle_pneumonia = np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v3/kaggle_pneumonia_images.npy')\n",
        "kaggle_normal = np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v3/kaggle_normal_images.npy')\n",
        "ieee_corona=np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v3/ieee8023_covid_images.npy')\n",
        "ieee_pneumonia=np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v3/ieee8023_pneumonia_images.npy')\n",
        "figure1 = np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v2/figure1_covid.npy')\n",
        "figure1_mask = np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v2/figure1_covid_mask.npy')\n",
        "figure1_corona = figure1[figure1_mask]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1UnegslHSkj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "covid_folds = np.array([[186, 168, 167, 59, 57, 158, 157, 93, 85, 56, 55, 32, 24],\n",
        " [178, 169, 164, 67, 51, 159, 156, 94, 84, 60, 54, 33, 23],\n",
        " [173, 183, 163, 68, 37, 161, 155, 99, 83, 61, 53, 34, 22],\n",
        " [142, 185, 150, 69, 36, 162, 154, 118, 82, 62, 52, 35, 21],\n",
        " [19, 15, 144, 70, 6, 165, 153, 137, 81, 63, 50, 39, 20],\n",
        " [13, 17, 143, 73, 4, 166, 152, 138, 80, 64, 49, 40, 18],\n",
        " [2, 58, 139, 86, 184, 174, 151, 140, 79, 65, 48, 41, 16],\n",
        " [187, 71, 115, 95, 182, 175, 149, 141, 78, 72, 47, 42, 14],\n",
        " [179, 116, 114, 98, 181, 176, 148, 145, 77, 74, 46, 43, 12],\n",
        " [132, 117, 113, 112, 180, 177, 147, 146, 76, 75, 45, 44, 11]]\n",
        ")\n",
        "pnemonia_folds = np.array([[31],\n",
        " [171],\n",
        " [87, 29],\n",
        " [10, 30],\n",
        " [3, 88],\n",
        " [92, 89],\n",
        " [9, 90],\n",
        " [7, 100],\n",
        " [172, 110],\n",
        " [170, 111, 8]]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IexLK3XQoZ7R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "covid_patient_id=np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v3/ieee8023_covid_images_patient_ids.npy')\n",
        "pneumonia_patient_id=np.load('/content/drive/My Drive/Corona_Image/Pavel/Data/v3/ieee8023_pneumonia_images_patient_ids.npy')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fv5fwFnsuATu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# fold_no=10\n",
        "# mask = np.isin(covid_patient_id, covid_folds[fold_no-1])\n",
        "# test = ieee_corona[mask]\n",
        "# mask2 = np.isin(covid_patient_id, covid_folds[fold_no%10])\n",
        "# validation = ieee_corona[mask2]\n",
        "# arr = np.concatenate((covid_folds[fold_no-1], covid_folds[fold_no%10]), axis=0)\n",
        "# mask3 = np.isin(covid_patient_id, arr, invert=True)\n",
        "# train = ieee_corona[mask3]\n",
        "# print(train.shape,validation.shape,test.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Km5AMci5uejL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# num_folds = 10\n",
        "# kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=1)\n",
        "# label = np.empty(figure1_corona.shape[0],dtype='int32')\n",
        "# label.fill(2)\n",
        "# fold_no=1\n",
        "# for train,test in kfold.split(figure1_corona,label): \n",
        "#    np.save(f'/content/drive/My Drive/Corona_Image/Final_data/figure1/train{fold_no}.npy',train)\n",
        "#    np.save(f'/content/drive/My Drive/Corona_Image/Final_data/figure1/test{fold_no}.npy',test)\n",
        "#    fold_no=fold_no+1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eU_F4y2jLzH8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def augment(images,num):\n",
        "  aug = []\n",
        "  datagens = []\n",
        "  datagens.append(ImageDataGenerator(width_shift_range=0.15))\n",
        "  datagens.append(ImageDataGenerator(height_shift_range=0.15))\n",
        "  datagens.append(ImageDataGenerator(horizontal_flip=True))\n",
        "  datagens.append(ImageDataGenerator(rotation_range=45))\n",
        "  datagens.append(ImageDataGenerator(brightness_range=[0.6,1.0]))\n",
        "  datagens.append(ImageDataGenerator(zoom_range=0.15))\n",
        "  for data in images:\n",
        "    samples = expand_dims(data, 0)\n",
        "    for datagen in datagens:\n",
        "      it = datagen.flow(samples, batch_size=1)\n",
        "      for i in range(num):\n",
        "        batch = it.next()\n",
        "        image = batch[0].astype('uint8')\n",
        "        aug.append(image)\n",
        "  return aug"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BAU_DSp1L9TW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_train(fold_no):\n",
        "    k_pneumonia = np.load(f'/content/drive/My Drive/Corona_Image/Final_data/kaggle_pneumonia/train{fold_no}.npy')\n",
        "    nor = np.load(f'/content/drive/My Drive/Corona_Image/Final_data/normal/train{fold_no}.npy')\n",
        "    fig = np.load(f'/content/drive/My Drive/Corona_Image/Final_data/figure1/train{fold_no}.npy')\n",
        "\n",
        "    mask = np.isin(covid_patient_id, covid_folds[fold_no%10])\n",
        "    val_data = ieee_corona[mask]\n",
        "    arr = np.concatenate((covid_folds[fold_no-1], covid_folds[fold_no%10]), axis=0)\n",
        "    mask = np.isin(covid_patient_id, arr, invert=True)\n",
        "    train_data = ieee_corona[mask]\n",
        "    \n",
        "    figure1 = figure1_corona[fig]\n",
        "    train_data2, val_data2 = train_test_split(figure1, test_size=0.1, random_state=42)\n",
        "    train_data = np.concatenate((train_data,train_data2), axis=0)\n",
        "    aug = np.array(augment(train_data,9))\n",
        "    train_data = np.concatenate((train_data,aug), axis=0)\n",
        "    train_label = np.empty(train_data.shape[0],dtype='int32')\n",
        "    train_label.fill(2)\n",
        "    val_data = np.concatenate((val_data,val_data2), axis=0)\n",
        "    aug = np.array(augment(val_data,9))\n",
        "    val_data = np.concatenate((val_data,aug), axis=0)\n",
        "    val_label = np.empty(val_data.shape[0],dtype='int32')\n",
        "    val_label.fill(2)\n",
        "\n",
        "    mask = np.isin(pneumonia_patient_id, pnemonia_folds[fold_no%10])\n",
        "    val_data2 = ieee_pneumonia[mask]\n",
        "    val_data = np.concatenate((val_data,val_data2), axis=0)\n",
        "    l = np.empty(val_data2.shape[0],dtype='int32')\n",
        "    l.fill(1)\n",
        "    val_label = np.concatenate((val_label, l), axis=0)\n",
        "    arr = np.concatenate((pnemonia_folds[fold_no-1], pnemonia_folds[fold_no%10]), axis=0)\n",
        "    mask = np.isin(pneumonia_patient_id, arr, invert=True)\n",
        "    train_data2 = ieee_pneumonia[mask]\n",
        "    train_data = np.concatenate((train_data,train_data2), axis=0)\n",
        "    l = np.empty(train_data2.shape[0],dtype='int32')\n",
        "    l.fill(1)\n",
        "    train_label = np.concatenate((train_label, l), axis=0)\n",
        "\n",
        "    pneumonia = kaggle_pneumonia[k_pneumonia]\n",
        "    train_data2, val_data2 = train_test_split(pneumonia, test_size=0.1, random_state=42)\n",
        "    train_data = np.concatenate((train_data,train_data2), axis=0)\n",
        "    l = np.empty(train_data2.shape[0],dtype='int32')\n",
        "    l.fill(1)\n",
        "    train_label = np.concatenate((train_label, l), axis=0)\n",
        "    val_data = np.concatenate((val_data,val_data2), axis=0)\n",
        "    l = np.empty(val_data2.shape[0],dtype='int32')\n",
        "    l.fill(1)\n",
        "    val_label = np.concatenate((val_label, l), axis=0)\n",
        "\n",
        "    normal = kaggle_normal[nor]\n",
        "    train_data2, val_data2 = train_test_split(normal, test_size=0.1, random_state=42)\n",
        "    train_data = np.concatenate((train_data,train_data2), axis=0)\n",
        "    l = np.empty(train_data2.shape[0],dtype='int32')\n",
        "    l.fill(0)\n",
        "    train_label = np.concatenate((train_label, l), axis=0)\n",
        "    val_data = np.concatenate((val_data,val_data2), axis=0)\n",
        "    l = np.empty(val_data2.shape[0],dtype='int32')\n",
        "    l.fill(0)\n",
        "    val_label = np.concatenate((val_label, l), axis=0)\n",
        "    \n",
        "    train_data,train_label = shuffle(train_data,train_label)\n",
        "    val_data,val_label = shuffle(val_data,val_label)\n",
        "    return train_data, val_data, train_label, val_label"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ut6d61kPKNn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_test(fold_no):\n",
        "    k_pneumonia = np.load(f'/content/drive/My Drive/Corona_Image/Final_data/kaggle_pneumonia/test{fold_no}.npy')\n",
        "    nor = np.load(f'/content/drive/My Drive/Corona_Image/Final_data/normal/test{fold_no}.npy')\n",
        "    fig = np.load(f'/content/drive/My Drive/Corona_Image/Final_data/figure1/test{fold_no}.npy')\n",
        "\n",
        "    mask = np.isin(covid_patient_id, covid_folds[fold_no-1])\n",
        "    data = ieee_corona[mask]\n",
        "    figure1 = figure1_corona[fig]\n",
        "    data = np.concatenate((data,figure1), axis=0)\n",
        "    label = np.empty(data.shape[0],dtype='int32')\n",
        "    label.fill(2)\n",
        "\n",
        "    mask = np.isin(pneumonia_patient_id, pnemonia_folds[fold_no-1])\n",
        "    pneumonia = ieee_pneumonia[mask]\n",
        "    data = np.concatenate((data, pneumonia), axis=0)\n",
        "    l = np.empty(pneumonia.shape[0],dtype='int32')\n",
        "    l.fill(1)\n",
        "    label = np.concatenate((label, l), axis=0)\n",
        "\n",
        "    pneumonia = kaggle_pneumonia[k_pneumonia]\n",
        "    data = np.concatenate((data, pneumonia), axis=0)\n",
        "    l = np.empty(pneumonia.shape[0],dtype='int32')\n",
        "    l.fill(1)\n",
        "    label = np.concatenate((label, l), axis=0)\n",
        "\n",
        "    normal = kaggle_normal[nor]\n",
        "    data = np.concatenate((data, normal), axis=0)\n",
        "    l = np.empty(normal.shape[0],dtype='int32')\n",
        "    l.fill(0)\n",
        "    label = np.concatenate((label, l), axis=0)\n",
        "    \n",
        "    data,label=shuffle(data,label)\n",
        "\n",
        "    return data,label"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wORgFPRtRcc2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def define_mode(l,batch):\n",
        "    # f.write(str(l)+\",\"+str(batch)+\",\"+str(drop)+\",\")\n",
        "    # Define per-fold score containers\n",
        "    acc_per_fold = []\n",
        "    loss_per_fold = []\n",
        "    corona_acc=[]\n",
        "    corona_loss=[]\n",
        "    pneumonia_acc=[]\n",
        "    pneumonia_loss=[]\n",
        "    normal_acc=[]\n",
        "    normal_loss=[]\n",
        "    #Merge Training and Test Data\n",
        "    # x = np.concatenate((x_train, x_test), axis=0)\n",
        "    # y = np.concatenate((y_train, y_test), axis=0)\n",
        "\n",
        "    # Define the K-fold Cross Validator\n",
        "    num_folds = 10\n",
        "    # kfold = KFold(n_splits=num_folds, shuffle=True)\n",
        "    # kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=1)\n",
        "\n",
        "    # K-fold Cross Validation model evaluation\n",
        "    fold_no = 10\n",
        "    for i in range(fold_no,11):\n",
        "\n",
        "      f = open(\"/content/drive/My Drive/Corona_Image/E3_patient_wise/samll_model_custom_weight.txt\", \"a\")\n",
        "\n",
        "      densenet = DenseNet121(\n",
        "        weights='/content/drive/My Drive/Corona_Image/chexnet_weights.h5',\n",
        "        # weights = None ,\n",
        "        include_top=True,\n",
        "        input_shape=(224,224,3),classes=14\n",
        "      )\n",
        "      \n",
        "      m = densenet.layers[-2].output\n",
        "      # y=BatchNormalization(axis=1, momentum=0.99, epsilon=0.001, center=True, \n",
        "      #                               scale=True, beta_initializer='zeros', gamma_initializer='ones', \n",
        "      #                               moving_mean_initializer='zeros', moving_variance_initializer='ones',\n",
        "      #                               beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(m)\n",
        "      # z=Dropout(drop)(y)\n",
        "      predictions = Dense(3, activation='softmax')(m)\n",
        "      model = Model(inputs=densenet.inputs, outputs=predictions)\n",
        "\n",
        "      adam = Adamax(lr=l, beta_1=0.9, beta_2=0.999)\n",
        "      model.compile(loss='categorical_crossentropy',optimizer=adam,metrics=['accuracy'])\n",
        "      reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5)\n",
        "\n",
        "      es = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto', baseline=None, restore_best_weights=True)\n",
        "\n",
        "\n",
        "      # Generate a print\n",
        "      print('------------------------------------------------------------------------')\n",
        "      print(f'Training for fold {fold_no} ...')\n",
        "      \n",
        "      class_weights = {0: 2, 1: 3, 2: 5}\n",
        "\n",
        "      # Train data load\n",
        "      train_data, val_data, train_label, val_label = load_train(fold_no)\n",
        "      #class_weights = class_weight.compute_class_weight('balanced', np.unique(train_label), train_label)\n",
        "      train_label = np_utils.to_categorical(train_label, 3)\n",
        "      val_label = np_utils.to_categorical(val_label, 3)\n",
        "      \n",
        "      history = model.fit(train_data, train_label, batch_size=batch,\n",
        "                          epochs=100,verbose=1, validation_data=(val_data,val_label), shuffle=True, class_weight=class_weights,\n",
        "                            callbacks=[reduce_lr, es])\n",
        "      \n",
        "      # Generate generalization metrics\n",
        "      x_test, y_test_new = load_test(fold_no)\n",
        "\n",
        "\n",
        "      y_test = np_utils.to_categorical(y_test_new, 3)\n",
        "      scores = model.evaluate(x_test, y_test, verbose=0)\n",
        "      print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')\n",
        "      f.write(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')\n",
        "      acc_per_fold.append(scores[1] * 100)\n",
        "      loss_per_fold.append(scores[0])\n",
        "      \n",
        "      corona_test = x_test[y_test_new == 2]\n",
        "      corona = model.predict(corona_test)\n",
        "      corona_test_label = y_test[y_test_new == 2]\n",
        "      scores=model.evaluate(corona_test,corona_test_label)\n",
        "      print(scores)\n",
        "      corona_acc.append(scores[1] * 100)\n",
        "      corona_loss.append(scores[0])\n",
        "      f.write(f'> Fold {fold_no} - corona_loss: {scores[0]} - corona_acc: {scores[1] * 100}%')\n",
        "\n",
        "\n",
        "      corona_test = x_test[y_test_new == 1]\n",
        "      pneumonia = model.predict(corona_test)\n",
        "      corona_test_label = y_test[y_test_new == 1]\n",
        "      scores=model.evaluate(corona_test,corona_test_label)\n",
        "      print(scores)\n",
        "      pneumonia_acc.append(scores[1] * 100)\n",
        "      pneumonia_loss.append(scores[0])\n",
        "      f.write(f'> Fold {fold_no} - pneumonia_loss: {scores[0]} - pneumonia_acc: {scores[1] * 100}%')\n",
        "\n",
        "      corona_test = x_test[y_test_new == 0]\n",
        "      normal = model.predict(corona_test)\n",
        "      corona_test_label = y_test[y_test_new == 0]\n",
        "      scores=model.evaluate(corona_test,corona_test_label)\n",
        "      print(scores)\n",
        "      normal_acc.append(scores[1] * 100)\n",
        "      normal_loss.append(scores[0])\n",
        "      f.write(f'> Fold {fold_no} - normal_loss: {scores[0]} - normal_acc: {scores[1] * 100}%')\n",
        "      \n",
        "\n",
        "      corona_classes = np.unique([np.argmax(y, axis=None, out=None) for y in corona], return_counts=True)\n",
        "      pneumonia_classes = np.unique([np.argmax(y, axis=None, out=None) for y in pneumonia], return_counts=True)\n",
        "      normal_classes = np.unique([np.argmax(y, axis=None, out=None) for y in normal], return_counts=True)\n",
        "      print(corona_classes)\n",
        "      print(pneumonia_classes)\n",
        "      print(normal_classes)\n",
        "      f.write(\"corona_classes :\"+str(corona_classes)+\"\\n\")\n",
        "      f.write(\"pneumonia_classes :\"+str(pneumonia_classes)+\"\\n\")\n",
        "      f.write(\"normal_classes :\"+str(normal_classes)+\"\\n\")\n",
        "\n",
        "      f.close()\n",
        "\n",
        "      #model.save(f'/content/drive/My Drive/Corona_Image/E3_patient_wise/Models/{fold_no}.h5')\n",
        "\n",
        "      # Increase fold number\n",
        "      fold_no = fold_no + 1\n",
        "\n",
        "      #del model,x_train,y_train,x_test,y_test,y_test_new,train_label\n",
        "\n",
        "      # models.append(model)\n",
        "\n",
        "    # == Provide average scores ==\n",
        "    f = open(\"/content/drive/My Drive/Corona_Image/E3_patient_wise/samll_model_custom_weight.txt\", \"a\")\n",
        "    print('------------------------------------------------------------------------')\n",
        "    print('Score per fold')\n",
        "    for i in range(0, len(acc_per_fold)):\n",
        "      print('------------------------------------------------------------------------')\n",
        "      print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%')\n",
        "      print(f'> Fold {i+1} - corona_loss: {corona_loss[i]} - corona_acc: {corona_acc[i]}%')\n",
        "      print(f'> Fold {i+1} - pneumonia_loss: {pneumonia_loss[i]} - pneumonia_acc: {pneumonia_acc[i]}%')\n",
        "      print(f'> Fold {i+1} - normal_loss: {normal_loss[i]} - normal_acc: {normal_acc[i]}%')\n",
        "    print('------------------------------------------------------------------------')\n",
        "    print('Average scores for all folds:')\n",
        "    print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')\n",
        "    print(f'> Loss: {np.mean(loss_per_fold)}')\n",
        "\n",
        "    print(f'> corona_acc: {np.mean(corona_acc)} (+- {np.std(corona_acc)})')\n",
        "    print(f'> corona_loss: {np.mean(corona_loss)}')\n",
        "\n",
        "    print(f'> pneumonia_acc: {np.mean(pneumonia_acc)} (+- {np.std(pneumonia_acc)})')\n",
        "    print(f'> pneumonia_loss: {np.mean(pneumonia_loss)}')\n",
        "\n",
        "    print(f'> normal_acc: {np.mean(normal_acc)} (+- {np.std(normal_acc)})')\n",
        "    print(f'> normal_loss: {np.mean(normal_loss)}')\n",
        "    print('------------------------------------------------------------------------')\n",
        "\n",
        "    f.write('Average scores for all folds:')\n",
        "    f.write(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')\n",
        "    f.write(f'> Loss: {np.mean(loss_per_fold)}')\n",
        "\n",
        "    f.write(f'> corona_acc: {np.mean(corona_acc)} (+- {np.std(corona_acc)})')\n",
        "    f.write(f'> corona_loss: {np.mean(corona_loss)}')\n",
        "\n",
        "    f.write(f'> pneumonia_acc: {np.mean(pneumonia_acc)} (+- {np.std(pneumonia_acc)})')\n",
        "    f.write(f'> pneumonia_loss: {np.mean(pneumonia_loss)}')\n",
        "\n",
        "    f.write(f'> normal_acc: {np.mean(normal_acc)} (+- {np.std(normal_acc)})')\n",
        "    f.write(f'> normal_loss: {np.mean(normal_loss)}')\n",
        "    f.write('------------------------------------------------------------------------')\n",
        "    f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g4ctEa1OUgMj",
        "colab_type": "code",
        "outputId": "b8493f74-4711-466f-fdb3-b325bb23994b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "define_mode(.00001,16)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4074: The name tf.nn.avg_pool is deprecated. Please use tf.nn.avg_pool2d instead.\n",
            "\n",
            "------------------------------------------------------------------------\n",
            "Training for fold 10 ...\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "Train on 22074 samples, validate on 2608 samples\n",
            "Epoch 1/100\n",
            "22074/22074 [==============================] - 246s 11ms/step - loss: 3.4923 - accuracy: 0.4634 - val_loss: 0.9432 - val_accuracy: 0.5054\n",
            "Epoch 2/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 2.8074 - accuracy: 0.5193 - val_loss: 0.8060 - val_accuracy: 0.6472\n",
            "Epoch 3/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 2.4905 - accuracy: 0.6240 - val_loss: 0.6958 - val_accuracy: 0.7063\n",
            "Epoch 4/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 2.2398 - accuracy: 0.6677 - val_loss: 0.6016 - val_accuracy: 0.7389\n",
            "Epoch 5/100\n",
            "22074/22074 [==============================] - 221s 10ms/step - loss: 1.9849 - accuracy: 0.6923 - val_loss: 0.5160 - val_accuracy: 0.7588\n",
            "Epoch 6/100\n",
            "22074/22074 [==============================] - 221s 10ms/step - loss: 1.7399 - accuracy: 0.7365 - val_loss: 0.4419 - val_accuracy: 0.8275\n",
            "Epoch 7/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 1.5282 - accuracy: 0.8013 - val_loss: 0.3793 - val_accuracy: 0.8738\n",
            "Epoch 8/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 1.3530 - accuracy: 0.8379 - val_loss: 0.3264 - val_accuracy: 0.8873\n",
            "Epoch 9/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 1.2167 - accuracy: 0.8592 - val_loss: 0.2967 - val_accuracy: 0.8953\n",
            "Epoch 10/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 1.0991 - accuracy: 0.8722 - val_loss: 0.2714 - val_accuracy: 0.9030\n",
            "Epoch 11/100\n",
            "22074/22074 [==============================] - 217s 10ms/step - loss: 1.0150 - accuracy: 0.8843 - val_loss: 0.2583 - val_accuracy: 0.9053\n",
            "Epoch 12/100\n",
            "22074/22074 [==============================] - 217s 10ms/step - loss: 0.9283 - accuracy: 0.8926 - val_loss: 0.2419 - val_accuracy: 0.9095\n",
            "Epoch 13/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.8827 - accuracy: 0.9000 - val_loss: 0.2333 - val_accuracy: 0.9145\n",
            "Epoch 14/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.8428 - accuracy: 0.9045 - val_loss: 0.2296 - val_accuracy: 0.9153\n",
            "Epoch 15/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.7935 - accuracy: 0.9094 - val_loss: 0.2227 - val_accuracy: 0.9164\n",
            "Epoch 16/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.7546 - accuracy: 0.9142 - val_loss: 0.2195 - val_accuracy: 0.9179\n",
            "Epoch 17/100\n",
            "22074/22074 [==============================] - 217s 10ms/step - loss: 0.7290 - accuracy: 0.9150 - val_loss: 0.2149 - val_accuracy: 0.9176\n",
            "Epoch 18/100\n",
            "22074/22074 [==============================] - 216s 10ms/step - loss: 0.7038 - accuracy: 0.9185 - val_loss: 0.2130 - val_accuracy: 0.9183\n",
            "Epoch 19/100\n",
            "22074/22074 [==============================] - 217s 10ms/step - loss: 0.6601 - accuracy: 0.9243 - val_loss: 0.2102 - val_accuracy: 0.9149\n",
            "Epoch 20/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 0.6463 - accuracy: 0.9257 - val_loss: 0.2066 - val_accuracy: 0.9160\n",
            "Epoch 21/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 0.6257 - accuracy: 0.9269 - val_loss: 0.2071 - val_accuracy: 0.9176\n",
            "Epoch 22/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 0.6266 - accuracy: 0.9276 - val_loss: 0.2037 - val_accuracy: 0.9187\n",
            "Epoch 23/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.5696 - accuracy: 0.9357 - val_loss: 0.2048 - val_accuracy: 0.9164\n",
            "Epoch 24/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.5628 - accuracy: 0.9351 - val_loss: 0.2032 - val_accuracy: 0.9183\n",
            "Epoch 25/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.5478 - accuracy: 0.9354 - val_loss: 0.2046 - val_accuracy: 0.9122\n",
            "Epoch 26/100\n",
            "22074/22074 [==============================] - 218s 10ms/step - loss: 0.5351 - accuracy: 0.9363 - val_loss: 0.2009 - val_accuracy: 0.9179\n",
            "Epoch 27/100\n",
            "22074/22074 [==============================] - 217s 10ms/step - loss: 0.5160 - accuracy: 0.9383 - val_loss: 0.2051 - val_accuracy: 0.9107\n",
            "Epoch 28/100\n",
            "22074/22074 [==============================] - 216s 10ms/step - loss: 0.5274 - accuracy: 0.9383 - val_loss: 0.1997 - val_accuracy: 0.9187\n",
            "Epoch 29/100\n",
            "22074/22074 [==============================] - 216s 10ms/step - loss: 0.4869 - accuracy: 0.9421 - val_loss: 0.2015 - val_accuracy: 0.9153\n",
            "Epoch 30/100\n",
            "22074/22074 [==============================] - 217s 10ms/step - loss: 0.4825 - accuracy: 0.9434 - val_loss: 0.2018 - val_accuracy: 0.9133\n",
            "Epoch 31/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 0.4772 - accuracy: 0.9421 - val_loss: 0.2038 - val_accuracy: 0.9110\n",
            "Epoch 32/100\n",
            "22074/22074 [==============================] - 221s 10ms/step - loss: 0.4828 - accuracy: 0.9427 - val_loss: 0.1985 - val_accuracy: 0.9168\n",
            "Epoch 33/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 0.4652 - accuracy: 0.9438 - val_loss: 0.2033 - val_accuracy: 0.9145\n",
            "Epoch 34/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 0.4376 - accuracy: 0.9486 - val_loss: 0.2029 - val_accuracy: 0.9145\n",
            "Epoch 35/100\n",
            "22074/22074 [==============================] - 219s 10ms/step - loss: 0.4487 - accuracy: 0.9465 - val_loss: 0.2031 - val_accuracy: 0.9145\n",
            "Epoch 36/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 0.4409 - accuracy: 0.9473 - val_loss: 0.2059 - val_accuracy: 0.9126\n",
            "Epoch 37/100\n",
            "22074/22074 [==============================] - 220s 10ms/step - loss: 0.4296 - accuracy: 0.9485 - val_loss: 0.2058 - val_accuracy: 0.9137\n",
            "Score for fold 10: loss of 0.2115239499987948; accuracy of 92.91390776634216%\n",
            "21/21 [==============================] - 0s 3ms/step\n",
            "[0.24596218764781952, 0.8571428656578064]\n",
            "604/604 [==============================] - 2s 3ms/step\n",
            "[0.29497348171788335, 0.9023178815841675]\n",
            "885/885 [==============================] - 2s 3ms/step\n",
            "[0.15375364427007523, 0.9491525292396545]\n",
            "(array([1, 2]), array([ 3, 18]))\n",
            "(array([0, 1, 2]), array([ 34, 545,  25]))\n",
            "(array([0, 1, 2]), array([840,  31,  14]))\n",
            "------------------------------------------------------------------------\n",
            "Score per fold\n",
            "------------------------------------------------------------------------\n",
            "> Fold 1 - Loss: 0.2115239499987948 - Accuracy: 92.91390776634216%\n",
            "> Fold 1 - corona_loss: 0.24596218764781952 - corona_acc: 85.71428656578064%\n",
            "> Fold 1 - pneumonia_loss: 0.29497348171788335 - pneumonia_acc: 90.23178815841675%\n",
            "> Fold 1 - normal_loss: 0.15375364427007523 - normal_acc: 94.91525292396545%\n",
            "------------------------------------------------------------------------\n",
            "Average scores for all folds:\n",
            "> Accuracy: 92.91390776634216 (+- 0.0)\n",
            "> Loss: 0.2115239499987948\n",
            "> corona_acc: 85.71428656578064 (+- 0.0)\n",
            "> corona_loss: 0.24596218764781952\n",
            "> pneumonia_acc: 90.23178815841675 (+- 0.0)\n",
            "> pneumonia_loss: 0.29497348171788335\n",
            "> normal_acc: 94.91525292396545 (+- 0.0)\n",
            "> normal_loss: 0.15375364427007523\n",
            "------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}