Deep NN for image recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/Users/farshad/anaconda3/envs/Coursera/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.models  import Sequential \n",
    "from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras import optimizers\n",
    "from keras.datasets import mnist\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "num_classes = 10\n",
    "epochs = 30\n",
    "\n",
    "train_data= pd.read_csv('optdigits-training.txt', header = None)\n",
    "test_data = pd.read_csv('optdigits-test.txt', header = None)\n",
    "y_train = np.array(train_data.iloc[:,-1])\n",
    "y_test = np.array(test_data.iloc[:,-1])\n",
    "x_train = np.array(train_data.iloc[:,:-1])\n",
    "x_test = np.array(test_data.iloc[:,:-1])\n",
    "\n",
    "#rescale\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (3058, 64)\n",
      "x_valid shape: (765, 64)\n",
      "3058 train samples\n",
      "765 validation sample\n"
     ]
    }
   ],
   "source": [
    "seed = 10\n",
    "np.random.seed(seed)\n",
    "x_train,x_valid,train_label,valid_label = train_test_split(x_train, y_train, test_size=0.2, random_state=13)\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_valid shape:', x_valid.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_valid.shape[0], 'validation sample')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## One layer, cross-entropy with reLU and stochastic gradient descent\n",
    "\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.03, 0.05, 0.07, 0.09]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 1')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model11 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model11.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model11.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model11.add(Dense(units, activation='relu'))\n",
    "            model11.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model11.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model11.summary()\n",
    "\n",
    "            model11.compile(loss = 'categorical_crossentropy',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model11.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model11.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model11.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model11.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Two layers, Cross-entropy with reLU and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.03, 0.05, 0.07, 0.09]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 2')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model12 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model12.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model12.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model12.add(Dense(units, activation='relu'))\n",
    "            model12.add(Dropout(0.2))\n",
    "            model12.add(Dense(units, activation='relu'))\n",
    "            model12.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model12.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model12.summary()\n",
    "\n",
    "            model12.compile(loss = 'categorical_crossentropy',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model12.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model12.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model12.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model12.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Three layers, Cross-entropy with reLU and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.003, 0.007, 0.01, 0.03]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 3')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model13 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model13.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model13.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model13.add(Dense(units, activation='relu'))\n",
    "            model13.add(Dropout(0.2))\n",
    "            model13.add(Dense(units, activation='relu'))\n",
    "            model13.add(Dropout(0.2))\n",
    "            model13.add(Dense(units, activation='relu'))\n",
    "            model13.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model13.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model13.summary()\n",
    "\n",
    "            model13.compile(loss = 'categorical_crossentropy',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model13.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model13.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model13.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model13.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## One layer, Sum of squared with reLU and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.3, 0.5, 0.7, 0.9]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 1')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model21 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model21.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model21.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model21.add(Dense(units, activation='relu'))\n",
    "            model21.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model21.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model21.summary()\n",
    "\n",
    "            model21.compile(loss = 'mean_squared_error',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model21.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model21.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model21.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model21.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Two layers, sum of squared with reLU and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.3, 0.5, 0.7, 0.9]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 2')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model22 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model22.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model22.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model22.add(Dense(units, activation='relu'))\n",
    "            model22.add(Dropout(0.2))\n",
    "            model22.add(Dense(units, activation='relu'))\n",
    "            model22.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model22.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model22.summary()\n",
    "\n",
    "            model22.compile(loss = 'mean_squared_error',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model22.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model22.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model22.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model22.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Three layers, sum of squared with reLU and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.3, 0.5, 0.7, 0.9]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 2')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model23 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model23.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model23.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model23.add(Dense(units, activation='relu'))\n",
    "            model23.add(Dropout(0.2))\n",
    "            model23.add(Dense(units, activation='relu'))\n",
    "            model23.add(Dropout(0.2))\n",
    "            model23.add(Dense(units, activation='relu'))\n",
    "            model23.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model23.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model23.summary()\n",
    "\n",
    "            model23.compile(loss = 'mean_squared_error',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model23.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model23.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model23.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model23.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## One layer, Cross-entropy with tanh and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.03, 0.05, 0.07, 0.09]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 1')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model31 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model31.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model31.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model31.add(Dense(units, activation='tanh'))\n",
    "            model31.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model31.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model31.summary()\n",
    "\n",
    "            model31.compile(loss = 'categorical_crossentropy',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model31.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model31.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model31.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model31.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Two layers, Cross-entropy with tanh and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.03, 0.05, 0.07, 0.09]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 2')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model32 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model32.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model32.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model32.add(Dense(units, activation='tanh'))\n",
    "            model32.add(Dropout(0.2))\n",
    "            model32.add(Dense(units, activation='tanh'))\n",
    "            model32.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model32.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model32.summary()\n",
    "\n",
    "            model32.compile(loss = 'categorical_crossentropy',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model32.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model32.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                y_train_fit = model32.predict_classes(x_train)\n",
    "                print(classification_report(Y_train_label, y_train_fit))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                print('Class accuracy and confusion matrix for test')\n",
    "                Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                y_pred = model32.predict_classes(x_test)\n",
    "                print(classification_report(Y_test, y_pred))\n",
    "                print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Three layers, Cross-entropy with tanh and stochastic gradient descent\n",
    "train = []\n",
    "test = []\n",
    "learning_rate = [0.005, 0.007, 0.01, 0.09]\n",
    "momentum = [0.95, 0.97, 0.99]\n",
    "number_of_units = [128, 256, 512]\n",
    "print('number of layers: 3')\n",
    "for LR in learning_rate:\n",
    "    for MT in momentum:\n",
    "        for units in number_of_units:\n",
    "            sgd = optimizers.SGD(lr = LR, decay = 5e-6, momentum = MT, nesterov = True)\n",
    "            model33 = Sequential()\n",
    "\n",
    "#This is the first layer\n",
    "#The first layer in a Sequential model (and only the first, \n",
    "#because following layers can do automatic shape inference) needs \n",
    "#to receive information about its input shape\n",
    "            model33.add(Dense(units, activation='relu', input_shape=(64,)))\n",
    "#helps prevent overfitting\n",
    "            model33.add(Dropout(0.2))\n",
    "#hidden units activation function should be reLU or tanh\n",
    "            model33.add(Dense(units, activation='tanh'))\n",
    "            model33.add(Dropout(0.2))\n",
    "            model33.add(Dense(units, activation='tanh'))\n",
    "            model33.add(Dropout(0.2))\n",
    "            model33.add(Dense(units, activation='tanh'))\n",
    "            model33.add(Dropout(0.2))\n",
    "#Output layer, stays the same\n",
    "            model33.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "            #model33.summary()\n",
    "\n",
    "            model33.compile(loss = 'categorical_crossentropy',\n",
    "                      optimizer = sgd,\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "            early_stopping_monitor = EarlyStopping(patience=2)\n",
    "\n",
    "            history = model33.fit(x_train, train_label,\n",
    "                      batch_size=batch_size,\n",
    "                      epochs=epochs,\n",
    "                      verbose = 0,\n",
    "                      validation_data=(x_valid, valid_label),\n",
    "                      callbacks=[early_stopping_monitor])\n",
    "            \n",
    "            test_eval = model33.evaluate(x_test, y_test, verbose=0)\n",
    "            \n",
    "            accuracy = history.history['acc']\n",
    "            val_accuracy = history.history['val_acc']\n",
    "            loss = history.history['loss']\n",
    "            val_loss = history.history['val_loss']\n",
    "            \n",
    "            \n",
    "            \n",
    "            #print('Test loss:', test_eval[0])\n",
    "            if accuracy[-1] > test_eval[1]:\n",
    "                train.append(accuracy[-1])\n",
    "                test.append(test_eval[1])\n",
    "                print('Learning rate: ' + str(LR) + \n",
    "                  ', momentum: ' + str(MT) + \n",
    "                  ', number of units: ' + str(units) + \n",
    "                  ', train accuracy: '+ str(round(accuracy[-1], 4)) + \n",
    "                  ' and Test accuracy:', round(test_eval[1], 4))\n",
    "                #training class accuracy and confusion matrix\n",
    "                print(' ')\n",
    "                #print('Class accuracy and confusion matrix for training'.format('foo'))\n",
    "                #Y_train_label = np.argmax(train_label, axis = 1)\n",
    "                #y_train_fit = model33.predict_classes(x_train)\n",
    "                #print(classification_report(Y_train_label, y_train_fit))\n",
    "                #print(pd.DataFrame(confusion_matrix(Y_train_label, y_train_fit)))\n",
    "                #test class accuracy and confusion matrix \n",
    "                #print('Class accuracy and confusion matrix for test')\n",
    "                #Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "                #y_pred = model33.predict_classes(x_test)\n",
    "                #print(classification_report(Y_test, y_pred))\n",
    "                #print(pd.DataFrame(confusion_matrix(Y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = history.history['acc']\n",
    "val_accuracy = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}