CV_Final_Project.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BzWwKNf8ZrPt",
        "outputId": "92c06411-d553-47b7-fbac-100f60412a2e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading cv-final-project.zip to /content\n",
            "100% 1.73G/1.73G [01:18<00:00, 23.8MB/s]\n",
            "100% 1.73G/1.73G [01:18<00:00, 23.7MB/s]\n"
          ]
        }
      ],
      "source": [
        "!pip install -U -q kaggle\n",
        "!mkdir -p ~/.kaggle\n",
        "!echo '{\"username\":\"sjzheng24\", \"key\":\"c927cde8b5feb02f75a21b8b84d629ca\"}' > ~/.kaggle/kaggle.json\n",
        "!chmod 600 ~/.kaggle/kaggle.json\n",
        "!kaggle competitions download -c cv-final-project"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "Zu3b5fCRecnb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7465f0f8-99f6-42b7-95ec-f952ba1fbfb3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  cv-final-project.zip\n",
            "  inflating: data/eye_tracker_train_and_val.npz  \n"
          ]
        }
      ],
      "source": [
        "!unzip cv-final-project.zip -d data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "igvXhzxsPUHG"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import argparse\n",
        "import timeit\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "tf.compat.v1.disable_eager_execution()\n",
        "\n",
        "\n",
        "# Network Parameters\n",
        "img_size = 64\n",
        "n_channel = 3\n",
        "mask_size = 25\n",
        "\n",
        "# pathway: eye_left and eye_right\n",
        "conv1_eye_size = 11\n",
        "conv1_eye_out = 96\n",
        "pool1_eye_size = 2\n",
        "pool1_eye_stride = 2\n",
        "\n",
        "conv2_eye_size = 5\n",
        "conv2_eye_out = 256\n",
        "pool2_eye_size = 2\n",
        "pool2_eye_stride = 2\n",
        "\n",
        "conv3_eye_size = 3\n",
        "conv3_eye_out = 384\n",
        "pool3_eye_size = 2\n",
        "pool3_eye_stride = 2\n",
        "\n",
        "conv4_eye_size = 1\n",
        "conv4_eye_out = 64\n",
        "pool4_eye_size = 2\n",
        "pool4_eye_stride = 2\n",
        "\n",
        "eye_size = 2 * 2 * 2 * conv4_eye_out\n",
        "\n",
        "# pathway: face\n",
        "conv1_face_size = 11\n",
        "conv1_face_out = 96\n",
        "pool1_face_size = 2\n",
        "pool1_face_stride = 2\n",
        "\n",
        "conv2_face_size = 5\n",
        "conv2_face_out = 256\n",
        "pool2_face_size = 2\n",
        "pool2_face_stride = 2\n",
        "\n",
        "conv3_face_size = 3\n",
        "conv3_face_out = 384\n",
        "pool3_face_size = 2\n",
        "pool3_face_stride = 2\n",
        "\n",
        "conv4_face_size = 1\n",
        "conv4_face_out = 64\n",
        "pool4_face_size = 2\n",
        "pool4_face_stride = 2\n",
        "\n",
        "face_size = 2 * 2 * conv4_face_out\n",
        "\n",
        "# fc layer\n",
        "fc_eye_size = 128\n",
        "fc_face_size = 128\n",
        "fc2_face_size = 64\n",
        "fc_face_mask_size = 256\n",
        "fc2_face_mask_size = 128\n",
        "fc_size = 128\n",
        "fc2_size = 2\n",
        "\n",
        "\n",
        "# Import data\n",
        "def load_data(file):\n",
        "    npzfile = np.load(file)\n",
        "    train_eye_left = npzfile[\"train_eye_left\"]\n",
        "    train_eye_right = npzfile[\"train_eye_right\"]\n",
        "    train_face = npzfile[\"train_face\"]\n",
        "    train_face_mask = npzfile[\"train_face_mask\"]\n",
        "    train_y = npzfile[\"train_y\"]\n",
        "    val_eye_left = npzfile[\"val_eye_left\"]\n",
        "    val_eye_right = npzfile[\"val_eye_right\"]\n",
        "    val_face = npzfile[\"val_face\"]\n",
        "    val_face_mask = npzfile[\"val_face_mask\"]\n",
        "    val_y = npzfile[\"val_y\"]\n",
        "    return [train_eye_left, train_eye_right, train_face, train_face_mask, train_y], [val_eye_left, val_eye_right, val_face, val_face_mask, val_y]\n",
        "\n",
        "def normalize(data):\n",
        "    shape = data.shape\n",
        "    data = np.reshape(data, (shape[0], -1))\n",
        "    data = data.astype('float32') / 255. # scaling\n",
        "    data = data - np.mean(data, axis=0) # normalizing\n",
        "    return np.reshape(data, shape)\n",
        "\n",
        "def prepare_data(data):\n",
        "    eye_left, eye_right, face, face_mask, y = data\n",
        "    eye_left = normalize(eye_left)\n",
        "    eye_right = normalize(eye_right)\n",
        "    face = normalize(face)\n",
        "    face_mask = np.reshape(face_mask, (face_mask.shape[0], -1)).astype('float32')\n",
        "    y = y.astype('float32')\n",
        "    return [eye_left, eye_right, face, face_mask, y]\n",
        "\n",
        "def shuffle_data(data):\n",
        "    idx = np.arange(data[0].shape[0])\n",
        "    np.random.shuffle(idx)\n",
        "    for i in range(len(data)):\n",
        "        data[i] = data[i][idx]\n",
        "    return data\n",
        "\n",
        "def next_batch(data, batch_size):\n",
        "    for i in np.arange(0, data[0].shape[0], batch_size):\n",
        "        # yield a tuple of the current batched data\n",
        "        yield [each[i: i + batch_size] for each in data]\n",
        "\n",
        "class EyeTracker(object):\n",
        "    def __init__(self):\n",
        "        # tf Graph input\n",
        "        self.eye_left = tf.compat.v1.placeholder(tf.float32, [None, img_size, img_size, n_channel], name='eye_left')\n",
        "        self.eye_right = tf.compat.v1.placeholder(tf.float32, [None, img_size, img_size, n_channel], name='eye_right')\n",
        "        self.face = tf.compat.v1.placeholder(tf.float32, [None, img_size, img_size, n_channel], name='face')\n",
        "        self.face_mask = tf.compat.v1.placeholder(tf.float32, [None, mask_size * mask_size], name='face_mask')\n",
        "        self.y = tf.compat.v1.placeholder(tf.float32, [None, 2], name='pos')\n",
        "        # Store layers weight & bias\n",
        "        self.weights = {\n",
        "            'conv1_eye': tf.compat.v1.get_variable('conv1_eye_w', shape=(conv1_eye_size, conv1_eye_size, n_channel, conv1_eye_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv2_eye': tf.compat.v1.get_variable('conv2_eye_w', shape=(conv2_eye_size, conv2_eye_size, conv1_eye_out, conv2_eye_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv3_eye': tf.compat.v1.get_variable('conv3_eye_w', shape=(conv3_eye_size, conv3_eye_size, conv2_eye_out, conv3_eye_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv4_eye': tf.compat.v1.get_variable('conv4_eye_w', shape=(conv4_eye_size, conv4_eye_size, conv3_eye_out, conv4_eye_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv1_face': tf.compat.v1.get_variable('conv1_face_w', shape=(conv1_face_size, conv1_face_size, n_channel, conv1_face_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv2_face': tf.compat.v1.get_variable('conv2_face_w', shape=(conv2_face_size, conv2_face_size, conv1_face_out, conv2_face_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv3_face': tf.compat.v1.get_variable('conv3_face_w', shape=(conv3_face_size, conv3_face_size, conv2_face_out, conv3_face_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'conv4_face': tf.compat.v1.get_variable('conv4_face_w', shape=(conv4_face_size, conv4_face_size, conv3_face_out, conv4_face_out), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc_eye': tf.compat.v1.get_variable('fc_eye_w', shape=(eye_size, fc_eye_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc_face': tf.compat.v1.get_variable('fc_face_w', shape=(face_size, fc_face_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc2_face': tf.compat.v1.get_variable('fc2_face_w', shape=(fc_face_size, fc2_face_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc_face_mask': tf.compat.v1.get_variable('fc_face_mask_w', shape=(mask_size * mask_size, fc_face_mask_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc2_face_mask': tf.compat.v1.get_variable('fc2_face_mask_w', shape=(fc_face_mask_size, fc2_face_mask_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc': tf.compat.v1.get_variable('fc_w', shape=(fc_eye_size + fc2_face_size + fc2_face_mask_size, fc_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\")),\n",
        "            'fc2': tf.compat.v1.get_variable('fc2_w', shape=(fc_size, fc2_size), initializer=tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode=\"fan_avg\", distribution=\"uniform\"))\n",
        "        }\n",
        "        self.biases = {\n",
        "            'conv1_eye': tf.Variable(tf.constant(0.1, shape=[conv1_eye_out])),\n",
        "            'conv2_eye': tf.Variable(tf.constant(0.1, shape=[conv2_eye_out])),\n",
        "            'conv3_eye': tf.Variable(tf.constant(0.1, shape=[conv3_eye_out])),\n",
        "            'conv4_eye': tf.Variable(tf.constant(0.1, shape=[conv4_eye_out])),\n",
        "            'conv1_face': tf.Variable(tf.constant(0.1, shape=[conv1_face_out])),\n",
        "            'conv2_face': tf.Variable(tf.constant(0.1, shape=[conv2_face_out])),\n",
        "            'conv3_face': tf.Variable(tf.constant(0.1, shape=[conv3_face_out])),\n",
        "            'conv4_face': tf.Variable(tf.constant(0.1, shape=[conv4_face_out])),\n",
        "            'fc_eye': tf.Variable(tf.constant(0.1, shape=[fc_eye_size])),\n",
        "            'fc_face': tf.Variable(tf.constant(0.1, shape=[fc_face_size])),\n",
        "            'fc2_face': tf.Variable(tf.constant(0.1, shape=[fc2_face_size])),\n",
        "            'fc_face_mask': tf.Variable(tf.constant(0.1, shape=[fc_face_mask_size])),\n",
        "            'fc2_face_mask': tf.Variable(tf.constant(0.1, shape=[fc2_face_mask_size])),\n",
        "            'fc': tf.Variable(tf.constant(0.1, shape=[fc_size])),\n",
        "            'fc2': tf.Variable(tf.constant(0.1, shape=[fc2_size]))\n",
        "        }\n",
        "\n",
        "        # Construct model\n",
        "        self.pred = self.itracker_nets(self.eye_left, self.eye_right, self.face, self.face_mask, self.weights, self.biases)\n",
        "\n",
        "    # Create some wrappers for simplicity\n",
        "    def conv2d(self, x, W, b, strides=1):\n",
        "        # Conv2D wrapper, with bias and relu activation\n",
        "        x = tf.nn.conv2d(x, filters=W, strides=[1, strides, strides, 1], padding='VALID')\n",
        "        x = tf.nn.bias_add(x, b)\n",
        "        return tf.nn.relu(x)\n",
        "\n",
        "    def maxpool2d(self, x, k, strides):\n",
        "        # MaxPool2D wrapper\n",
        "        return tf.nn.max_pool2d(input=x, ksize=[1, k, k, 1], strides=[1, strides, strides, 1],\n",
        "                              padding='VALID')\n",
        "\n",
        "    # Create model\n",
        "    def itracker_nets(self, eye_left, eye_right, face, face_mask, weights, biases):\n",
        "        # pathway: left eye\n",
        "        eye_left = self.conv2d(eye_left, weights['conv1_eye'], biases['conv1_eye'], strides=1)\n",
        "        eye_left = self.maxpool2d(eye_left, k=pool1_eye_size, strides=pool1_eye_stride)\n",
        "\n",
        "        eye_left = self.conv2d(eye_left, weights['conv2_eye'], biases['conv2_eye'], strides=1)\n",
        "        eye_left = self.maxpool2d(eye_left, k=pool2_eye_size, strides=pool2_eye_stride)\n",
        "\n",
        "        eye_left = self.conv2d(eye_left, weights['conv3_eye'], biases['conv3_eye'], strides=1)\n",
        "        eye_left = self.maxpool2d(eye_left, k=pool3_eye_size, strides=pool3_eye_stride)\n",
        "\n",
        "        eye_left = self.conv2d(eye_left, weights['conv4_eye'], biases['conv4_eye'], strides=1)\n",
        "        eye_left = self.maxpool2d(eye_left, k=pool4_eye_size, strides=pool4_eye_stride)\n",
        "\n",
        "        # pathway: right eye\n",
        "        eye_right = self.conv2d(eye_right, weights['conv1_eye'], biases['conv1_eye'], strides=1)\n",
        "        eye_right = self.maxpool2d(eye_right, k=pool1_eye_size, strides=pool1_eye_stride)\n",
        "\n",
        "        eye_right = self.conv2d(eye_right, weights['conv2_eye'], biases['conv2_eye'], strides=1)\n",
        "        eye_right = self.maxpool2d(eye_right, k=pool2_eye_size, strides=pool2_eye_stride)\n",
        "\n",
        "        eye_right = self.conv2d(eye_right, weights['conv3_eye'], biases['conv3_eye'], strides=1)\n",
        "        eye_right = self.maxpool2d(eye_right, k=pool3_eye_size, strides=pool3_eye_stride)\n",
        "\n",
        "        eye_right = self.conv2d(eye_right, weights['conv4_eye'], biases['conv4_eye'], strides=1)\n",
        "        eye_right = self.maxpool2d(eye_right, k=pool4_eye_size, strides=pool4_eye_stride)\n",
        "\n",
        "        # pathway: face\n",
        "        face = self.conv2d(face, weights['conv1_face'], biases['conv1_face'], strides=1)\n",
        "        face = self.maxpool2d(face, k=pool1_face_size, strides=pool1_face_stride)\n",
        "\n",
        "        face = self.conv2d(face, weights['conv2_face'], biases['conv2_face'], strides=1)\n",
        "        face = self.maxpool2d(face, k=pool2_face_size, strides=pool2_face_stride)\n",
        "\n",
        "        face = self.conv2d(face, weights['conv3_face'], biases['conv3_face'], strides=1)\n",
        "        face = self.maxpool2d(face, k=pool3_face_size, strides=pool3_face_stride)\n",
        "\n",
        "        face = self.conv2d(face, weights['conv4_face'], biases['conv4_face'], strides=1)\n",
        "        face = self.maxpool2d(face, k=pool4_face_size, strides=pool4_face_stride)\n",
        "\n",
        "        # fc layer\n",
        "        # eye\n",
        "        eye_left = tf.reshape(eye_left, [-1, int(np.prod(eye_left.get_shape()[1:]))])\n",
        "        eye_right = tf.reshape(eye_right, [-1, int(np.prod(eye_right.get_shape()[1:]))])\n",
        "        eye = tf.concat([eye_left, eye_right], 1)\n",
        "        eye = tf.nn.relu(tf.add(tf.matmul(eye, weights['fc_eye']), biases['fc_eye']))\n",
        "\n",
        "        # face\n",
        "        face = tf.reshape(face, [-1, int(np.prod(face.get_shape()[1:]))])\n",
        "        face = tf.nn.relu(tf.add(tf.matmul(face, weights['fc_face']), biases['fc_face']))\n",
        "        face = tf.nn.relu(tf.add(tf.matmul(face, weights['fc2_face']), biases['fc2_face']))\n",
        "\n",
        "        # face mask\n",
        "        face_mask = tf.nn.relu(tf.add(tf.matmul(face_mask, weights['fc_face_mask']), biases['fc_face_mask']))\n",
        "        face_mask = tf.nn.relu(tf.add(tf.matmul(face_mask, weights['fc2_face_mask']), biases['fc2_face_mask']))\n",
        "\n",
        "        # all\n",
        "        fc = tf.concat([eye, face, face_mask], 1)\n",
        "        fc = tf.nn.relu(tf.add(tf.matmul(fc, weights['fc']), biases['fc']))\n",
        "        out = tf.add(tf.matmul(fc, weights['fc2']), biases['fc2'])\n",
        "        return out\n",
        "\n",
        "    def train(self, train_data, val_data, lr=.01, batch_size=32, max_epoch=10, min_delta=1e-4, patience=10, print_per_epoch=1, out_model='my_model'):\n",
        "\n",
        "        print('Train on %s samples, validate on %s samples' % (train_data[0].shape[0], val_data[0].shape[0]))\n",
        "        # Define loss and optimizer\n",
        "        self.cost = tf.compat.v1.losses.mean_squared_error(self.y, self.pred)\n",
        "        self.optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=lr).minimize(self.cost)\n",
        "\n",
        "        # Evaluate model\n",
        "        self.err = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.math.squared_difference(self.pred, self.y), axis=1)))\n",
        "        train_loss_history = []\n",
        "        train_err_history = []\n",
        "        val_loss_history = []\n",
        "        val_err_history = []\n",
        "        n_incr_error = 0  # nb. of consecutive increase in error\n",
        "        best_loss = np.Inf\n",
        "        n_batches = train_data[0].shape[0] / batch_size + (train_data[0].shape[0] % batch_size != 0)\n",
        "\n",
        "        # Create the collection\n",
        "        tf.compat.v1.get_collection(\"validation_nodes\")\n",
        "        # Add stuff to the collection.\n",
        "        tf.compat.v1.add_to_collection(\"validation_nodes\", self.eye_left)\n",
        "        tf.compat.v1.add_to_collection(\"validation_nodes\", self.eye_right)\n",
        "        tf.compat.v1.add_to_collection(\"validation_nodes\", self.face)\n",
        "        tf.compat.v1.add_to_collection(\"validation_nodes\", self.face_mask)\n",
        "        tf.compat.v1.add_to_collection(\"validation_nodes\", self.pred)\n",
        "        saver = tf.compat.v1.train.Saver(max_to_keep=1)\n",
        "\n",
        "        # Initializing the variables\n",
        "        init = tf.compat.v1.global_variables_initializer()\n",
        "        # Launch the graph\n",
        "        with tf.compat.v1.Session() as sess:\n",
        "            sess.run(init)\n",
        "            # Keep training until reach max iterations\n",
        "            for n_epoch in range(1, max_epoch + 1):\n",
        "                n_incr_error += 1\n",
        "                train_loss = 0.\n",
        "                val_loss = 0.\n",
        "                train_err = 0.\n",
        "                val_err = 0.\n",
        "                train_data = shuffle_data(train_data)\n",
        "                for batch_train_data in next_batch(train_data, batch_size):\n",
        "                    # Run optimization op (backprop)\n",
        "                    sess.run(self.optimizer, feed_dict={self.eye_left: batch_train_data[0], \\\n",
        "                                self.eye_right: batch_train_data[1], self.face: batch_train_data[2], \\\n",
        "                                self.face_mask: batch_train_data[3], self.y: batch_train_data[4]})\n",
        "                    train_batch_loss, train_batch_err = sess.run([self.cost, self.err], feed_dict={self.eye_left: batch_train_data[0], \\\n",
        "                                self.eye_right: batch_train_data[1], self.face: batch_train_data[2], \\\n",
        "                                self.face_mask: batch_train_data[3], self.y: batch_train_data[4]})\n",
        "                    train_loss += train_batch_loss / n_batches\n",
        "                    train_err += train_batch_err / n_batches\n",
        "                val_loss, val_err = sess.run([self.cost, self.err], feed_dict={self.eye_left: val_data[0], \\\n",
        "                                self.eye_right: val_data[1], self.face: val_data[2], \\\n",
        "                                self.face_mask: val_data[3], self.y: val_data[4]})\n",
        "\n",
        "                train_loss_history.append(train_loss)\n",
        "                train_err_history.append(train_err)\n",
        "                val_loss_history.append(val_loss)\n",
        "                val_err_history.append(val_err)\n",
        "                if val_loss - min_delta < best_loss:\n",
        "                    best_loss = val_loss\n",
        "                    save_path = saver.save(sess, out_model, global_step=n_epoch)\n",
        "                    print(\"Model saved in file: %s\" % save_path)\n",
        "                    n_incr_error = 0\n",
        "\n",
        "                if n_epoch % print_per_epoch == 0:\n",
        "                    print('Epoch %s/%s, train loss: %.5f, train error: %.5f, val loss: %.5f, val error: %.5f' % \\\n",
        "                                                (n_epoch, max_epoch, train_loss, train_err, val_loss, val_err))\n",
        "\n",
        "                if n_incr_error >= patience:\n",
        "                    print('Early stopping occured. Optimization Finished!')\n",
        "                    return train_loss_history, train_err_history, val_loss_history, val_err_history\n",
        "\n",
        "            return train_loss_history, train_err_history, val_loss_history, val_err_history\n",
        "\n",
        "def extract_validation_handles(session):\n",
        "    \"\"\" Extracts the input and predict_op handles that we use for validation.\n",
        "    Args:\n",
        "        session: The session with the loaded graph.\n",
        "    Returns:\n",
        "        validation handles.\n",
        "    \"\"\"\n",
        "    valid_nodes = tf.compat.v1.get_collection_ref(\"validation_nodes\")\n",
        "    if len(valid_nodes) != 5:\n",
        "        print(valid_nodes)\n",
        "        raise Exception(\"ERROR: Expected 5 items in validation_nodes, got %d.\" % len(valid_nodes))\n",
        "    return valid_nodes\n",
        "\n",
        "def load_model(session, save_path):\n",
        "    \"\"\" Loads a saved TF model from a file.\n",
        "    Args:\n",
        "        session: The tf.Session to use.\n",
        "        save_path: The save path for the saved session, returned by Saver.save().\n",
        "    Returns:\n",
        "        The inputs placehoder and the prediction operation.\n",
        "    \"\"\"\n",
        "    print(\"Loading model from file '%s'...\" % save_path)\n",
        "\n",
        "    meta_file = save_path + \".meta\"\n",
        "    if not os.path.exists(meta_file):\n",
        "        raise Exception(\"ERROR: Expected .meta file '%s', but could not find it.\" % meta_file)\n",
        "\n",
        "    saver = tf.compat.v1.train.import_meta_graph(meta_file)\n",
        "    # It's finicky about the save path.\n",
        "    save_path = os.path.join(\"./\", save_path)\n",
        "    saver.restore(session, save_path)\n",
        "\n",
        "    # Check that we have the handles we expected.\n",
        "    return extract_validation_handles(session)\n",
        "\n",
        "def validate_model(session, val_data, val_ops):\n",
        "    \"\"\" Validates the model stored in a session.\n",
        "    Args:\n",
        "        session: The session where the model is loaded.\n",
        "        val_data: The validation data to use for evaluating the model.\n",
        "        val_ops: The validation operations.\n",
        "    Returns:\n",
        "        The overall validation error for the model. \"\"\"\n",
        "    print(\"Validating model...\")\n",
        "\n",
        "    eye_left, eye_right, face, face_mask, pred = val_ops\n",
        "    val_eye_left, val_eye_right, val_face, val_face_mask, val_y = val_data\n",
        "    y = tf.compat.v1.placeholder(tf.float32, [None, 2], name='pos')\n",
        "    err = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.math.squared_difference(pred, y), axis=1)))\n",
        "    # Validate the model.\n",
        "    error = session.run(err, feed_dict={eye_left: val_eye_left, \\\n",
        "                                eye_right: val_eye_right, face: val_face, \\\n",
        "                                face_mask: val_face_mask, y: val_y})\n",
        "    return error\n",
        "\n",
        "def plot_loss(train_loss, train_err, test_err, start=0, per=1, save_file='loss.png'):\n",
        "    assert len(train_err) == len(test_err)\n",
        "    idx = np.arange(start, len(train_loss), per)\n",
        "    fig, ax1 = plt.subplots()\n",
        "    lns1 = ax1.plot(idx, train_loss[idx], 'b-', alpha=1.0, label='train loss')\n",
        "    ax1.set_xlabel('epochs')\n",
        "    # Make the y-axis label, ticks and tick labels match the line color.\n",
        "    ax1.set_ylabel('loss', color='b')\n",
        "    ax1.tick_params('y', colors='b')\n",
        "\n",
        "    ax2 = ax1.twinx()\n",
        "    lns2 = ax2.plot(idx, train_err[idx], 'r-', alpha=1.0, label='train error')\n",
        "    lns3 = ax2.plot(idx, test_err[idx], 'g-', alpha=1.0, label='test error')\n",
        "    ax2.set_ylabel('error', color='r')\n",
        "    ax2.tick_params('y', colors='r')\n",
        "\n",
        "    # added these three lines\n",
        "    lns = lns1 + lns2 + lns3\n",
        "    labs = [l.get_label() for l in lns]\n",
        "    ax1.legend(lns, labs, loc=0)\n",
        "\n",
        "    fig.tight_layout()\n",
        "    plt.savefig(save_file)\n",
        "    # plt.show()\n",
        "\n",
        "def train(train, input, max_epoch, learning_rate, batch_size, patience, print_per_epoch, save_model, load_model, plot_filter, plot_loss, save_loss):\n",
        "\n",
        "    train_data, val_data = load_data(input)\n",
        "\n",
        "    train_data = prepare_data(train_data)\n",
        "    val_data = prepare_data(val_data)\n",
        "\n",
        "    start = timeit.default_timer()\n",
        "    et = EyeTracker()\n",
        "    train_loss_history, train_err_history, val_loss_history, val_err_history = et.train(train_data, val_data, lr=learning_rate, batch_size=batch_size, max_epoch=max_epoch, min_delta=1e-4, patience=patience, print_per_epoch=print_per_epoch, out_model=save_model)\n",
        "\n",
        "    print('runtime: %.1fs' % (timeit.default_timer() - start))\n",
        "\n",
        "    if save_loss:\n",
        "        with open(save_loss, 'w') as outfile:\n",
        "            np.savez(outfile, train_loss_history=train_loss_history, train_err_history=train_err_history, \\\n",
        "                                    val_loss_history=val_loss_history, val_err_history=val_err_history)\n",
        "\n",
        "    if plot_loss:\n",
        "        plot_loss(np.array(train_loss_history), np.array(train_err_history), np.array(val_err_history), start=0, per=1, save_file=plot_loss)\n",
        "\n",
        "def test(input, model_path):\n",
        "    _, val_data = load_data(input)\n",
        "\n",
        "    val_data = prepare_data(val_data)\n",
        "\n",
        "    # Load and validate the network.\n",
        "    with tf.compat.v1.Session() as sess:\n",
        "        # reset validation nodes on each run\n",
        "        tf.compat.v1.get_collection_ref(\"validation_nodes\").clear()\n",
        "        val_ops = load_model(sess, model_path)\n",
        "        error = validate_model(sess, val_data, val_ops)\n",
        "        print('Overall validation error: %f' % error)\n",
        "\n",
        "# def main():\n",
        "#     parser = argparse.ArgumentParser()\n",
        "#     parser.add_argument('--train', action='store_true', help='train flag')\n",
        "#     parser.add_argument('-i', '--input', required=True, type=str, help='path to the input data')\n",
        "#     parser.add_argument('-max_epoch', '--max_epoch', type=int, default=10, help='max number of iterations (default 100)')\n",
        "#     parser.add_argument('-lr', '--learning_rate', type=float, default=0.01, help='learning rate (default 1e-3)')\n",
        "#     parser.add_argument('-bs', '--batch_size', type=int, default=128, help='batch size (default 50)')\n",
        "#     parser.add_argument('-p', '--patience', type=int, default=5, help='early stopping patience (default 10)')\n",
        "#     parser.add_argument('-pp_iter', '--print_per_epoch', type=int, default=1, help='print per iteration (default 10)')\n",
        "#     parser.add_argument('-sm', '--save_model', type=str, default='my_model', help='path to the output model (default my_model)')\n",
        "#     parser.add_argument('-lm', '--load_model', type=str, help='path to the loaded model')\n",
        "#     parser.add_argument('-pf', '--plot_filter', type=str, default='filter.png', help='plot filters')\n",
        "#     parser.add_argument('-pl', '--plot_loss', type=str, default='loss.png', help='plot loss')\n",
        "#     parser.add_argument('-sl', '--save_loss', type=str, default='loss.npz', help='save loss')\n",
        "#     args = parser.parse_args()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Preprocessing\n",
        "\n",
        "Splitting the original dataset into training, testing, and validation data. Original training data of 40k is split into 80% training / 20% testing with 5k validation datapoints."
      ],
      "metadata": {
        "id": "KT5Vofwh0cft"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# splitting the dataset into training and test data\n",
        "\n",
        "data = np.load('data/eye_tracker_train_and_val.npz')\n",
        "# for key in data.files:\n",
        "#   print(key)\n",
        "\n",
        "train_y = data['train_y']\n",
        "train_eye_right = data['train_eye_right']\n",
        "train_face = data['train_face']\n",
        "train_eye_left = data['train_eye_left']\n",
        "train_face_mask = data['train_face_mask']\n",
        "val_eye_left = data[\"val_eye_left\"]\n",
        "val_eye_right = data[\"val_eye_right\"]\n",
        "val_face = data[\"val_face\"]\n",
        "val_face_mask = data[\"val_face_mask\"]\n",
        "val_y = data[\"val_y\"]\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Split the data into train and test sets, adjust the test_size and random_state as needed\n",
        "y_train, y_test, eye_right_train, eye_right_test, face_train, face_test, eye_left_train, eye_left_test, face_mask_train, face_mask_test = train_test_split(train_y, train_eye_right, train_face, train_eye_left, train_face_mask, test_size=0.2, random_state=42)\n",
        "\n",
        "# save the split datasets into separate NPZ files\n",
        "np.savez('train_data.npz', train_y=y_train, train_eye_right=eye_right_train, train_face=face_train, train_eye_left=eye_left_train, train_face_mask=face_mask_train, val_eye_left=val_eye_left, val_eye_right=val_eye_right, val_face=val_face, val_face_mask=val_face_mask, val_y=val_y)\n",
        "np.savez('test_data.npz', train_y=y_test, train_eye_right=eye_right_test, train_face=face_test, train_eye_left=eye_left_test, train_face_mask=face_mask_test, val_eye_left=val_eye_left, val_eye_right=val_eye_right, val_face=val_face, val_face_mask=val_face_mask, val_y=val_y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "id": "l9lxDwPIv5Ic",
        "outputId": "83e105c7-fb25-40de-9653-fb2f0cd0d2d4"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-0b9eed86bde8>\u001b[0m in \u001b[0;36m<cell line: 21>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;31m# Split the data into train and test sets, adjust the test_size and random_state as needed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meye_right_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meye_right_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mface_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mface_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meye_left_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meye_left_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mface_mask_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mface_mask_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_eye_right\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_face\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_eye_left\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_face_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m42\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;31m# save the split datasets into separate NPZ files\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_split.py\u001b[0m in \u001b[0;36mtrain_test_split\u001b[0;34m(test_size, train_size, random_state, shuffle, stratify, *arrays)\u001b[0m\n\u001b[1;32m   2583\u001b[0m         \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstratify\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2584\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2585\u001b[0;31m     return list(\n\u001b[0m\u001b[1;32m   2586\u001b[0m         chain.from_iterable(\n\u001b[1;32m   2587\u001b[0m             \u001b[0;34m(\u001b[0m\u001b[0m_safe_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_safe_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_split.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   2585\u001b[0m     return list(\n\u001b[1;32m   2586\u001b[0m         chain.from_iterable(\n\u001b[0;32m-> 2587\u001b[0;31m             \u001b[0;34m(\u001b[0m\u001b[0m_safe_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_safe_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2588\u001b[0m         )\n\u001b[1;32m   2589\u001b[0m     )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/utils/__init__.py\u001b[0m in \u001b[0;36m_safe_indexing\u001b[0;34m(X, indices, axis)\u001b[0m\n\u001b[1;32m    354\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_pandas_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"shape\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_array_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_list_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training the model\n",
        "\n",
        "Executing this cell will train the model and save the parameters as my-model-9.meta. After this, the model can be loaded and and used to validate the test data."
      ],
      "metadata": {
        "id": "yImhhsxX6QSh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E_plWuta5jS5"
      },
      "outputs": [],
      "source": [
        "train(train=True, input='train_data.npz', max_epoch=10, learning_rate=0.01, batch_size=32, patience=5, print_per_epoch=1, save_model='my_model', load_model=None, plot_filter='filter.png', plot_loss='loss.png', save_loss='loss.npz')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Testing the model"
      ],
      "metadata": {
        "id": "_V8VEVtG6oQe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test(input='test_data.npz', model_path='my_model-9')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q6NCIkp6yxaN",
        "outputId": "9f1d824e-ec20-4de9-a0cb-06579e6d0a79"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading model from file 'my_model-9'...\n",
            "Validating model...\n",
            "Overall validation error: 6.830703\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ak4ICS2B2uwv"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}