face_filter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.models import load_model\n",
    "from pandas.io.parsers import read_csv\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "def load_data(test=False):\n",
    "    \"\"\"\n",
    "    Loads data from FTEST if *test* is True, otherwise from FTRAIN.\n",
    "    Important that the files are in a `data` directory\n",
    "    \"\"\"\n",
    "    FTRAIN = 'data/training.csv'\n",
    "    FTEST = 'data/test.csv'\n",
    "    fname = FTEST if test else FTRAIN\n",
    "    df = read_csv(os.path.expanduser(fname))  # load dataframes\n",
    "\n",
    "    # The Image column has pixel values separated by space; convert\n",
    "    # the values to numpy arrays:\n",
    "    df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep=' '))\n",
    "\n",
    "    df = df.dropna()  # drop all rows that have missing values in them\n",
    "\n",
    "    X = np.vstack(df['Image'].values) / 255.  # scale pixel values to [0, 1] (Normalizing)\n",
    "    X = X.astype(np.float32)\n",
    "    X = X.reshape(-1, 96, 96, 1) # return each images as 96 x 96 x 1\n",
    "\n",
    "    if not test:  # only FTRAIN has target columns\n",
    "        y = df[df.columns[:-1]].values\n",
    "        y = (y - 48) / 48  # scale target coordinates to [-1, 1] (Normalizing)\n",
    "        X, y = shuffle(X, y, random_state=42)  # shuffle train data\n",
    "        y = y.astype(np.float32)\n",
    "    else:\n",
    "        y = None\n",
    "\n",
    "    return X, y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.models import load_model\n",
    "from keras.layers import Convolution2D, MaxPooling2D, Dropout\n",
    "from keras.layers import Flatten, Dense\n",
    "from keras.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax, Nadam\n",
    "\n",
    "def get_my_CNN_model_architecture():\n",
    "    '''\n",
    "    The network should accept a 96x96 grayscale image as input, and it should output a vector with 30 entries,\n",
    "    corresponding to the predicted (horizontal and vertical) locations of 15 facial keypoints.\n",
    "    '''\n",
    "    model = Sequential()\n",
    "    model.add(Convolution2D(32, (5, 5), input_shape=(96,96,1), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Convolution2D(64, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.1))\n",
    "\n",
    "    model.add(Convolution2D(128, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.2))\n",
    "\n",
    "    model.add(Convolution2D(30, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.3))\n",
    "\n",
    "    model.add(Flatten())\n",
    "\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(Dense(30))\n",
    "    \n",
    "\n",
    "    return model;\n",
    "\n",
    "def compile_my_CNN_model(model, optimizer, loss, metrics):\n",
    "    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n",
    "\n",
    "def train_my_CNN_model(model, X_train, y_train):\n",
    "    return model.fit(X_train, y_train, epochs=100, batch_size=200, verbose=1, validation_split=0.2)\n",
    "\n",
    "def save_my_CNN_model(model, fileName):\n",
    "    model.save(fileName + '.h5')\n",
    "\n",
    "def load_my_CNN_model(fileName):\n",
    "    return load_model(fileName + '.h5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1712 samples, validate on 428 samples\n",
      "Epoch 1/100\n",
      "1712/1712 [==============================] - 97s 57ms/step - loss: 0.0932 - accuracy: 0.0000e+00 - val_loss: 0.0594 - val_accuracy: 0.0000e+00\n",
      "Epoch 2/100\n",
      "1712/1712 [==============================] - 45s 26ms/step - loss: 0.0341 - accuracy: 0.2208 - val_loss: 0.0294 - val_accuracy: 0.0514\n",
      "Epoch 3/100\n"
     ]
    }
   ],
   "source": [
    "from utils import load_data\n",
    "from my_CNN_model import *\n",
    "import cv2\n",
    "\n",
    "\n",
    "# Load training set\n",
    "X_train, y_train = load_data()\n",
    "\n",
    "# NOTE: Please check the load_data() method in utils.py to see how the data is preprocessed (normalizations and stuff)\n",
    "\n",
    "\n",
    "# Setting the CNN architecture\n",
    "my_model = get_my_CNN_model_architecture()\n",
    "\n",
    "# Compiling the CNN model with an appropriate optimizer and loss and metrics\n",
    "compile_my_CNN_model(my_model, optimizer = 'adam', loss = 'mean_squared_error', metrics = ['accuracy'])\n",
    "\n",
    "# Training the model\n",
    "hist = train_my_CNN_model(my_model, X_train, y_train)\n",
    "\n",
    "# train_my_CNN_model returns a History object. History.history attribute is a record of training loss values and metrics\n",
    "# values at successive epochs, as well as validation loss values and validation metrics values (if applicable).\n",
    "\n",
    "# Saving the model\n",
    "save_my_CNN_model(model,'model')\n",
    "\n",
    "'''\n",
    "# You can skip all the steps above (from 'Setting the CNN architecture') after running the script for the first time.\n",
    "# Just load the recent model using load_my_CNN_model and use it to predict keypoints on any face data\n",
    "my_model = load_my_CNN_model('my_model')\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_CNN_model import *\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "# Load the model built in the previous step\n",
    "my_model = load_my_CNN_model('my_model')\n",
    "\n",
    "# Face cascade to detect faces\n",
    "face_cascade = cv2.CascadeClassifier('cascades/haarcascade_frontalface_default.xml')\n",
    "\n",
    "# Define the upper and lower boundaries for a color to be considered \"red\"\n",
    "blueLower = np.array([100, 60, 60])\n",
    "blueUpper = np.array([140, 255, 255])\n",
    "  \n",
    "# Define a 5x5 kernel for erosion and dilation\n",
    "kernel = np.ones((5, 5), np.uint8)\n",
    "\n",
    "# Define filters\n",
    "filters = ['images/sunglasses.png','images/sunglasses_1.png' ,'images/sunglasses_2.png', 'images/sunglasses_3.jpg', 'images/sunglasses_4.png', 'images/sunglasses_5.jpg', 'images/sunglasses_6.png','images/sunglasses_7.png','images/sunglasses_8.png','images/sunglasses_9.png']\n",
    "filterIndex = 0\n",
    "\n",
    "# Load the video\n",
    "camera = cv2.VideoCapture(0)\n",
    "\n",
    "# Keep looping\n",
    "while True:\n",
    "    # Grab the current paintWindow\n",
    "    (grabbed, frame) = camera.read()\n",
    "    frame = cv2.flip(frame, 1)\n",
    "    frame2 = np.copy(frame)\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "    # Add the 'Next Filter' button to the frame\n",
    "    frame = cv2.rectangle(frame,(620,65), (500,10), (0,0,0), -1)\n",
    "    cv2.putText(frame, \"NEXT FILTER\", (512, 37), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA)\n",
    "\n",
    "    # Detect faces\n",
    "    faces = face_cascade.detectMultiScale(gray, 1.25, 6)\n",
    "\n",
    "    # Determine which pixels fall within the blue boundaries and then blur the binary image\n",
    "    blueMask = cv2.inRange(hsv, blueLower, blueUpper)\n",
    "    blueMask = cv2.erode(blueMask, kernel, iterations=2)\n",
    "    blueMask = cv2.morphologyEx(blueMask, cv2.MORPH_OPEN, kernel)\n",
    "    blueMask = cv2.dilate(blueMask, kernel, iterations=1)\n",
    "\n",
    "    # Find contours (bottle cap in my case) in the image\n",
    "    (_, cnts, _) = cv2.findContours(blueMask.copy(), cv2.RETR_EXTERNAL,\n",
    "    \tcv2.CHAIN_APPROX_SIMPLE)\n",
    "    center = None\n",
    "\n",
    "    # Check to see if any contours were found\n",
    "    if len(cnts) > 0:\n",
    "    \t# Sort the contours and find the largest one -- we\n",
    "    \t# will assume this contour correspondes to the area of the bottle cap\n",
    "        cnt = sorted(cnts, key = cv2.contourArea, reverse = True)[0]\n",
    "        # Get the radius of the enclosing circle around the found contour\n",
    "        ((x, y), radius) = cv2.minEnclosingCircle(cnt)\n",
    "        # Draw the circle around the contour\n",
    "        cv2.circle(frame, (int(x), int(y)), int(radius), (0, 0, 255), 2)\n",
    "        # Get the moments to calculate the center of the contour (in this case Circle)\n",
    "        M = cv2.moments(cnt)\n",
    "        center = (int(M['m10'] / M['m00']), int(M['m01'] / M['m00']))\n",
    "\n",
    "        if center[1] <= 65:\n",
    "            if 500 <= center[0] <= 620: # Next Filter\n",
    "                filterIndex += 1\n",
    "                filterIndex %= 6\n",
    "                continue\n",
    "\n",
    "    for (x, y, w, h) in faces:\n",
    "\n",
    "        # Grab the face\n",
    "        gray_face = gray[y:y+h, x:x+w]\n",
    "        color_face = frame[y:y+h, x:x+w]\n",
    "\n",
    "        # Normalize to match the input format of the model - Range of pixel to [0, 1]\n",
    "        gray_normalized = gray_face / 255\n",
    "\n",
    "        # Resize it to 96x96 to match the input format of the model\n",
    "        original_shape = gray_face.shape # A Copy for future reference\n",
    "        face_resized = cv2.resize(gray_normalized, (96, 96), interpolation = cv2.INTER_AREA)\n",
    "        face_resized_copy = face_resized.copy()\n",
    "        face_resized = face_resized.reshape(1, 96, 96, 1)\n",
    "\n",
    "        # Predicting the keypoints using the model\n",
    "        keypoints = my_model.predict(face_resized)\n",
    "\n",
    "        # De-Normalize the keypoints values\n",
    "        keypoints = keypoints * 48 + 48\n",
    "\n",
    "        # Map the Keypoints back to the original image\n",
    "        face_resized_color = cv2.resize(color_face, (96, 96), interpolation = cv2.INTER_AREA)\n",
    "        face_resized_color2 = np.copy(face_resized_color)\n",
    "\n",
    "        # Pair them together\n",
    "        points = []\n",
    "        for i, co in enumerate(keypoints[0][0::2]):\n",
    "            points.append((co, keypoints[0][1::2][i]))\n",
    "\n",
    "        # Add FILTER to the frame\n",
    "        sunglasses = cv2.imread(filters[filterIndex], cv2.IMREAD_UNCHANGED)\n",
    "        sunglass_width = int((points[7][0]-points[9][0])*1.1)\n",
    "        sunglass_height = int((points[10][1]-points[8][1])/1.1)\n",
    "        sunglass_resized = cv2.resize(sunglasses, (sunglass_width, sunglass_height), interpolation = cv2.INTER_CUBIC)\n",
    "        transparent_region = sunglass_resized[:,:,:3] != 0\n",
    "        face_resized_color[int(points[9][1]):int(points[9][1])+sunglass_height, int(points[9][0]):int(points[9][0])+sunglass_width,:][transparent_region] = sunglass_resized[:,:,:3][transparent_region]\n",
    "\n",
    "        # Resize the face_resized_color image back to its original shape\n",
    "        frame[y:y+h, x:x+w] = cv2.resize(face_resized_color, original_shape, interpolation = cv2.INTER_CUBIC)\n",
    "\n",
    "        # Add KEYPOINTS to the frame2\n",
    "        for keypoint in points:\n",
    "            cv2.circle(face_resized_color2, keypoint, 1, (255,255,255), 1)\n",
    "\n",
    "        frame2[y:y+h, x:x+w] = cv2.resize(face_resized_color2, original_shape, interpolation = cv2.INTER_CUBIC)\n",
    "\n",
    "        # Show the frame and the frame2\n",
    "        cv2.imshow(\"Selfie Filters\", frame)\n",
    "        cv2.imshow(\"Facial Keypoints\", frame2)\n",
    "\n",
    "    # If the 'q' key is pressed, stop the loop\n",
    "    if cv2.waitKey(1) & 0xFF == 27:\n",
    "        break\n",
    "\n",
    "# Cleanup the camera and close any open windows\n",
    "camera.release()\n",
    "cv2.destroyAllWindows()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}