notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import io\n",
    "import urllib\n",
    "import requests\n",
    "import json\n",
    "\n",
    "from PIL import Image\n",
    "import cv2 \n",
    "from skimage.transform import resize\n",
    "import matplotlib.pyplot as plt\n",
    "import ipywidgets as ipy\n",
    "import numpy as np\n",
    "from torchvision import models, transforms, utils\n",
    "from torch.autograd import Variable, Function\n",
    "import torch\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_image(img):\n",
    "    means = [0.485, 0.456, 0.406]\n",
    "    stds = [0.229, 0.224, 0.225]\n",
    "\n",
    "    preprocessed_img = np.expand_dims((img - means)/stds, axis=0)\n",
    "    preprocessed_tensor =torch.from_numpy(preprocessed_img).permute(0, 3, 1, 2).type(torch.FloatTensor)\n",
    "\n",
    "    preprocessed_var = Variable(preprocessed_tensor, requires_grad=True)\n",
    "    return preprocessed_var\n",
    "\n",
    "def deprocess_image(image_tensor):\n",
    "    means = [0.485, 0.456, 0.406]\n",
    "    stds = [0.229, 0.224, 0.225]\n",
    "    \n",
    "    img = np.transpose(image_tensor.numpy(), (1, 2, 0))\n",
    "    img = (img*stds)+means\n",
    "    img = np.clip(img, 0.0, 1.0)\n",
    "    img = img.astype(np.float32)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class IgnoreNeptune(object):\n",
    "    def channel_send(self, *args):\n",
    "        pass\n",
    "    \n",
    "    def channel_reset(self, *args):\n",
    "        pass\n",
    "    \n",
    "# ctx = IgnoreNeptune() # uncomment if you are running it without neptune"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ImageNet labels\n",
    "\n",
    "[ImageNet](http://image-net.org/) is a large database with images pertaining to 1 of 1000 classes. We load the class names into a dictionary in the cell below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABELS_URL = 'https://gist.githubusercontent.com/yrevar/942d3a0ac09ec9e5eb3a/raw/c2c91c8e767d04621020c30ed31192724b863041/imagenet1000_clsid_to_human.txt'\n",
    "labels = eval(requests.get(LABELS_URL).text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download image from URL\n",
    "\n",
    "The function below retrieves a file by URL and tries to read it as an instance of `PIL.Image` class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_read_image(url):\n",
    "    \"\"\"Download and read image as an instance of PIL.Image class.\"\"\"\n",
    "    image_url = urllib.request.urlopen(url)\n",
    "    image = Image.open(io.BytesIO(image_url.read()))\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = download_and_read_image(\n",
    "    'https://i.ytimg.com/vi/UXOt4LRLajY/hqdefault.jpg'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 10))\n",
    "plt.imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.RandomCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean = [ 0.485, 0.456, 0.406 ],\n",
    "                          std = [ 0.229, 0.224, 0.225 ]),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_processed = transform(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(deprocess_image(image_processed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a pretrained network from PyTorch library\n",
    "\n",
    "Here we will use pretrained [VGG](https://arxiv.org/abs/1409.1556) network to generate label for a given image. You can choose this model (among other ones) from PyTorch [model library](http://pytorch.org/docs/master/torchvision/models.html).\n",
    "\n",
    "First, we load the model and its pretrained weights. We also set it in evaluation state by calling `eval()` function on it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = models.vgg19_bn()\n",
    "net.load_state_dict(torch.load('/public/models/pytorch/vgg/vgg19_bn-c79401a0.pth'))\n",
    "net.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations = net(Variable(image_processed.unsqueeze(0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.max(activations, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, i = torch.max(activations, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = int(i.data.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 10))\n",
    "plt.imshow(deprocess_image(image_processed))\n",
    "plt.title(\"{}: {}\".format(i, labels[i].upper()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise\n",
    "* Get five most likely labels for the given image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## But why did you say that?\n",
    "\n",
    "As neural networks are considered black box models, there was some effort to increase their transparency. One example is [Grad-CAM](https://arxiv.org/pdf/1610.02391v1.pdf) method for generating class activation heatmaps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "CONVO_EXTRACTOR_NR = '52'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building calculation graph\n",
    "* Building feature extractor\n",
    "* Building gradient and activation extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeatureExtractor():\n",
    "    \"\"\" Class for extracting activations and \n",
    "    registering gradients from targetted intermediate layers \"\"\"\n",
    "\n",
    "    def __init__(self, model, target_layers):\n",
    "        self.model = model\n",
    "        self.model.eval()\n",
    "        self.target_layers = target_layers\n",
    "        self.gradients = []\n",
    "\n",
    "    def save_gradient(self, grad):\n",
    "        self.gradients.append(grad)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        outputs = []\n",
    "        self.gradients = []\n",
    "        for name, module in self.model._modules.items():\n",
    "            x = module(x)\n",
    "            if name in self.target_layers:\n",
    "                x.register_hook(self.save_gradient)\n",
    "                outputs += [x]\n",
    "        return outputs, x\n",
    "    \n",
    "class ModelOutputs():\n",
    "    \"\"\" Class for making a forward pass, and getting:\n",
    "    1. The network output.\n",
    "    2. Activations from intermeddiate targetted layers.\n",
    "    3. Gradients from intermeddiate targetted layers. \"\"\"\n",
    "\n",
    "    def __init__(self, model, target_layers):\n",
    "        self.model = model\n",
    "        self.model.eval()\n",
    "        self.feature_extractor = FeatureExtractor(self.model.features, target_layers)\n",
    "\n",
    "    def get_gradients(self):\n",
    "        return self.feature_extractor.gradients\n",
    "\n",
    "    def __call__(self, x):\n",
    "        target_activations, output = self.feature_extractor(x)\n",
    "        output = output.view(output.size(0), -1)\n",
    "        output = self.model.classifier(output)\n",
    "        return target_activations, output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting heatmap\n",
    "* Building one hot objective function for chosen class \n",
    "* calculating grads and output activations for a given model and image \n",
    "* building heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_cam_on_image(img, mask):\n",
    "    heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)\n",
    "    heatmap = np.float32(heatmap) / 255\n",
    "    cam = heatmap + np.float32(img)\n",
    "    cam = cam / np.max(cam)\n",
    "    return cam\n",
    "\n",
    "\n",
    "class GradCam:\n",
    "    def __init__(self, model, target_layer_names, use_cuda):\n",
    "        self.model = model\n",
    "        self.model.eval()\n",
    "        self.cuda = use_cuda\n",
    "        if self.cuda:\n",
    "            self.model = model.cuda()\n",
    "\n",
    "        self.extractor = ModelOutputs(self.model, target_layer_names)\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.model(input)\n",
    "\n",
    "    def __call__(self, input, index=None):\n",
    "        if self.cuda:\n",
    "            features, output = self.extractor(input.cuda())\n",
    "        else:\n",
    "            features, output = self.extractor(input)\n",
    "        \n",
    "        index_max = np.argmax(output.cpu().data.numpy())\n",
    "        if index == None:\n",
    "            index = index_max\n",
    "\n",
    "        one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)\n",
    "        one_hot[0][index] = 1\n",
    "        one_hot = Variable(torch.from_numpy(one_hot), requires_grad=True)\n",
    "        if self.cuda:\n",
    "            one_hot = torch.sum(one_hot.cuda() * output)\n",
    "        else:\n",
    "            one_hot = torch.sum(one_hot * output)\n",
    "\n",
    "        self.model.features.zero_grad()\n",
    "        self.model.classifier.zero_grad()\n",
    "        one_hot.backward(retain_graph=True)\n",
    "\n",
    "        grads_val = self.extractor.get_gradients()[-1].cpu().data.numpy()\n",
    "\n",
    "        target = features[-1]\n",
    "        target = target.cpu().data.numpy()[0, :]\n",
    "\n",
    "        weights = np.mean(grads_val, axis=(2, 3))[0, :]\n",
    "        cam = np.ones(target.shape[1:], dtype=np.float32)\n",
    "\n",
    "        for i, w in enumerate(weights):\n",
    "            cam += w * target[i, :, :]\n",
    "\n",
    "        cam = np.maximum(cam, 0)\n",
    "        cam = cv2.resize(cam, (224, 224))\n",
    "        cam = cam - np.min(cam)\n",
    "        cam = cam / np.max(cam)\n",
    "        return cam, index, index_max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def category_search(categories, text=None, index=None, by_text=True):\n",
    "    if by_text:\n",
    "        options = {category:index for index,category in categories.items() if text in category}\n",
    "        return options\n",
    "    else:\n",
    "        return categories[index]\n",
    "    \n",
    "def plot_grad_cam(img, model, categories, target_index=None):\n",
    "    grad_cam = GradCam(model = model, target_layer_names = [CONVO_EXTRACTOR_NR], use_cuda=USE_CUDA)\n",
    "\n",
    "    input = preprocess_image(img)\n",
    "\n",
    "    mask, target_index, target_max = grad_cam(input, target_index)\n",
    "    reversed_mask = 1.0 - mask\n",
    "    img_cam = show_cam_on_image(img, reversed_mask)\n",
    "    \n",
    "    plt.figure(figsize=(16,10))\n",
    "    plt.subplot(1,2,1)\n",
    "    plt.title((category_search(categories, index=target_max, by_text=False)))\n",
    "    plt.imshow(img)\n",
    "    plt.subplot(1,2,2)\n",
    "    plt.title((category_search(categories, index=target_index, by_text=False)))\n",
    "    plt.imshow(img_cam)    \n",
    "    plt.show()\n",
    "    \n",
    "def visualize_heatmaps(net, image_numpy, labels):\n",
    "    @ipy.interact(text='leop')\n",
    "    def explore_heatmaps(text):\n",
    "        options = category_search(labels, text=text)\n",
    "        dropdown = ipy.Dropdown(\n",
    "            options=list(options.keys()),\n",
    "            value=list(options.keys())[0]\n",
    "        )\n",
    "        button = ipy.ToggleButton(\n",
    "            description='Generate Heatmap',\n",
    "            value=False\n",
    "            )      \n",
    "        def get_index(chosen, generate):\n",
    "            if generate:\n",
    "                index = [k for k,v in labels.items() if v == chosen][0]\n",
    "                plot_grad_cam(image_numpy, net, labels, target_index=index)\n",
    "\n",
    "        display(ipy.interactive(get_index, chosen=dropdown, generate=button))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_numpy = deprocess_image(image_processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_heatmaps(net, image_numpy, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 2: DIY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n",
    "We will work with [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html): a benchmark dataset for image recognition. There are ten classes of objects on the images with 50K examples in the training set and 10K in the testset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = torchvision.datasets.CIFAR10('/public/cifar/', train=True, download=True, transform=transform)\n",
    "testset = torchvision.datasets.CIFAR10('/public/cifar/', train=False, transform=transform)\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining your own network\n",
    "\n",
    "In PyTorch, each model needs to inherit from the `nn.Module` class. Thanks to it, we can use powerful set of PyTorch's methods for differentation (and, in turn, model training)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    \"\"\"Neural network model.\"\"\"\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc_size = 16 * 5 * 5\n",
    "        self.fc1 = nn.Linear(self.fc_size, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "        \n",
    "    def forward(self, x):               # Height and width and of x:\n",
    "        x = F.relu(self.conv1(x))       # 32 - 5 + 1 = 28\n",
    "        x = self.pool(x)                # 28 / 2 = 14\n",
    "        x = F.relu(self.conv2(x))       # 14 - 5 + 1 = 10\n",
    "        x = self.pool(x)                # 10 / 2 = 5\n",
    "        x = x.view(-1, self.fc_size)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def performance_stats(net, dataloader, use_gpu=False):\n",
    "    \"\"\"Computes performance statistics - logloss and accuracy.\"\"\"\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    logloss = 0.0\n",
    "    for i, data in enumerate(dataloader, 0):\n",
    "        images, labels = data\n",
    "        if use_gpu:\n",
    "            images, labels = images.cuda(), labels.cuda()\n",
    "        outputs = net(Variable(images, volatile=True))\n",
    "        logloss += F.cross_entropy(outputs, Variable(labels, volatile=True), size_average=False)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        correct += (predicted == labels).sum()\n",
    "        total += labels.size(0)\n",
    "    acc = float(correct) / total\n",
    "    logloss /= total\n",
    "    return acc, logloss.data[0], total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def highest_loss_img(images, labels, outputs, use_gpu):\n",
    "    \"\"\"Determines the image with the highest logloss in a given batch.\"\"\"\n",
    "    probs = F.softmax(outputs)\n",
    "    if use_gpu:\n",
    "        np_probs = probs.data.cpu().numpy()\n",
    "        np_labels = labels.data.cpu().numpy()\n",
    "    else:\n",
    "        np_probs = probs.data.numpy()\n",
    "        np_labels = labels.data.numpy()\n",
    "    predicted_prob = np.array([prob[label] for prob, label in zip(np_probs, np_labels)])\n",
    "    i = np.argmin(predicted_prob)\n",
    "    label_pred = np.argmax(np_probs[i])\n",
    "    img = images[i] / 2 + 0.5 # Rescale to the original values\n",
    "    if use_gpu:\n",
    "        img = img.cpu()\n",
    "    pil_img = transforms.ToPILImage()(img.data)\n",
    "    return pil_img, np_labels[i], predicted_prob[i], label_pred, np_probs[i,label_pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training(net, optimizer, criterion, trainloader, testloader, num_epoch, logging_window=1000, use_gpu=True, experiment_name=''):\n",
    "    n_iter = 0\n",
    "    \"\"\"Main function for model training.\"\"\"\n",
    "    # Channels need to be reset on another invocation of training() function.\n",
    "    # At first, a warning will be printed as these channels do not exists.\n",
    "    ctx.channel_reset(experiment_name + ' training set running loss')\n",
    "    ctx.channel_reset(experiment_name + ' training set accuracy')\n",
    "    ctx.channel_reset(experiment_name + ' training set logloss')\n",
    "    ctx.channel_reset(experiment_name + ' test set accuracy')\n",
    "    ctx.channel_reset(experiment_name + ' test set logloss')\n",
    "    ctx.channel_reset(experiment_name + ' image with the highest logloss in a batch')\n",
    "    for epoch in range(num_epoch):    \n",
    "        running_loss = 0.0\n",
    "        for i, data in enumerate(trainloader, 0):\n",
    "            images, labels = data\n",
    "            if use_gpu:\n",
    "                images, labels = images.cuda(), labels.cuda()\n",
    "            images, labels = Variable(images), Variable(labels)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = net(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.data[0]\n",
    "            if i % logging_window == (logging_window - 1):\n",
    "                n_iter += logging_window\n",
    "                ctx.channel_send(experiment_name + ' training set running loss', n_iter, running_loss / logging_window)\n",
    "                running_loss = 0.0\n",
    "                img, label, prob, label_pred, prob_pred = highest_loss_img(images, labels, outputs, use_gpu=use_gpu)\n",
    "                ctx.channel_send(experiment_name + ' image with the highest logloss in a batch', neptune.Image(\n",
    "                  name=classes[label],\n",
    "                  description='True label: %s (p = %.4f)\\nPredicted:  %s (p = %.4f)' % \n",
    "                    (classes[label], prob, classes[label_pred], prob_pred),\n",
    "                  data=img.resize((128, 128))))\n",
    "        \n",
    "        # Post epoch statistics\n",
    "        acc_train, logloss_train, total_train = performance_stats(net, trainloader, use_gpu)\n",
    "        print('------------------------------')\n",
    "        print(epoch, acc_train, logloss_train, total_train)\n",
    "        ctx.channel_send(experiment_name + ' training set accuracy', epoch, acc_train)\n",
    "        ctx.channel_send(experiment_name + ' training set logloss', epoch, logloss_train)\n",
    "\n",
    "        acc_test, logloss_test, total_test = performance_stats(net, testloader, use_gpu)\n",
    "        print(epoch, acc_test, logloss_test, total_test)\n",
    "        ctx.channel_send(experiment_name + ' test set accuracy', epoch, acc_test)\n",
    "        ctx.channel_send(experiment_name + ' test set logloss', epoch, logloss_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n",
    "Each model needs some fuel to work. Here, it is images that we feed the network. \n",
    "\n",
    "You can display some examples using code below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size * len(testloader), batch_size * len(trainloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # Unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()\n",
    "plt.figure(figsize=(8, 8))\n",
    "\n",
    "# Show images\n",
    "imshow(torchvision.utils.make_grid(images, nrow=8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizing parameters\n",
    "\n",
    "Now we are going to use backpropagation to train the model using data. First, we need a loss function and an optimization routine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Net()\n",
    "net.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training(net, optimizer, criterion, trainloader, testloader, num_epoch=20, logging_window=100,\n",
    "         experiment_name='Basic', use_gpu=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercises\n",
    "* Play with `batch_size` parameter and see how it ifluences training.\n",
    "* Replace the first 5x5 convolutional filters with two 3x3 filters. How does it affect the number of the model's parameters?\n",
    "* Augment training data by randomly flipping images before passing them to your network.\n",
    "* Modify the network architecture to achieve accuracy over 80% on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 3: Transfer learning\n",
    "In a typical computer vision project one uses some model pretrained on a large set of data like we saw in the first part. \n",
    "\n",
    "We will use such a model and finetune it for our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = models.vgg19_bn()\n",
    "net.load_state_dict(torch.load('/public/models/pytorch/vgg/vgg19_bn-c79401a0.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Essentials\n",
    "* Substitute the classifier part of the network\n",
    "* Train with lower learning rate\n",
    "* Freeze lower layers if needed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise\n",
    "Implement a classifier that works with Cifar 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.classifier = nn.Sequential(\n",
    "    nn.Linear(512, 1024),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(0.5),\n",
    "    nn.Linear(1024, 10)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(net.parameters(), lr=0.000025)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training(net, optimizer, criterion, trainloader, testloader, num_epoch=1, logging_window=100, experiment_name=\"Transfer learning\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Neptune",
   "language": "",
   "name": "neptune-kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "name": "_merged"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}