NetWork Visualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network Visualization\n",
    "在这份Notebook里面我们会来探索图像梯度对于生成新图片的用法。\n",
    "当训练模型的时候，我们会定义一个损失函数来表示我们对当前模型性能的不满意程度(和标准答案的差异程度)；\n",
    "接着我们用反向传播来计算损失函数对于模型参数的梯度，接着在模型参数上用梯度下降来最小化损失(loss)。\n",
    "这里我们会做一些不同的事情。我们首先用一个在ImageNet数据集上做了预训练的卷积神经网络模型，接着用这\n",
    "个模型来定义一个损失函数，表示我们对于当前图片的不满意程度(和标准答案的差异程度)。接着用反向传播来计\n",
    "算损失函数对于图片像素的梯度。接着保持模型不变，对图片进行梯度下降，从而生成一张新图片来降低损失\n",
    "(loss)。(笔者注: 模型里面的参数是不参与梯度下降的，只有输入的图片像素点会根据梯度做调整)\n",
    "\n",
    "我们在这次作业里面主要探索三种图片生成的技巧:\n",
    "1. Saliency Maps: Saliency maps 是一个很快的方法来说明图片中的哪些部分影响了模型对于最后那个分类\n",
    "label的判断。\n",
    "2. Fooling Images: 我们可以扰乱一个输入的图片，使它对于人类来说看起来是一样的，但是会被我们的与训\n",
    "练的模型分错类。\n",
    "3. Class Visualization: 我们可以合成一张图片来最大化一个特定类的打分;这可以给我们一些直观感受，来看\n",
    "看模型在判断图片是当前这个类的时候它在关注的是图片的哪些部分\n",
    "\n",
    "我们预先训练过的模型是在经过预处理的图像上进行训练的，这些图像通过减去每个颜色的均值和除以每个颜色的标准偏差来进行预处理。我们定义了一些用于执行和取消此预处理的helper函数。在这个单元格里你不需要做任何事情"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torchvision\n",
    "import torchvision.transforms as T\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "from scipy.ndimage.filters import gaussian_filter1d\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess(img, size=224):\n",
    "    transform = T.Compose([\n",
    "        T.Scale(size),\n",
    "        T.ToTensor(),\n",
    "        T.Normalize(mean=SQUEEZENET_MEAN.tolist(),\n",
    "                    std=SQUEEZENET_STD.tolist()),\n",
    "        T.Lambda(lambda x: x[None]),\n",
    "    ])\n",
    "    return transform(img)\n",
    "\n",
    "def deprocess(img, should_rescale=True):\n",
    "    transform = T.Compose([\n",
    "        T.Lambda(lambda x: x[0]),\n",
    "        T.Normalize(mean=[0, 0, 0], std=(1.0 / SQUEEZENET_STD).tolist()),\n",
    "        T.Normalize(mean=(-SQUEEZENET_MEAN).tolist(), std=[1, 1, 1]),\n",
    "        T.Lambda(rescale) if should_rescale else T.Lambda(lambda x: x),\n",
    "        T.ToPILImage(),\n",
    "    ])\n",
    "    return transform(img)\n",
    "\n",
    "def rescale(x):\n",
    "    low, high = x.min(), x.max()\n",
    "    x_rescaled = (x - low) / (high - low)\n",
    "    return x_rescaled\n",
    "    \n",
    "def blur_image(X, sigma=1):\n",
    "    X_np = X.cpu().clone().numpy()\n",
    "    X_np = gaussian_filter1d(X_np, sigma, axis=2)\n",
    "    X_np = gaussian_filter1d(X_np, sigma, axis=3)\n",
    "    X.copy_(torch.Tensor(X_np).type_as(X))\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Download and load the pretrained SqueezeNet model.\n",
    "model = torchvision.models.squeezenet1_1(pretrained=True)\n",
    "\n",
    "# We don't want to train the model, so tell PyTorch not to compute gradients\n",
    "# with respect to model parameters.\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SqueezeNet(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d (3, 64, kernel_size=(3, 3), stride=(2, 2))\n",
       "    (1): ReLU(inplace)\n",
       "    (2): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1))\n",
       "    (3): Fire(\n",
       "      (squeeze): Conv2d (64, 16, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (16, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (4): Fire(\n",
       "      (squeeze): Conv2d (128, 16, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (16, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (5): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1))\n",
       "    (6): Fire(\n",
       "      (squeeze): Conv2d (128, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (32, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (7): Fire(\n",
       "      (squeeze): Conv2d (256, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (32, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (8): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1))\n",
       "    (9): Fire(\n",
       "      (squeeze): Conv2d (256, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (48, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (10): Fire(\n",
       "      (squeeze): Conv2d (384, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (48, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (11): Fire(\n",
       "      (squeeze): Conv2d (384, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "    (12): Fire(\n",
       "      (squeeze): Conv2d (512, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (squeeze_activation): ReLU(inplace)\n",
       "      (expand1x1): Conv2d (64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand1x1_activation): ReLU(inplace)\n",
       "      (expand3x3): Conv2d (64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (expand3x3_activation): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Dropout(p=0.5)\n",
       "    (1): Conv2d (512, 1000, kernel_size=(1, 1), stride=(1, 1))\n",
       "    (2): ReLU(inplace)\n",
       "    (3): AvgPool2d(kernel_size=13, stride=1, padding=0, ceil_mode=False, count_include_pad=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input=Variable(torch.Tensor(1,3,224,224))\n",
    "out=model(input)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- X: numpy array with shape [num, 224, 224, 3]\n",
    "- y: numpy array of integer image labels, shape [num]\n",
    "- class_names: dict mapping integer label to class name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saliency Maps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一张saliency map告诉了我们在图片中的每个像素点对于这张图片最后的预测得分的影响程度。为了计算它，我们\n",
    "要计算正确的那个类的未归一化的打分对于图片中每个像素点的梯度。如果图片的尺寸是(H,W,3),那么梯度的尺寸\n",
    "也应该是(H,W,3);对于图片中的每个像素点,梯度值反映了如果某个像素点的值改变一点点，分类的打分(score)会改\n",
    "变的程度大小。为了计算saliency map, 我们用梯度的绝对值，然后在3个channel上面求最大值，因此最后的\n",
    "saliency map的形状应该是(H,W)，并且所有的值都是非负数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.5036  0.4709 -0.9879  0.1580  0.4326\n",
      "-1.1120  0.0444 -0.9021  2.2720 -1.7631\n",
      " 0.8349 -0.9070 -1.7829  0.0620 -0.0225\n",
      "-0.3118 -1.4760  0.2954  0.7395 -0.3158\n",
      "[torch.FloatTensor of size 4x5]\n",
      "\n",
      "\n",
      " 1\n",
      " 2\n",
      " 1\n",
      " 3\n",
      "[torch.LongTensor of size 4]\n",
      "\n",
      "\n",
      " 0.4709\n",
      "-0.9021\n",
      "-0.9070\n",
      " 0.7395\n",
      "[torch.FloatTensor of size 4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example of using gather to select one entry from each row in PyTorch\n",
    "def gather_example():\n",
    "    N, C = 4, 5\n",
    "    s = torch.randn(N, C)\n",
    "    y = torch.LongTensor([1, 2, 1, 3])\n",
    "    print(s)\n",
    "    print(y)\n",
    "    print(s.gather(1, y.view(-1, 1)).squeeze())\n",
    "gather_example()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_saliency_maps(X, y, model):\n",
    "    \"\"\"\n",
    "    Compute a class saliency map using the model for images X and labels y.\n",
    "\n",
    "    Input:\n",
    "    - X: Input images; Tensor of shape (N, 3, H, W)\n",
    "    - y: Labels for X; LongTensor of shape (N,)\n",
    "    - model: A pretrained CNN that will be used to compute the saliency map.\n",
    "\n",
    "    Returns:\n",
    "    - saliency: A Tensor of shape (N, H, W) giving the saliency maps for the input\n",
    "    images.\n",
    "    \"\"\"\n",
    "    # Make sure the model is in \"test\" mode\n",
    "    model.eval()\n",
    "    \n",
    "    # Wrap the input tensors in Variables\n",
    "    X_var = Variable(X, requires_grad=True)\n",
    "    y_var = Variable(y)\n",
    "    saliency = None\n",
    "    ##############################################################################\n",
    "    # TODO: Implement this function. Perform a forward and backward pass through #\n",
    "    # the model to compute the gradient of the correct class score with respect  #\n",
    "    # to each input image. You first want to compute the loss over the correct   #\n",
    "    # scores, and then compute the gradients with a backward pass.               #\n",
    "    ##############################################################################\n",
    "    pred = model(X_var)\n",
    "    #进行softmax\n",
    "    pred = torch.exp(pred - torch.max(pred,dim=1)[0].expand_as(pred) )\n",
    "    grad = torch.sum(pred.gather(1,y_var.view(-1,1)) / torch.sum(pred,dim=1))\n",
    "    grad.backward()\n",
    "    saliency = torch.abs(X_var.grad).data\n",
    "    saliency = torch.max(saliency,dim=1)[0].squeeze()\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    ##############################################################################\n",
    "    #                             END OF YOUR CODE                               #\n",
    "    ##############################################################################\n",
    "    return saliency"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 运行进行显示"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-ba12047e71e8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m \u001b[0mshow_saliency_maps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'X' is not defined"
     ]
    }
   ],
   "source": [
    "def show_saliency_maps(X, y):\n",
    "    # Convert X and y from numpy arrays to Torch Tensors\n",
    "    X_tensor = torch.cat([preprocess(Image.fromarray(x)) for x in X], dim=0)\n",
    "    y_tensor = torch.LongTensor(y)\n",
    "\n",
    "    # Compute saliency maps for images in X\n",
    "    saliency = compute_saliency_maps(X_tensor, y_tensor, model)\n",
    "\n",
    "    # Convert the saliency map from Torch Tensor to numpy array and show images\n",
    "    # and saliency maps together.\n",
    "    saliency = saliency.numpy()\n",
    "    N = X.shape[0]\n",
    "    for i in range(N):\n",
    "        plt.subplot(2, N, i + 1)\n",
    "        plt.imshow(X[i])\n",
    "        plt.axis('off')\n",
    "        plt.title(class_names[y[i]])\n",
    "        plt.subplot(2, N, N + i + 1)\n",
    "        plt.imshow(saliency[i], cmap=plt.cm.hot)\n",
    "        plt.axis('off')\n",
    "        plt.gcf().set_size_inches(12, 5)\n",
    "    plt.show()\n",
    "\n",
    "show_saliency_maps(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fooling Images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们也可以用图像梯度来生成一些\"fooling images\"，正如[3]中讨论的那样。 给定了一张图片和一个目标的类，我\n",
    "们可以在图片上做梯度上升来最大化目标类的分数，直到神经网络把这个图片预测为目标类位置。 实现下面的函数\n",
    "来生成fooling images。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_fooling_image(X, target_y, model):\n",
    "    \"\"\"\n",
    "    Generate a fooling image that is close to X, but that the model classifies\n",
    "    as target_y.\n",
    "\n",
    "    Inputs:\n",
    "    - X: Input image; Tensor of shape (1, 3, 224, 224)\n",
    "    - target_y: An integer in the range [0, 1000)\n",
    "    - model: A pretrained CNN\n",
    "\n",
    "    Returns:\n",
    "    - X_fooling: An image that is close to X, but that is classifed as target_y\n",
    "    by the model.\n",
    "    \"\"\"\n",
    "    # Initialize our fooling image to the input image, and wrap it in a Variable.\n",
    "    X_fooling = X.clone()\n",
    "    X_fooling_var = Variable(X_fooling, requires_grad=True)\n",
    "    \n",
    "    learning_rate = 1\n",
    "    ##############################################################################\n",
    "    # TODO: Generate a fooling image X_fooling that the model will classify as   #\n",
    "    # the class target_y. You should perform gradient ascent on the score of the #\n",
    "    # target class, stopping when the model is fooled.                           #\n",
    "    # When computing an update step, first normalize the gradient:               #\n",
    "    #   dX = learning_rate * g / ||g||_2                                         #\n",
    "    #                                                                            #\n",
    "    # You should write a training loop.                                          #\n",
    "    #                                                                            #\n",
    "    # HINT: For most examples, you should be able to generate a fooling image    #\n",
    "    # in fewer than 100 iterations of gradient ascent.                           #\n",
    "    # You can print your progress over iterations to check your algorithm.       #\n",
    "    ##############################################################################\n",
    "    for i in range(100):\n",
    "        pred=model(X_fooling_var)#1xclasses\n",
    "        pred=(pred-pred.max(1)[0].expand_as(pred))\n",
    "        grad = (pred[0,target_y]/torch.sum(pred,1)).squeeze()\n",
    "        grad.backward()\n",
    "        norm=torch.sqrt(torch.sum(X_fooling_var.grad.data**2))\n",
    "        X_fooling_var.data+=learning_rate*X_fooling_var.grad.data/norm\n",
    "        X_fooling_var.grad.data.zero_()\n",
    "        if torch.max(pred,1)[0]==target_y:\n",
    "            break\n",
    "        \n",
    "    ##############################################################################\n",
    "    #                             END OF YOUR CODE                               #\n",
    "    ##############################################################################\n",
    "    return X_fooling_var.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idx = 0\n",
    "target_y = 6\n",
    "\n",
    "X_tensor = torch.cat([preprocess(Image.fromarray(x)) for x in X], dim=0)\n",
    "X_fooling = make_fooling_image(X_tensor[idx:idx+1], target_y, model)\n",
    "\n",
    "scores = model(Variable(X_fooling))\n",
    "assert target_y == scores.data.max(1)[1][0, 0], 'The model is not fooled!'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_fooling_np = deprocess(X_fooling.clone())\n",
    "X_fooling_np = np.asarray(X_fooling_np).astype(np.uint8)\n",
    "\n",
    "plt.subplot(1, 4, 1)\n",
    "plt.imshow(X[idx])\n",
    "plt.title(class_names[y[idx]])\n",
    "plt.axis('off')\n",
    "\n",
    "plt.subplot(1, 4, 2)\n",
    "plt.imshow(X_fooling_np)\n",
    "plt.title(class_names[target_y])\n",
    "plt.axis('off')\n",
    "\n",
    "plt.subplot(1, 4, 3)\n",
    "X_pre = preprocess(Image.fromarray(X[idx]))\n",
    "diff = np.asarray(deprocess(X_fooling - X_pre, should_rescale=False))\n",
    "plt.imshow(diff)\n",
    "plt.title('Difference')\n",
    "plt.axis('off')\n",
    "\n",
    "plt.subplot(1, 4, 4)\n",
    "diff = np.asarray(deprocess(10 * (X_fooling - X_pre), should_rescale=False))\n",
    "plt.imshow(diff)\n",
    "plt.title('Magnified difference (10x)')\n",
    "plt.axis('off')\n",
    "\n",
    "plt.gcf().set_size_inches(12, 5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 类别可视化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过产生一个随机噪声的图片，然后在目标类上做梯度上升，我们就可以生成一张模型会认为是目标类的图片了。\n",
    "这个想法第一次是在[2]中发表的；[3]又拓展了这个想法通过加入一些正则的技巧来提升生成图片的质量。\n",
    "具体来说，假设 是一张图片， 是目标类. 假设 是卷积网络在图片 是目标类 上面的打分; 注意这些都是未归\n",
    "一化的打分, 并不是类的概率.我们希望通过解决下面这个公式来可以生成一张图片 能够在目标类 上得分高。\n",
    "$$\n",
    "I^* = \\arg\\max_I s_y(I) - R(I)\n",
    "$$\n",
    "其中 是一个正则项(注意 在argmax中的符号: 我们希望减小这个正则项)。我们可以用梯度上升来解决这个优\n",
    "化问题, 计算对于生成图片上的梯度. 我们用明确的L2正则项:\n",
    "$$\n",
    "R(I) = \\lambda \\|I\\|_2^2\n",
    "$$  \n",
    "以及 [3]中建议的隐式的正则向，通过周期性的模糊图片。我们可以对生成图片做梯度上升来解决这个问题。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def jitter(X, ox, oy):\n",
    "    \"\"\"\n",
    "    Helper function to randomly jitter an image.\n",
    "    \n",
    "    Inputs\n",
    "    - X: PyTorch Tensor of shape (N, C, H, W)\n",
    "    - ox, oy: Integers giving number of pixels to jitter along W and H axes\n",
    "    \n",
    "    Returns: A new PyTorch Tensor of shape (N, C, H, W)\n",
    "    \"\"\"\n",
    "    if ox != 0:\n",
    "        left = X[:, :, :, :-ox]\n",
    "        right = X[:, :, :, -ox:]\n",
    "        X = torch.cat([right, left], dim=3)\n",
    "    if oy != 0:\n",
    "        top = X[:, :, :-oy]\n",
    "        bottom = X[:, :, -oy:]\n",
    "        X = torch.cat([bottom, top], dim=2)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SQUEEZENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)\n",
    "SQUEEZENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)\n",
    "def create_class_visualization(target_y, model, dtype, **kwargs):\n",
    "    \"\"\"\n",
    "    Generate an image to maximize the score of target_y under a pretrained model.\n",
    "    \n",
    "    Inputs:\n",
    "    - target_y: Integer in the range [0, 1000) giving the index of the class\n",
    "    - model: A pretrained CNN that will be used to generate the image\n",
    "    - dtype: Torch datatype to use for computations\n",
    "    \n",
    "    Keyword arguments:\n",
    "    - l2_reg: Strength of L2 regularization on the image\n",
    "    - learning_rate: How big of a step to take\n",
    "    - num_iterations: How many iterations to use\n",
    "    - blur_every: How often to blur the image as an implicit regularizer\n",
    "    - max_jitter: How much to gjitter the image as an implicit regularizer\n",
    "    - show_every: How often to show the intermediate result\n",
    "    \"\"\"\n",
    "    model.type(dtype)\n",
    "    l2_reg = kwargs.pop('l2_reg', 1e-3)\n",
    "    learning_rate = kwargs.pop('learning_rate', 25)\n",
    "    num_iterations = kwargs.pop('num_iterations', 100)\n",
    "    blur_every = kwargs.pop('blur_every', 10)\n",
    "    max_jitter = kwargs.pop('max_jitter', 16)\n",
    "    show_every = kwargs.pop('show_every', 25)\n",
    "\n",
    "    # Randomly initialize the image as a PyTorch Tensor, and also wrap it in\n",
    "    # a PyTorch Variable.\n",
    "    img = torch.randn(1, 3, 224, 224).mul_(1.0).type(dtype)\n",
    "    img_var = Variable(img, requires_grad=True)\n",
    "\n",
    "    for t in range(num_iterations):\n",
    "        # Randomly jitter the image a bit; this gives slightly nicer results\n",
    "        ox, oy = random.randint(0, max_jitter), random.randint(0, max_jitter)\n",
    "        img.copy_(jitter(img, ox, oy))\n",
    "\n",
    "        ########################################################################\n",
    "        # TODO: Use the model to compute the gradient of the score for the     #\n",
    "        # class target_y with respect to the pixels of the image, and make a   #\n",
    "        # gradient step on the image using the learning rate. Don't forget the #\n",
    "        # L2 regularization term!                                              #\n",
    "        # Be very careful about the signs of elements in your code.            #\n",
    "        ########################################################################\n",
    "        img_var.data.copy_(img)\n",
    "        pred=model(img_var)\n",
    "        loss=pred[0,target_y]-l2_reg*torch.sum(img*img)\n",
    "        loss.backward()\n",
    "        img=img+learning_rate*img_var.grad.data\n",
    "        img_var.grad.data.zero_()\n",
    "        ########################################################################\n",
    "        #                             END OF YOUR CODE                         #\n",
    "        ########################################################################\n",
    "        \n",
    "        # Undo the random jitter\n",
    "        img.copy_(jitter(img, -ox, -oy))\n",
    "\n",
    "        # As regularizer, clamp and periodically blur the image\n",
    "        for c in range(3):\n",
    "            lo = float(-SQUEEZENET_MEAN[c] / SQUEEZENET_STD[c])\n",
    "            hi = float((1.0 - SQUEEZENET_MEAN[c]) / SQUEEZENET_STD[c])\n",
    "            img[:, c].clamp_(min=lo, max=hi)\n",
    "        if t % blur_every == 0:\n",
    "            blur_image(img, sigma=0.5)\n",
    "        \n",
    "        # Periodically show the image\n",
    "        if t == 0 or (t + 1) % show_every == 0 or t == num_iterations - 1:\n",
    "            plt.imshow(deprocess(img.clone().cpu()))\n",
    "            class_name = class_names[target_y]\n",
    "            plt.title('%s\\nIteration %d / %d' % (class_name, t + 1, num_iterations))\n",
    "            plt.gcf().set_size_inches(4, 4)\n",
    "            plt.axis('off')\n",
    "            plt.show()\n",
    "\n",
    "    return deprocess(img.cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'SQUEEZENET_MEAN' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-7feb18930c71>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;31m# target_y = 366 # Gorilla\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;31m# target_y = 604 # Hourglass\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcreate_class_visualization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-12-ee056232d253>\u001b[0m in \u001b[0;36mcreate_class_visualization\u001b[1;34m(target_y, model, dtype, **kwargs)\u001b[0m\n\u001b[0;32m     56\u001b[0m         \u001b[1;31m# As regularizer, clamp and periodically blur the image\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 58\u001b[1;33m             \u001b[0mlo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mSQUEEZENET_MEAN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mSQUEEZENET_STD\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     59\u001b[0m             \u001b[0mhi\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1.0\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mSQUEEZENET_MEAN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mSQUEEZENET_STD\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     60\u001b[0m             \u001b[0mimg\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclamp_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlo\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mhi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'SQUEEZENET_MEAN' is not defined"
     ]
    }
   ],
   "source": [
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to use GPU\n",
    "model.type(dtype)\n",
    "\n",
    "target_y = 76 # Tarantula\n",
    "# target_y = 78 # Tick\n",
    "# target_y = 187 # Yorkshire Terrier\n",
    "# target_y = 683 # Oboe\n",
    "# target_y = 366 # Gorilla\n",
    "# target_y = 604 # Hourglass\n",
    "out = create_class_visualization(target_y, model, dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# target_y = 78 # Tick\n",
    "# target_y = 187 # Yorkshire Terrier\n",
    "# target_y = 683 # Oboe\n",
    "# target_y = 366 # Gorilla\n",
    "# target_y = 604 # Hourglass\n",
    "target_y = np.random.randint(1000)\n",
    "print(class_names[target_y])\n",
    "X = create_class_visualization(target_y, model, dtype)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}