examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PyTorchViz examples\n",
    "=========="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "%pip install -U git+https://github.com/szagoruyko/pytorchviz.git@master\n",
    "from torchviz import make_dot, make_dot_from_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize gradients of simple MLP\n",
    "\n",
    "The method below is for building directed graphs of PyTorch operations, built during forward propagation and showing which operations will be called on backward. It omits subgraphs which do not require gradients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"333pt\" height=\"413pt\"\n",
       " viewBox=\"0.00 0.00 333.00 413.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 409)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-409 329,-409 329,4 -4,4\"/>\n",
       "<!-- 140336274140720 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140336274140720</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"193,-31 134,-31 134,0 193,0 193,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"163.5\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> (1, 1)</text>\n",
       "</g>\n",
       "<!-- 140327258376704 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140327258376704</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"211,-86 116,-86 116,-67 211,-67 211,-86\"/>\n",
       "<text text-anchor=\"middle\" x=\"163.5\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258376704&#45;&gt;140336274140720 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140327258376704&#45;&gt;140336274140720</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.5,-66.79C163.5,-60.07 163.5,-50.4 163.5,-41.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"167,-41.19 163.5,-31.19 160,-41.19 167,-41.19\"/>\n",
       "</g>\n",
       "<!-- 140327258376864 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140327258376864</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"101,-141 0,-141 0,-122 101,-122 101,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327258376864&#45;&gt;140327258376704 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140327258376864&#45;&gt;140327258376704</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68.66,-121.98C86.75,-113.5 114.78,-100.35 135.69,-90.54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.45,-93.59 145.01,-86.17 134.47,-87.25 137.45,-93.59\"/>\n",
       "</g>\n",
       "<!-- 140327258874320 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140327258874320</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"80,-207 21,-207 21,-177 80,-177 80,-207\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-195\" font-family=\"monospace\" font-size=\"10.00\">W1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\"> (1)</text>\n",
       "</g>\n",
       "<!-- 140327258874320&#45;&gt;140327258376864 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140327258874320&#45;&gt;140327258376864</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-176.84C50.5,-169.21 50.5,-159.7 50.5,-151.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-151.27 50.5,-141.27 47,-151.27 54,-151.27\"/>\n",
       "</g>\n",
       "<!-- 140327258376784 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140327258376784</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"208,-141 119,-141 119,-122 208,-122 208,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"163.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 140327258376784&#45;&gt;140327258376704 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140327258376784&#45;&gt;140327258376704</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.5,-121.75C163.5,-114.8 163.5,-104.85 163.5,-96.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"167,-96.09 163.5,-86.09 160,-96.09 167,-96.09\"/>\n",
       "</g>\n",
       "<!-- 140336656532016 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140336656532016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"206,-201.5 111,-201.5 111,-182.5 206,-182.5 206,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.5\" y=\"-189.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140336656532016&#45;&gt;140327258376784 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140336656532016&#45;&gt;140327258376784</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M159.24,-182.37C159.93,-174.25 161,-161.81 161.89,-151.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"165.39,-151.43 162.76,-141.17 158.42,-150.83 165.39,-151.43\"/>\n",
       "</g>\n",
       "<!-- 140336656533216 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140336656533216</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"118,-267.5 17,-267.5 17,-248.5 118,-248.5 118,-267.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-255.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140336656533216&#45;&gt;140336656532016 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140336656533216&#45;&gt;140336656532016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M79.77,-248.37C94.56,-237.97 119.65,-220.32 137.53,-207.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"139.96,-210.31 146.13,-201.7 135.94,-204.59 139.96,-210.31\"/>\n",
       "</g>\n",
       "<!-- 140327258844912 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140327258844912</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"97,-339 38,-339 38,-309 97,-309 97,-339\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-327\" font-family=\"monospace\" font-size=\"10.00\">W0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-316\" font-family=\"monospace\" font-size=\"10.00\"> (16)</text>\n",
       "</g>\n",
       "<!-- 140327258844912&#45;&gt;140336656533216 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140327258844912&#45;&gt;140336656533216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M67.5,-308.8C67.5,-299.7 67.5,-287.79 67.5,-277.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"71,-277.84 67.5,-267.84 64,-277.84 71,-277.84\"/>\n",
       "</g>\n",
       "<!-- 140336656533696 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140336656533696</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"207,-267.5 136,-267.5 136,-248.5 207,-248.5 207,-267.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"171.5\" y=\"-255.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140336656533696&#45;&gt;140336656532016 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140336656533696&#45;&gt;140336656532016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.75,-248.37C167.86,-239.07 164.79,-223.98 162.34,-211.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"165.73,-211.01 160.31,-201.91 158.87,-212.4 165.73,-211.01\"/>\n",
       "</g>\n",
       "<!-- 140327289353872 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140327289353872</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"222,-333.5 121,-333.5 121,-314.5 222,-314.5 222,-333.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"171.5\" y=\"-321.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327289353872&#45;&gt;140336656533696 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140327289353872&#45;&gt;140336656533696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M171.5,-314.37C171.5,-305.16 171.5,-290.29 171.5,-278.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"175,-277.91 171.5,-267.91 168,-277.91 175,-277.91\"/>\n",
       "</g>\n",
       "<!-- 140327258843952 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140327258843952</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"207,-405 136,-405 136,-375 207,-375 207,-405\"/>\n",
       "<text text-anchor=\"middle\" x=\"171.5\" y=\"-393\" font-family=\"monospace\" font-size=\"10.00\">W0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"171.5\" y=\"-382\" font-family=\"monospace\" font-size=\"10.00\"> (16, 8)</text>\n",
       "</g>\n",
       "<!-- 140327258843952&#45;&gt;140327289353872 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140327258843952&#45;&gt;140327289353872</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M171.5,-374.8C171.5,-365.7 171.5,-353.79 171.5,-343.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"175,-343.84 171.5,-333.84 168,-343.84 175,-343.84\"/>\n",
       "</g>\n",
       "<!-- 140327258375984 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140327258375984</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"301,-141 230,-141 230,-122 301,-122 301,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"265.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258375984&#45;&gt;140327258376704 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140327258375984&#45;&gt;140327258376704</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M249.11,-121.98C232.93,-113.57 207.93,-100.59 189.12,-90.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"190.67,-87.68 180.19,-86.17 187.45,-93.89 190.67,-87.68\"/>\n",
       "</g>\n",
       "<!-- 140336656576016 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140336656576016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"325,-201.5 224,-201.5 224,-182.5 325,-182.5 325,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"274.5\" y=\"-189.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140336656576016&#45;&gt;140327258375984 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140336656576016&#45;&gt;140327258375984</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M273.17,-182.37C271.91,-174.16 269.97,-161.54 268.35,-151.05\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"271.81,-150.52 266.83,-141.17 264.89,-151.58 271.81,-150.52\"/>\n",
       "</g>\n",
       "<!-- 140336274101840 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140336274101840</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"310,-273 239,-273 239,-243 310,-243 310,-273\"/>\n",
       "<text text-anchor=\"middle\" x=\"274.5\" y=\"-261\" font-family=\"monospace\" font-size=\"10.00\">W1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"274.5\" y=\"-250\" font-family=\"monospace\" font-size=\"10.00\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 140336274101840&#45;&gt;140336656576016 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140336274101840&#45;&gt;140336656576016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M274.5,-242.8C274.5,-233.7 274.5,-221.79 274.5,-211.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"278,-211.84 274.5,-201.84 271,-211.84 278,-211.84\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa07c634960>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = nn.Sequential()\n",
    "model.add_module('W0', nn.Linear(8, 16))\n",
    "model.add_module('tanh', nn.Tanh())\n",
    "model.add_module('W1', nn.Linear(16, 1))\n",
    "\n",
    "x = torch.randn(1,8)\n",
    "\n",
    "make_dot(model(x), params=dict(model.named_parameters()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specify `show_attrs=True` and `show_saved=True` to see what autograd saves for the backward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"457pt\" height=\"622pt\"\n",
       " viewBox=\"0.00 0.00 457.00 622.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 618)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-618 453,-618 453,4 -4,4\"/>\n",
       "<!-- 140327258437872 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140327258437872</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"104,-31 45,-31 45,0 104,0 104,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"74.5\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> (1, 1)</text>\n",
       "</g>\n",
       "<!-- 140327258397024 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140327258397024</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"247,-185 62,-185 62,-67 247,-67 247,-185\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-173\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-162\" font-family=\"monospace\" font-size=\"10.00\">&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-151\" font-family=\"monospace\" font-size=\"10.00\">alpha &#160;&#160;&#160;&#160;&#160;&#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;1</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-140\" font-family=\"monospace\" font-size=\"10.00\">beta &#160;&#160;&#160;&#160;&#160;&#160;&#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;1</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">mat1 &#160;&#160;&#160;&#160;&#160;&#160;&#160;: [saved tensor]</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-118\" font-family=\"monospace\" font-size=\"10.00\">mat1_sizes &#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;(1, 16)</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-107\" font-family=\"monospace\" font-size=\"10.00\">mat1_strides: &#160;&#160;&#160;&#160;&#160;&#160;&#160;(16, 1)</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-96\" font-family=\"monospace\" font-size=\"10.00\">mat2 &#160;&#160;&#160;&#160;&#160;&#160;&#160;: [saved tensor]</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-85\" font-family=\"monospace\" font-size=\"10.00\">mat2_sizes &#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;(16, 1)</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">mat2_strides: &#160;&#160;&#160;&#160;&#160;&#160;&#160;(1, 16)</text>\n",
       "</g>\n",
       "<!-- 140327258397024&#45;&gt;140327258437872 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140327258397024&#45;&gt;140327258437872</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M111.69,-66.94C104.49,-57.17 97.41,-47.57 91.39,-39.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"94.07,-37.15 85.32,-31.18 88.44,-41.3 94.07,-37.15\"/>\n",
       "</g>\n",
       "<!-- 140327258439504 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140327258439504</title>\n",
       "<polygon fill=\"orange\" stroke=\"black\" points=\"187,-30.5 122,-30.5 122,-0.5 187,-0.5 187,-30.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-18.5\" font-family=\"monospace\" font-size=\"10.00\">mat1</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-7.5\" font-family=\"monospace\" font-size=\"10.00\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 140327258397024&#45;&gt;140327258439504 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140327258397024&#45;&gt;140327258439504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-66.94C154.5,-53.44 154.5,-40.25 154.5,-30.75\"/>\n",
       "</g>\n",
       "<!-- 140327258439600 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140327258439600</title>\n",
       "<polygon fill=\"orange\" stroke=\"black\" points=\"270,-30.5 205,-30.5 205,-0.5 270,-0.5 270,-30.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"237.5\" y=\"-18.5\" font-family=\"monospace\" font-size=\"10.00\">mat2</text>\n",
       "<text text-anchor=\"middle\" x=\"237.5\" y=\"-7.5\" font-family=\"monospace\" font-size=\"10.00\"> (16, 1)</text>\n",
       "</g>\n",
       "<!-- 140327258397024&#45;&gt;140327258439600 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140327258397024&#45;&gt;140327258439600</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.92,-66.94C209.25,-53.44 219.33,-40.25 226.6,-30.75\"/>\n",
       "</g>\n",
       "<!-- 140327258376864 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140327258376864</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"205,-251 104,-251 104,-232 205,-232 205,-251\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-239\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327258376864&#45;&gt;140327258397024 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140327258376864&#45;&gt;140327258397024</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-231.97C154.5,-223.63 154.5,-210.09 154.5,-195.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"158,-195.22 154.5,-185.22 151,-195.22 158,-195.22\"/>\n",
       "</g>\n",
       "<!-- 140327258874320 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140327258874320</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"184,-372 125,-372 125,-342 184,-342 184,-372\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-360\" font-family=\"monospace\" font-size=\"10.00\">W1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-349\" font-family=\"monospace\" font-size=\"10.00\"> (1)</text>\n",
       "</g>\n",
       "<!-- 140327258874320&#45;&gt;140327258376864 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140327258874320&#45;&gt;140327258376864</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-341.63C154.5,-321.21 154.5,-283.81 154.5,-261.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"158,-261.23 154.5,-251.23 151,-261.23 158,-261.23\"/>\n",
       "</g>\n",
       "<!-- 140327258374544 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140327258374544</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"449,-262 300,-262 300,-221 449,-221 449,-262\"/>\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-250\" font-family=\"monospace\" font-size=\"10.00\">TanhBackward</text>\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-239\" font-family=\"monospace\" font-size=\"10.00\">&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-228\" font-family=\"monospace\" font-size=\"10.00\">result: [saved tensor]</text>\n",
       "</g>\n",
       "<!-- 140327258374544&#45;&gt;140327258397024 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140327258374544&#45;&gt;140327258397024</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M336.68,-220.99C314.4,-209.49 285.04,-194.35 256.42,-179.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"257.65,-176.28 247.16,-174.8 254.44,-182.5 257.65,-176.28\"/>\n",
       "</g>\n",
       "<!-- 140327258456144 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140327258456144</title>\n",
       "<polygon fill=\"orange\" stroke=\"black\" points=\"407,-141 342,-141 342,-111 407,-111 407,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">result</text>\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-118\" font-family=\"monospace\" font-size=\"10.00\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 140327258374544&#45;&gt;140327258456144 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140327258374544&#45;&gt;140327258456144</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M374.5,-220.99C374.5,-198.3 374.5,-161.39 374.5,-141.24\"/>\n",
       "</g>\n",
       "<!-- 140327258375904 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140327258375904</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"427,-416 242,-416 242,-298 427,-298 427,-416\"/>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-404\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-393\" font-family=\"monospace\" font-size=\"10.00\">&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-382\" font-family=\"monospace\" font-size=\"10.00\">alpha &#160;&#160;&#160;&#160;&#160;&#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;1</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-371\" font-family=\"monospace\" font-size=\"10.00\">beta &#160;&#160;&#160;&#160;&#160;&#160;&#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;1</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-360\" font-family=\"monospace\" font-size=\"10.00\">mat1 &#160;&#160;&#160;&#160;&#160;&#160;&#160;: [saved tensor]</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-349\" font-family=\"monospace\" font-size=\"10.00\">mat1_sizes &#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;(1, 8)</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-338\" font-family=\"monospace\" font-size=\"10.00\">mat1_strides: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;()</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-327\" font-family=\"monospace\" font-size=\"10.00\">mat2 &#160;&#160;&#160;&#160;&#160;&#160;&#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;None</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-316\" font-family=\"monospace\" font-size=\"10.00\">mat2_sizes &#160;: &#160;&#160;&#160;&#160;&#160;&#160;&#160;(8, 16)</text>\n",
       "<text text-anchor=\"middle\" x=\"334.5\" y=\"-305\" font-family=\"monospace\" font-size=\"10.00\">mat2_strides: &#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;(1, 8)</text>\n",
       "</g>\n",
       "<!-- 140327258375904&#45;&gt;140327258374544 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140327258375904&#45;&gt;140327258374544</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M354.97,-297.92C358.16,-288.87 361.33,-279.86 364.16,-271.83\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"367.52,-272.82 367.55,-262.23 360.92,-270.5 367.52,-272.82\"/>\n",
       "</g>\n",
       "<!-- 140327258456912 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140327258456912</title>\n",
       "<polygon fill=\"orange\" stroke=\"black\" points=\"282,-256.5 223,-256.5 223,-226.5 282,-226.5 282,-256.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"252.5\" y=\"-244.5\" font-family=\"monospace\" font-size=\"10.00\">mat1</text>\n",
       "<text text-anchor=\"middle\" x=\"252.5\" y=\"-233.5\" font-family=\"monospace\" font-size=\"10.00\"> (1, 8)</text>\n",
       "</g>\n",
       "<!-- 140327258375904&#45;&gt;140327258456912 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140327258375904&#45;&gt;140327258456912</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M292.54,-297.92C281.51,-282.65 270.57,-267.51 262.92,-256.92\"/>\n",
       "</g>\n",
       "<!-- 140336656533216 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140336656533216</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"437,-476.5 336,-476.5 336,-457.5 437,-457.5 437,-476.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"386.5\" y=\"-464.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140336656533216&#45;&gt;140327258375904 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140336656533216&#45;&gt;140327258375904</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M382.24,-457.16C378.61,-449.62 373.03,-438.02 366.94,-425.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"369.92,-423.51 362.43,-416.01 363.62,-426.54 369.92,-423.51\"/>\n",
       "</g>\n",
       "<!-- 140327258844912 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140327258844912</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"416,-548 357,-548 357,-518 416,-518 416,-548\"/>\n",
       "<text text-anchor=\"middle\" x=\"386.5\" y=\"-536\" font-family=\"monospace\" font-size=\"10.00\">W0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"386.5\" y=\"-525\" font-family=\"monospace\" font-size=\"10.00\"> (16)</text>\n",
       "</g>\n",
       "<!-- 140327258844912&#45;&gt;140336656533216 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140327258844912&#45;&gt;140336656533216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M386.5,-517.8C386.5,-508.7 386.5,-496.79 386.5,-486.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"390,-486.84 386.5,-476.84 383,-486.84 390,-486.84\"/>\n",
       "</g>\n",
       "<!-- 140327258863408 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140327258863408</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"318,-476.5 247,-476.5 247,-457.5 318,-457.5 318,-476.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"282.5\" y=\"-464.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258863408&#45;&gt;140327258375904 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140327258863408&#45;&gt;140327258375904</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M286.76,-457.16C290.39,-449.62 295.97,-438.02 302.06,-425.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"305.38,-426.54 306.57,-416.01 299.08,-423.51 305.38,-426.54\"/>\n",
       "</g>\n",
       "<!-- 140327289353872 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140327289353872</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"333,-542.5 232,-542.5 232,-523.5 333,-523.5 333,-542.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"282.5\" y=\"-530.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327289353872&#45;&gt;140327258863408 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140327289353872&#45;&gt;140327258863408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M282.5,-523.37C282.5,-514.16 282.5,-499.29 282.5,-487.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"286,-486.91 282.5,-476.91 279,-486.91 286,-486.91\"/>\n",
       "</g>\n",
       "<!-- 140327258843952 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140327258843952</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"318,-614 247,-614 247,-584 318,-584 318,-614\"/>\n",
       "<text text-anchor=\"middle\" x=\"282.5\" y=\"-602\" font-family=\"monospace\" font-size=\"10.00\">W0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"282.5\" y=\"-591\" font-family=\"monospace\" font-size=\"10.00\"> (16, 8)</text>\n",
       "</g>\n",
       "<!-- 140327258843952&#45;&gt;140327289353872 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140327258843952&#45;&gt;140327289353872</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M282.5,-583.8C282.5,-574.7 282.5,-562.79 282.5,-552.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"286,-552.84 282.5,-542.84 279,-552.84 286,-552.84\"/>\n",
       "</g>\n",
       "<!-- 140327258374224 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140327258374224</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"86,-251 15,-251 15,-232 86,-232 86,-251\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-239\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258374224&#45;&gt;140327258397024 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140327258374224&#45;&gt;140327258397024</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M58.32,-231.97C66.4,-223.15 79.81,-208.51 94.09,-192.93\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"96.98,-194.96 101.15,-185.22 91.82,-190.23 96.98,-194.96\"/>\n",
       "</g>\n",
       "<!-- 140336656576016 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140336656576016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"101,-366.5 0,-366.5 0,-347.5 101,-347.5 101,-366.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-354.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140336656576016&#45;&gt;140327258374224 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140336656576016&#45;&gt;140327258374224</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-347.47C50.5,-329.23 50.5,-286.07 50.5,-261.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-261.13 50.5,-251.13 47,-261.13 54,-261.13\"/>\n",
       "</g>\n",
       "<!-- 140336274101840 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140336274101840</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"86,-482 15,-482 15,-452 86,-452 86,-482\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-470\" font-family=\"monospace\" font-size=\"10.00\">W1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-459\" font-family=\"monospace\" font-size=\"10.00\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 140336274101840&#45;&gt;140336656576016 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140336274101840&#45;&gt;140336656576016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-451.88C50.5,-432.59 50.5,-397.96 50.5,-376.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-376.51 50.5,-366.51 47,-376.51 54,-376.51\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa07c639a00>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_dot(model(x), params=dict(model.named_parameters()), show_attrs=True, show_saved=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Double Backpropagation\n",
    "\n",
    "Implements \"Double Backpropagation\" from [Drucker and Lecun](http://yann.lecun.com/exdb/publis/pdf/drucker-lecun-92.pdf). The idea is to minimize the loss:\n",
    "\n",
    "$$f(x, \\theta) = f(x, \\theta) + g(\\frac{\\partial f(x, \\theta)}{\\partial x})$$\n",
    "\n",
    "where $x$ and $\\theta$ are input and parameter vectors, $f(x, \\theta)$ is the original loss function, and $g$ is a function of gradient w.r.t. input.\n",
    "\n",
    "This is used in [Improved Wasserstein GAN](https://arxiv.org/abs/1704.00028) and [Attention Transfer](https://arxiv.org/abs/1612.03928)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"452pt\" height=\"622pt\"\n",
       " viewBox=\"0.00 0.00 452.00 622.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 618)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-618 448,-618 448,4 -4,4\"/>\n",
       "<!-- 140327258845008 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140327258845008</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"284.5,-31 230.5,-31 230.5,0 284.5,0 284.5,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"257.5\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> ()</text>\n",
       "</g>\n",
       "<!-- 140327258846064 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140327258846064</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"302,-86 213,-86 213,-67 302,-67 302,-86\"/>\n",
       "<text text-anchor=\"middle\" x=\"257.5\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258846064&#45;&gt;140327258845008 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140327258846064&#45;&gt;140327258845008</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M257.5,-66.79C257.5,-60.07 257.5,-50.4 257.5,-41.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"261,-41.19 257.5,-31.19 254,-41.19 261,-41.19\"/>\n",
       "</g>\n",
       "<!-- 140327258528816 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140327258528816</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"267,-141 172,-141 172,-122 267,-122 267,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"219.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">MeanBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258528816&#45;&gt;140327258846064 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140327258528816&#45;&gt;140327258846064</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M225.77,-121.75C231.09,-114.34 238.86,-103.5 245.38,-94.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"248.36,-96.26 251.34,-86.09 242.67,-92.18 248.36,-96.26\"/>\n",
       "</g>\n",
       "<!-- 140327258528656 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140327258528656</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"247,-196 158,-196 158,-177 247,-177 247,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"202.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\">PowBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258528656&#45;&gt;140327258528816 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140327258528656&#45;&gt;140327258528816</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M205.31,-176.75C207.56,-169.72 210.8,-159.62 213.62,-150.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.02,-151.68 216.75,-141.09 210.36,-149.54 217.02,-151.68\"/>\n",
       "</g>\n",
       "<!-- 140327258528976 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140327258528976</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"229,-251 152,-251 152,-232 229,-232 229,-251\"/>\n",
       "<text text-anchor=\"middle\" x=\"190.5\" y=\"-239\" font-family=\"monospace\" font-size=\"10.00\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258528976&#45;&gt;140327258528656 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140327258528976&#45;&gt;140327258528656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M192.48,-231.75C194.07,-224.72 196.36,-214.62 198.35,-205.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"201.76,-206.62 200.56,-196.09 194.93,-205.07 201.76,-206.62\"/>\n",
       "</g>\n",
       "<!-- 140327258529136 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140327258529136</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"259,-306 122,-306 122,-287 259,-287 259,-306\"/>\n",
       "<text text-anchor=\"middle\" x=\"190.5\" y=\"-294\" font-family=\"monospace\" font-size=\"10.00\">TanhBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140327258529136&#45;&gt;140327258528976 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140327258529136&#45;&gt;140327258528976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.5,-286.75C190.5,-279.8 190.5,-269.85 190.5,-261.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194,-261.09 190.5,-251.09 187,-261.09 194,-261.09\"/>\n",
       "</g>\n",
       "<!-- 140327258529376 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140327258529376</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"156,-361 79,-361 79,-342 156,-342 156,-361\"/>\n",
       "<text text-anchor=\"middle\" x=\"117.5\" y=\"-349\" font-family=\"monospace\" font-size=\"10.00\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258529376&#45;&gt;140327258529136 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140327258529376&#45;&gt;140327258529136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M129.23,-341.98C140.23,-334 156.92,-321.88 170.1,-312.31\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"172.52,-314.88 178.56,-306.17 168.41,-309.21 172.52,-314.88\"/>\n",
       "</g>\n",
       "<!-- 140327258529616 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140327258529616</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"291,-421.5 220,-421.5 220,-402.5 291,-402.5 291,-421.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"255.5\" y=\"-409.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258529616&#45;&gt;140327258529376 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140327258529616&#45;&gt;140327258529376</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M235.43,-402.49C212.13,-392.62 173.49,-376.23 146.82,-364.93\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"148.16,-361.7 137.59,-361.02 145.43,-368.14 148.16,-361.7\"/>\n",
       "</g>\n",
       "<!-- 140327258484816 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140327258484816</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"429,-482 358,-482 358,-463 429,-463 429,-482\"/>\n",
       "<text text-anchor=\"middle\" x=\"393.5\" y=\"-470\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258484816&#45;&gt;140327258529616 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140327258484816&#45;&gt;140327258529616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M373.43,-462.99C350.13,-453.12 311.49,-436.73 284.82,-425.43\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"286.16,-422.2 275.59,-421.52 283.43,-428.64 286.16,-422.2\"/>\n",
       "</g>\n",
       "<!-- 140327258528896 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140327258528896</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"375,-306 280,-306 280,-287 375,-287 375,-306\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-294\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258484816&#45;&gt;140327258528896 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140327258484816&#45;&gt;140327258528896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M395.77,-462.97C401.06,-441.25 412.18,-382.69 390.5,-342 383.31,-328.5 369.98,-318.18 357.43,-310.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"359,-307.77 348.53,-306.13 355.7,-313.94 359,-307.77\"/>\n",
       "</g>\n",
       "<!-- 140336656576016 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140336656576016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"444,-542.5 343,-542.5 343,-523.5 444,-523.5 444,-542.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"393.5\" y=\"-530.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140336656576016&#45;&gt;140327258484816 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140336656576016&#45;&gt;140327258484816</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M393.5,-523.37C393.5,-515.25 393.5,-502.81 393.5,-492.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"397,-492.17 393.5,-482.17 390,-492.17 397,-492.17\"/>\n",
       "</g>\n",
       "<!-- 140336274101840 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140336274101840</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"429,-614 358,-614 358,-584 429,-584 429,-614\"/>\n",
       "<text text-anchor=\"middle\" x=\"393.5\" y=\"-602\" font-family=\"monospace\" font-size=\"10.00\">W1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"393.5\" y=\"-591\" font-family=\"monospace\" font-size=\"10.00\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 140336274101840&#45;&gt;140336656576016 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140336274101840&#45;&gt;140336656576016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M393.5,-583.8C393.5,-574.7 393.5,-562.79 393.5,-552.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"397,-552.84 393.5,-542.84 390,-552.84 397,-552.84\"/>\n",
       "</g>\n",
       "<!-- 140327258529456 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140327258529456</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"263,-361 174,-361 174,-342 263,-342 263,-361\"/>\n",
       "<text text-anchor=\"middle\" x=\"218.5\" y=\"-349\" font-family=\"monospace\" font-size=\"10.00\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 140327258529456&#45;&gt;140327258529136 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140327258529456&#45;&gt;140327258529136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M213.88,-341.75C210.04,-334.49 204.48,-323.95 199.74,-314.98\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"202.8,-313.3 195.04,-306.09 196.61,-316.57 202.8,-313.3\"/>\n",
       "</g>\n",
       "<!-- 140327258529456&#45;&gt;140327258528896 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140327258529456&#45;&gt;140327258528896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M236.01,-341.98C253.47,-333.5 280.51,-320.35 300.67,-310.54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"302.21,-313.69 309.67,-306.17 299.14,-307.4 302.21,-313.69\"/>\n",
       "</g>\n",
       "<!-- 140327258529696 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140327258529696</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"202,-421.5 107,-421.5 107,-402.5 202,-402.5 202,-421.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-409.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258529696&#45;&gt;140327258529456 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140327258529696&#45;&gt;140327258529456</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.94,-402.37C173.77,-393.38 189.39,-379.11 201.39,-368.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"204,-370.5 209.02,-361.17 199.28,-365.33 204,-370.5\"/>\n",
       "</g>\n",
       "<!-- 140336656533216 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140336656533216</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"205,-482 104,-482 104,-463 205,-463 205,-482\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-470\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140336656533216&#45;&gt;140327258529696 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140336656533216&#45;&gt;140327258529696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-462.87C154.5,-454.75 154.5,-442.31 154.5,-431.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"158,-431.67 154.5,-421.67 151,-431.67 158,-431.67\"/>\n",
       "</g>\n",
       "<!-- 140327258844912 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140327258844912</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"184,-548 125,-548 125,-518 184,-518 184,-548\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-536\" font-family=\"monospace\" font-size=\"10.00\">W0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-525\" font-family=\"monospace\" font-size=\"10.00\"> (16)</text>\n",
       "</g>\n",
       "<!-- 140327258844912&#45;&gt;140336656533216 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140327258844912&#45;&gt;140336656533216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-517.84C154.5,-510.21 154.5,-500.7 154.5,-492.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"158,-492.27 154.5,-482.27 151,-492.27 158,-492.27\"/>\n",
       "</g>\n",
       "<!-- 140327258484896 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140327258484896</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"324,-482 223,-482 223,-463 324,-463 324,-482\"/>\n",
       "<text text-anchor=\"middle\" x=\"273.5\" y=\"-470\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327258484896&#45;&gt;140327258529696 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140327258484896&#45;&gt;140327258529696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M256.2,-462.99C236.45,-453.29 203.95,-437.31 181.01,-426.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"182.34,-422.79 171.82,-421.52 179.25,-429.07 182.34,-422.79\"/>\n",
       "</g>\n",
       "<!-- 140327258459024 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140327258459024</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"303,-548 244,-548 244,-518 303,-518 303,-548\"/>\n",
       "<text text-anchor=\"middle\" x=\"273.5\" y=\"-536\" font-family=\"monospace\" font-size=\"10.00\">x</text>\n",
       "<text text-anchor=\"middle\" x=\"273.5\" y=\"-525\" font-family=\"monospace\" font-size=\"10.00\"> (1, 8)</text>\n",
       "</g>\n",
       "<!-- 140327258459024&#45;&gt;140327258484896 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140327258459024&#45;&gt;140327258484896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M273.5,-517.84C273.5,-510.21 273.5,-500.7 273.5,-492.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"277,-492.27 273.5,-482.27 270,-492.27 277,-492.27\"/>\n",
       "</g>\n",
       "<!-- 140327258484976 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140327258484976</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"86,-482 15,-482 15,-463 86,-463 86,-482\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-470\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258484976&#45;&gt;140327258529696 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140327258484976&#45;&gt;140327258529696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M65.85,-462.87C82.82,-453.32 110.4,-437.81 130.29,-426.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"132.09,-429.62 139.09,-421.67 128.66,-423.52 132.09,-429.62\"/>\n",
       "</g>\n",
       "<!-- 140327258529216 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140327258529216</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"98,-306 27,-306 27,-287 98,-287 98,-306\"/>\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-294\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140327258484976&#45;&gt;140327258529216 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140327258484976&#45;&gt;140327258529216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.11,-462.63C52.98,-435.52 58.65,-353.32 61.22,-316.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"64.71,-316.36 61.9,-306.14 57.72,-315.88 64.71,-316.36\"/>\n",
       "</g>\n",
       "<!-- 140327289353872 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140327289353872</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"101,-542.5 0,-542.5 0,-523.5 101,-523.5 101,-542.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-530.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327289353872&#45;&gt;140327258484976 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140327289353872&#45;&gt;140327258484976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-523.37C50.5,-515.25 50.5,-502.81 50.5,-492.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-492.17 50.5,-482.17 47,-492.17 54,-492.17\"/>\n",
       "</g>\n",
       "<!-- 140327258843952 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140327258843952</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"86,-614 15,-614 15,-584 86,-584 86,-614\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-602\" font-family=\"monospace\" font-size=\"10.00\">W0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-591\" font-family=\"monospace\" font-size=\"10.00\"> (16, 8)</text>\n",
       "</g>\n",
       "<!-- 140327258843952&#45;&gt;140327289353872 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140327258843952&#45;&gt;140327289353872</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-583.8C50.5,-574.7 50.5,-562.79 50.5,-552.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-552.84 50.5,-542.84 47,-552.84 54,-552.84\"/>\n",
       "</g>\n",
       "<!-- 140327258529216&#45;&gt;140327258528976 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140327258529216&#45;&gt;140327258528976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M83.07,-286.98C103.93,-278.34 136.47,-264.87 160.27,-255.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"161.66,-258.23 169.56,-251.17 158.98,-251.76 161.66,-258.23\"/>\n",
       "</g>\n",
       "<!-- 140327258528496 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140327258528496</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"366,-196 271,-196 271,-177 366,-177 366,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"318.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\">MeanBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258528496&#45;&gt;140327258846064 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140327258528496&#45;&gt;140327258846064</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M313.51,-176.66C303.45,-158.85 280.58,-118.37 267.39,-95.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"270.36,-93.14 262.39,-86.16 264.26,-96.59 270.36,-93.14\"/>\n",
       "</g>\n",
       "<!-- 140327258528896&#45;&gt;140327258528496 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140327258528896&#45;&gt;140327258528496</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M326.76,-286.66C325.31,-269.17 322.03,-229.8 320.06,-206.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"323.54,-205.83 319.22,-196.16 316.56,-206.41 323.54,-205.83\"/>\n",
       "</g>\n",
       "<!-- 140327258376864 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140327258376864</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"382,-361 281,-361 281,-342 382,-342 382,-361\"/>\n",
       "<text text-anchor=\"middle\" x=\"331.5\" y=\"-349\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140327258376864&#45;&gt;140327258528896 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140327258376864&#45;&gt;140327258528896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M330.84,-341.75C330.32,-334.8 329.56,-324.85 328.91,-316.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"332.39,-315.8 328.15,-306.09 325.41,-316.32 332.39,-315.8\"/>\n",
       "</g>\n",
       "<!-- 140327258874320 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140327258874320</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"368,-427 309,-427 309,-397 368,-397 368,-427\"/>\n",
       "<text text-anchor=\"middle\" x=\"338.5\" y=\"-415\" font-family=\"monospace\" font-size=\"10.00\">W1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"338.5\" y=\"-404\" font-family=\"monospace\" font-size=\"10.00\"> (1)</text>\n",
       "</g>\n",
       "<!-- 140327258874320&#45;&gt;140327258376864 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140327258874320&#45;&gt;140327258376864</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M336.81,-396.84C335.89,-389.21 334.76,-379.7 333.77,-371.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"337.21,-370.78 332.55,-361.27 330.26,-371.61 337.21,-370.78\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa07c6ab6e0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(1,8).requires_grad_(True)\n",
    "\n",
    "def double_backprop(inputs, net):\n",
    "    y = net(x).mean()\n",
    "    grad,  = torch.autograd.grad(y, x, create_graph=True, retain_graph=True)\n",
    "    return grad.pow(2).mean() + y\n",
    "\n",
    "make_dot(double_backprop(x, model), params=dict(list(model.named_parameters()) + [('x', x)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"999pt\" height=\"1285pt\"\n",
       " viewBox=\"0.00 0.00 999.35 1285.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1.15 1.15) rotate(0) translate(4 1476)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-1476 1147,-1476 1147,4 -4,4\"/>\n",
       "<!-- 140327258459024 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140327258459024</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"948,-31 871,-31 871,0 948,0 948,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"909.5\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> (1, 1000)</text>\n",
       "</g>\n",
       "<!-- 140327258485056 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140327258485056</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"957,-86 862,-86 862,-67 957,-67 957,-86\"/>\n",
       "<text text-anchor=\"middle\" x=\"909.5\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258485056&#45;&gt;140327258459024 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>140327258485056&#45;&gt;140327258459024</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M909.5,-66.79C909.5,-60.07 909.5,-50.4 909.5,-41.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"913,-41.19 909.5,-31.19 906,-41.19 913,-41.19\"/>\n",
       "</g>\n",
       "<!-- 140331691488352 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140331691488352</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"840,-141 739,-141 739,-122 840,-122 840,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"789.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691488352&#45;&gt;140327258485056 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140331691488352&#45;&gt;140327258485056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M808.78,-121.98C828.17,-113.42 858.31,-100.11 880.57,-90.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"882.14,-93.41 889.87,-86.17 879.31,-87.01 882.14,-93.41\"/>\n",
       "</g>\n",
       "<!-- 140331691483504 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140331691483504</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"844,-207 725,-207 725,-177 844,-177 844,-207\"/>\n",
       "<text text-anchor=\"middle\" x=\"784.5\" y=\"-195\" font-family=\"monospace\" font-size=\"10.00\">classifier.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"784.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\"> (1000)</text>\n",
       "</g>\n",
       "<!-- 140331691483504&#45;&gt;140331691488352 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140331691483504&#45;&gt;140331691488352</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M785.71,-176.84C786.36,-169.21 787.17,-159.7 787.88,-151.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"791.39,-151.53 788.75,-141.27 784.41,-150.93 791.39,-151.53\"/>\n",
       "</g>\n",
       "<!-- 140331691488432 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140331691488432</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"957,-141 862,-141 862,-122 957,-122 957,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"909.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691488432&#45;&gt;140327258485056 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140331691488432&#45;&gt;140327258485056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M909.5,-121.75C909.5,-114.8 909.5,-104.85 909.5,-96.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"913,-96.09 909.5,-86.09 906,-96.09 913,-96.09\"/>\n",
       "</g>\n",
       "<!-- 140331691488272 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140331691488272</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"957,-201.5 862,-201.5 862,-182.5 957,-182.5 957,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"909.5\" y=\"-189.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331691488272&#45;&gt;140331691488432 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140331691488272&#45;&gt;140331691488432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M909.5,-182.37C909.5,-174.25 909.5,-161.81 909.5,-151.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"913,-151.17 909.5,-141.17 906,-151.17 913,-151.17\"/>\n",
       "</g>\n",
       "<!-- 140331691488912 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140331691488912</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"792,-267.5 691,-267.5 691,-248.5 792,-248.5 792,-267.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"741.5\" y=\"-255.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691488912&#45;&gt;140331691488272 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140331691488912&#45;&gt;140331691488272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M764.16,-248.37C793.3,-237.27 844.09,-217.92 877.25,-205.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"878.84,-208.43 886.93,-201.6 876.34,-201.89 878.84,-208.43\"/>\n",
       "</g>\n",
       "<!-- 140331691405136 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140331691405136</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"795,-339 676,-339 676,-309 795,-309 795,-339\"/>\n",
       "<text text-anchor=\"middle\" x=\"735.5\" y=\"-327\" font-family=\"monospace\" font-size=\"10.00\">classifier.4.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"735.5\" y=\"-316\" font-family=\"monospace\" font-size=\"10.00\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 140331691405136&#45;&gt;140331691488912 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140331691405136&#45;&gt;140331691488912</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M736.83,-308.8C737.68,-299.7 738.8,-287.79 739.73,-277.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"743.22,-278.13 740.67,-267.84 736.25,-277.47 743.22,-278.13\"/>\n",
       "</g>\n",
       "<!-- 140331691488992 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140331691488992</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"905,-267.5 816,-267.5 816,-248.5 905,-248.5 905,-267.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"860.5\" y=\"-255.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331691488992&#45;&gt;140331691488272 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140331691488992&#45;&gt;140331691488272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M867.11,-248.37C874.59,-238.59 886.97,-222.42 896.43,-210.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"899.38,-211.97 902.68,-201.91 893.82,-207.72 899.38,-211.97\"/>\n",
       "</g>\n",
       "<!-- 140331691488752 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140331691488752</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"908,-333.5 813,-333.5 813,-314.5 908,-314.5 908,-333.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"860.5\" y=\"-321.5\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691488752&#45;&gt;140331691488992 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140331691488752&#45;&gt;140331691488992</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M860.5,-314.37C860.5,-305.16 860.5,-290.29 860.5,-278.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"864,-277.91 860.5,-267.91 857,-277.91 864,-277.91\"/>\n",
       "</g>\n",
       "<!-- 140331691489392 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140331691489392</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"904,-399.5 809,-399.5 809,-380.5 904,-380.5 904,-399.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"856.5\" y=\"-387.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331691489392&#45;&gt;140331691488752 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140331691489392&#45;&gt;140331691488752</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M857.04,-380.37C857.62,-371.07 858.56,-355.98 859.32,-343.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"862.81,-344.1 859.94,-333.91 855.83,-343.67 862.81,-344.1\"/>\n",
       "</g>\n",
       "<!-- 140331691489552 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140331691489552</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"790,-460 689,-460 689,-441 790,-441 790,-460\"/>\n",
       "<text text-anchor=\"middle\" x=\"739.5\" y=\"-448\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691489552&#45;&gt;140331691489392 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140331691489552&#45;&gt;140331691489392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M756.51,-440.99C775.92,-431.29 807.89,-415.31 830.44,-404.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"832.09,-407.12 839.47,-399.52 828.96,-400.86 832.09,-407.12\"/>\n",
       "</g>\n",
       "<!-- 140331691404560 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140331691404560</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"794,-526 675,-526 675,-496 794,-496 794,-526\"/>\n",
       "<text text-anchor=\"middle\" x=\"734.5\" y=\"-514\" font-family=\"monospace\" font-size=\"10.00\">classifier.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"734.5\" y=\"-503\" font-family=\"monospace\" font-size=\"10.00\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 140331691404560&#45;&gt;140331691489552 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140331691404560&#45;&gt;140331691489552</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M735.71,-495.84C736.36,-488.21 737.17,-478.7 737.88,-470.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"741.39,-470.53 738.75,-460.27 734.41,-469.93 741.39,-470.53\"/>\n",
       "</g>\n",
       "<!-- 140331691489632 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140331691489632</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"901,-460 812,-460 812,-441 901,-441 901,-460\"/>\n",
       "<text text-anchor=\"middle\" x=\"856.5\" y=\"-448\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331691489632&#45;&gt;140331691489392 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140331691489632&#45;&gt;140331691489392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M856.5,-440.87C856.5,-432.75 856.5,-420.31 856.5,-409.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"860,-409.67 856.5,-399.67 853,-409.67 860,-409.67\"/>\n",
       "</g>\n",
       "<!-- 140331691489232 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140331691489232</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"901,-520.5 812,-520.5 812,-501.5 901,-501.5 901,-520.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"856.5\" y=\"-508.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward</text>\n",
       "</g>\n",
       "<!-- 140331691489232&#45;&gt;140331691489632 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140331691489232&#45;&gt;140331691489632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M856.5,-501.37C856.5,-493.25 856.5,-480.81 856.5,-470.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"860,-470.17 856.5,-460.17 853,-470.17 860,-470.17\"/>\n",
       "</g>\n",
       "<!-- 140331691490032 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140331691490032</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"927,-586.5 760,-586.5 760,-567.5 927,-567.5 927,-586.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-574.5\" font-family=\"monospace\" font-size=\"10.00\">AdaptiveAvgPool2DBackward</text>\n",
       "</g>\n",
       "<!-- 140331691490032&#45;&gt;140331691489232 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140331691490032&#45;&gt;140331691489232</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M845.25,-567.37C847.14,-558.07 850.21,-542.98 852.66,-530.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"856.13,-531.4 854.69,-520.91 849.27,-530.01 856.13,-531.4\"/>\n",
       "</g>\n",
       "<!-- 140331691490192 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140331691490192</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"936,-647 751,-647 751,-628 936,-628 936,-647\"/>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-635\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140331691490192&#45;&gt;140331691490032 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140331691490192&#45;&gt;140331691490032</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M843.5,-627.87C843.5,-619.75 843.5,-607.31 843.5,-596.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"847,-596.67 843.5,-586.67 840,-596.67 847,-596.67\"/>\n",
       "</g>\n",
       "<!-- 140331691490432 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140331691490432</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"891,-702 796,-702 796,-683 891,-683 891,-702\"/>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-690\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691490432&#45;&gt;140331691490192 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140331691490432&#45;&gt;140331691490192</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M843.5,-682.75C843.5,-675.8 843.5,-665.85 843.5,-657.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"847,-657.09 843.5,-647.09 840,-657.09 847,-657.09\"/>\n",
       "</g>\n",
       "<!-- 140331691490672 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140331691490672</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"927,-757 760,-757 760,-738 927,-738 927,-757\"/>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-745\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331691490672&#45;&gt;140331691490432 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140331691490672&#45;&gt;140331691490432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M843.5,-737.75C843.5,-730.8 843.5,-720.85 843.5,-712.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"847,-712.09 843.5,-702.09 840,-712.09 847,-712.09\"/>\n",
       "</g>\n",
       "<!-- 140331691490912 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140331691490912</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"751,-812 656,-812 656,-793 751,-793 751,-812\"/>\n",
       "<text text-anchor=\"middle\" x=\"703.5\" y=\"-800\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691490912&#45;&gt;140331691490672 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140331691490912&#45;&gt;140331691490672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M725.99,-792.98C749.02,-784.27 785.04,-770.63 811.13,-760.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"812.48,-763.98 820.6,-757.17 810.01,-757.44 812.48,-763.98\"/>\n",
       "</g>\n",
       "<!-- 140331691503856 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140331691503856</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"763,-872.5 596,-872.5 596,-853.5 763,-853.5 763,-872.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"679.5\" y=\"-860.5\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331691503856&#45;&gt;140331691490912 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140331691503856&#45;&gt;140331691490912</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M683.04,-853.37C686.44,-845.07 691.69,-832.28 696.02,-821.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"699.39,-822.75 699.94,-812.17 692.91,-820.09 699.39,-822.75\"/>\n",
       "</g>\n",
       "<!-- 140331691504096 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140331691504096</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"589,-933 494,-933 494,-914 589,-914 589,-933\"/>\n",
       "<text text-anchor=\"middle\" x=\"541.5\" y=\"-921\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691504096&#45;&gt;140331691503856 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140331691504096&#45;&gt;140331691503856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M561.57,-913.99C584.87,-904.12 623.51,-887.73 650.18,-876.43\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"651.57,-879.64 659.41,-872.52 648.84,-873.2 651.57,-879.64\"/>\n",
       "</g>\n",
       "<!-- 140331691504656 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140331691504656</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"602,-993.5 435,-993.5 435,-974.5 602,-974.5 602,-993.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"518.5\" y=\"-981.5\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331691504656&#45;&gt;140331691504096 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140331691504656&#45;&gt;140331691504096</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M521.89,-974.37C525.16,-966.07 530.19,-953.28 534.34,-942.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"537.69,-943.75 538.09,-933.17 531.18,-941.19 537.69,-943.75\"/>\n",
       "</g>\n",
       "<!-- 140331691504896 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140331691504896</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"450,-1054 265,-1054 265,-1035 450,-1035 450,-1054\"/>\n",
       "<text text-anchor=\"middle\" x=\"357.5\" y=\"-1042\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140331691504896&#45;&gt;140331691504656 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140331691504896&#45;&gt;140331691504656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M380.91,-1034.99C408.45,-1024.99 454.37,-1008.3 485.51,-996.99\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"486.86,-1000.22 495.06,-993.52 484.47,-993.64 486.86,-1000.22\"/>\n",
       "</g>\n",
       "<!-- 140331691505456 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140331691505456</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"405,-1114.5 310,-1114.5 310,-1095.5 405,-1095.5 405,-1114.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"357.5\" y=\"-1102.5\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691505456&#45;&gt;140331691504896 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140331691505456&#45;&gt;140331691504896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M357.5,-1095.37C357.5,-1087.25 357.5,-1074.81 357.5,-1064.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"361,-1064.17 357.5,-1054.17 354,-1064.17 361,-1064.17\"/>\n",
       "</g>\n",
       "<!-- 140331691505696 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>140331691505696</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"441,-1175 274,-1175 274,-1156 441,-1156 441,-1175\"/>\n",
       "<text text-anchor=\"middle\" x=\"357.5\" y=\"-1163\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331691505696&#45;&gt;140331691505456 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140331691505696&#45;&gt;140331691505456</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M357.5,-1155.87C357.5,-1147.75 357.5,-1135.31 357.5,-1124.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"361,-1124.67 357.5,-1114.67 354,-1124.67 361,-1124.67\"/>\n",
       "</g>\n",
       "<!-- 140331691505936 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>140331691505936</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"289,-1230 104,-1230 104,-1211 289,-1211 289,-1230\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.5\" y=\"-1218\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140331691505936&#45;&gt;140331691505696 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140331691505936&#45;&gt;140331691505696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M222.37,-1210.98C249.28,-1202.13 291.62,-1188.19 321.73,-1178.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"323.03,-1181.53 331.43,-1175.08 320.84,-1174.88 323.03,-1181.53\"/>\n",
       "</g>\n",
       "<!-- 140331691506496 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>140331691506496</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"244,-1290.5 149,-1290.5 149,-1271.5 244,-1271.5 244,-1290.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.5\" y=\"-1278.5\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331691506496&#45;&gt;140331691505936 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140331691506496&#45;&gt;140331691505936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.5,-1271.37C196.5,-1263.25 196.5,-1250.81 196.5,-1240.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"200,-1240.17 196.5,-1230.17 193,-1240.17 200,-1240.17\"/>\n",
       "</g>\n",
       "<!-- 140331691506736 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>140331691506736</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"280,-1351 113,-1351 113,-1332 280,-1332 280,-1351\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.5\" y=\"-1339\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331691506736&#45;&gt;140331691506496 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140331691506736&#45;&gt;140331691506496</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.5,-1331.87C196.5,-1323.75 196.5,-1311.31 196.5,-1300.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"200,-1300.67 196.5,-1290.67 193,-1300.67 200,-1300.67\"/>\n",
       "</g>\n",
       "<!-- 140331691506976 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>140331691506976</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"119,-1406 18,-1406 18,-1387 119,-1387 119,-1406\"/>\n",
       "<text text-anchor=\"middle\" x=\"68.5\" y=\"-1394\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691506976&#45;&gt;140331691506736 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>140331691506976&#45;&gt;140331691506736</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M89.07,-1386.98C109.93,-1378.34 142.47,-1364.87 166.27,-1355.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"167.66,-1358.23 175.56,-1351.17 164.98,-1351.76 167.66,-1358.23\"/>\n",
       "</g>\n",
       "<!-- 140336274101840 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>140336274101840</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"119,-1472 0,-1472 0,-1442 119,-1442 119,-1472\"/>\n",
       "<text text-anchor=\"middle\" x=\"59.5\" y=\"-1460\" font-family=\"monospace\" font-size=\"10.00\">x</text>\n",
       "<text text-anchor=\"middle\" x=\"59.5\" y=\"-1449\" font-family=\"monospace\" font-size=\"10.00\"> (1, 3, 227, 227)</text>\n",
       "</g>\n",
       "<!-- 140336274101840&#45;&gt;140331691506976 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>140336274101840&#45;&gt;140331691506976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M61.68,-1441.84C62.87,-1434.13 64.35,-1424.49 65.63,-1416.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"69.09,-1416.68 67.15,-1406.27 62.17,-1415.62 69.09,-1416.68\"/>\n",
       "</g>\n",
       "<!-- 140331691507136 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>140331691507136</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"247,-1406 146,-1406 146,-1387 247,-1387 247,-1406\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.5\" y=\"-1394\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691507136&#45;&gt;140331691506736 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>140331691507136&#45;&gt;140331691506736</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.5,-1386.75C196.5,-1379.8 196.5,-1369.85 196.5,-1361.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"200,-1361.09 196.5,-1351.09 193,-1361.09 200,-1361.09\"/>\n",
       "</g>\n",
       "<!-- 140336758072048 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>140336758072048</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"256,-1472 137,-1472 137,-1442 256,-1442 256,-1472\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.5\" y=\"-1460\" font-family=\"monospace\" font-size=\"10.00\">features.0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"196.5\" y=\"-1449\" font-family=\"monospace\" font-size=\"10.00\"> (64, 3, 11, 11)</text>\n",
       "</g>\n",
       "<!-- 140336758072048&#45;&gt;140331691507136 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>140336758072048&#45;&gt;140331691507136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.5,-1441.84C196.5,-1434.21 196.5,-1424.7 196.5,-1416.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"200,-1416.27 196.5,-1406.27 193,-1416.27 200,-1416.27\"/>\n",
       "</g>\n",
       "<!-- 140331691507296 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>140331691507296</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"372,-1406 271,-1406 271,-1387 372,-1387 372,-1406\"/>\n",
       "<text text-anchor=\"middle\" x=\"321.5\" y=\"-1394\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691507296&#45;&gt;140331691506736 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>140331691507296&#45;&gt;140331691506736</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M301.42,-1386.98C281.13,-1378.38 249.54,-1364.99 226.33,-1355.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"227.52,-1351.85 216.95,-1351.17 224.79,-1358.3 227.52,-1351.85\"/>\n",
       "</g>\n",
       "<!-- 140331691402544 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>140331691402544</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"381,-1472 274,-1472 274,-1442 381,-1442 381,-1472\"/>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-1460\" font-family=\"monospace\" font-size=\"10.00\">features.0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"327.5\" y=\"-1449\" font-family=\"monospace\" font-size=\"10.00\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140331691402544&#45;&gt;140331691507296 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>140331691402544&#45;&gt;140331691507296</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M326.05,-1441.84C325.27,-1434.21 324.29,-1424.7 323.44,-1416.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"326.9,-1415.86 322.4,-1406.27 319.94,-1416.57 326.9,-1415.86\"/>\n",
       "</g>\n",
       "<!-- 140331691506096 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>140331691506096</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"408,-1230 307,-1230 307,-1211 408,-1211 408,-1230\"/>\n",
       "<text text-anchor=\"middle\" x=\"357.5\" y=\"-1218\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691506096&#45;&gt;140331691505696 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>140331691506096&#45;&gt;140331691505696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M357.5,-1210.75C357.5,-1203.8 357.5,-1193.85 357.5,-1185.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"361,-1185.09 357.5,-1175.09 354,-1185.09 361,-1185.09\"/>\n",
       "</g>\n",
       "<!-- 140327265511184 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>140327265511184</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"414,-1296 295,-1296 295,-1266 414,-1266 414,-1296\"/>\n",
       "<text text-anchor=\"middle\" x=\"354.5\" y=\"-1284\" font-family=\"monospace\" font-size=\"10.00\">features.3.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"354.5\" y=\"-1273\" font-family=\"monospace\" font-size=\"10.00\"> (192, 64, 5, 5)</text>\n",
       "</g>\n",
       "<!-- 140327265511184&#45;&gt;140331691506096 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>140327265511184&#45;&gt;140331691506096</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M355.23,-1265.84C355.62,-1258.21 356.1,-1248.7 356.53,-1240.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"360.03,-1240.43 357.05,-1230.27 353.04,-1240.07 360.03,-1240.43\"/>\n",
       "</g>\n",
       "<!-- 140331691506256 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>140331691506256</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"533,-1230 432,-1230 432,-1211 533,-1211 533,-1230\"/>\n",
       "<text text-anchor=\"middle\" x=\"482.5\" y=\"-1218\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691506256&#45;&gt;140331691505696 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>140331691506256&#45;&gt;140331691505696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M462.42,-1210.98C442.13,-1202.38 410.54,-1188.99 387.33,-1179.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"388.52,-1175.85 377.95,-1175.17 385.79,-1182.3 388.52,-1175.85\"/>\n",
       "</g>\n",
       "<!-- 140331691402640 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>140331691402640</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"539,-1296 432,-1296 432,-1266 539,-1266 539,-1296\"/>\n",
       "<text text-anchor=\"middle\" x=\"485.5\" y=\"-1284\" font-family=\"monospace\" font-size=\"10.00\">features.3.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"485.5\" y=\"-1273\" font-family=\"monospace\" font-size=\"10.00\"> (192)</text>\n",
       "</g>\n",
       "<!-- 140331691402640&#45;&gt;140331691506256 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>140331691402640&#45;&gt;140331691506256</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M484.77,-1265.84C484.38,-1258.21 483.9,-1248.7 483.47,-1240.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"486.96,-1240.07 482.95,-1230.27 479.97,-1240.43 486.96,-1240.07\"/>\n",
       "</g>\n",
       "<!-- 140331691505056 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>140331691505056</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"569,-1054 468,-1054 468,-1035 569,-1035 569,-1054\"/>\n",
       "<text text-anchor=\"middle\" x=\"518.5\" y=\"-1042\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691505056&#45;&gt;140331691504656 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>140331691505056&#45;&gt;140331691504656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M518.5,-1034.87C518.5,-1026.75 518.5,-1014.31 518.5,-1003.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"522,-1003.67 518.5,-993.67 515,-1003.67 522,-1003.67\"/>\n",
       "</g>\n",
       "<!-- 140331691402832 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>140331691402832</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"575,-1120 456,-1120 456,-1090 575,-1090 575,-1120\"/>\n",
       "<text text-anchor=\"middle\" x=\"515.5\" y=\"-1108\" font-family=\"monospace\" font-size=\"10.00\">features.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"515.5\" y=\"-1097\" font-family=\"monospace\" font-size=\"10.00\"> (384, 192, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140331691402832&#45;&gt;140331691505056 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>140331691402832&#45;&gt;140331691505056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M516.23,-1089.84C516.62,-1082.21 517.1,-1072.7 517.53,-1064.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"521.03,-1064.43 518.05,-1054.27 514.04,-1064.07 521.03,-1064.43\"/>\n",
       "</g>\n",
       "<!-- 140331691505216 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>140331691505216</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"694,-1054 593,-1054 593,-1035 694,-1035 694,-1054\"/>\n",
       "<text text-anchor=\"middle\" x=\"643.5\" y=\"-1042\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691505216&#45;&gt;140331691504656 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>140331691505216&#45;&gt;140331691504656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M625.32,-1034.99C604.49,-1025.25 570.13,-1009.16 546.02,-997.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"547.24,-994.59 536.7,-993.52 544.27,-1000.93 547.24,-994.59\"/>\n",
       "</g>\n",
       "<!-- 140331691403024 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>140331691403024</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"700,-1120 593,-1120 593,-1090 700,-1090 700,-1120\"/>\n",
       "<text text-anchor=\"middle\" x=\"646.5\" y=\"-1108\" font-family=\"monospace\" font-size=\"10.00\">features.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"646.5\" y=\"-1097\" font-family=\"monospace\" font-size=\"10.00\"> (384)</text>\n",
       "</g>\n",
       "<!-- 140331691403024&#45;&gt;140331691505216 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>140331691403024&#45;&gt;140331691505216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M645.77,-1089.84C645.38,-1082.21 644.9,-1072.7 644.47,-1064.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"647.96,-1064.07 643.95,-1054.27 640.97,-1064.43 647.96,-1064.07\"/>\n",
       "</g>\n",
       "<!-- 140331691504256 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>140331691504256</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"730,-933 629,-933 629,-914 730,-914 730,-933\"/>\n",
       "<text text-anchor=\"middle\" x=\"679.5\" y=\"-921\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691504256&#45;&gt;140331691503856 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>140331691504256&#45;&gt;140331691503856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M679.5,-913.87C679.5,-905.75 679.5,-893.31 679.5,-882.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"683,-882.67 679.5,-872.67 676,-882.67 683,-882.67\"/>\n",
       "</g>\n",
       "<!-- 140331691403216 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>140331691403216</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"739,-999 620,-999 620,-969 739,-969 739,-999\"/>\n",
       "<text text-anchor=\"middle\" x=\"679.5\" y=\"-987\" font-family=\"monospace\" font-size=\"10.00\">features.8.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"679.5\" y=\"-976\" font-family=\"monospace\" font-size=\"10.00\"> (256, 384, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140331691403216&#45;&gt;140331691504256 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>140331691403216&#45;&gt;140331691504256</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M679.5,-968.84C679.5,-961.21 679.5,-951.7 679.5,-943.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"683,-943.27 679.5,-933.27 676,-943.27 683,-943.27\"/>\n",
       "</g>\n",
       "<!-- 140331691504416 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>140331691504416</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"855,-933 754,-933 754,-914 855,-914 855,-933\"/>\n",
       "<text text-anchor=\"middle\" x=\"804.5\" y=\"-921\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691504416&#45;&gt;140331691503856 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>140331691504416&#45;&gt;140331691503856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M786.32,-913.99C765.49,-904.25 731.13,-888.16 707.02,-876.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"708.24,-873.59 697.7,-872.52 705.27,-879.93 708.24,-873.59\"/>\n",
       "</g>\n",
       "<!-- 140331691403408 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>140331691403408</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"864,-999 757,-999 757,-969 864,-969 864,-999\"/>\n",
       "<text text-anchor=\"middle\" x=\"810.5\" y=\"-987\" font-family=\"monospace\" font-size=\"10.00\">features.8.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"810.5\" y=\"-976\" font-family=\"monospace\" font-size=\"10.00\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140331691403408&#45;&gt;140331691504416 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>140331691403408&#45;&gt;140331691504416</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M809.05,-968.84C808.27,-961.21 807.29,-951.7 806.44,-943.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"809.9,-942.86 805.4,-933.27 802.94,-943.57 809.9,-942.86\"/>\n",
       "</g>\n",
       "<!-- 140331691491072 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>140331691491072</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"894,-812 793,-812 793,-793 894,-793 894,-812\"/>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-800\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691491072&#45;&gt;140331691490672 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>140331691491072&#45;&gt;140331691490672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M843.5,-792.75C843.5,-785.8 843.5,-775.85 843.5,-767.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"847,-767.09 843.5,-757.09 840,-767.09 847,-767.09\"/>\n",
       "</g>\n",
       "<!-- 140331691403600 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>140331691403600</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"906,-878 781,-878 781,-848 906,-848 906,-878\"/>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-866\" font-family=\"monospace\" font-size=\"10.00\">features.10.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"843.5\" y=\"-855\" font-family=\"monospace\" font-size=\"10.00\"> (256, 256, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140331691403600&#45;&gt;140331691491072 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>140331691403600&#45;&gt;140331691491072</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M843.5,-847.84C843.5,-840.21 843.5,-830.7 843.5,-822.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"847,-822.27 843.5,-812.27 840,-822.27 847,-822.27\"/>\n",
       "</g>\n",
       "<!-- 140331691491232 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>140331691491232</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1022,-812 921,-812 921,-793 1022,-793 1022,-812\"/>\n",
       "<text text-anchor=\"middle\" x=\"971.5\" y=\"-800\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691491232&#45;&gt;140331691490672 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>140331691491232&#45;&gt;140331691490672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M950.93,-792.98C930.07,-784.34 897.53,-770.87 873.73,-761.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"875.02,-757.76 864.44,-757.17 872.34,-764.23 875.02,-757.76\"/>\n",
       "</g>\n",
       "<!-- 140331691403792 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>140331691403792</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1037,-878 924,-878 924,-848 1037,-848 1037,-878\"/>\n",
       "<text text-anchor=\"middle\" x=\"980.5\" y=\"-866\" font-family=\"monospace\" font-size=\"10.00\">features.10.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"980.5\" y=\"-855\" font-family=\"monospace\" font-size=\"10.00\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140331691403792&#45;&gt;140331691491232 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>140331691403792&#45;&gt;140331691491232</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M978.32,-847.84C977.13,-840.13 975.65,-830.49 974.37,-822.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"977.83,-821.62 972.85,-812.27 970.91,-822.68 977.83,-821.62\"/>\n",
       "</g>\n",
       "<!-- 140331691489712 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>140331691489712</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1011,-460 940,-460 940,-441 1011,-441 1011,-460\"/>\n",
       "<text text-anchor=\"middle\" x=\"975.5\" y=\"-448\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331691489712&#45;&gt;140331691489392 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>140331691489712&#45;&gt;140331691489392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M958.2,-440.99C938.45,-431.29 905.95,-415.31 883.01,-404.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"884.34,-400.79 873.82,-399.52 881.25,-407.07 884.34,-400.79\"/>\n",
       "</g>\n",
       "<!-- 140331691489952 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>140331691489952</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1047,-520.5 946,-520.5 946,-501.5 1047,-501.5 1047,-520.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"996.5\" y=\"-508.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691489952&#45;&gt;140331691489712 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>140331691489952&#45;&gt;140331691489712</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M993.4,-501.37C990.42,-493.07 985.83,-480.28 982.04,-469.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"985.28,-468.4 978.61,-460.17 978.7,-470.76 985.28,-468.4\"/>\n",
       "</g>\n",
       "<!-- 140331691404368 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>140331691404368</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1076,-592 945,-592 945,-562 1076,-562 1076,-592\"/>\n",
       "<text text-anchor=\"middle\" x=\"1010.5\" y=\"-580\" font-family=\"monospace\" font-size=\"10.00\">classifier.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1010.5\" y=\"-569\" font-family=\"monospace\" font-size=\"10.00\"> (4096, 9216)</text>\n",
       "</g>\n",
       "<!-- 140331691404368&#45;&gt;140331691489952 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>140331691404368&#45;&gt;140331691489952</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1007.39,-561.8C1005.4,-552.7 1002.8,-540.79 1000.63,-530.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1003.99,-529.87 998.43,-520.84 997.15,-531.36 1003.99,-529.87\"/>\n",
       "</g>\n",
       "<!-- 140331691489072 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>140331691489072</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"994,-267.5 923,-267.5 923,-248.5 994,-248.5 994,-267.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"958.5\" y=\"-255.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331691489072&#45;&gt;140331691488272 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>140331691489072&#45;&gt;140331691488272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M951.89,-248.37C944.41,-238.59 932.03,-222.42 922.57,-210.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"925.18,-207.72 916.32,-201.91 919.62,-211.97 925.18,-207.72\"/>\n",
       "</g>\n",
       "<!-- 140331691489312 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>140331691489312</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1034,-333.5 933,-333.5 933,-314.5 1034,-314.5 1034,-333.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"983.5\" y=\"-321.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691489312&#45;&gt;140331691489072 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>140331691489312&#45;&gt;140331691489072</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M980.13,-314.37C976.46,-304.97 970.48,-289.67 965.74,-277.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"968.88,-275.95 961.98,-267.91 962.36,-278.49 968.88,-275.95\"/>\n",
       "</g>\n",
       "<!-- 140331691404944 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>140331691404944</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1053,-405 922,-405 922,-375 1053,-375 1053,-405\"/>\n",
       "<text text-anchor=\"middle\" x=\"987.5\" y=\"-393\" font-family=\"monospace\" font-size=\"10.00\">classifier.4.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"987.5\" y=\"-382\" font-family=\"monospace\" font-size=\"10.00\"> (4096, 4096)</text>\n",
       "</g>\n",
       "<!-- 140331691404944&#45;&gt;140331691489312 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>140331691404944&#45;&gt;140331691489312</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M986.61,-374.8C986.04,-365.7 985.3,-353.79 984.68,-343.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"988.17,-343.61 984.05,-333.84 981.18,-344.04 988.17,-343.61\"/>\n",
       "</g>\n",
       "<!-- 140331691488592 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>140331691488592</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1055,-141 984,-141 984,-122 1055,-122 1055,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"1019.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331691488592&#45;&gt;140327258485056 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>140331691488592&#45;&gt;140327258485056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1001.83,-121.98C984.21,-113.5 956.92,-100.35 936.57,-90.54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"938.02,-87.36 927.49,-86.17 934.98,-93.66 938.02,-87.36\"/>\n",
       "</g>\n",
       "<!-- 140331691488832 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>140331691488832</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1127,-201.5 1026,-201.5 1026,-182.5 1127,-182.5 1127,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1076.5\" y=\"-189.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331691488832&#45;&gt;140331691488592 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>140331691488832&#45;&gt;140331691488592</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1068.09,-182.37C1059.42,-173.47 1045.7,-159.39 1035.04,-148.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1037.43,-145.89 1027.95,-141.17 1032.42,-150.77 1037.43,-145.89\"/>\n",
       "</g>\n",
       "<!-- 140331691483312 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>140331691483312</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1143,-273 1012,-273 1012,-243 1143,-243 1143,-273\"/>\n",
       "<text text-anchor=\"middle\" x=\"1077.5\" y=\"-261\" font-family=\"monospace\" font-size=\"10.00\">classifier.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1077.5\" y=\"-250\" font-family=\"monospace\" font-size=\"10.00\"> (1000, 4096)</text>\n",
       "</g>\n",
       "<!-- 140331691483312&#45;&gt;140331691488832 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>140331691483312&#45;&gt;140331691488832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1077.28,-242.8C1077.14,-233.7 1076.95,-221.79 1076.8,-211.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1080.29,-211.79 1076.64,-201.84 1073.29,-211.9 1080.29,-211.79\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa1849f3500>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchvision.models import AlexNet\n",
    "\n",
    "model = AlexNet()\n",
    "\n",
    "x = torch.randn(1, 3, 227, 227).requires_grad_(True)\n",
    "y = model(x)\n",
    "make_dot(y, params=dict(list(model.named_parameters()) + [('x', x)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And AlexNet double backprop for fun:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"1106pt\" height=\"2171pt\"\n",
       " viewBox=\"0.00 0.00 1105.75 2171.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1.26 1.26) rotate(0) translate(4 2730)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-2730 1388.5,-2730 1388.5,4 -4,4\"/>\n",
       "<!-- 140327258495408 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140327258495408</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"256,-31 202,-31 202,0 256,0 256,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"229\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> ()</text>\n",
       "</g>\n",
       "<!-- 140331691490112 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140331691490112</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"273.5,-86 184.5,-86 184.5,-67 273.5,-67 273.5,-86\"/>\n",
       "<text text-anchor=\"middle\" x=\"229\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140331691490112&#45;&gt;140327258495408 -->\n",
       "<g id=\"edge112\" class=\"edge\">\n",
       "<title>140331691490112&#45;&gt;140327258495408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M229,-66.79C229,-60.07 229,-50.4 229,-41.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"232.5,-41.19 229,-31.19 225.5,-41.19 232.5,-41.19\"/>\n",
       "</g>\n",
       "<!-- 140327258484400 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140327258484400</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"506.5,-141 411.5,-141 411.5,-122 506.5,-122 506.5,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"459\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">MeanBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258484400&#45;&gt;140331691490112 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140327258484400&#45;&gt;140331691490112</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M422.05,-121.98C382.43,-112.86 319.41,-98.33 276.23,-88.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"276.77,-84.92 266.24,-86.08 275.2,-91.74 276.77,-84.92\"/>\n",
       "</g>\n",
       "<!-- 140327258484320 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140327258484320</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"694.5,-196 605.5,-196 605.5,-177 694.5,-177 694.5,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"650\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\">PowBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258484320&#45;&gt;140327258484400 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140327258484320&#45;&gt;140327258484400</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M619.31,-176.98C586.83,-167.97 535.4,-153.7 499.59,-143.76\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"500.5,-140.38 489.93,-141.08 498.63,-147.13 500.5,-140.38\"/>\n",
       "</g>\n",
       "<!-- 140327258483360 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140327258483360</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"949.5,-251 734.5,-251 734.5,-232 949.5,-232 949.5,-251\"/>\n",
       "<text text-anchor=\"middle\" x=\"842\" y=\"-239\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140327258483360&#45;&gt;140327258484320 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140327258483360&#45;&gt;140327258484320</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M811.15,-231.98C778.5,-222.97 726.8,-208.7 690.8,-198.76\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"691.66,-195.37 681.09,-196.08 689.8,-202.12 691.66,-195.37\"/>\n",
       "</g>\n",
       "<!-- 140331401085248 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140331401085248</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1366.5,-2660 1265.5,-2660 1265.5,-2641 1366.5,-2641 1366.5,-2660\"/>\n",
       "<text text-anchor=\"middle\" x=\"1316\" y=\"-2648\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401085248&#45;&gt;140327258483360 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140331401085248&#45;&gt;140327258483360</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1316,-2640.96C1316,-2622.05 1316,-2575.21 1316,-2536 1316,-2536 1316,-2536 1316,-350.5 1316,-277.63 1100.34,-253.93 959.95,-246.22\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"959.73,-242.7 949.56,-245.67 959.36,-249.69 959.73,-242.7\"/>\n",
       "</g>\n",
       "<!-- 140331401086528 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140331401086528</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1202.5,-2605 1035.5,-2605 1035.5,-2586 1202.5,-2586 1202.5,-2605\"/>\n",
       "<text text-anchor=\"middle\" x=\"1119\" y=\"-2593\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331401085248&#45;&gt;140331401086528 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140331401085248&#45;&gt;140331401086528</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1284.35,-2640.98C1250.84,-2631.97 1197.8,-2617.7 1160.86,-2607.76\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1161.47,-2604.3 1150.9,-2605.08 1159.65,-2611.06 1161.47,-2604.3\"/>\n",
       "</g>\n",
       "<!-- 140336274101840 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140336274101840</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1384.5,-2726 1265.5,-2726 1265.5,-2696 1384.5,-2696 1384.5,-2726\"/>\n",
       "<text text-anchor=\"middle\" x=\"1325\" y=\"-2714\" font-family=\"monospace\" font-size=\"10.00\">x</text>\n",
       "<text text-anchor=\"middle\" x=\"1325\" y=\"-2703\" font-family=\"monospace\" font-size=\"10.00\"> (1, 3, 227, 227)</text>\n",
       "</g>\n",
       "<!-- 140336274101840&#45;&gt;140331401085248 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140336274101840&#45;&gt;140331401085248</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1322.82,-2695.84C1321.63,-2688.13 1320.15,-2678.49 1318.87,-2670.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1322.33,-2669.62 1317.35,-2660.27 1315.41,-2670.68 1322.33,-2669.62\"/>\n",
       "</g>\n",
       "<!-- 140331401085328 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140331401085328</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"489.5,-306 322.5,-306 322.5,-287 489.5,-287 489.5,-306\"/>\n",
       "<text text-anchor=\"middle\" x=\"406\" y=\"-294\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401085328&#45;&gt;140327258483360 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140331401085328&#45;&gt;140327258483360</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M476.05,-286.98C554.04,-277.5 679.86,-262.21 761.75,-252.26\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"762.26,-255.72 771.76,-251.04 761.41,-248.77 762.26,-255.72\"/>\n",
       "</g>\n",
       "<!-- 140327258483520 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140327258483520</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"522.5,-361 289.5,-361 289.5,-342 522.5,-342 522.5,-361\"/>\n",
       "<text text-anchor=\"middle\" x=\"406\" y=\"-349\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140327258483520&#45;&gt;140331401085328 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140327258483520&#45;&gt;140331401085328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M406,-341.75C406,-334.8 406,-324.85 406,-316.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"409.5,-316.09 406,-306.09 402.5,-316.09 409.5,-316.09\"/>\n",
       "</g>\n",
       "<!-- 140331401085888 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140331401085888</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"735.5,-416 520.5,-416 520.5,-397 735.5,-397 735.5,-416\"/>\n",
       "<text text-anchor=\"middle\" x=\"628\" y=\"-404\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401085888&#45;&gt;140327258483520 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140331401085888&#45;&gt;140327258483520</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M592.33,-396.98C554.17,-387.87 493.53,-373.4 451.85,-363.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"452.49,-360 441.95,-361.08 450.86,-366.81 452.49,-360\"/>\n",
       "</g>\n",
       "<!-- 140331401086128 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140331401086128</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"250.5,-2484 65.5,-2484 65.5,-2465 250.5,-2465 250.5,-2484\"/>\n",
       "<text text-anchor=\"middle\" x=\"158\" y=\"-2472\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140331401086128&#45;&gt;140331401085888 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140331401086128&#45;&gt;140331401085888</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M145.55,-2464.79C122.61,-2447.6 76,-2406.92 76,-2360 76,-2360 76,-2360 76,-680.5 76,-488.34 348.1,-431.44 510.22,-414.59\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"510.66,-418.06 520.26,-413.58 509.96,-411.1 510.66,-418.06\"/>\n",
       "</g>\n",
       "<!-- 140331401087408 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140331401087408</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1183.5,-2429 1016.5,-2429 1016.5,-2410 1183.5,-2410 1183.5,-2429\"/>\n",
       "<text text-anchor=\"middle\" x=\"1100\" y=\"-2417\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331401086128&#45;&gt;140331401087408 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140331401086128&#45;&gt;140331401087408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M250.9,-2468.27C430.96,-2458.14 825.58,-2435.94 1006.29,-2425.77\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1006.51,-2429.27 1016.3,-2425.21 1006.12,-2422.28 1006.51,-2429.27\"/>\n",
       "</g>\n",
       "<!-- 140331401085808 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140331401085808</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"145.5,-2544.5 50.5,-2544.5 50.5,-2525.5 145.5,-2525.5 145.5,-2544.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"98\" y=\"-2532.5\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401085808&#45;&gt;140331401085328 -->\n",
       "<g id=\"edge104\" class=\"edge\">\n",
       "<title>140331401085808&#45;&gt;140331401085328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M82.25,-2525.38C54.55,-2508.93 0,-2470.22 0,-2420.5 0,-2420.5 0,-2420.5 0,-570.5 0,-404.53 236.53,-332.87 348.52,-308.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.59,-311.6 358.63,-306.08 348.12,-304.76 349.59,-311.6\"/>\n",
       "</g>\n",
       "<!-- 140331401085808&#45;&gt;140327258483520 -->\n",
       "<g id=\"edge103\" class=\"edge\">\n",
       "<title>140331401085808&#45;&gt;140327258483520</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M88.65,-2525.29C79.14,-2515.94 64.6,-2500.23 56,-2484 42.26,-2458.08 38,-2449.83 38,-2420.5 38,-2420.5 38,-2420.5 38,-625.5 38,-467.26 260.19,-390.1 359.27,-363.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"360.24,-366.93 369.03,-361.01 358.47,-360.16 360.24,-366.93\"/>\n",
       "</g>\n",
       "<!-- 140331401085808&#45;&gt;140331401086128 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140331401085808&#45;&gt;140331401086128</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M106.85,-2525.37C115.98,-2516.47 130.42,-2502.39 141.64,-2491.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"144.39,-2493.65 149.11,-2484.17 139.51,-2488.64 144.39,-2493.65\"/>\n",
       "</g>\n",
       "<!-- 140331401086528&#45;&gt;140331401085808 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140331401086528&#45;&gt;140331401085808</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.45,-2589.71C834.29,-2578.19 327.06,-2549.12 155.63,-2539.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155.74,-2535.8 145.55,-2538.72 155.34,-2542.79 155.74,-2535.8\"/>\n",
       "</g>\n",
       "<!-- 140331401085408 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140331401085408</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1238.5,-2660 1137.5,-2660 1137.5,-2641 1238.5,-2641 1238.5,-2660\"/>\n",
       "<text text-anchor=\"middle\" x=\"1188\" y=\"-2648\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401085408&#45;&gt;140327258483360 -->\n",
       "<g id=\"edge105\" class=\"edge\">\n",
       "<title>140331401085408&#45;&gt;140327258483360</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1202.06,-2640.82C1227.38,-2623.99 1278,-2584.28 1278,-2536 1278,-2536 1278,-2536 1278,-350.5 1278,-284.82 1089.25,-258.82 960.13,-248.71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"960.01,-245.19 949.78,-247.92 959.48,-252.17 960.01,-245.19\"/>\n",
       "</g>\n",
       "<!-- 140331401085408&#45;&gt;140331401086528 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140331401085408&#45;&gt;140331401086528</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1176.91,-2640.98C1166.51,-2633 1150.74,-2620.88 1138.28,-2611.31\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1140.35,-2608.49 1130.29,-2605.17 1136.09,-2614.04 1140.35,-2608.49\"/>\n",
       "</g>\n",
       "<!-- 140336758072048 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140336758072048</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1247.5,-2726 1128.5,-2726 1128.5,-2696 1247.5,-2696 1247.5,-2726\"/>\n",
       "<text text-anchor=\"middle\" x=\"1188\" y=\"-2714\" font-family=\"monospace\" font-size=\"10.00\">features.0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1188\" y=\"-2703\" font-family=\"monospace\" font-size=\"10.00\"> (64, 3, 11, 11)</text>\n",
       "</g>\n",
       "<!-- 140336758072048&#45;&gt;140331401085408 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140336758072048&#45;&gt;140331401085408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1188,-2695.84C1188,-2688.21 1188,-2678.7 1188,-2670.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1191.5,-2670.27 1188,-2660.27 1184.5,-2670.27 1191.5,-2670.27\"/>\n",
       "</g>\n",
       "<!-- 140331401086688 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140331401086688</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1113.5,-2660 1012.5,-2660 1012.5,-2641 1113.5,-2641 1113.5,-2660\"/>\n",
       "<text text-anchor=\"middle\" x=\"1063\" y=\"-2648\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401086688&#45;&gt;140331401086528 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140331401086688&#45;&gt;140331401086528</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1072,-2640.98C1080.19,-2633.23 1092.51,-2621.58 1102.47,-2612.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1104.98,-2614.59 1109.84,-2605.17 1100.17,-2609.5 1104.98,-2614.59\"/>\n",
       "</g>\n",
       "<!-- 140331691402544 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140331691402544</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1110.5,-2726 1003.5,-2726 1003.5,-2696 1110.5,-2696 1110.5,-2726\"/>\n",
       "<text text-anchor=\"middle\" x=\"1057\" y=\"-2714\" font-family=\"monospace\" font-size=\"10.00\">features.0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"1057\" y=\"-2703\" font-family=\"monospace\" font-size=\"10.00\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140331691402544&#45;&gt;140331401086688 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140331691402544&#45;&gt;140331401086688</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1058.45,-2695.84C1059.23,-2688.21 1060.21,-2678.7 1061.06,-2670.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1064.56,-2670.57 1062.1,-2660.27 1057.6,-2669.86 1064.56,-2670.57\"/>\n",
       "</g>\n",
       "<!-- 140331401086208 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140331401086208</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1126.5,-471 959.5,-471 959.5,-452 1126.5,-452 1126.5,-471\"/>\n",
       "<text text-anchor=\"middle\" x=\"1043\" y=\"-459\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401086208&#45;&gt;140331401085888 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140331401086208&#45;&gt;140331401085888</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M976.32,-451.98C902.32,-442.53 783.06,-427.3 705.1,-417.35\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"705.22,-413.83 694.86,-416.04 704.33,-420.78 705.22,-413.83\"/>\n",
       "</g>\n",
       "<!-- 140331401086448 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140331401086448</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1159.5,-526 926.5,-526 926.5,-507 1159.5,-507 1159.5,-526\"/>\n",
       "<text text-anchor=\"middle\" x=\"1043\" y=\"-514\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401086448&#45;&gt;140331401086208 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140331401086448&#45;&gt;140331401086208</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1043,-506.75C1043,-499.8 1043,-489.85 1043,-481.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1046.5,-481.09 1043,-471.09 1039.5,-481.09 1046.5,-481.09\"/>\n",
       "</g>\n",
       "<!-- 140331401085488 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140331401085488</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1110.5,-581 895.5,-581 895.5,-562 1110.5,-562 1110.5,-581\"/>\n",
       "<text text-anchor=\"middle\" x=\"1003\" y=\"-569\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401085488&#45;&gt;140331401086448 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140331401085488&#45;&gt;140331401086448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1009.61,-561.75C1015.26,-554.26 1023.54,-543.28 1030.45,-534.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1033.29,-536.18 1036.52,-526.09 1027.7,-531.96 1033.29,-536.18\"/>\n",
       "</g>\n",
       "<!-- 140331401087008 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140331401087008</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1136.5,-2308 951.5,-2308 951.5,-2289 1136.5,-2289 1136.5,-2308\"/>\n",
       "<text text-anchor=\"middle\" x=\"1044\" y=\"-2296\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140331401087008&#45;&gt;140331401085488 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140331401087008&#45;&gt;140331401085488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1062.47,-2288.99C1076.72,-2281.48 1095.84,-2269.22 1107,-2253 1126.37,-2224.85 1126,-2212.67 1126,-2178.5 1126,-2178.5 1126,-2178.5 1126,-680.5 1126,-632.63 1072.75,-601 1036.32,-584.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1037.58,-581.69 1027.01,-581.03 1034.86,-588.14 1037.58,-581.69\"/>\n",
       "</g>\n",
       "<!-- 140331401087968 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>140331401087968</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"497.5,-2247.5 330.5,-2247.5 330.5,-2228.5 497.5,-2228.5 497.5,-2247.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"414\" y=\"-2235.5\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331401087008&#45;&gt;140331401087968 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>140331401087008&#45;&gt;140331401087968</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M952.39,-2288.99C833.68,-2277.97 627.75,-2258.85 507.92,-2247.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"507.95,-2244.21 497.66,-2246.77 507.3,-2251.18 507.95,-2244.21\"/>\n",
       "</g>\n",
       "<!-- 140331401086608 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140331401086608</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1179.5,-2368.5 1084.5,-2368.5 1084.5,-2349.5 1179.5,-2349.5 1179.5,-2368.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1132\" y=\"-2356.5\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401086608&#45;&gt;140331401086208 -->\n",
       "<g id=\"edge101\" class=\"edge\">\n",
       "<title>140331401086608&#45;&gt;140331401086208</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1141.44,-2349.48C1160.55,-2331.23 1202,-2286.22 1202,-2239 1202,-2239 1202,-2239 1202,-570.5 1202,-538.49 1192.26,-527.88 1168,-507 1149.22,-490.84 1124.15,-480.4 1101.39,-473.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1102.03,-470.28 1091.46,-471.01 1100.18,-477.03 1102.03,-470.28\"/>\n",
       "</g>\n",
       "<!-- 140331401086608&#45;&gt;140331401086448 -->\n",
       "<g id=\"edge100\" class=\"edge\">\n",
       "<title>140331401086608&#45;&gt;140331401086448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1136.05,-2349.26C1144.66,-2329.87 1164,-2281.57 1164,-2239 1164,-2239 1164,-2239 1164,-625.5 1164,-578.14 1111.77,-546.4 1075.94,-530.19\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1076.9,-526.79 1066.33,-526.03 1074.12,-533.21 1076.9,-526.79\"/>\n",
       "</g>\n",
       "<!-- 140331401086608&#45;&gt;140331401087008 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140331401086608&#45;&gt;140331401087008</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1119.02,-2349.37C1104.98,-2340.04 1082.37,-2325.01 1065.64,-2313.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1067.3,-2310.79 1057.04,-2308.17 1063.43,-2316.62 1067.3,-2310.79\"/>\n",
       "</g>\n",
       "<!-- 140331401087408&#45;&gt;140331401086608 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140331401087408&#45;&gt;140331401086608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1104.72,-2409.87C1109.35,-2401.4 1116.55,-2388.24 1122.39,-2377.57\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1125.53,-2379.12 1127.26,-2368.67 1119.39,-2375.76 1125.53,-2379.12\"/>\n",
       "</g>\n",
       "<!-- 140331401086288 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140331401086288</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1240.5,-2484 1139.5,-2484 1139.5,-2465 1240.5,-2465 1240.5,-2484\"/>\n",
       "<text text-anchor=\"middle\" x=\"1190\" y=\"-2472\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401086288&#45;&gt;140331401085888 -->\n",
       "<g id=\"edge102\" class=\"edge\">\n",
       "<title>140331401086288&#45;&gt;140331401085888</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1196.74,-2464.95C1210.39,-2446.76 1240,-2402.42 1240,-2360 1240,-2360 1240,-2360 1240,-515.5 1240,-460.96 1186.84,-468.93 1135,-452 1065.23,-429.21 872.51,-417.26 745.79,-411.68\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"745.79,-408.17 735.65,-411.24 745.49,-415.17 745.79,-408.17\"/>\n",
       "</g>\n",
       "<!-- 140331401086288&#45;&gt;140331401087408 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140331401086288&#45;&gt;140331401087408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1175.54,-2464.98C1161.58,-2456.77 1140.21,-2444.18 1123.75,-2434.49\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1125.12,-2431.23 1114.72,-2429.17 1121.56,-2437.26 1125.12,-2431.23\"/>\n",
       "</g>\n",
       "<!-- 140327265511184 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140327265511184</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1249.5,-2550 1130.5,-2550 1130.5,-2520 1249.5,-2520 1249.5,-2550\"/>\n",
       "<text text-anchor=\"middle\" x=\"1190\" y=\"-2538\" font-family=\"monospace\" font-size=\"10.00\">features.3.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1190\" y=\"-2527\" font-family=\"monospace\" font-size=\"10.00\"> (192, 64, 5, 5)</text>\n",
       "</g>\n",
       "<!-- 140327265511184&#45;&gt;140331401086288 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140327265511184&#45;&gt;140331401086288</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1190,-2519.84C1190,-2512.21 1190,-2502.7 1190,-2494.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1193.5,-2494.27 1190,-2484.27 1186.5,-2494.27 1193.5,-2494.27\"/>\n",
       "</g>\n",
       "<!-- 140331401087568 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>140331401087568</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1115.5,-2484 1014.5,-2484 1014.5,-2465 1115.5,-2465 1115.5,-2484\"/>\n",
       "<text text-anchor=\"middle\" x=\"1065\" y=\"-2472\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401087568&#45;&gt;140331401087408 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140331401087568&#45;&gt;140331401087408</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1070.78,-2464.75C1075.62,-2457.42 1082.68,-2446.73 1088.64,-2437.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1091.74,-2439.36 1094.33,-2429.09 1085.9,-2435.5 1091.74,-2439.36\"/>\n",
       "</g>\n",
       "<!-- 140331691402640 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>140331691402640</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1112.5,-2550 1005.5,-2550 1005.5,-2520 1112.5,-2520 1112.5,-2550\"/>\n",
       "<text text-anchor=\"middle\" x=\"1059\" y=\"-2538\" font-family=\"monospace\" font-size=\"10.00\">features.3.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"1059\" y=\"-2527\" font-family=\"monospace\" font-size=\"10.00\"> (192)</text>\n",
       "</g>\n",
       "<!-- 140331691402640&#45;&gt;140331401087568 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140331691402640&#45;&gt;140331401087568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1060.45,-2519.84C1061.23,-2512.21 1062.21,-2502.7 1063.06,-2494.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1066.56,-2494.57 1064.1,-2484.27 1059.6,-2493.86 1066.56,-2494.57\"/>\n",
       "</g>\n",
       "<!-- 140331401086848 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>140331401086848</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1029.5,-636 862.5,-636 862.5,-617 1029.5,-617 1029.5,-636\"/>\n",
       "<text text-anchor=\"middle\" x=\"946\" y=\"-624\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401086848&#45;&gt;140331401085488 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>140331401086848&#45;&gt;140331401085488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M955.16,-616.98C963.5,-609.23 976.03,-597.58 986.18,-588.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"988.74,-590.54 993.68,-581.17 983.97,-585.42 988.74,-590.54\"/>\n",
       "</g>\n",
       "<!-- 140331401087328 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>140331401087328</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1053.5,-691 838.5,-691 838.5,-672 1053.5,-672 1053.5,-691\"/>\n",
       "<text text-anchor=\"middle\" x=\"946\" y=\"-679\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401087328&#45;&gt;140331401086848 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>140331401087328&#45;&gt;140331401086848</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M946,-671.75C946,-664.8 946,-654.85 946,-646.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"949.5,-646.09 946,-636.09 942.5,-646.09 949.5,-646.09\"/>\n",
       "</g>\n",
       "<!-- 140331401087488 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>140331401087488</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"461.5,-2187 366.5,-2187 366.5,-2168 461.5,-2168 461.5,-2187\"/>\n",
       "<text text-anchor=\"middle\" x=\"414\" y=\"-2175\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401087488&#45;&gt;140331401086848 -->\n",
       "<g id=\"edge98\" class=\"edge\">\n",
       "<title>140331401087488&#45;&gt;140331401086848</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M409.19,-2167.85C398.96,-2148.63 376,-2100.66 376,-2057.5 376,-2057.5 376,-2057.5 376,-1577 376,-1495.07 418,-1479.43 418,-1397.5 418,-1397.5 418,-1397.5 418,-735.5 418,-647.92 700.33,-630.68 852.09,-627.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"852.4,-631.23 862.34,-627.55 852.28,-624.23 852.4,-631.23\"/>\n",
       "</g>\n",
       "<!-- 140331401087488&#45;&gt;140331401087328 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>140331401087488&#45;&gt;140331401087328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414,-2167.84C414,-2148.21 414,-2098.82 414,-2057.5 414,-2057.5 414,-2057.5 414,-1577 414,-1495.07 456,-1479.43 456,-1397.5 456,-1397.5 456,-1397.5 456,-790.5 456,-714.58 683.19,-691.97 828.17,-685.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"828.63,-688.76 838.47,-684.83 828.32,-681.77 828.63,-688.76\"/>\n",
       "</g>\n",
       "<!-- 140331401088608 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>140331401088608</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1060.5,-2126.5 893.5,-2126.5 893.5,-2107.5 1060.5,-2107.5 1060.5,-2126.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"977\" y=\"-2114.5\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331401087488&#45;&gt;140331401088608 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>140331401087488&#45;&gt;140331401088608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M461.8,-2172.85C546.86,-2166.16 730.04,-2150.86 884,-2132 893.72,-2130.81 904,-2129.41 914.03,-2127.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"914.57,-2131.42 923.95,-2126.5 913.55,-2124.49 914.57,-2131.42\"/>\n",
       "</g>\n",
       "<!-- 140331401087968&#45;&gt;140331401087488 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>140331401087968&#45;&gt;140331401087488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414,-2228.37C414,-2220.25 414,-2207.81 414,-2197.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"417.5,-2197.17 414,-2187.17 410.5,-2197.17 417.5,-2197.17\"/>\n",
       "</g>\n",
       "<!-- 140331401087088 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>140331401087088</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"274.5,-2308 173.5,-2308 173.5,-2289 274.5,-2289 274.5,-2308\"/>\n",
       "<text text-anchor=\"middle\" x=\"224\" y=\"-2296\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401087088&#45;&gt;140331401085488 -->\n",
       "<g id=\"edge99\" class=\"edge\">\n",
       "<title>140331401087088&#45;&gt;140331401085488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M206.33,-2288.77C175.23,-2272.03 114,-2232.24 114,-2178.5 114,-2178.5 114,-2178.5 114,-845.5 114,-682.53 666.82,-606.82 900.58,-582.1\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"901.21,-585.55 910.79,-581.03 900.48,-578.59 901.21,-585.55\"/>\n",
       "</g>\n",
       "<!-- 140331401087088&#45;&gt;140331401087968 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>140331401087088&#45;&gt;140331401087968</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M251.63,-2288.99C284.55,-2278.86 339.73,-2261.87 376.49,-2250.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"377.81,-2253.8 386.34,-2247.52 375.75,-2247.11 377.81,-2253.8\"/>\n",
       "</g>\n",
       "<!-- 140331691402832 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>140331691402832</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"283.5,-2374 164.5,-2374 164.5,-2344 283.5,-2344 283.5,-2374\"/>\n",
       "<text text-anchor=\"middle\" x=\"224\" y=\"-2362\" font-family=\"monospace\" font-size=\"10.00\">features.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"224\" y=\"-2351\" font-family=\"monospace\" font-size=\"10.00\"> (384, 192, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140331691402832&#45;&gt;140331401087088 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>140331691402832&#45;&gt;140331401087088</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M224,-2343.84C224,-2336.21 224,-2326.7 224,-2318.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"227.5,-2318.27 224,-2308.27 220.5,-2318.27 227.5,-2318.27\"/>\n",
       "</g>\n",
       "<!-- 140331401088128 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>140331401088128</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"464.5,-2308 363.5,-2308 363.5,-2289 464.5,-2289 464.5,-2308\"/>\n",
       "<text text-anchor=\"middle\" x=\"414\" y=\"-2296\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401088128&#45;&gt;140331401087968 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>140331401088128&#45;&gt;140331401087968</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414,-2288.87C414,-2280.75 414,-2268.31 414,-2257.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"417.5,-2257.67 414,-2247.67 410.5,-2257.67 417.5,-2257.67\"/>\n",
       "</g>\n",
       "<!-- 140331691403024 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>140331691403024</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"467.5,-2374 360.5,-2374 360.5,-2344 467.5,-2344 467.5,-2374\"/>\n",
       "<text text-anchor=\"middle\" x=\"414\" y=\"-2362\" font-family=\"monospace\" font-size=\"10.00\">features.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"414\" y=\"-2351\" font-family=\"monospace\" font-size=\"10.00\"> (384)</text>\n",
       "</g>\n",
       "<!-- 140331691403024&#45;&gt;140331401088128 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>140331691403024&#45;&gt;140331401088128</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M414,-2343.84C414,-2336.21 414,-2326.7 414,-2318.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"417.5,-2318.27 414,-2308.27 410.5,-2318.27 417.5,-2318.27\"/>\n",
       "</g>\n",
       "<!-- 140331401087808 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>140331401087808</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1029.5,-746 862.5,-746 862.5,-727 1029.5,-727 1029.5,-746\"/>\n",
       "<text text-anchor=\"middle\" x=\"946\" y=\"-734\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401087808&#45;&gt;140331401087328 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>140331401087808&#45;&gt;140331401087328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M946,-726.75C946,-719.8 946,-709.85 946,-701.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"949.5,-701.09 946,-691.09 942.5,-701.09 949.5,-701.09\"/>\n",
       "</g>\n",
       "<!-- 140331401088048 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>140331401088048</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1021.5,-801 806.5,-801 806.5,-782 1021.5,-782 1021.5,-801\"/>\n",
       "<text text-anchor=\"middle\" x=\"914\" y=\"-789\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401088048&#45;&gt;140331401087808 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>140331401088048&#45;&gt;140331401087808</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M919.28,-781.75C923.71,-774.42 930.16,-763.73 935.62,-754.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"938.64,-756.46 940.81,-746.09 932.65,-752.84 938.64,-756.46\"/>\n",
       "</g>\n",
       "<!-- 140331401088208 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>140331401088208</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1043.5,-2066 948.5,-2066 948.5,-2047 1043.5,-2047 1043.5,-2066\"/>\n",
       "<text text-anchor=\"middle\" x=\"996\" y=\"-2054\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401088208&#45;&gt;140331401087808 -->\n",
       "<g id=\"edge96\" class=\"edge\">\n",
       "<title>140331401088208&#45;&gt;140331401087808</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1003.74,-2046.99C1018.67,-2029.64 1050,-1988.38 1050,-1947.5 1050,-1947.5 1050,-1947.5 1050,-845.5 1050,-815.91 1048.93,-804.74 1030,-782 1017.91,-767.48 999.82,-757.04 983.53,-749.94\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"984.78,-746.67 974.2,-746.14 982.14,-753.16 984.78,-746.67\"/>\n",
       "</g>\n",
       "<!-- 140331401088208&#45;&gt;140331401088048 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>140331401088208&#45;&gt;140331401088048</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M998.16,-2046.88C1002.53,-2028.67 1012,-1984.84 1012,-1947.5 1012,-1947.5 1012,-1947.5 1012,-900.5 1012,-857.55 968.97,-824.1 939.93,-806.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"941.47,-803.26 931.07,-801.24 937.94,-809.3 941.47,-803.26\"/>\n",
       "</g>\n",
       "<!-- 140331401103632 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>140331401103632</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"946.5,-2011 779.5,-2011 779.5,-1992 946.5,-1992 946.5,-2011\"/>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-1999\" font-family=\"monospace\" font-size=\"10.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140331401088208&#45;&gt;140331401103632 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>140331401088208&#45;&gt;140331401103632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M974.63,-2046.98C952.85,-2038.3 918.84,-2024.75 894.08,-2014.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"895.34,-2011.62 884.76,-2011.17 892.75,-2018.12 895.34,-2011.62\"/>\n",
       "</g>\n",
       "<!-- 140331401088608&#45;&gt;140331401088208 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>140331401088608&#45;&gt;140331401088208</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M979.8,-2107.37C982.5,-2099.07 986.65,-2086.28 990.08,-2075.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"993.42,-2076.76 993.18,-2066.17 986.77,-2074.6 993.42,-2076.76\"/>\n",
       "</g>\n",
       "<!-- 140331401087888 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>140331401087888</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1088.5,-2187 987.5,-2187 987.5,-2168 1088.5,-2168 1088.5,-2187\"/>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-2175\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401087888&#45;&gt;140331401087328 -->\n",
       "<g id=\"edge97\" class=\"edge\">\n",
       "<title>140331401087888&#45;&gt;140331401087328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1045.43,-2167.88C1052.45,-2159.31 1062.81,-2145.53 1069,-2132 1083.22,-2100.93 1088,-2091.67 1088,-2057.5 1088,-2057.5 1088,-2057.5 1088,-790.5 1088,-739.32 1028.83,-709.21 986.88,-694.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"987.76,-690.9 977.16,-691 985.51,-697.53 987.76,-690.9\"/>\n",
       "</g>\n",
       "<!-- 140331401087888&#45;&gt;140331401088608 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>140331401087888&#45;&gt;140331401088608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1029,-2167.87C1019.63,-2158.88 1004.75,-2144.61 993.3,-2133.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"995.68,-2131.06 986.04,-2126.67 990.83,-2136.12 995.68,-2131.06\"/>\n",
       "</g>\n",
       "<!-- 140331691403216 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>140331691403216</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1097.5,-2253 978.5,-2253 978.5,-2223 1097.5,-2223 1097.5,-2253\"/>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-2241\" font-family=\"monospace\" font-size=\"10.00\">features.8.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-2230\" font-family=\"monospace\" font-size=\"10.00\"> (256, 384, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140331691403216&#45;&gt;140331401087888 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>140331691403216&#45;&gt;140331401087888</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1038,-2222.84C1038,-2215.21 1038,-2205.7 1038,-2197.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1041.5,-2197.27 1038,-2187.27 1034.5,-2197.27 1041.5,-2197.27\"/>\n",
       "</g>\n",
       "<!-- 140331401088688 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>140331401088688</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"963.5,-2187 862.5,-2187 862.5,-2168 963.5,-2168 963.5,-2187\"/>\n",
       "<text text-anchor=\"middle\" x=\"913\" y=\"-2175\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401088688&#45;&gt;140331401088608 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>140331401088688&#45;&gt;140331401088608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M922.44,-2167.87C932.27,-2158.88 947.89,-2144.61 959.89,-2133.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"962.5,-2136 967.52,-2126.67 957.78,-2130.83 962.5,-2136\"/>\n",
       "</g>\n",
       "<!-- 140331691403408 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>140331691403408</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"960.5,-2253 853.5,-2253 853.5,-2223 960.5,-2223 960.5,-2253\"/>\n",
       "<text text-anchor=\"middle\" x=\"907\" y=\"-2241\" font-family=\"monospace\" font-size=\"10.00\">features.8.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"907\" y=\"-2230\" font-family=\"monospace\" font-size=\"10.00\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140331691403408&#45;&gt;140331401088688 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>140331691403408&#45;&gt;140331401088688</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M908.45,-2222.84C909.23,-2215.21 910.21,-2205.7 911.06,-2197.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"914.56,-2197.57 912.1,-2187.27 907.6,-2196.86 914.56,-2197.57\"/>\n",
       "</g>\n",
       "<!-- 140331401088448 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>140331401088448</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"946.5,-856 779.5,-856 779.5,-837 946.5,-837 946.5,-856\"/>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-844\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401088448&#45;&gt;140331401088048 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>140331401088448&#45;&gt;140331401088048</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M871.42,-836.75C878.77,-829.11 889.63,-817.82 898.54,-808.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"901.32,-810.72 905.74,-801.09 896.28,-805.87 901.32,-810.72\"/>\n",
       "</g>\n",
       "<!-- 140331401087248 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>140331401087248</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"907.5,-911 674.5,-911 674.5,-892 907.5,-892 907.5,-911\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-899\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401087248&#45;&gt;140331401088448 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>140331401087248&#45;&gt;140331401088448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M802.57,-891.98C813.42,-884 829.88,-871.88 842.88,-862.31\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"845.24,-864.92 851.22,-856.17 841.09,-859.28 845.24,-864.92\"/>\n",
       "</g>\n",
       "<!-- 140331401086048 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>140331401086048</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"869.5,-966 654.5,-966 654.5,-947 869.5,-947 869.5,-966\"/>\n",
       "<text text-anchor=\"middle\" x=\"762\" y=\"-954\" font-family=\"monospace\" font-size=\"10.00\">AdaptiveAvgPool2DBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401086048&#45;&gt;140331401087248 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>140331401086048&#45;&gt;140331401087248</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M766.79,-946.75C770.76,-939.49 776.53,-928.95 781.43,-919.98\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"784.57,-921.54 786.3,-911.09 778.43,-918.18 784.57,-921.54\"/>\n",
       "</g>\n",
       "<!-- 140327258487936 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>140327258487936</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"806.5,-1021 717.5,-1021 717.5,-1002 806.5,-1002 806.5,-1021\"/>\n",
       "<text text-anchor=\"middle\" x=\"762\" y=\"-1009\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward</text>\n",
       "</g>\n",
       "<!-- 140327258487936&#45;&gt;140331401086048 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>140327258487936&#45;&gt;140331401086048</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M762,-1001.75C762,-994.8 762,-984.85 762,-976.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"765.5,-976.09 762,-966.09 758.5,-976.09 765.5,-976.09\"/>\n",
       "</g>\n",
       "<!-- 140327258486416 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>140327258486416</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"806.5,-1076 717.5,-1076 717.5,-1057 806.5,-1057 806.5,-1076\"/>\n",
       "<text text-anchor=\"middle\" x=\"762\" y=\"-1064\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258486416&#45;&gt;140327258487936 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>140327258486416&#45;&gt;140327258487936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M762,-1056.75C762,-1049.8 762,-1039.85 762,-1031.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"765.5,-1031.09 762,-1021.09 758.5,-1031.09 765.5,-1031.09\"/>\n",
       "</g>\n",
       "<!-- 140327258485376 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>140327258485376</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"800.5,-1131 723.5,-1131 723.5,-1112 800.5,-1112 800.5,-1131\"/>\n",
       "<text text-anchor=\"middle\" x=\"762\" y=\"-1119\" font-family=\"monospace\" font-size=\"10.00\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 140327258485376&#45;&gt;140327258486416 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>140327258485376&#45;&gt;140327258486416</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M762,-1111.75C762,-1104.8 762,-1094.85 762,-1086.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"765.5,-1086.09 762,-1076.09 758.5,-1086.09 765.5,-1086.09\"/>\n",
       "</g>\n",
       "<!-- 140327258486016 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>140327258486016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"768.5,-1186 601.5,-1186 601.5,-1167 768.5,-1167 768.5,-1186\"/>\n",
       "<text text-anchor=\"middle\" x=\"685\" y=\"-1174\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140327258486016&#45;&gt;140327258485376 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>140327258486016&#45;&gt;140327258485376</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M697.37,-1166.98C709.09,-1158.92 726.92,-1146.65 740.89,-1137.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"743.15,-1139.72 749.4,-1131.17 739.18,-1133.96 743.15,-1139.72\"/>\n",
       "</g>\n",
       "<!-- 140327258486176 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>140327258486176</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"703.5,-1241 614.5,-1241 614.5,-1222 703.5,-1222 703.5,-1241\"/>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-1229\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258486176&#45;&gt;140327258486016 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>140327258486176&#45;&gt;140327258486016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M663.29,-1221.75C666.81,-1214.57 671.91,-1204.18 676.28,-1195.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"679.52,-1196.61 680.79,-1186.09 673.24,-1193.53 679.52,-1196.61\"/>\n",
       "</g>\n",
       "<!-- 140331691507616 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>140331691507616</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"697.5,-1296 620.5,-1296 620.5,-1277 697.5,-1277 697.5,-1296\"/>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-1284\" font-family=\"monospace\" font-size=\"10.00\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 140331691507616&#45;&gt;140327258486176 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>140331691507616&#45;&gt;140327258486176</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M659,-1276.75C659,-1269.8 659,-1259.85 659,-1251.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"662.5,-1251.09 659,-1241.09 655.5,-1251.09 662.5,-1251.09\"/>\n",
       "</g>\n",
       "<!-- 140331401101392 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>140331401101392</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"650.5,-1351 483.5,-1351 483.5,-1332 650.5,-1332 650.5,-1351\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-1339\" font-family=\"monospace\" font-size=\"10.00\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 140331401101392&#45;&gt;140331691507616 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>140331401101392&#45;&gt;140331691507616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M581.78,-1331.98C596.18,-1323.69 618.31,-1310.94 635.19,-1301.22\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"637.03,-1304.19 643.95,-1296.17 633.54,-1298.13 637.03,-1304.19\"/>\n",
       "</g>\n",
       "<!-- 140331401101632 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>140331401101632</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"389.5,-1406 312.5,-1406 312.5,-1387 389.5,-1387 389.5,-1406\"/>\n",
       "<text text-anchor=\"middle\" x=\"351\" y=\"-1394\" font-family=\"monospace\" font-size=\"10.00\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 140331401101632&#45;&gt;140331401101392 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>140331401101632&#45;&gt;140331401101392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M385.7,-1386.98C422.75,-1377.89 481.59,-1363.46 522.13,-1353.51\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"523.15,-1356.86 532.02,-1351.08 521.48,-1350.07 523.15,-1356.86\"/>\n",
       "</g>\n",
       "<!-- 140331401101872 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>140331401101872</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"389.5,-1466.5 318.5,-1466.5 318.5,-1447.5 389.5,-1447.5 389.5,-1466.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"354\" y=\"-1454.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401101872&#45;&gt;140331401101632 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>140331401101872&#45;&gt;140331401101632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M353.56,-1447.37C353.14,-1439.25 352.5,-1426.81 351.97,-1416.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"355.45,-1415.97 351.44,-1406.17 348.46,-1416.33 355.45,-1415.97\"/>\n",
       "</g>\n",
       "<!-- 140331401102032 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>140331401102032</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"325.5,-1527 254.5,-1527 254.5,-1508 325.5,-1508 325.5,-1527\"/>\n",
       "<text text-anchor=\"middle\" x=\"290\" y=\"-1515\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102032&#45;&gt;140331401101872 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>140331401102032&#45;&gt;140331401101872</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M299.44,-1507.87C309.27,-1498.88 324.89,-1484.61 336.89,-1473.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"339.5,-1476 344.52,-1466.67 334.78,-1470.83 339.5,-1476\"/>\n",
       "</g>\n",
       "<!-- 140327258482960 -->\n",
       "<g id=\"node87\" class=\"node\">\n",
       "<title>140327258482960</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"290.5,-1351 195.5,-1351 195.5,-1332 290.5,-1332 290.5,-1351\"/>\n",
       "<text text-anchor=\"middle\" x=\"243\" y=\"-1339\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102032&#45;&gt;140327258482960 -->\n",
       "<g id=\"edge111\" class=\"edge\">\n",
       "<title>140331401102032&#45;&gt;140327258482960</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M288.72,-1507.87C285.47,-1486.85 276.17,-1431.49 262,-1387 259.13,-1377.99 255.12,-1368.31 251.51,-1360.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"254.67,-1358.78 247.29,-1351.18 248.32,-1361.72 254.67,-1358.78\"/>\n",
       "</g>\n",
       "<!-- 140331401102192 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>140331401102192</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"336.5,-1587.5 235.5,-1587.5 235.5,-1568.5 336.5,-1568.5 336.5,-1587.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"286\" y=\"-1575.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401102192&#45;&gt;140331401102032 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>140331401102192&#45;&gt;140331401102032</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M286.59,-1568.37C287.15,-1560.25 288,-1547.81 288.71,-1537.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"292.22,-1537.38 289.41,-1527.17 285.23,-1536.91 292.22,-1537.38\"/>\n",
       "</g>\n",
       "<!-- 140331691483312 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>140331691483312</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"347.5,-1659 216.5,-1659 216.5,-1629 347.5,-1629 347.5,-1659\"/>\n",
       "<text text-anchor=\"middle\" x=\"282\" y=\"-1647\" font-family=\"monospace\" font-size=\"10.00\">classifier.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"282\" y=\"-1636\" font-family=\"monospace\" font-size=\"10.00\"> (1000, 4096)</text>\n",
       "</g>\n",
       "<!-- 140331691483312&#45;&gt;140331401102192 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>140331691483312&#45;&gt;140331401102192</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M282.89,-1628.8C283.46,-1619.7 284.2,-1607.79 284.82,-1597.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"288.32,-1598.04 285.45,-1587.84 281.33,-1597.61 288.32,-1598.04\"/>\n",
       "</g>\n",
       "<!-- 140331401101712 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>140331401101712</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"614.5,-1406 519.5,-1406 519.5,-1387 614.5,-1387 614.5,-1406\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-1394\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401101712&#45;&gt;140331401101392 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>140331401101712&#45;&gt;140331401101392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M567,-1386.75C567,-1379.8 567,-1369.85 567,-1361.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"570.5,-1361.09 567,-1351.09 563.5,-1361.09 570.5,-1361.09\"/>\n",
       "</g>\n",
       "<!-- 140331401101712&#45;&gt;140327258482960 -->\n",
       "<g id=\"edge110\" class=\"edge\">\n",
       "<title>140331401101712&#45;&gt;140327258482960</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M519.19,-1387.68C461.08,-1378.17 363.09,-1362.14 300.8,-1351.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"301.17,-1348.47 290.74,-1350.31 300.04,-1355.38 301.17,-1348.47\"/>\n",
       "</g>\n",
       "<!-- 140331401085728 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>140331401085728</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"633.5,-1466.5 538.5,-1466.5 538.5,-1447.5 633.5,-1447.5 633.5,-1466.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"586\" y=\"-1454.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331401085728&#45;&gt;140331401101712 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>140331401085728&#45;&gt;140331401101712</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M583.2,-1447.37C580.5,-1439.07 576.35,-1426.28 572.92,-1415.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"576.23,-1414.6 569.82,-1406.17 569.58,-1416.76 576.23,-1414.6\"/>\n",
       "</g>\n",
       "<!-- 140331401102112 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>140331401102112</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"543.5,-1527 442.5,-1527 442.5,-1508 543.5,-1508 543.5,-1527\"/>\n",
       "<text text-anchor=\"middle\" x=\"493\" y=\"-1515\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401102112&#45;&gt;140331401085728 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>140331401102112&#45;&gt;140331401085728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M506.72,-1507.87C521.7,-1498.45 545.89,-1483.23 563.62,-1472.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"565.62,-1474.95 572.22,-1466.67 561.89,-1469.03 565.62,-1474.95\"/>\n",
       "</g>\n",
       "<!-- 140331691405136 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>140331691405136</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"561.5,-1593 442.5,-1593 442.5,-1563 561.5,-1563 561.5,-1593\"/>\n",
       "<text text-anchor=\"middle\" x=\"502\" y=\"-1581\" font-family=\"monospace\" font-size=\"10.00\">classifier.4.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"502\" y=\"-1570\" font-family=\"monospace\" font-size=\"10.00\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 140331691405136&#45;&gt;140331401102112 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>140331691405136&#45;&gt;140331401102112</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M499.82,-1562.84C498.63,-1555.13 497.15,-1545.49 495.87,-1537.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"499.33,-1536.62 494.35,-1527.27 492.41,-1537.68 499.33,-1536.62\"/>\n",
       "</g>\n",
       "<!-- 140331401102272 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>140331401102272</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"650.5,-1527 561.5,-1527 561.5,-1508 650.5,-1508 650.5,-1527\"/>\n",
       "<text text-anchor=\"middle\" x=\"606\" y=\"-1515\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331401102272&#45;&gt;140331401085728 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>140331401102272&#45;&gt;140331401085728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M603.05,-1507.87C600.21,-1499.57 595.84,-1486.78 592.23,-1476.22\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"595.51,-1475 588.96,-1466.67 588.89,-1477.26 595.51,-1475\"/>\n",
       "</g>\n",
       "<!-- 140327258485136 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>140327258485136</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"674.5,-1587.5 579.5,-1587.5 579.5,-1568.5 674.5,-1568.5 674.5,-1587.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"627\" y=\"-1575.5\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140327258485136&#45;&gt;140327258486016 -->\n",
       "<g id=\"edge89\" class=\"edge\">\n",
       "<title>140327258485136&#45;&gt;140327258486016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M655.27,-1568.47C686.84,-1558.41 735.22,-1541.26 748,-1527 769.31,-1503.22 768,-1489.93 768,-1458 768,-1458 768,-1458 768,-1395.5 768,-1316.28 719.54,-1230.85 696.68,-1194.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"699.36,-1192.65 690.97,-1186.18 693.49,-1196.47 699.36,-1192.65\"/>\n",
       "</g>\n",
       "<!-- 140327258485136&#45;&gt;140331401102272 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>140327258485136&#45;&gt;140331401102272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M623.9,-1568.37C620.92,-1560.07 616.33,-1547.28 612.54,-1536.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"615.78,-1535.4 609.11,-1527.17 609.2,-1537.76 615.78,-1535.4\"/>\n",
       "</g>\n",
       "<!-- 140331401102672 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>140331401102672</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"644.5,-1653.5 549.5,-1653.5 549.5,-1634.5 644.5,-1634.5 644.5,-1653.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"597\" y=\"-1641.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102672&#45;&gt;140327258485136 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>140331401102672&#45;&gt;140327258485136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M601.05,-1634.37C605.49,-1624.88 612.77,-1609.36 618.49,-1597.16\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"621.75,-1598.45 622.83,-1587.91 615.41,-1595.47 621.75,-1598.45\"/>\n",
       "</g>\n",
       "<!-- 140331401102832 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>140331401102832</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"554.5,-1714 453.5,-1714 453.5,-1695 554.5,-1695 554.5,-1714\"/>\n",
       "<text text-anchor=\"middle\" x=\"504\" y=\"-1702\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401102832&#45;&gt;140331401102672 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>140331401102832&#45;&gt;140331401102672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M517.72,-1694.87C532.7,-1685.45 556.89,-1670.23 574.62,-1659.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"576.62,-1661.95 583.22,-1653.67 572.89,-1656.03 576.62,-1661.95\"/>\n",
       "</g>\n",
       "<!-- 140331691404560 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>140331691404560</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"561.5,-1780 442.5,-1780 442.5,-1750 561.5,-1750 561.5,-1780\"/>\n",
       "<text text-anchor=\"middle\" x=\"502\" y=\"-1768\" font-family=\"monospace\" font-size=\"10.00\">classifier.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"502\" y=\"-1757\" font-family=\"monospace\" font-size=\"10.00\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 140331691404560&#45;&gt;140331401102832 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>140331691404560&#45;&gt;140331401102832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M502.48,-1749.84C502.74,-1742.21 503.07,-1732.7 503.35,-1724.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"506.86,-1724.38 503.7,-1714.27 499.86,-1724.14 506.86,-1724.38\"/>\n",
       "</g>\n",
       "<!-- 140331401102912 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>140331401102912</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"664.5,-1714 575.5,-1714 575.5,-1695 664.5,-1695 664.5,-1714\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-1702\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331401102912&#45;&gt;140331401102672 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>140331401102912&#45;&gt;140331401102672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M616.61,-1694.87C613.34,-1686.57 608.31,-1673.78 604.16,-1663.22\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"607.32,-1661.69 600.41,-1653.67 600.81,-1664.25 607.32,-1661.69\"/>\n",
       "</g>\n",
       "<!-- 140331401102592 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>140331401102592</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"668.5,-1774.5 579.5,-1774.5 579.5,-1755.5 668.5,-1755.5 668.5,-1774.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"624\" y=\"-1762.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102592&#45;&gt;140331401102912 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>140331401102592&#45;&gt;140331401102912</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M623.41,-1755.37C622.85,-1747.25 622,-1734.81 621.29,-1724.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"624.77,-1723.91 620.59,-1714.17 617.78,-1724.38 624.77,-1723.91\"/>\n",
       "</g>\n",
       "<!-- 140331401103312 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>140331401103312</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"682.5,-1840.5 515.5,-1840.5 515.5,-1821.5 682.5,-1821.5 682.5,-1840.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"599\" y=\"-1828.5\" font-family=\"monospace\" font-size=\"10.00\">AdaptiveAvgPool2DBackward</text>\n",
       "</g>\n",
       "<!-- 140331401103312&#45;&gt;140331401102592 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>140331401103312&#45;&gt;140331401102592</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M602.37,-1821.37C606.04,-1811.97 612.02,-1796.67 616.76,-1784.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"620.14,-1785.49 620.52,-1774.91 613.62,-1782.95 620.14,-1785.49\"/>\n",
       "</g>\n",
       "<!-- 140327258486256 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>140327258486256</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"870.5,-1901 685.5,-1901 685.5,-1882 870.5,-1882 870.5,-1901\"/>\n",
       "<text text-anchor=\"middle\" x=\"778\" y=\"-1889\" font-family=\"monospace\" font-size=\"10.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140327258486256&#45;&gt;140331401086048 -->\n",
       "<g id=\"edge92\" class=\"edge\">\n",
       "<title>140327258486256&#45;&gt;140331401086048</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M796.2,-1881.93C810.24,-1874.39 829.06,-1862.11 840,-1846 860.59,-1815.68 860,-1802.65 860,-1766 860,-1766 860,-1766 860,-1065.5 860,-1022.55 816.97,-989.1 787.93,-971.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"789.47,-968.26 779.07,-966.24 785.94,-974.3 789.47,-968.26\"/>\n",
       "</g>\n",
       "<!-- 140327258486256&#45;&gt;140331401103312 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>140327258486256&#45;&gt;140331401103312</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M751.97,-1881.99C721.08,-1871.9 669.42,-1855.01 634.78,-1843.69\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"635.65,-1840.3 625.06,-1840.52 633.48,-1846.95 635.65,-1840.3\"/>\n",
       "</g>\n",
       "<!-- 140331401088768 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>140331401088768</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"927.5,-1956 832.5,-1956 832.5,-1937 927.5,-1937 927.5,-1956\"/>\n",
       "<text text-anchor=\"middle\" x=\"880\" y=\"-1944\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401088768&#45;&gt;140331401088448 -->\n",
       "<g id=\"edge94\" class=\"edge\">\n",
       "<title>140331401088768&#45;&gt;140331401088448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M887.79,-1936.82C903.17,-1918.72 936,-1875.07 936,-1832 936,-1832 936,-1832 936,-955.5 936,-925.91 932.43,-916.61 916,-892 908.08,-880.13 896.1,-869.7 885.54,-861.93\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"887.32,-858.9 877.11,-856.05 883.31,-864.64 887.32,-858.9\"/>\n",
       "</g>\n",
       "<!-- 140331401088768&#45;&gt;140331401087248 -->\n",
       "<g id=\"edge93\" class=\"edge\">\n",
       "<title>140331401088768&#45;&gt;140331401087248</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M882.35,-1936.75C887.23,-1917.8 898,-1871.5 898,-1832 898,-1832 898,-1832 898,-1010.5 898,-980.91 897.1,-969.6 878,-947 865.55,-932.27 847,-921.84 830.22,-914.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"831.2,-911.43 820.61,-911.05 828.65,-917.95 831.2,-911.43\"/>\n",
       "</g>\n",
       "<!-- 140331401088768&#45;&gt;140327258486256 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>140331401088768&#45;&gt;140327258486256</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M863.61,-1936.98C847.43,-1928.57 822.43,-1915.59 803.62,-1905.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"805.17,-1902.68 794.69,-1901.17 801.95,-1908.89 805.17,-1902.68\"/>\n",
       "</g>\n",
       "<!-- 140331401103632&#45;&gt;140331401088768 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>140331401103632&#45;&gt;140331401088768</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M865.81,-1991.75C868.06,-1984.72 871.3,-1974.62 874.12,-1965.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"877.52,-1966.68 877.25,-1956.09 870.86,-1964.54 877.52,-1966.68\"/>\n",
       "</g>\n",
       "<!-- 140331401088288 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>140331401088288</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"913.5,-2066 812.5,-2066 812.5,-2047 913.5,-2047 913.5,-2066\"/>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-2054\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401088288&#45;&gt;140331401088048 -->\n",
       "<g id=\"edge95\" class=\"edge\">\n",
       "<title>140331401088288&#45;&gt;140331401088048</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M895.71,-2046.99C915.46,-2040.27 939.7,-2028.96 955,-2011 974.1,-1988.57 974,-1976.96 974,-1947.5 974,-1947.5 974,-1947.5 974,-900.5 974,-871.04 969.58,-862.6 955,-837 948.86,-826.21 939.64,-815.89 931.51,-807.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"933.86,-805.33 924.17,-801.05 929.08,-810.44 933.86,-805.33\"/>\n",
       "</g>\n",
       "<!-- 140331401088288&#45;&gt;140331401103632 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>140331401088288&#45;&gt;140331401103632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M863,-2046.75C863,-2039.8 863,-2029.85 863,-2021.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"866.5,-2021.09 863,-2011.09 859.5,-2021.09 866.5,-2021.09\"/>\n",
       "</g>\n",
       "<!-- 140331691403600 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>140331691403600</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"875.5,-2132 750.5,-2132 750.5,-2102 875.5,-2102 875.5,-2132\"/>\n",
       "<text text-anchor=\"middle\" x=\"813\" y=\"-2120\" font-family=\"monospace\" font-size=\"10.00\">features.10.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"813\" y=\"-2109\" font-family=\"monospace\" font-size=\"10.00\"> (256, 256, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140331691403600&#45;&gt;140331401088288 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>140331691403600&#45;&gt;140331401088288</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M825.1,-2101.84C832.29,-2093.43 841.43,-2082.74 848.93,-2073.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"851.67,-2076.14 855.51,-2066.27 846.35,-2071.59 851.67,-2076.14\"/>\n",
       "</g>\n",
       "<!-- 140331401103792 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>140331401103792</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"760.5,-2066 659.5,-2066 659.5,-2047 760.5,-2047 760.5,-2066\"/>\n",
       "<text text-anchor=\"middle\" x=\"710\" y=\"-2054\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401103792&#45;&gt;140331401103632 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>140331401103792&#45;&gt;140331401103632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M734.58,-2046.98C760.05,-2038.16 800.04,-2024.31 828.63,-2014.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"829.92,-2017.66 838.23,-2011.08 827.63,-2011.05 829.92,-2017.66\"/>\n",
       "</g>\n",
       "<!-- 140331691403792 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>140331691403792</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"732.5,-2132 619.5,-2132 619.5,-2102 732.5,-2102 732.5,-2132\"/>\n",
       "<text text-anchor=\"middle\" x=\"676\" y=\"-2120\" font-family=\"monospace\" font-size=\"10.00\">features.10.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"676\" y=\"-2109\" font-family=\"monospace\" font-size=\"10.00\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140331691403792&#45;&gt;140331401103792 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>140331691403792&#45;&gt;140331401103792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.23,-2101.84C688.91,-2093.78 694.82,-2083.62 699.8,-2075.06\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"702.91,-2076.67 704.91,-2066.27 696.85,-2073.15 702.91,-2076.67\"/>\n",
       "</g>\n",
       "<!-- 140331401102992 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>140331401102992</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"801.5,-1714 730.5,-1714 730.5,-1695 801.5,-1695 801.5,-1714\"/>\n",
       "<text text-anchor=\"middle\" x=\"766\" y=\"-1702\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102992&#45;&gt;140331401102672 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>140331401102992&#45;&gt;140331401102672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M741.43,-1694.99C712.39,-1684.94 663.9,-1668.16 631.2,-1656.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"632.2,-1653.48 621.6,-1653.52 629.91,-1660.1 632.2,-1653.48\"/>\n",
       "</g>\n",
       "<!-- 140327258485856 -->\n",
       "<g id=\"node85\" class=\"node\">\n",
       "<title>140327258485856</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"831.5,-1241 760.5,-1241 760.5,-1222 831.5,-1222 831.5,-1241\"/>\n",
       "<text text-anchor=\"middle\" x=\"796\" y=\"-1229\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102992&#45;&gt;140327258485856 -->\n",
       "<g id=\"edge91\" class=\"edge\">\n",
       "<title>140331401102992&#45;&gt;140327258485856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M775.02,-1694.78C783.23,-1686.36 795.08,-1672.87 802,-1659 818.36,-1626.2 822,-1615.65 822,-1579 822,-1579 822,-1579 822,-1340.5 822,-1308.15 810.83,-1271.82 803.14,-1250.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"806.32,-1249.13 799.51,-1241.01 799.77,-1251.6 806.32,-1249.13\"/>\n",
       "</g>\n",
       "<!-- 140331401103232 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>140331401103232</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"816.5,-1774.5 715.5,-1774.5 715.5,-1755.5 816.5,-1755.5 816.5,-1774.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"766\" y=\"-1762.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401103232&#45;&gt;140331401102992 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>140331401103232&#45;&gt;140331401102992</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M766,-1755.37C766,-1747.25 766,-1734.81 766,-1724.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"769.5,-1724.17 766,-1714.17 762.5,-1724.17 769.5,-1724.17\"/>\n",
       "</g>\n",
       "<!-- 140331691404368 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>140331691404368</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"831.5,-1846 700.5,-1846 700.5,-1816 831.5,-1816 831.5,-1846\"/>\n",
       "<text text-anchor=\"middle\" x=\"766\" y=\"-1834\" font-family=\"monospace\" font-size=\"10.00\">classifier.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"766\" y=\"-1823\" font-family=\"monospace\" font-size=\"10.00\"> (4096, 9216)</text>\n",
       "</g>\n",
       "<!-- 140331691404368&#45;&gt;140331401103232 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>140331691404368&#45;&gt;140331401103232</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M766,-1815.8C766,-1806.7 766,-1794.79 766,-1784.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"769.5,-1784.84 766,-1774.84 762.5,-1784.84 769.5,-1784.84\"/>\n",
       "</g>\n",
       "<!-- 140331401102512 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>140331401102512</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"739.5,-1527 668.5,-1527 668.5,-1508 739.5,-1508 739.5,-1527\"/>\n",
       "<text text-anchor=\"middle\" x=\"704\" y=\"-1515\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102512&#45;&gt;140331401085728 -->\n",
       "<g id=\"edge84\" class=\"edge\">\n",
       "<title>140331401102512&#45;&gt;140331401085728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M686.84,-1507.99C667.27,-1498.29 635.03,-1482.31 612.28,-1471.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"613.69,-1467.82 603.18,-1466.52 610.58,-1474.09 613.69,-1467.82\"/>\n",
       "</g>\n",
       "<!-- 140331401101472 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>140331401101472</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"739.5,-1351 668.5,-1351 668.5,-1332 739.5,-1332 739.5,-1351\"/>\n",
       "<text text-anchor=\"middle\" x=\"704\" y=\"-1339\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401102512&#45;&gt;140331401101472 -->\n",
       "<g id=\"edge88\" class=\"edge\">\n",
       "<title>140331401102512&#45;&gt;140331401101472</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M704,-1507.63C704,-1480.64 704,-1399.03 704,-1361.61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"707.5,-1361.14 704,-1351.14 700.5,-1361.14 707.5,-1361.14\"/>\n",
       "</g>\n",
       "<!-- 140331401102432 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>140331401102432</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"793.5,-1587.5 692.5,-1587.5 692.5,-1568.5 793.5,-1568.5 793.5,-1587.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"743\" y=\"-1575.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401102432&#45;&gt;140331401102512 -->\n",
       "<g id=\"edge85\" class=\"edge\">\n",
       "<title>140331401102432&#45;&gt;140331401102512</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M737.25,-1568.37C731.54,-1559.81 722.65,-1546.47 715.49,-1535.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"718.24,-1533.55 709.78,-1527.17 712.41,-1537.43 718.24,-1533.55\"/>\n",
       "</g>\n",
       "<!-- 140331691404944 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>140331691404944</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"793.5,-1659 662.5,-1659 662.5,-1629 793.5,-1629 793.5,-1659\"/>\n",
       "<text text-anchor=\"middle\" x=\"728\" y=\"-1647\" font-family=\"monospace\" font-size=\"10.00\">classifier.4.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"728\" y=\"-1636\" font-family=\"monospace\" font-size=\"10.00\"> (4096, 4096)</text>\n",
       "</g>\n",
       "<!-- 140331691404944&#45;&gt;140331401102432 -->\n",
       "<g id=\"edge86\" class=\"edge\">\n",
       "<title>140331691404944&#45;&gt;140331401102432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M731.33,-1628.8C733.46,-1619.7 736.25,-1607.79 738.57,-1597.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"742.05,-1598.38 740.93,-1587.84 735.24,-1596.78 742.05,-1598.38\"/>\n",
       "</g>\n",
       "<!-- 140331401101472&#45;&gt;140331691507616 -->\n",
       "<g id=\"edge87\" class=\"edge\">\n",
       "<title>140331401101472&#45;&gt;140331691507616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M696.57,-1331.75C690.15,-1324.18 680.69,-1313.05 672.88,-1303.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"675.43,-1301.45 666.29,-1296.09 670.1,-1305.98 675.43,-1301.45\"/>\n",
       "</g>\n",
       "<!-- 140327258485856&#45;&gt;140327258485376 -->\n",
       "<g id=\"edge90\" class=\"edge\">\n",
       "<title>140327258485856&#45;&gt;140327258485376</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M793.51,-1221.84C789.97,-1209.59 783.21,-1186.52 777,-1167 774.22,-1158.25 770.98,-1148.58 768.22,-1140.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"771.52,-1139.32 764.96,-1131 764.9,-1141.6 771.52,-1139.32\"/>\n",
       "</g>\n",
       "<!-- 140327258484000 -->\n",
       "<g id=\"node86\" class=\"node\">\n",
       "<title>140327258484000</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"240.5,-1186 145.5,-1186 145.5,-1167 240.5,-1167 240.5,-1186\"/>\n",
       "<text text-anchor=\"middle\" x=\"193\" y=\"-1174\" font-family=\"monospace\" font-size=\"10.00\">MeanBackward0</text>\n",
       "</g>\n",
       "<!-- 140327258484000&#45;&gt;140331691490112 -->\n",
       "<g id=\"edge106\" class=\"edge\">\n",
       "<title>140327258484000&#45;&gt;140331691490112</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.3,-1166.96C176.03,-1149.23 152,-1106.81 152,-1067.5 152,-1067.5 152,-1067.5 152,-845.5 152,-639.44 0,-613.56 0,-407.5 0,-407.5 0,-407.5 0,-185.5 0,-108.78 106.45,-86.47 174.2,-80.04\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"174.64,-83.52 184.3,-79.17 174.04,-76.54 174.64,-83.52\"/>\n",
       "</g>\n",
       "<!-- 140327258482960&#45;&gt;140327258484000 -->\n",
       "<g id=\"edge107\" class=\"edge\">\n",
       "<title>140327258482960&#45;&gt;140327258484000</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M240.31,-1331.74C232.48,-1306.2 209.54,-1231.43 198.75,-1196.23\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"201.98,-1194.84 195.7,-1186.31 195.29,-1196.89 201.98,-1194.84\"/>\n",
       "</g>\n",
       "<!-- 140331401085568 -->\n",
       "<g id=\"node88\" class=\"node\">\n",
       "<title>140331401085568</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"253.5,-1406 152.5,-1406 152.5,-1387 253.5,-1387 253.5,-1406\"/>\n",
       "<text text-anchor=\"middle\" x=\"203\" y=\"-1394\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401085568&#45;&gt;140327258482960 -->\n",
       "<g id=\"edge108\" class=\"edge\">\n",
       "<title>140331401085568&#45;&gt;140327258482960</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M209.61,-1386.75C215.26,-1379.26 223.54,-1368.28 230.45,-1359.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.29,-1361.18 236.52,-1351.09 227.7,-1356.96 233.29,-1361.18\"/>\n",
       "</g>\n",
       "<!-- 140331691483504 -->\n",
       "<g id=\"node89\" class=\"node\">\n",
       "<title>140331691483504</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"261.5,-1472 142.5,-1472 142.5,-1442 261.5,-1442 261.5,-1472\"/>\n",
       "<text text-anchor=\"middle\" x=\"202\" y=\"-1460\" font-family=\"monospace\" font-size=\"10.00\">classifier.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"202\" y=\"-1449\" font-family=\"monospace\" font-size=\"10.00\"> (1000)</text>\n",
       "</g>\n",
       "<!-- 140331691483504&#45;&gt;140331401085568 -->\n",
       "<g id=\"edge109\" class=\"edge\">\n",
       "<title>140331691483504&#45;&gt;140331401085568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M202.24,-1441.84C202.37,-1434.21 202.53,-1424.7 202.68,-1416.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"206.18,-1416.32 202.85,-1406.27 199.18,-1416.2 206.18,-1416.32\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa07c64ed70>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_dot(double_backprop(x, model), params=dict(list(model.named_parameters()) + [('x', x)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"439pt\" height=\"678pt\"\n",
       " viewBox=\"0.00 0.00 439.00 678.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 674)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-674 435,-674 435,4 -4,4\"/>\n",
       "<!-- 140331401151024 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140331401151024</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"135,-31 64,-31 64,0 135,0 135,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"99.5\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> (1, 128)</text>\n",
       "</g>\n",
       "<!-- 140331401085648 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140331401085648</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"144,-86 55,-86 55,-67 144,-67 144,-86\"/>\n",
       "<text text-anchor=\"middle\" x=\"99.5\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331401085648&#45;&gt;140331401151024 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140331401085648&#45;&gt;140331401151024</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M99.5,-66.79C99.5,-60.07 99.5,-50.4 99.5,-41.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103,-41.19 99.5,-31.19 96,-41.19 103,-41.19\"/>\n",
       "</g>\n",
       "<!-- 140331401140416 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140331401140416</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"117,-208 10,-208 10,-189 117,-189 117,-208\"/>\n",
       "<text text-anchor=\"middle\" x=\"63.5\" y=\"-196\" font-family=\"monospace\" font-size=\"10.00\">SigmoidBackward</text>\n",
       "</g>\n",
       "<!-- 140331401140416&#45;&gt;140331401085648 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140331401140416&#45;&gt;140331401085648</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M65.39,-188.57C68.47,-174.34 75.01,-145.72 82.5,-122 85.27,-113.23 88.84,-103.67 92,-95.68\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"95.3,-96.84 95.8,-86.26 88.81,-94.22 95.3,-96.84\"/>\n",
       "</g>\n",
       "<!-- 140331401140016 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140331401140016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"273,-373 142,-373 142,-354 273,-354 273,-373\"/>\n",
       "<text text-anchor=\"middle\" x=\"207.5\" y=\"-361\" font-family=\"monospace\" font-size=\"10.00\">UnsafeSplitBackward</text>\n",
       "</g>\n",
       "<!-- 140331401140016&#45;&gt;140331401140416 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140331401140016&#45;&gt;140331401140416</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M145.45,-353.99C122.46,-347.73 98.19,-336.86 82.5,-318 58.93,-289.66 58.86,-243.9 61.07,-218.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"64.58,-218.45 62.17,-208.14 57.62,-217.7 64.58,-218.45\"/>\n",
       "</g>\n",
       "<!-- 140331401141936 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140331401141936</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"199,-318 92,-318 92,-299 199,-299 199,-318\"/>\n",
       "<text text-anchor=\"middle\" x=\"145.5\" y=\"-306\" font-family=\"monospace\" font-size=\"10.00\">SigmoidBackward</text>\n",
       "</g>\n",
       "<!-- 140331401140016&#45;&gt;140331401141936 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140331401140016&#45;&gt;140331401141936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.54,-353.98C188.37,-346.15 174.56,-334.34 163.47,-324.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"165.52,-322.01 155.64,-318.17 160.97,-327.33 165.52,-322.01\"/>\n",
       "</g>\n",
       "<!-- 140331401142176 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140331401142176</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"324,-318 217,-318 217,-299 324,-299 324,-318\"/>\n",
       "<text text-anchor=\"middle\" x=\"270.5\" y=\"-306\" font-family=\"monospace\" font-size=\"10.00\">SigmoidBackward</text>\n",
       "</g>\n",
       "<!-- 140331401140016&#45;&gt;140331401142176 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140331401140016&#45;&gt;140331401142176</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M217.62,-353.98C226.94,-346.15 240.97,-334.34 252.24,-324.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"254.79,-327.29 260.19,-318.17 250.29,-321.93 254.79,-327.29\"/>\n",
       "</g>\n",
       "<!-- 140331401142016 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140331401142016</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"431,-318 342,-318 342,-299 431,-299 431,-318\"/>\n",
       "<text text-anchor=\"middle\" x=\"386.5\" y=\"-306\" font-family=\"monospace\" font-size=\"10.00\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 140331401140016&#45;&gt;140331401142016 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140331401140016&#45;&gt;140331401142016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M236.26,-353.98C266.44,-345.05 314.07,-330.94 347.6,-321.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"348.92,-324.28 357.52,-318.08 346.93,-317.57 348.92,-324.28\"/>\n",
       "</g>\n",
       "<!-- 140331401140976 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140331401140976</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"252,-428 163,-428 163,-409 252,-409 252,-428\"/>\n",
       "<text text-anchor=\"middle\" x=\"207.5\" y=\"-416\" font-family=\"monospace\" font-size=\"10.00\">AddBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401140976&#45;&gt;140331401140016 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140331401140976&#45;&gt;140331401140016</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M207.5,-408.75C207.5,-401.8 207.5,-391.85 207.5,-383.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"211,-383.09 207.5,-373.09 204,-383.09 211,-383.09\"/>\n",
       "</g>\n",
       "<!-- 140331401141136 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140331401141136</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"199,-483 104,-483 104,-464 199,-464 199,-483\"/>\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-471\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331401141136&#45;&gt;140331401140976 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140331401141136&#45;&gt;140331401140976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M160.5,-463.98C168.69,-456.23 181.01,-444.58 190.97,-435.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"193.48,-437.59 198.34,-428.17 188.67,-432.5 193.48,-437.59\"/>\n",
       "</g>\n",
       "<!-- 140331401141376 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140331401141376</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"101,-538 0,-538 0,-519 101,-519 101,-538\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-526\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401141376&#45;&gt;140331401141136 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140331401141376&#45;&gt;140331401141136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M66.73,-518.98C82.68,-510.61 107.28,-497.7 125.88,-487.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"127.75,-490.92 134.98,-483.17 124.5,-484.72 127.75,-490.92\"/>\n",
       "</g>\n",
       "<!-- 140331401150544 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140331401150544</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"80,-604 21,-604 21,-574 80,-574 80,-604\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-592\" font-family=\"monospace\" font-size=\"10.00\">bias_hh</text>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-581\" font-family=\"monospace\" font-size=\"10.00\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140331401150544&#45;&gt;140331401141376 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140331401150544&#45;&gt;140331401141376</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-573.84C50.5,-566.21 50.5,-556.7 50.5,-548.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-548.27 50.5,-538.27 47,-548.27 54,-548.27\"/>\n",
       "</g>\n",
       "<!-- 140331401141456 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140331401141456</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"190,-538 119,-538 119,-519 190,-519 190,-538\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-526\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401141456&#45;&gt;140331401141136 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140331401141456&#45;&gt;140331401141136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154,-518.75C153.61,-511.8 153.05,-501.85 152.55,-493.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"156.05,-492.88 151.99,-483.09 149.06,-493.27 156.05,-492.88\"/>\n",
       "</g>\n",
       "<!-- 140331401139856 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140331401139856</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"205,-598.5 104,-598.5 104,-579.5 205,-579.5 205,-598.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-586.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401139856&#45;&gt;140331401141456 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140331401139856&#45;&gt;140331401141456</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-579.37C154.5,-571.25 154.5,-558.81 154.5,-548.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"158,-548.17 154.5,-538.17 151,-548.17 158,-548.17\"/>\n",
       "</g>\n",
       "<!-- 140331401129808 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140331401129808</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"196,-670 113,-670 113,-640 196,-640 196,-670\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-658\" font-family=\"monospace\" font-size=\"10.00\">weight_hh</text>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-647\" font-family=\"monospace\" font-size=\"10.00\"> (512, 128)</text>\n",
       "</g>\n",
       "<!-- 140331401129808&#45;&gt;140331401139856 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140331401129808&#45;&gt;140331401139856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.5,-639.8C154.5,-630.7 154.5,-618.79 154.5,-608.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"158,-608.84 154.5,-598.84 151,-608.84 158,-608.84\"/>\n",
       "</g>\n",
       "<!-- 140331401141216 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140331401141216</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"312,-483 217,-483 217,-464 312,-464 312,-483\"/>\n",
       "<text text-anchor=\"middle\" x=\"264.5\" y=\"-471\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 140331401141216&#45;&gt;140331401140976 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140331401141216&#45;&gt;140331401140976</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M255.34,-463.98C247,-456.23 234.47,-444.58 224.32,-435.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"226.53,-432.42 216.82,-428.17 221.76,-437.54 226.53,-432.42\"/>\n",
       "</g>\n",
       "<!-- 140331401141536 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140331401141536</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"313,-538 212,-538 212,-519 313,-519 313,-538\"/>\n",
       "<text text-anchor=\"middle\" x=\"262.5\" y=\"-526\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401141536&#45;&gt;140331401141216 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140331401141536&#45;&gt;140331401141216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M262.83,-518.75C263.09,-511.8 263.47,-501.85 263.8,-493.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"267.3,-493.21 264.18,-483.09 260.3,-492.95 267.3,-493.21\"/>\n",
       "</g>\n",
       "<!-- 140331401129904 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140331401129904</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"292,-604 233,-604 233,-574 292,-574 292,-604\"/>\n",
       "<text text-anchor=\"middle\" x=\"262.5\" y=\"-592\" font-family=\"monospace\" font-size=\"10.00\">bias_ih</text>\n",
       "<text text-anchor=\"middle\" x=\"262.5\" y=\"-581\" font-family=\"monospace\" font-size=\"10.00\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140331401129904&#45;&gt;140331401141536 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140331401129904&#45;&gt;140331401141536</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M262.5,-573.84C262.5,-566.21 262.5,-556.7 262.5,-548.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"266,-548.27 262.5,-538.27 259,-548.27 266,-548.27\"/>\n",
       "</g>\n",
       "<!-- 140331401141696 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140331401141696</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"402,-538 331,-538 331,-519 402,-519 402,-538\"/>\n",
       "<text text-anchor=\"middle\" x=\"366.5\" y=\"-526\" font-family=\"monospace\" font-size=\"10.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 140331401141696&#45;&gt;140331401141216 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140331401141696&#45;&gt;140331401141216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M350.11,-518.98C333.93,-510.57 308.93,-497.59 290.12,-487.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"291.67,-484.68 281.19,-483.17 288.45,-490.89 291.67,-484.68\"/>\n",
       "</g>\n",
       "<!-- 140331401141856 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140331401141856</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"417,-598.5 316,-598.5 316,-579.5 417,-579.5 417,-598.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"366.5\" y=\"-586.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 140331401141856&#45;&gt;140331401141696 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140331401141856&#45;&gt;140331401141696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M366.5,-579.37C366.5,-571.25 366.5,-558.81 366.5,-548.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370,-548.17 366.5,-538.17 363,-548.17 370,-548.17\"/>\n",
       "</g>\n",
       "<!-- 140327258456240 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140327258456240</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"408,-670 325,-670 325,-640 408,-640 408,-670\"/>\n",
       "<text text-anchor=\"middle\" x=\"366.5\" y=\"-658\" font-family=\"monospace\" font-size=\"10.00\">weight_ih</text>\n",
       "<text text-anchor=\"middle\" x=\"366.5\" y=\"-647\" font-family=\"monospace\" font-size=\"10.00\"> (512, 128)</text>\n",
       "</g>\n",
       "<!-- 140327258456240&#45;&gt;140331401141856 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140327258456240&#45;&gt;140331401141856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M366.5,-639.8C366.5,-630.7 366.5,-618.79 366.5,-608.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370,-608.84 366.5,-598.84 363,-608.84 370,-608.84\"/>\n",
       "</g>\n",
       "<!-- 140331401140336 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140331401140336</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"181,-147 92,-147 92,-128 181,-128 181,-147\"/>\n",
       "<text text-anchor=\"middle\" x=\"136.5\" y=\"-135\" font-family=\"monospace\" font-size=\"10.00\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 140331401140336&#45;&gt;140331401085648 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140331401140336&#45;&gt;140331401085648</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M131.04,-127.79C125.63,-119.17 117.19,-105.71 110.4,-94.89\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"113.26,-92.85 104.98,-86.24 107.33,-96.57 113.26,-92.85\"/>\n",
       "</g>\n",
       "<!-- 140331401140896 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140331401140896</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"251,-208 162,-208 162,-189 251,-189 251,-208\"/>\n",
       "<text text-anchor=\"middle\" x=\"206.5\" y=\"-196\" font-family=\"monospace\" font-size=\"10.00\">AddBackward1</text>\n",
       "</g>\n",
       "<!-- 140331401140896&#45;&gt;140331401140336 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140331401140896&#45;&gt;140331401140336</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.17,-188.79C185.31,-179.64 168.01,-165.06 154.83,-153.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"156.77,-151.01 146.87,-147.24 152.26,-156.36 156.77,-151.01\"/>\n",
       "</g>\n",
       "<!-- 140331401151216 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140331401151216</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"270,-153 199,-153 199,-122 270,-122 270,-153\"/>\n",
       "<text text-anchor=\"middle\" x=\"234.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\"> (1, 128)</text>\n",
       "</g>\n",
       "<!-- 140331401140896&#45;&gt;140331401151216 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140331401140896&#45;&gt;140331401151216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M210.63,-188.79C213.93,-181.85 218.71,-171.77 223.13,-162.46\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"226.4,-163.73 227.53,-153.19 220.08,-160.73 226.4,-163.73\"/>\n",
       "</g>\n",
       "<!-- 140331401141296 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140331401141296</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"199,-263 110,-263 110,-244 199,-244 199,-263\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.5\" y=\"-251\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331401141296&#45;&gt;140331401140896 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140331401141296&#45;&gt;140331401140896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.09,-243.75C170.66,-236.03 181.88,-224.6 191.01,-215.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"193.57,-217.68 198.07,-208.09 188.57,-212.78 193.57,-217.68\"/>\n",
       "</g>\n",
       "<!-- 140331401141936&#45;&gt;140331401141296 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140331401141936&#45;&gt;140331401141296</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M146.99,-298.75C148.17,-291.8 149.86,-281.85 151.34,-273.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"154.82,-273.53 153.04,-263.09 147.92,-272.36 154.82,-273.53\"/>\n",
       "</g>\n",
       "<!-- 140331401141616 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140331401141616</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"315,-263 226,-263 226,-244 315,-244 315,-263\"/>\n",
       "<text text-anchor=\"middle\" x=\"270.5\" y=\"-251\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 140331401141616&#45;&gt;140331401140896 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140331401141616&#45;&gt;140331401140896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M260.22,-243.98C250.66,-236.07 236.22,-224.11 224.72,-214.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"226.9,-211.85 216.97,-208.17 222.44,-217.24 226.9,-211.85\"/>\n",
       "</g>\n",
       "<!-- 140331401142176&#45;&gt;140331401141616 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140331401142176&#45;&gt;140331401141616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M270.5,-298.75C270.5,-291.8 270.5,-281.85 270.5,-273.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"274,-273.09 270.5,-263.09 267,-273.09 274,-273.09\"/>\n",
       "</g>\n",
       "<!-- 140331401142016&#45;&gt;140331401141616 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140331401142016&#45;&gt;140331401141616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M367.86,-298.98C349.2,-290.46 320.25,-277.23 298.76,-267.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"300.03,-264.14 289.48,-263.17 297.12,-270.51 300.03,-264.14\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fa17350da50>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_cell = nn.LSTMCell(128, 128)\n",
    "x = torch.randn(1, 128)\n",
    "make_dot(lstm_cell(x), params=dict(list(lstm_cell.named_parameters())))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}