11_train_imagenette_jcat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from exp.nb_10c import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torchviz for visualizing neural networks\n",
    "# first install in your fastai environment, with `pip install torchviz`\n",
    "from torchviz import make_dot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XResNets and Imagenet(te) training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=1681)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Get the imagenette data set \n",
    "Download and preprocess the data, create a databunch object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = datasets.untar_data(datasets.URLs.IMAGENETTE_160)\n",
    "\n",
    "size = 128\n",
    "tfms = [make_rgb, RandomResizedCrop(size, scale=(0.35,1)), np_to_float, PilRandomFlip()]\n",
    "\n",
    "bs = 64\n",
    "\n",
    "il = ImageList.from_files(path, tfms=tfms)\n",
    "sd = SplitData.split_by_func(il, partial(grandparent_splitter, valid_name='val'))\n",
    "ll = label_by_func(sd, parent_labeler, proc_y=CategoryProcessor())\n",
    "\n",
    "ll.valid.x.tfms = [make_rgb, CenterCrop(size), np_to_float]\n",
    "\n",
    "data = ll.to_databunch(bs, c_in=3, c_out=10, num_workers=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. XResNet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Some helper functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=1701)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "# set activation function\n",
    "act_fn = nn.ReLU(inplace=True)\n",
    "\n",
    "# pass-through method returns input\n",
    "def noop(x): return x\n",
    "\n",
    "# convert image to vector for each channel, so output is of size (n_batch,n_channels)\n",
    "class Flatten(nn.Module):\n",
    "    def forward(self, x): return x.view(x.size(0), -1)\n",
    "\n",
    "# preset all but ni and nf inputs for Conv2d: ks=3 stride=1, bias = False\n",
    "#      ks is the kernel size of the convolution filter\n",
    "def conv(ni, nf, ks=3, stride=1, bias=False):\n",
    "    return nn.Conv2d(ni, nf, kernel_size=ks, stride=stride, padding=ks//2, bias=bias)\n",
    "\n",
    "# initialize a neural network recursively \n",
    "def init_cnn(m):\n",
    "    # initialize bias to zero if 'bias' is not None\n",
    "    if getattr(m, 'bias', None) is not None: nn.init.constant_(m.bias, 0)\n",
    "    # if the model is a convolutional or linear layer, do kaiming initialization of weights\n",
    "    if isinstance(m, (nn.Conv2d,nn.Linear)): nn.init.kaiming_normal_(m.weight)\n",
    "    # recursively apply init_cnn to all children\n",
    "    for l in m.children(): init_cnn(l)\n",
    "\n",
    "# make a convolution layer, consisting of a convolution, a batchnorm, and optionally an activation function\n",
    "def conv_layer(ni, nf, ks=3, stride=1, zero_bn=False, act=True):\n",
    "    bn = nn.BatchNorm2d(nf)\n",
    "    nn.init.constant_(bn.weight, 0. if zero_bn else 1.)\n",
    "    layers = [conv(ni, nf, ks, stride=stride), bn]\n",
    "    if act: layers.append(act_fn)\n",
    "    return nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 A class to create ResNet building blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "# Class to creat a resnet block\n",
    "class ResBlock(nn.Module):\n",
    "    # expansion = 1 for resnet 18 and 34\n",
    "    # expansion = 4 for resnet 50, 101, and 152\n",
    "    def __init__(self, expansion, ni, nh, stride=1):\n",
    "        super().__init__()\n",
    "        nf,ni = nh*expansion,ni*expansion\n",
    "        # two conv_layers for resnet 18 and 43\n",
    "        # three conv_layers for resnet 50, 101, 152\n",
    "        # final layer has batchnorm weights initialized to zero, and no activation applied\n",
    "        layers  = [conv_layer(ni, nh, ks=3, stride=stride),\n",
    "                   conv_layer(nh, nf, ks=3, zero_bn=True, act=False)\n",
    "        ] if expansion == 1 else [\n",
    "                   conv_layer(ni, nh, ks=1),\n",
    "                   conv_layer(nh, nh, ks=3, stride=stride),\n",
    "                   conv_layer(nh, nf, ks=1, zero_bn=True, act=False)\n",
    "        ]\n",
    "        \n",
    "        \n",
    "        # left branch of resnet\n",
    "        self.convs = nn.Sequential(*layers)\n",
    "        \n",
    "        # right branch of resnet\n",
    "        self.idconv = noop if ni==nf else conv_layer(ni, nf, ks=1, act=False)\n",
    "        self.pool = noop if stride==1 else nn.AvgPool2d(2, ceil_mode=True)\n",
    "\n",
    "    # Apply the activation function to the concatenation of the left and right branches of the resnet\n",
    "    #     and return the output\n",
    "    def forward(self, x):\n",
    "        return act_fn(self.convs(x) + self.idconv(self.pool(x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 An XResNet class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "# Class to build XResNet\n",
    "class XResNet(nn.Sequential):\n",
    "    # output: XResNet model\n",
    "    # Inputs:\n",
    "    # expansion = 1 for resnet 18 and 34\n",
    "    # expansion = 4 for resnet 50, 101, and 152\n",
    "    # layers: a list of positive integers specifying the number of resblocks in each layer of res_layers\n",
    "    # c_in: the number of input channels\n",
    "    # c_out: the number of output channels\n",
    "    \n",
    "    # a classmethod can access and change the class state; cls is the class\n",
    "    @classmethod \n",
    "    def create(cls, expansion, layers, c_in=3, c_out=1000):\n",
    "        \n",
    "        \n",
    "        # number of filters in each layer for the 'stem' part of the network\n",
    "        nfs = [c_in, (c_in+1)*8, 64, 64]\n",
    "        \n",
    "        # build a sequential list of three conv_layers with numbers of input and output channels specified by nfs\n",
    "        stem = [conv_layer(nfs[i], nfs[i+1], stride=2 if i==0 else 1)\n",
    "            for i in range(3)]\n",
    "\n",
    "        # build a sequential list of resblock layers\n",
    "        #      nfs is a list of the number of filters in each layer for the 'res_layers' part of the network\n",
    "        #      note that len(layers) must be less than len(nfs)\n",
    "        nfs = [64//expansion,64,128,256,512]\n",
    "        res_layers = [cls._make_layer(expansion, nfs[i], nfs[i+1],\n",
    "                                      n_blocks=l, stride=1 if i==0 else 2)\n",
    "                  for i,l in enumerate(layers)]\n",
    "        \n",
    "        # build the XResNet model from the list [stem, MaxPool, res_layers, AdaptiveAvgPool, flatten, linear] layers \n",
    "        res = cls(\n",
    "            *stem,\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),\n",
    "            *res_layers,\n",
    "            nn.AdaptiveAvgPool2d(1), Flatten(),\n",
    "            nn.Linear(nfs[-1]*expansion, c_out),\n",
    "        )\n",
    "        # return the initialized XResNet\n",
    "        init_cnn(res)\n",
    "        return res\n",
    "    \n",
    "    # a static method is a utility function that cannot access or change the class state\n",
    "    # create a network model consisting of a set of n_blocks ResBlock layers\n",
    "    # Q: what does the * in front of the list do?\n",
    "    @staticmethod \n",
    "    def _make_layer(expansion, ni, nf, n_blocks, stride):\n",
    "        return nn.Sequential(\n",
    "            *[ResBlock(expansion, ni if i==0 else nf, nf, stride if i==0 else 1)\n",
    "              for i in range(n_blocks)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 Commonly used XResNet models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def xresnet18 (**kwargs): return XResNet.create(1, [2, 2,  2, 2], **kwargs)\n",
    "def xresnet34 (**kwargs): return XResNet.create(1, [3, 4,  6, 3], **kwargs)\n",
    "def xresnet50 (**kwargs): return XResNet.create(4, [3, 4,  6, 3], **kwargs)\n",
    "def xresnet101(**kwargs): return XResNet.create(4, [3, 4, 23, 3], **kwargs)\n",
    "def xresnet152(**kwargs): return XResNet.create(4, [3, 8, 36, 3], **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Visualize a ResBlock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResBlock(\n",
      "  (convs): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(3, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (idconv): Sequential(\n",
      "    (0): Conv2d(3, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      ")\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: %3 Pages: 1 -->\r\n",
       "<svg width=\"864pt\" height=\"476pt\"\r\n",
       " viewBox=\"0.00 0.00 864.00 475.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.990826 0.990826) rotate(0) translate(4 476)\">\r\n",
       "<title>%3</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-476 868,-476 868,4 -4,4\"/>\r\n",
       "<!-- 1230937806832 -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>1230937806832</title>\r\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"561.5,-21 471.5,-21 471.5,-0 561.5,-0 561.5,-21\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"516.5\" y=\"-7.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894207832 -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>1230894207832</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"561.5,-78 471.5,-78 471.5,-57 561.5,-57 561.5,-78\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"516.5\" y=\"-64.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894207832&#45;&gt;1230937806832 -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>1230894207832&#45;&gt;1230937806832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M516.5,-56.9197C516.5,-49.9083 516.5,-40.1442 516.5,-31.4652\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"520,-31.3408 516.5,-21.3408 513,-31.3409 520,-31.3408\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230894210184 -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>1230894210184</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"476,-135 329,-135 329,-114 476,-114 476,-135\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-121.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894210184&#45;&gt;1230894207832 -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>1230894210184&#45;&gt;1230894207832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M422.357,-113.92C440.31,-105.258 466.971,-92.3957 487.342,-82.5675\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"489.002,-85.6527 496.488,-78.155 485.96,-79.3481 489.002,-85.6527\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230894207496 -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>1230894207496</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"307,-198.5 182,-198.5 182,-177.5 307,-177.5 307,-198.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"244.5\" y=\"-184.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894207496&#45;&gt;1230894210184 -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>1230894207496&#45;&gt;1230894210184</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M269.188,-177.391C295.856,-167.01 338.528,-150.4 368.391,-138.777\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"369.936,-141.931 377.986,-135.042 367.397,-135.408 369.936,-141.931\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606408 -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>1230892606408</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"234.5,-268.5 144.5,-268.5 144.5,-247.5 234.5,-247.5 234.5,-268.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"189.5\" y=\"-254.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606408&#45;&gt;1230894207496 -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>1230892606408&#45;&gt;1230894207496</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.38,-247.257C205.957,-236.653 219.846,-219.481 230.328,-206.521\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.067,-208.701 236.634,-198.725 227.624,-204.299 233.067,-208.701\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606296 -->\r\n",
       "<g id=\"node6\" class=\"node\"><title>1230892606296</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"263,-332 116,-332 116,-311 263,-311 263,-332\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"189.5\" y=\"-318.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606296&#45;&gt;1230892606408 -->\r\n",
       "<g id=\"edge5\" class=\"edge\"><title>1230892606296&#45;&gt;1230892606408</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M189.5,-310.891C189.5,-302.366 189.5,-289.639 189.5,-278.923\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"193,-278.745 189.5,-268.745 186,-278.745 193,-278.745\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892605904 -->\r\n",
       "<g id=\"node7\" class=\"node\"><title>1230892605904</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"125,-395.5 0,-395.5 0,-374.5 125,-374.5 125,-395.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-381.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892605904&#45;&gt;1230892606296 -->\r\n",
       "<g id=\"edge6\" class=\"edge\"><title>1230892605904&#45;&gt;1230892606296</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M82.3438,-374.391C103.314,-364.236 136.594,-348.12 160.504,-336.542\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"162.321,-339.55 169.796,-332.042 159.27,-333.25 162.321,-339.55\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606016 -->\r\n",
       "<g id=\"node8\" class=\"node\"><title>1230892606016</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"109,-472 16,-472 16,-438 109,-438 109,-472\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-458.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-445.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 3, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606016&#45;&gt;1230892605904 -->\r\n",
       "<g id=\"edge7\" class=\"edge\"><title>1230892606016&#45;&gt;1230892605904</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M62.5,-437.885C62.5,-428.309 62.5,-416.088 62.5,-405.912\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"66.0001,-405.895 62.5,-395.895 59.0001,-405.895 66.0001,-405.895\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892605680 -->\r\n",
       "<g id=\"node9\" class=\"node\"><title>1230892605680</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"236,-402 143,-402 143,-368 236,-368 236,-402\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"189.5\" y=\"-388.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"189.5\" y=\"-375.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892605680&#45;&gt;1230892606296 -->\r\n",
       "<g id=\"edge8\" class=\"edge\"><title>1230892605680&#45;&gt;1230892606296</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M189.5,-367.842C189.5,-360.012 189.5,-350.54 189.5,-342.282\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"193,-342.042 189.5,-332.042 186,-342.042 193,-342.042\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606072 -->\r\n",
       "<g id=\"node10\" class=\"node\"><title>1230892606072</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"336.5,-402 254.5,-402 254.5,-368 336.5,-368 336.5,-402\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"295.5\" y=\"-388.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"295.5\" y=\"-375.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606072&#45;&gt;1230892606296 -->\r\n",
       "<g id=\"edge9\" class=\"edge\"><title>1230892606072&#45;&gt;1230892606296</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M267.65,-367.842C251.241,-358.322 230.646,-346.373 214.718,-337.131\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.352,-334.033 205.946,-332.042 212.839,-340.088 216.352,-334.033\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606352 -->\r\n",
       "<g id=\"node11\" class=\"node\"><title>1230892606352</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"346,-275 253,-275 253,-241 346,-241 346,-275\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"299.5\" y=\"-261.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"299.5\" y=\"-248.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 128, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606352&#45;&gt;1230894207496 -->\r\n",
       "<g id=\"edge10\" class=\"edge\"><title>1230892606352&#45;&gt;1230894207496</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M286.466,-240.885C278.144,-230.596 267.352,-217.253 258.796,-206.674\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"261.513,-204.469 252.503,-198.895 256.07,-208.871 261.513,-204.469\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230894207440 -->\r\n",
       "<g id=\"node12\" class=\"node\"><title>1230894207440</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"418,-205 325,-205 325,-171 418,-171 418,-205\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"371.5\" y=\"-191.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"371.5\" y=\"-178.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894207440&#45;&gt;1230894210184 -->\r\n",
       "<g id=\"edge11\" class=\"edge\"><title>1230894207440&#45;&gt;1230894210184</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M379.645,-170.842C383.771,-162.656 388.802,-152.675 393.092,-144.163\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"396.314,-145.547 397.69,-135.042 390.064,-142.396 396.314,-145.547\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230894207272 -->\r\n",
       "<g id=\"node13\" class=\"node\"><title>1230894207272</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"518.5,-205 436.5,-205 436.5,-171 518.5,-171 518.5,-205\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"477.5\" y=\"-191.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"477.5\" y=\"-178.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894207272&#45;&gt;1230894210184 -->\r\n",
       "<g id=\"edge12\" class=\"edge\"><title>1230894207272&#45;&gt;1230894210184</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M457.795,-170.842C446.727,-161.766 432.968,-150.484 421.944,-141.444\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"424.088,-138.676 414.136,-135.042 419.65,-144.089 424.088,-138.676\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230894210856 -->\r\n",
       "<g id=\"node14\" class=\"node\"><title>1230894210856</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"735,-135 588,-135 588,-114 735,-114 735,-135\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"661.5\" y=\"-121.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 1230894210856&#45;&gt;1230894207832 -->\r\n",
       "<g id=\"edge13\" class=\"edge\"><title>1230894210856&#45;&gt;1230894207832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.243,-113.92C612.788,-105.023 577.648,-91.6942 551.495,-81.7739\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"552.545,-78.4291 541.954,-78.155 550.063,-84.9741 552.545,-78.4291\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606464 -->\r\n",
       "<g id=\"node15\" class=\"node\"><title>1230892606464</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"662,-198.5 537,-198.5 537,-177.5 662,-177.5 662,-198.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"599.5\" y=\"-184.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606464&#45;&gt;1230894210856 -->\r\n",
       "<g id=\"edge14\" class=\"edge\"><title>1230892606464&#45;&gt;1230894210856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M609.188,-177.391C618.596,-168.058 633.081,-153.69 644.425,-142.437\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"647.041,-144.772 651.676,-135.245 642.111,-139.802 647.041,-144.772\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892605848 -->\r\n",
       "<g id=\"node16\" class=\"node\"><title>1230892605848</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"643.5,-275 555.5,-275 555.5,-241 643.5,-241 643.5,-275\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"599.5\" y=\"-261.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">idconv.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"599.5\" y=\"-248.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 3, 1, 1)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892605848&#45;&gt;1230892606464 -->\r\n",
       "<g id=\"edge15\" class=\"edge\"><title>1230892605848&#45;&gt;1230892606464</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M599.5,-240.885C599.5,-231.309 599.5,-219.088 599.5,-208.912\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"603,-208.895 599.5,-198.895 596,-208.895 603,-208.895\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892606184 -->\r\n",
       "<g id=\"node17\" class=\"node\"><title>1230892606184</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"768.5,-205 680.5,-205 680.5,-171 768.5,-171 768.5,-205\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"724.5\" y=\"-191.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">idconv.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"724.5\" y=\"-178.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892606184&#45;&gt;1230894210856 -->\r\n",
       "<g id=\"edge16\" class=\"edge\"><title>1230892606184&#45;&gt;1230894210856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M707.947,-170.842C698.924,-162.033 687.771,-151.146 678.656,-142.247\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"680.875,-139.523 671.275,-135.042 675.986,-144.532 680.875,-139.523\"/>\r\n",
       "</g>\r\n",
       "<!-- 1230892605568 -->\r\n",
       "<g id=\"node18\" class=\"node\"><title>1230892605568</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"864,-205 787,-205 787,-171 864,-171 864,-205\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"825.5\" y=\"-191.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">idconv.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"825.5\" y=\"-178.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 1230892605568&#45;&gt;1230894210856 -->\r\n",
       "<g id=\"edge17\" class=\"edge\"><title>1230892605568&#45;&gt;1230894210856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M786.634,-172.425C759.421,-162.22 723.348,-148.693 696.926,-138.785\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"697.903,-135.413 687.311,-135.179 695.445,-141.968 697.903,-135.413\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x11e999c5860>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_block = ResBlock(1,3,128,1)\n",
    "print(res_block)\n",
    "learn0 = Learner(res_block, data, loss_func, lr=1, cb_funcs=cbfs, opt_func=opt_func)\n",
    "model=learn0.model.cpu()\n",
    "x = torch.randn([1, 3, 128, 128])\n",
    "graph = make_dot(model(x), params=dict(model.named_parameters()))\n",
    "graph.format = 'png'\n",
    "graph.render('bn')\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=2515)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Helper functions\n",
    "We need to replace the old `model_summary` since it used to take a `Runner`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_batch(dl, learn):\n",
    "    learn.xb,learn.yb = next(iter(dl))\n",
    "    learn.do_begin_fit(0)\n",
    "    learn('begin_batch')\n",
    "    learn('after_fit')\n",
    "    return learn.xb,learn.yb\n",
    "\n",
    "def model_summary(model, data, find_all=False, print_mod=False):\n",
    "    xb,yb = get_batch(data.valid_dl, learn)\n",
    "    mods = find_modules(model, is_lin_layer) if find_all else model.children()\n",
    "    f = lambda hook,mod,inp,out: print(f\"====\\n{mod}\\n\" if print_mod else \"\", out.shape)\n",
    "    with Hooks(mods, f) as hooks: learn.model(xb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Visualize the full XResNet-18"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define callbacks, loss function, model architecture, and optimization function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbfs = [partial(AvgStatsCallback,accuracy), ProgressCallback, CudaCallback,\n",
    "        partial(BatchTransformXCallback, norm_imagenette),\n",
    "#         partial(MixUp, alpha=0.2)\n",
    "       ]\n",
    "loss_func = LabelSmoothingCrossEntropy()\n",
    "arch = partial(xresnet18, c_out=10)\n",
    "opt_func = adam_opt(mom=0.9, mom_sqr=0.99, eps=1e-6, wd=1e-2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a Learner object with XResNet-18 for the imagenette data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(arch(), data, loss_func, lr=1, cb_funcs=cbfs, opt_func=opt_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### List the model layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " torch.Size([128, 32, 64, 64])\n",
      " torch.Size([128, 64, 64, 64])\n",
      " torch.Size([128, 64, 64, 64])\n",
      " torch.Size([128, 64, 32, 32])\n",
      " torch.Size([128, 64, 32, 32])\n",
      " torch.Size([128, 128, 16, 16])\n",
      " torch.Size([128, 256, 8, 8])\n",
      " torch.Size([128, 512, 4, 4])\n",
      " torch.Size([128, 512, 1, 1])\n",
      " torch.Size([128, 512])\n",
      " torch.Size([128, 10])\n"
     ]
    }
   ],
   "source": [
    "learn.model\n",
    "model_summary(learn.model, data, print_mod=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize the model architecture of XResNet-18 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: %3 Pages: 1 -->\r\n",
       "<svg width=\"866pt\" height=\"3294pt\"\r\n",
       " viewBox=\"0.00 0.00 866.32 3294.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.71068 0.71068) rotate(0) translate(4 4631)\">\r\n",
       "<title>%3</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-4631 1215,-4631 1215,4 -4,4\"/>\r\n",
       "<!-- 2422722663760 -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>2422722663760</title>\r\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"202,-21 101,-21 101,-0 202,-0 202,-21\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-7.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddmmBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722663928 -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>2422722663928</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"89.5,-91 35.5,-91 35.5,-57 89.5,-57 89.5,-91\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-77.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">10.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-64.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (10)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722663928&#45;&gt;2422722663760 -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>2422722663928&#45;&gt;2422722663760</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M85.8837,-56.8416C99.2743,-47.5885 115.986,-36.0407 129.192,-26.9149\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"131.454,-29.6063 137.691,-21.0419 127.475,-23.8474 131.454,-29.6063\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722663984 -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>2422722663984</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"195.5,-84.5 107.5,-84.5 107.5,-63.5 195.5,-63.5 195.5,-84.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-70.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ViewBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722663984&#45;&gt;2422722663760 -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>2422722663984&#45;&gt;2422722663760</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M151.5,-63.3906C151.5,-54.8657 151.5,-42.1392 151.5,-31.4235\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155,-31.2448 151.5,-21.2449 148,-31.2449 155,-31.2448\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664264 -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>2422722664264</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"195.5,-154.5 107.5,-154.5 107.5,-133.5 195.5,-133.5 195.5,-154.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-140.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ViewBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664264&#45;&gt;2422722663984 -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>2422722664264&#45;&gt;2422722663984</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M151.5,-133.257C151.5,-123.259 151.5,-107.422 151.5,-94.7821\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155,-94.7246 151.5,-84.7247 148,-94.7247 155,-94.7246\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664376 -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>2422722664376</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"199.5,-218 103.5,-218 103.5,-197 199.5,-197 199.5,-218\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-204.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">MeanBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664376&#45;&gt;2422722664264 -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>2422722664376&#45;&gt;2422722664264</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M151.5,-196.891C151.5,-188.366 151.5,-175.639 151.5,-164.923\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155,-164.745 151.5,-154.745 148,-164.745 155,-164.745\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664488 -->\r\n",
       "<g id=\"node6\" class=\"node\"><title>2422722664488</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"195.5,-275 107.5,-275 107.5,-254 195.5,-254 195.5,-275\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-261.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ViewBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664488&#45;&gt;2422722664376 -->\r\n",
       "<g id=\"edge5\" class=\"edge\"><title>2422722664488&#45;&gt;2422722664376</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M151.5,-253.92C151.5,-246.908 151.5,-237.144 151.5,-228.465\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155,-228.341 151.5,-218.341 148,-228.341 155,-228.341\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664600 -->\r\n",
       "<g id=\"node7\" class=\"node\"><title>2422722664600</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"196.5,-332 106.5,-332 106.5,-311 196.5,-311 196.5,-332\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-318.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664600&#45;&gt;2422722664488 -->\r\n",
       "<g id=\"edge6\" class=\"edge\"><title>2422722664600&#45;&gt;2422722664488</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M151.5,-310.92C151.5,-303.908 151.5,-294.144 151.5,-285.465\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155,-285.341 151.5,-275.341 148,-285.341 155,-285.341\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664712 -->\r\n",
       "<g id=\"node8\" class=\"node\"><title>2422722664712</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"196.5,-389 106.5,-389 106.5,-368 196.5,-368 196.5,-389\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-375.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664712&#45;&gt;2422722664600 -->\r\n",
       "<g id=\"edge7\" class=\"edge\"><title>2422722664712&#45;&gt;2422722664600</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M151.5,-367.92C151.5,-360.908 151.5,-351.144 151.5,-342.465\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155,-342.341 151.5,-332.341 148,-342.341 155,-342.341\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664824 -->\r\n",
       "<g id=\"node9\" class=\"node\"><title>2422722664824</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"329,-446 182,-446 182,-425 329,-425 329,-446\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"255.5\" y=\"-432.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664824&#45;&gt;2422722664712 -->\r\n",
       "<g id=\"edge8\" class=\"edge\"><title>2422722664824&#45;&gt;2422722664712</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M237.385,-424.92C221.155,-416.337 197.124,-403.628 178.606,-393.835\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"180.233,-390.736 169.757,-389.155 176.96,-396.924 180.233,-390.736\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664992 -->\r\n",
       "<g id=\"node10\" class=\"node\"><title>2422722664992</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"502,-509.5 377,-509.5 377,-488.5 502,-488.5 502,-509.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"439.5\" y=\"-495.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664992&#45;&gt;2422722664824 -->\r\n",
       "<g id=\"edge9\" class=\"edge\"><title>2422722664992&#45;&gt;2422722664824</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M410.75,-488.391C379.288,-477.875 328.699,-460.966 293.869,-449.324\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"294.642,-445.892 284.048,-446.042 292.423,-452.531 294.642,-445.892\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665216 -->\r\n",
       "<g id=\"node11\" class=\"node\"><title>2422722665216</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"484.5,-579.5 394.5,-579.5 394.5,-558.5 484.5,-558.5 484.5,-579.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"439.5\" y=\"-565.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665216&#45;&gt;2422722664992 -->\r\n",
       "<g id=\"edge10\" class=\"edge\"><title>2422722665216&#45;&gt;2422722664992</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M439.5,-558.257C439.5,-548.259 439.5,-532.422 439.5,-519.782\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"443,-519.725 439.5,-509.725 436,-519.725 443,-519.725\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665384 -->\r\n",
       "<g id=\"node12\" class=\"node\"><title>2422722665384</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"480,-643 333,-643 333,-622 480,-622 480,-643\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"406.5\" y=\"-629.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665384&#45;&gt;2422722665216 -->\r\n",
       "<g id=\"edge11\" class=\"edge\"><title>2422722665384&#45;&gt;2422722665216</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M411.656,-621.891C416.423,-613.007 423.639,-599.56 429.53,-588.581\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"432.627,-590.211 434.271,-579.745 426.459,-586.902 432.627,-590.211\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665496 -->\r\n",
       "<g id=\"node13\" class=\"node\"><title>2422722665496</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"535,-706.5 410,-706.5 410,-685.5 535,-685.5 535,-706.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"472.5\" y=\"-692.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665496&#45;&gt;2422722665384 -->\r\n",
       "<g id=\"edge12\" class=\"edge\"><title>2422722665496&#45;&gt;2422722665384</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M462.188,-685.391C452.076,-675.968 436.456,-661.413 424.329,-650.114\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"426.66,-647.502 416.958,-643.245 421.888,-652.623 426.66,-647.502\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664880 -->\r\n",
       "<g id=\"node14\" class=\"node\"><title>2422722664880</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"271.5,-776.5 181.5,-776.5 181.5,-755.5 271.5,-755.5 271.5,-776.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"226.5\" y=\"-762.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664880&#45;&gt;2422722664712 -->\r\n",
       "<g id=\"edge154\" class=\"edge\"><title>2422722664880&#45;&gt;2422722664712</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M205.975,-755.409C170.272,-737.14 100.5,-693.733 100.5,-633.5 100.5,-633.5 100.5,-633.5 100.5,-498 100.5,-459.785 123.156,-419.87 138.267,-397.497\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"141.321,-399.237 144.193,-389.039 135.588,-395.22 141.321,-399.237\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664880&#45;&gt;2422722665496 -->\r\n",
       "<g id=\"edge13\" class=\"edge\"><title>2422722664880&#45;&gt;2422722665496</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M260.964,-755.473C304.666,-743.393 380.088,-722.545 428.071,-709.281\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"429.166,-712.61 437.872,-706.572 427.301,-705.863 429.166,-712.61\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665832 -->\r\n",
       "<g id=\"node15\" class=\"node\"><title>2422722665832</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"271.5,-840 181.5,-840 181.5,-819 271.5,-819 271.5,-840\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"226.5\" y=\"-826.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665832&#45;&gt;2422722664880 -->\r\n",
       "<g id=\"edge14\" class=\"edge\"><title>2422722665832&#45;&gt;2422722664880</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M226.5,-818.891C226.5,-810.366 226.5,-797.639 226.5,-786.923\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230,-786.745 226.5,-776.745 223,-786.745 230,-786.745\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665944 -->\r\n",
       "<g id=\"node16\" class=\"node\"><title>2422722665944</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"404,-897 257,-897 257,-876 404,-876 404,-897\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"330.5\" y=\"-883.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665944&#45;&gt;2422722665832 -->\r\n",
       "<g id=\"edge15\" class=\"edge\"><title>2422722665944&#45;&gt;2422722665832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M312.385,-875.92C296.155,-867.337 272.124,-854.628 253.606,-844.835\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"255.233,-841.736 244.757,-840.155 251.96,-847.924 255.233,-841.736\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666112 -->\r\n",
       "<g id=\"node17\" class=\"node\"><title>2422722666112</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"577,-960.5 452,-960.5 452,-939.5 577,-939.5 577,-960.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"514.5\" y=\"-946.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666112&#45;&gt;2422722665944 -->\r\n",
       "<g id=\"edge16\" class=\"edge\"><title>2422722666112&#45;&gt;2422722665944</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M485.75,-939.391C454.288,-928.875 403.699,-911.966 368.869,-900.324\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"369.642,-896.892 359.048,-897.042 367.423,-903.531 369.642,-896.892\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666336 -->\r\n",
       "<g id=\"node18\" class=\"node\"><title>2422722666336</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"584.5,-1030.5 494.5,-1030.5 494.5,-1009.5 584.5,-1009.5 584.5,-1030.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"539.5\" y=\"-1016.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666336&#45;&gt;2422722666112 -->\r\n",
       "<g id=\"edge17\" class=\"edge\"><title>2422722666336&#45;&gt;2422722666112</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M535.918,-1009.26C532.205,-999.158 526.302,-983.101 521.632,-970.4\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"524.811,-968.903 518.075,-960.725 518.241,-971.318 524.811,-968.903\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687048 -->\r\n",
       "<g id=\"node19\" class=\"node\"><title>2422722687048</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"613,-1100.5 466,-1100.5 466,-1079.5 613,-1079.5 613,-1100.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"539.5\" y=\"-1086.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687048&#45;&gt;2422722666336 -->\r\n",
       "<g id=\"edge18\" class=\"edge\"><title>2422722687048&#45;&gt;2422722666336</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M539.5,-1079.26C539.5,-1069.26 539.5,-1053.42 539.5,-1040.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"543,-1040.72 539.5,-1030.72 536,-1040.72 543,-1040.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687160 -->\r\n",
       "<g id=\"node20\" class=\"node\"><title>2422722687160</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"602,-1170.5 477,-1170.5 477,-1149.5 602,-1149.5 602,-1170.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"539.5\" y=\"-1156.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687160&#45;&gt;2422722687048 -->\r\n",
       "<g id=\"edge19\" class=\"edge\"><title>2422722687160&#45;&gt;2422722687048</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M539.5,-1149.26C539.5,-1139.26 539.5,-1123.42 539.5,-1110.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"543,-1110.72 539.5,-1100.72 536,-1110.72 543,-1110.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687384 -->\r\n",
       "<g id=\"node21\" class=\"node\"><title>2422722687384</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"395.5,-1240.5 305.5,-1240.5 305.5,-1219.5 395.5,-1219.5 395.5,-1240.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"350.5\" y=\"-1226.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687384&#45;&gt;2422722687160 -->\r\n",
       "<g id=\"edge20\" class=\"edge\"><title>2422722687384&#45;&gt;2422722687160</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M377.178,-1219.4C410.104,-1207.56 466.245,-1187.36 503.107,-1174.09\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"504.733,-1177.23 512.957,-1170.55 502.363,-1170.64 504.733,-1177.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687104 -->\r\n",
       "<g id=\"node138\" class=\"node\"><title>2422722687104</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"340.5,-1170.5 222.5,-1170.5 222.5,-1149.5 340.5,-1149.5 340.5,-1170.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"281.5\" y=\"-1156.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AvgPool2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687384&#45;&gt;2422722687104 -->\r\n",
       "<g id=\"edge144\" class=\"edge\"><title>2422722687384&#45;&gt;2422722687104</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M340.614,-1219.26C329.649,-1208.45 311.763,-1190.82 298.532,-1177.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"300.947,-1175.25 291.368,-1170.72 296.033,-1180.24 300.947,-1175.25\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687552 -->\r\n",
       "<g id=\"node22\" class=\"node\"><title>2422722687552</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"395.5,-1304 305.5,-1304 305.5,-1283 395.5,-1283 395.5,-1304\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"350.5\" y=\"-1290.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687552&#45;&gt;2422722687384 -->\r\n",
       "<g id=\"edge21\" class=\"edge\"><title>2422722687552&#45;&gt;2422722687384</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M350.5,-1282.89C350.5,-1274.37 350.5,-1261.64 350.5,-1250.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"354,-1250.74 350.5,-1240.74 347,-1250.74 354,-1250.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687664 -->\r\n",
       "<g id=\"node23\" class=\"node\"><title>2422722687664</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"528,-1361 381,-1361 381,-1340 528,-1340 528,-1361\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"454.5\" y=\"-1347.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687664&#45;&gt;2422722687552 -->\r\n",
       "<g id=\"edge22\" class=\"edge\"><title>2422722687664&#45;&gt;2422722687552</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M436.385,-1339.92C420.155,-1331.34 396.124,-1318.63 377.606,-1308.84\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"379.233,-1305.74 368.757,-1304.16 375.96,-1311.92 379.233,-1305.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687832 -->\r\n",
       "<g id=\"node24\" class=\"node\"><title>2422722687832</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"701,-1424.5 576,-1424.5 576,-1403.5 701,-1403.5 701,-1424.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"638.5\" y=\"-1410.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687832&#45;&gt;2422722687664 -->\r\n",
       "<g id=\"edge23\" class=\"edge\"><title>2422722687832&#45;&gt;2422722687664</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M609.75,-1403.39C578.288,-1392.87 527.699,-1375.97 492.869,-1364.32\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"493.642,-1360.89 483.048,-1361.04 491.423,-1367.53 493.642,-1360.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688056 -->\r\n",
       "<g id=\"node25\" class=\"node\"><title>2422722688056</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"683.5,-1494.5 593.5,-1494.5 593.5,-1473.5 683.5,-1473.5 683.5,-1494.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"638.5\" y=\"-1480.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688056&#45;&gt;2422722687832 -->\r\n",
       "<g id=\"edge24\" class=\"edge\"><title>2422722688056&#45;&gt;2422722687832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M638.5,-1473.26C638.5,-1463.26 638.5,-1447.42 638.5,-1434.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"642,-1434.72 638.5,-1424.72 635,-1434.72 642,-1434.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688224 -->\r\n",
       "<g id=\"node26\" class=\"node\"><title>2422722688224</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"679,-1558 532,-1558 532,-1537 679,-1537 679,-1558\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"605.5\" y=\"-1544.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688224&#45;&gt;2422722688056 -->\r\n",
       "<g id=\"edge25\" class=\"edge\"><title>2422722688224&#45;&gt;2422722688056</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M610.656,-1536.89C615.423,-1528.01 622.639,-1514.56 628.53,-1503.58\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"631.627,-1505.21 633.271,-1494.74 625.459,-1501.9 631.627,-1505.21\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688336 -->\r\n",
       "<g id=\"node27\" class=\"node\"><title>2422722688336</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"734,-1621.5 609,-1621.5 609,-1600.5 734,-1600.5 734,-1621.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"671.5\" y=\"-1607.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688336&#45;&gt;2422722688224 -->\r\n",
       "<g id=\"edge26\" class=\"edge\"><title>2422722688336&#45;&gt;2422722688224</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M661.188,-1600.39C651.076,-1590.97 635.456,-1576.41 623.329,-1565.11\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"625.66,-1562.5 615.958,-1558.24 620.888,-1567.62 625.66,-1562.5\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687720 -->\r\n",
       "<g id=\"node28\" class=\"node\"><title>2422722687720</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"470.5,-1691.5 380.5,-1691.5 380.5,-1670.5 470.5,-1670.5 470.5,-1691.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"425.5\" y=\"-1677.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687720&#45;&gt;2422722687552 -->\r\n",
       "<g id=\"edge134\" class=\"edge\"><title>2422722687720&#45;&gt;2422722687552</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M404.975,-1670.41C369.272,-1652.14 299.5,-1608.73 299.5,-1548.5 299.5,-1548.5 299.5,-1548.5 299.5,-1413 299.5,-1374.78 322.156,-1334.87 337.267,-1312.5\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"340.321,-1314.24 343.193,-1304.04 334.588,-1310.22 340.321,-1314.24\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687720&#45;&gt;2422722688336 -->\r\n",
       "<g id=\"edge27\" class=\"edge\"><title>2422722687720&#45;&gt;2422722688336</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M459.964,-1670.47C503.666,-1658.39 579.088,-1637.54 627.071,-1624.28\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"628.166,-1627.61 636.872,-1621.57 626.301,-1620.86 628.166,-1627.61\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688672 -->\r\n",
       "<g id=\"node29\" class=\"node\"><title>2422722688672</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"470.5,-1755 380.5,-1755 380.5,-1734 470.5,-1734 470.5,-1755\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"425.5\" y=\"-1741.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688672&#45;&gt;2422722687720 -->\r\n",
       "<g id=\"edge28\" class=\"edge\"><title>2422722688672&#45;&gt;2422722687720</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M425.5,-1733.89C425.5,-1725.37 425.5,-1712.64 425.5,-1701.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"429,-1701.74 425.5,-1691.74 422,-1701.74 429,-1701.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688784 -->\r\n",
       "<g id=\"node30\" class=\"node\"><title>2422722688784</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"603,-1812 456,-1812 456,-1791 603,-1791 603,-1812\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"529.5\" y=\"-1798.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688784&#45;&gt;2422722688672 -->\r\n",
       "<g id=\"edge29\" class=\"edge\"><title>2422722688784&#45;&gt;2422722688672</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M511.385,-1790.92C495.155,-1782.34 471.124,-1769.63 452.606,-1759.84\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"454.233,-1756.74 443.757,-1755.16 450.96,-1762.92 454.233,-1756.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688952 -->\r\n",
       "<g id=\"node31\" class=\"node\"><title>2422722688952</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"776,-1875.5 651,-1875.5 651,-1854.5 776,-1854.5 776,-1875.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"713.5\" y=\"-1861.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688952&#45;&gt;2422722688784 -->\r\n",
       "<g id=\"edge30\" class=\"edge\"><title>2422722688952&#45;&gt;2422722688784</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M684.75,-1854.39C653.288,-1843.87 602.699,-1826.97 567.869,-1815.32\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"568.642,-1811.89 558.048,-1812.04 566.423,-1818.53 568.642,-1811.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689176 -->\r\n",
       "<g id=\"node32\" class=\"node\"><title>2422722689176</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"783.5,-1945.5 693.5,-1945.5 693.5,-1924.5 783.5,-1924.5 783.5,-1945.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"738.5\" y=\"-1931.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689176&#45;&gt;2422722688952 -->\r\n",
       "<g id=\"edge31\" class=\"edge\"><title>2422722689176&#45;&gt;2422722688952</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M734.918,-1924.26C731.205,-1914.16 725.302,-1898.1 720.632,-1885.4\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"723.811,-1883.9 717.075,-1875.72 717.241,-1886.32 723.811,-1883.9\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689344 -->\r\n",
       "<g id=\"node33\" class=\"node\"><title>2422722689344</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"812,-2015.5 665,-2015.5 665,-1994.5 812,-1994.5 812,-2015.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"738.5\" y=\"-2001.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689344&#45;&gt;2422722689176 -->\r\n",
       "<g id=\"edge32\" class=\"edge\"><title>2422722689344&#45;&gt;2422722689176</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M738.5,-1994.26C738.5,-1984.26 738.5,-1968.42 738.5,-1955.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"742,-1955.72 738.5,-1945.72 735,-1955.72 742,-1955.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689456 -->\r\n",
       "<g id=\"node34\" class=\"node\"><title>2422722689456</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"801,-2085.5 676,-2085.5 676,-2064.5 801,-2064.5 801,-2085.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"738.5\" y=\"-2071.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689456&#45;&gt;2422722689344 -->\r\n",
       "<g id=\"edge33\" class=\"edge\"><title>2422722689456&#45;&gt;2422722689344</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M738.5,-2064.26C738.5,-2054.26 738.5,-2038.42 738.5,-2025.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"742,-2025.72 738.5,-2015.72 735,-2025.72 742,-2025.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689680 -->\r\n",
       "<g id=\"node35\" class=\"node\"><title>2422722689680</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"595.5,-2155.5 505.5,-2155.5 505.5,-2134.5 595.5,-2134.5 595.5,-2155.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"550.5\" y=\"-2141.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689680&#45;&gt;2422722689456 -->\r\n",
       "<g id=\"edge34\" class=\"edge\"><title>2422722689680&#45;&gt;2422722689456</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M577.037,-2134.4C609.788,-2122.56 665.632,-2102.36 702.3,-2089.09\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"703.885,-2092.24 712.098,-2085.55 701.504,-2085.66 703.885,-2092.24\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689288 -->\r\n",
       "<g id=\"node120\" class=\"node\"><title>2422722689288</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"539.5,-2085.5 421.5,-2085.5 421.5,-2064.5 539.5,-2064.5 539.5,-2085.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"480.5\" y=\"-2071.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AvgPool2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689680&#45;&gt;2422722689288 -->\r\n",
       "<g id=\"edge124\" class=\"edge\"><title>2422722689680&#45;&gt;2422722689288</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M540.471,-2134.26C529.347,-2123.45 511.201,-2105.82 497.779,-2092.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"500.122,-2090.18 490.511,-2085.72 495.245,-2095.2 500.122,-2090.18\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689848 -->\r\n",
       "<g id=\"node36\" class=\"node\"><title>2422722689848</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"595.5,-2219 505.5,-2219 505.5,-2198 595.5,-2198 595.5,-2219\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"550.5\" y=\"-2205.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689848&#45;&gt;2422722689680 -->\r\n",
       "<g id=\"edge35\" class=\"edge\"><title>2422722689848&#45;&gt;2422722689680</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M550.5,-2197.89C550.5,-2189.37 550.5,-2176.64 550.5,-2165.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"554,-2165.74 550.5,-2155.74 547,-2165.74 554,-2165.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689960 -->\r\n",
       "<g id=\"node37\" class=\"node\"><title>2422722689960</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"728,-2276 581,-2276 581,-2255 728,-2255 728,-2276\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"654.5\" y=\"-2262.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689960&#45;&gt;2422722689848 -->\r\n",
       "<g id=\"edge36\" class=\"edge\"><title>2422722689960&#45;&gt;2422722689848</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.385,-2254.92C620.155,-2246.34 596.124,-2233.63 577.606,-2223.84\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"579.233,-2220.74 568.757,-2219.16 575.96,-2226.92 579.233,-2220.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690128 -->\r\n",
       "<g id=\"node38\" class=\"node\"><title>2422722690128</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"901,-2339.5 776,-2339.5 776,-2318.5 901,-2318.5 901,-2339.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"838.5\" y=\"-2325.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690128&#45;&gt;2422722689960 -->\r\n",
       "<g id=\"edge37\" class=\"edge\"><title>2422722690128&#45;&gt;2422722689960</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M809.75,-2318.39C778.288,-2307.87 727.699,-2290.97 692.869,-2279.32\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"693.642,-2275.89 683.048,-2276.04 691.423,-2282.53 693.642,-2275.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422490290664 -->\r\n",
       "<g id=\"node39\" class=\"node\"><title>2422490290664</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"883.5,-2409.5 793.5,-2409.5 793.5,-2388.5 883.5,-2388.5 883.5,-2409.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"838.5\" y=\"-2395.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422490290664&#45;&gt;2422722690128 -->\r\n",
       "<g id=\"edge38\" class=\"edge\"><title>2422490290664&#45;&gt;2422722690128</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M838.5,-2388.26C838.5,-2378.26 838.5,-2362.42 838.5,-2349.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"842,-2349.72 838.5,-2339.72 835,-2349.72 842,-2349.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607700768 -->\r\n",
       "<g id=\"node40\" class=\"node\"><title>2422607700768</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"847,-2473 700,-2473 700,-2452 847,-2452 847,-2473\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"773.5\" y=\"-2459.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607700768&#45;&gt;2422490290664 -->\r\n",
       "<g id=\"edge39\" class=\"edge\"><title>2422607700768&#45;&gt;2422490290664</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M783.656,-2451.89C793.52,-2442.56 808.706,-2428.19 820.599,-2416.94\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"823.342,-2419.16 828.201,-2409.74 818.531,-2414.08 823.342,-2419.16\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607701496 -->\r\n",
       "<g id=\"node41\" class=\"node\"><title>2422607701496</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"934,-2536.5 809,-2536.5 809,-2515.5 934,-2515.5 934,-2536.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"871.5\" y=\"-2522.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607701496&#45;&gt;2422607700768 -->\r\n",
       "<g id=\"edge40\" class=\"edge\"><title>2422607701496&#45;&gt;2422607700768</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M856.188,-2515.39C840.438,-2505.51 815.688,-2489.97 797.366,-2478.48\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"799.036,-2475.39 788.705,-2473.04 795.315,-2481.32 799.036,-2475.39\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690016 -->\r\n",
       "<g id=\"node42\" class=\"node\"><title>2422722690016</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"671.5,-2606.5 581.5,-2606.5 581.5,-2585.5 671.5,-2585.5 671.5,-2606.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"626.5\" y=\"-2592.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690016&#45;&gt;2422722689848 -->\r\n",
       "<g id=\"edge114\" class=\"edge\"><title>2422722690016&#45;&gt;2422722689848</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M605.813,-2585.45C569.826,-2567.26 499.5,-2523.96 499.5,-2463.5 499.5,-2463.5 499.5,-2463.5 499.5,-2328 499.5,-2289.78 522.156,-2249.87 537.267,-2227.5\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"540.321,-2229.24 543.193,-2219.04 534.588,-2225.22 540.321,-2229.24\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690016&#45;&gt;2422607701496 -->\r\n",
       "<g id=\"edge41\" class=\"edge\"><title>2422722690016&#45;&gt;2422607701496</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M660.824,-2585.47C704.348,-2573.39 779.464,-2552.54 827.252,-2539.28\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"828.314,-2542.62 837.013,-2536.57 826.441,-2535.87 828.314,-2542.62\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607642808 -->\r\n",
       "<g id=\"node43\" class=\"node\"><title>2422607642808</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"671.5,-2670 581.5,-2670 581.5,-2649 671.5,-2649 671.5,-2670\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"626.5\" y=\"-2656.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607642808&#45;&gt;2422722690016 -->\r\n",
       "<g id=\"edge42\" class=\"edge\"><title>2422607642808&#45;&gt;2422722690016</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M626.5,-2648.89C626.5,-2640.37 626.5,-2627.64 626.5,-2616.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"630,-2616.74 626.5,-2606.74 623,-2616.74 630,-2616.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607646224 -->\r\n",
       "<g id=\"node44\" class=\"node\"><title>2422607646224</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"804,-2727 657,-2727 657,-2706 804,-2706 804,-2727\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"730.5\" y=\"-2713.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607646224&#45;&gt;2422607642808 -->\r\n",
       "<g id=\"edge43\" class=\"edge\"><title>2422607646224&#45;&gt;2422607642808</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M712.385,-2705.92C696.155,-2697.34 672.124,-2684.63 653.606,-2674.84\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"655.233,-2671.74 644.757,-2670.16 651.96,-2677.92 655.233,-2671.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690464 -->\r\n",
       "<g id=\"node45\" class=\"node\"><title>2422722690464</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"977,-2790.5 852,-2790.5 852,-2769.5 977,-2769.5 977,-2790.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"914.5\" y=\"-2776.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690464&#45;&gt;2422607646224 -->\r\n",
       "<g id=\"edge44\" class=\"edge\"><title>2422722690464&#45;&gt;2422607646224</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M885.75,-2769.39C854.288,-2758.87 803.699,-2741.97 768.869,-2730.32\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"769.642,-2726.89 759.048,-2727.04 767.423,-2733.53 769.642,-2726.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690688 -->\r\n",
       "<g id=\"node46\" class=\"node\"><title>2422722690688</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"984.5,-2860.5 894.5,-2860.5 894.5,-2839.5 984.5,-2839.5 984.5,-2860.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"939.5\" y=\"-2846.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690688&#45;&gt;2422722690464 -->\r\n",
       "<g id=\"edge45\" class=\"edge\"><title>2422722690688&#45;&gt;2422722690464</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M935.918,-2839.26C932.205,-2829.16 926.302,-2813.1 921.632,-2800.4\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"924.811,-2798.9 918.075,-2790.72 918.241,-2801.32 924.811,-2798.9\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690856 -->\r\n",
       "<g id=\"node47\" class=\"node\"><title>2422722690856</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1013,-2930.5 866,-2930.5 866,-2909.5 1013,-2909.5 1013,-2930.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"939.5\" y=\"-2916.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690856&#45;&gt;2422722690688 -->\r\n",
       "<g id=\"edge46\" class=\"edge\"><title>2422722690856&#45;&gt;2422722690688</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M939.5,-2909.26C939.5,-2899.26 939.5,-2883.42 939.5,-2870.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"943,-2870.72 939.5,-2860.72 936,-2870.72 943,-2870.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690968 -->\r\n",
       "<g id=\"node48\" class=\"node\"><title>2422722690968</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1002,-3000.5 877,-3000.5 877,-2979.5 1002,-2979.5 1002,-3000.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"939.5\" y=\"-2986.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690968&#45;&gt;2422722690856 -->\r\n",
       "<g id=\"edge47\" class=\"edge\"><title>2422722690968&#45;&gt;2422722690856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M939.5,-2979.26C939.5,-2969.26 939.5,-2953.42 939.5,-2940.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"943,-2940.72 939.5,-2930.72 936,-2940.72 943,-2940.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699448 -->\r\n",
       "<g id=\"node49\" class=\"node\"><title>2422722699448</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"796.5,-3070.5 706.5,-3070.5 706.5,-3049.5 796.5,-3049.5 796.5,-3070.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"751.5\" y=\"-3056.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699448&#45;&gt;2422722690968 -->\r\n",
       "<g id=\"edge48\" class=\"edge\"><title>2422722699448&#45;&gt;2422722690968</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M778.037,-3049.4C810.788,-3037.56 866.632,-3017.36 903.3,-3004.09\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"904.885,-3007.24 913.098,-3000.55 902.504,-3000.66 904.885,-3007.24\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699560 -->\r\n",
       "<g id=\"node102\" class=\"node\"><title>2422722699560</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"740.5,-3000.5 622.5,-3000.5 622.5,-2979.5 740.5,-2979.5 740.5,-3000.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"681.5\" y=\"-2986.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AvgPool2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699448&#45;&gt;2422722699560 -->\r\n",
       "<g id=\"edge104\" class=\"edge\"><title>2422722699448&#45;&gt;2422722699560</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M741.471,-3049.26C730.347,-3038.45 712.201,-3020.82 698.779,-3007.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"701.122,-3005.18 691.511,-3000.72 696.245,-3010.2 701.122,-3005.18\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699616 -->\r\n",
       "<g id=\"node50\" class=\"node\"><title>2422722699616</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"796.5,-3134 706.5,-3134 706.5,-3113 796.5,-3113 796.5,-3134\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"751.5\" y=\"-3120.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699616&#45;&gt;2422722699448 -->\r\n",
       "<g id=\"edge49\" class=\"edge\"><title>2422722699616&#45;&gt;2422722699448</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M751.5,-3112.89C751.5,-3104.37 751.5,-3091.64 751.5,-3080.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"755,-3080.74 751.5,-3070.74 748,-3080.74 755,-3080.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699728 -->\r\n",
       "<g id=\"node51\" class=\"node\"><title>2422722699728</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"929,-3191 782,-3191 782,-3170 929,-3170 929,-3191\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"855.5\" y=\"-3177.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699728&#45;&gt;2422722699616 -->\r\n",
       "<g id=\"edge50\" class=\"edge\"><title>2422722699728&#45;&gt;2422722699616</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M837.385,-3169.92C821.155,-3161.34 797.124,-3148.63 778.606,-3138.84\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"780.233,-3135.74 769.757,-3134.16 776.96,-3141.92 780.233,-3135.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699896 -->\r\n",
       "<g id=\"node52\" class=\"node\"><title>2422722699896</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1102,-3254.5 977,-3254.5 977,-3233.5 1102,-3233.5 1102,-3254.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1039.5\" y=\"-3240.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699896&#45;&gt;2422722699728 -->\r\n",
       "<g id=\"edge51\" class=\"edge\"><title>2422722699896&#45;&gt;2422722699728</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1010.75,-3233.39C979.288,-3222.87 928.699,-3205.97 893.869,-3194.32\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"894.642,-3190.89 884.048,-3191.04 892.423,-3197.53 894.642,-3190.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700120 -->\r\n",
       "<g id=\"node53\" class=\"node\"><title>2422722700120</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1084.5,-3324.5 994.5,-3324.5 994.5,-3303.5 1084.5,-3303.5 1084.5,-3324.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1039.5\" y=\"-3310.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700120&#45;&gt;2422722699896 -->\r\n",
       "<g id=\"edge52\" class=\"edge\"><title>2422722700120&#45;&gt;2422722699896</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1039.5,-3303.26C1039.5,-3293.26 1039.5,-3277.42 1039.5,-3264.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1043,-3264.72 1039.5,-3254.72 1036,-3264.72 1043,-3264.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700288 -->\r\n",
       "<g id=\"node54\" class=\"node\"><title>2422722700288</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1048,-3388 901,-3388 901,-3367 1048,-3367 1048,-3388\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"974.5\" y=\"-3374.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700288&#45;&gt;2422722700120 -->\r\n",
       "<g id=\"edge53\" class=\"edge\"><title>2422722700288&#45;&gt;2422722700120</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M984.656,-3366.89C994.52,-3357.56 1009.71,-3343.19 1021.6,-3331.94\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1024.34,-3334.16 1029.2,-3324.74 1019.53,-3329.08 1024.34,-3334.16\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700400 -->\r\n",
       "<g id=\"node55\" class=\"node\"><title>2422722700400</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1135,-3451.5 1010,-3451.5 1010,-3430.5 1135,-3430.5 1135,-3451.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1072.5\" y=\"-3437.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700400&#45;&gt;2422722700288 -->\r\n",
       "<g id=\"edge54\" class=\"edge\"><title>2422722700400&#45;&gt;2422722700288</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1057.19,-3430.39C1041.44,-3420.51 1016.69,-3404.97 998.366,-3393.48\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1000.04,-3390.39 989.705,-3388.04 996.315,-3396.32 1000.04,-3390.39\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699784 -->\r\n",
       "<g id=\"node56\" class=\"node\"><title>2422722699784</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"872.5,-3521.5 782.5,-3521.5 782.5,-3500.5 872.5,-3500.5 872.5,-3521.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"827.5\" y=\"-3507.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699784&#45;&gt;2422722699616 -->\r\n",
       "<g id=\"edge94\" class=\"edge\"><title>2422722699784&#45;&gt;2422722699616</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M806.813,-3500.45C770.826,-3482.26 700.5,-3438.96 700.5,-3378.5 700.5,-3378.5 700.5,-3378.5 700.5,-3243 700.5,-3204.78 723.156,-3164.87 738.267,-3142.5\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"741.321,-3144.24 744.193,-3134.04 735.588,-3140.22 741.321,-3144.24\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699784&#45;&gt;2422722700400 -->\r\n",
       "<g id=\"edge55\" class=\"edge\"><title>2422722699784&#45;&gt;2422722700400</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M861.824,-3500.47C905.348,-3488.39 980.464,-3467.54 1028.25,-3454.28\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1029.31,-3457.62 1038.01,-3451.57 1027.44,-3450.87 1029.31,-3457.62\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700736 -->\r\n",
       "<g id=\"node57\" class=\"node\"><title>2422722700736</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"872.5,-3585 782.5,-3585 782.5,-3564 872.5,-3564 872.5,-3585\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"827.5\" y=\"-3571.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">AddBackward0</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700736&#45;&gt;2422722699784 -->\r\n",
       "<g id=\"edge56\" class=\"edge\"><title>2422722700736&#45;&gt;2422722699784</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M827.5,-3563.89C827.5,-3555.37 827.5,-3542.64 827.5,-3531.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"831,-3531.74 827.5,-3521.74 824,-3531.74 831,-3531.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700848 -->\r\n",
       "<g id=\"node58\" class=\"node\"><title>2422722700848</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1005,-3642 858,-3642 858,-3621 1005,-3621 1005,-3642\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"931.5\" y=\"-3628.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700848&#45;&gt;2422722700736 -->\r\n",
       "<g id=\"edge57\" class=\"edge\"><title>2422722700848&#45;&gt;2422722700736</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M913.385,-3620.92C897.155,-3612.34 873.124,-3599.63 854.606,-3589.84\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"856.233,-3586.74 845.757,-3585.16 852.96,-3592.92 856.233,-3586.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701016 -->\r\n",
       "<g id=\"node59\" class=\"node\"><title>2422722701016</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1178,-3705.5 1053,-3705.5 1053,-3684.5 1178,-3684.5 1178,-3705.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1115.5\" y=\"-3691.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701016&#45;&gt;2422722700848 -->\r\n",
       "<g id=\"edge58\" class=\"edge\"><title>2422722701016&#45;&gt;2422722700848</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1086.75,-3684.39C1055.29,-3673.87 1004.7,-3656.97 969.869,-3645.32\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"970.642,-3641.89 960.048,-3642.04 968.423,-3648.53 970.642,-3641.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701240 -->\r\n",
       "<g id=\"node60\" class=\"node\"><title>2422722701240</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1160.5,-3775.5 1070.5,-3775.5 1070.5,-3754.5 1160.5,-3754.5 1160.5,-3775.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1115.5\" y=\"-3761.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701240&#45;&gt;2422722701016 -->\r\n",
       "<g id=\"edge59\" class=\"edge\"><title>2422722701240&#45;&gt;2422722701016</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1115.5,-3754.26C1115.5,-3744.26 1115.5,-3728.42 1115.5,-3715.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1119,-3715.72 1115.5,-3705.72 1112,-3715.72 1119,-3715.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701408 -->\r\n",
       "<g id=\"node61\" class=\"node\"><title>2422722701408</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1156,-3839 1009,-3839 1009,-3818 1156,-3818 1156,-3839\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-3825.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701408&#45;&gt;2422722701240 -->\r\n",
       "<g id=\"edge60\" class=\"edge\"><title>2422722701408&#45;&gt;2422722701240</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1087.66,-3817.89C1092.42,-3809.01 1099.64,-3795.56 1105.53,-3784.58\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1108.63,-3786.21 1110.27,-3775.74 1102.46,-3782.9 1108.63,-3786.21\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701520 -->\r\n",
       "<g id=\"node62\" class=\"node\"><title>2422722701520</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"1211,-3902.5 1086,-3902.5 1086,-3881.5 1211,-3881.5 1211,-3902.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1148.5\" y=\"-3888.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701520&#45;&gt;2422722701408 -->\r\n",
       "<g id=\"edge61\" class=\"edge\"><title>2422722701520&#45;&gt;2422722701408</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1138.19,-3881.39C1128.08,-3871.97 1112.46,-3857.41 1100.33,-3846.11\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1102.66,-3843.5 1092.96,-3839.24 1097.89,-3848.62 1102.66,-3843.5\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700904 -->\r\n",
       "<g id=\"node63\" class=\"node\"><title>2422722700904</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"969.5,-3972.5 795.5,-3972.5 795.5,-3951.5 969.5,-3951.5 969.5,-3972.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"882.5\" y=\"-3958.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">MaxPool2DWithIndicesBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700904&#45;&gt;2422722700736 -->\r\n",
       "<g id=\"edge87\" class=\"edge\"><title>2422722700904&#45;&gt;2422722700736</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M866.401,-3951.31C836.748,-3932.1 776.5,-3886.09 776.5,-3829.5 776.5,-3829.5 776.5,-3829.5 776.5,-3694 776.5,-3655.78 799.156,-3615.87 814.267,-3593.5\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"817.321,-3595.24 820.193,-3585.04 811.588,-3591.22 817.321,-3595.24\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700904&#45;&gt;2422722701520 -->\r\n",
       "<g id=\"edge62\" class=\"edge\"><title>2422722700904&#45;&gt;2422722701520</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M919.766,-3951.47C967.219,-3939.34 1049.26,-3918.37 1101.11,-3905.12\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1102.24,-3908.44 1111.06,-3902.57 1100.5,-3901.66 1102.24,-3908.44\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701856 -->\r\n",
       "<g id=\"node64\" class=\"node\"><title>2422722701856</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"927.5,-4036 837.5,-4036 837.5,-4015 927.5,-4015 927.5,-4036\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"882.5\" y=\"-4022.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701856&#45;&gt;2422722700904 -->\r\n",
       "<g id=\"edge63\" class=\"edge\"><title>2422722701856&#45;&gt;2422722700904</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M882.5,-4014.89C882.5,-4006.37 882.5,-3993.64 882.5,-3982.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"886,-3982.74 882.5,-3972.74 879,-3982.74 886,-3982.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701968 -->\r\n",
       "<g id=\"node65\" class=\"node\"><title>2422722701968</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"956,-4093 809,-4093 809,-4072 956,-4072 956,-4093\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"882.5\" y=\"-4079.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701968&#45;&gt;2422722701856 -->\r\n",
       "<g id=\"edge64\" class=\"edge\"><title>2422722701968&#45;&gt;2422722701856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M882.5,-4071.92C882.5,-4064.91 882.5,-4055.14 882.5,-4046.47\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"886,-4046.34 882.5,-4036.34 879,-4046.34 886,-4046.34\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702080 -->\r\n",
       "<g id=\"node66\" class=\"node\"><title>2422722702080</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"832,-4156.5 707,-4156.5 707,-4135.5 832,-4135.5 832,-4156.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"769.5\" y=\"-4142.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702080&#45;&gt;2422722701968 -->\r\n",
       "<g id=\"edge65\" class=\"edge\"><title>2422722702080&#45;&gt;2422722701968</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M787.156,-4135.39C805.649,-4125.33 834.901,-4109.41 856.131,-4097.85\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"857.857,-4100.9 864.968,-4093.04 854.511,-4094.75 857.857,-4100.9\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702304 -->\r\n",
       "<g id=\"node67\" class=\"node\"><title>2422722702304</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"762.5,-4226.5 672.5,-4226.5 672.5,-4205.5 762.5,-4205.5 762.5,-4226.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"717.5\" y=\"-4212.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702304&#45;&gt;2422722702080 -->\r\n",
       "<g id=\"edge66\" class=\"edge\"><title>2422722702304&#45;&gt;2422722702080</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M724.95,-4205.26C732.982,-4194.75 745.942,-4177.81 755.817,-4164.89\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"758.769,-4166.79 762.063,-4156.72 753.209,-4162.54 758.769,-4166.79\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702472 -->\r\n",
       "<g id=\"node68\" class=\"node\"><title>2422722702472</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"791,-4290 644,-4290 644,-4269 791,-4269 791,-4290\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"717.5\" y=\"-4276.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702472&#45;&gt;2422722702304 -->\r\n",
       "<g id=\"edge67\" class=\"edge\"><title>2422722702472&#45;&gt;2422722702304</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M717.5,-4268.89C717.5,-4260.37 717.5,-4247.64 717.5,-4236.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"721,-4236.74 717.5,-4226.74 714,-4236.74 721,-4236.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702584 -->\r\n",
       "<g id=\"node69\" class=\"node\"><title>2422722702584</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"667,-4353.5 542,-4353.5 542,-4332.5 667,-4332.5 667,-4353.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"604.5\" y=\"-4339.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702584&#45;&gt;2422722702472 -->\r\n",
       "<g id=\"edge68\" class=\"edge\"><title>2422722702584&#45;&gt;2422722702472</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M622.156,-4332.39C640.649,-4322.33 669.901,-4306.41 691.131,-4294.85\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"692.857,-4297.9 699.968,-4290.04 689.511,-4291.75 692.857,-4297.9\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702808 -->\r\n",
       "<g id=\"node70\" class=\"node\"><title>2422722702808</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"597.5,-4423.5 507.5,-4423.5 507.5,-4402.5 597.5,-4402.5 597.5,-4423.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"552.5\" y=\"-4409.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ReluBackward1</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702808&#45;&gt;2422722702584 -->\r\n",
       "<g id=\"edge69\" class=\"edge\"><title>2422722702808&#45;&gt;2422722702584</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M559.95,-4402.26C567.982,-4391.75 580.942,-4374.81 590.817,-4361.89\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"593.769,-4363.79 597.063,-4353.72 588.209,-4359.54 593.769,-4363.79\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702976 -->\r\n",
       "<g id=\"node71\" class=\"node\"><title>2422722702976</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"626,-4487 479,-4487 479,-4466 626,-4466 626,-4487\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"552.5\" y=\"-4473.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702976&#45;&gt;2422722702808 -->\r\n",
       "<g id=\"edge70\" class=\"edge\"><title>2422722702976&#45;&gt;2422722702808</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M552.5,-4465.89C552.5,-4457.37 552.5,-4444.64 552.5,-4433.92\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"556,-4433.74 552.5,-4423.74 549,-4433.74 556,-4433.74\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722703088 -->\r\n",
       "<g id=\"node72\" class=\"node\"><title>2422722703088</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"502,-4550.5 377,-4550.5 377,-4529.5 502,-4529.5 502,-4550.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"439.5\" y=\"-4536.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722703088&#45;&gt;2422722702976 -->\r\n",
       "<g id=\"edge71\" class=\"edge\"><title>2422722703088&#45;&gt;2422722702976</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M457.156,-4529.39C475.649,-4519.33 504.901,-4503.41 526.131,-4491.85\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"527.857,-4494.9 534.968,-4487.04 524.511,-4488.75 527.857,-4494.9\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722703312 -->\r\n",
       "<g id=\"node73\" class=\"node\"><title>2422722703312</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"477,-4627 402,-4627 402,-4593 477,-4593 477,-4627\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"439.5\" y=\"-4613.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"439.5\" y=\"-4600.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (32, 3, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722703312&#45;&gt;2422722703088 -->\r\n",
       "<g id=\"edge72\" class=\"edge\"><title>2422722703312&#45;&gt;2422722703088</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M439.5,-4592.88C439.5,-4583.31 439.5,-4571.09 439.5,-4560.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"443,-4560.89 439.5,-4550.89 436,-4560.89 443,-4560.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722703144 -->\r\n",
       "<g id=\"node74\" class=\"node\"><title>2422722703144</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"584.5,-4557 520.5,-4557 520.5,-4523 584.5,-4523 584.5,-4557\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"552.5\" y=\"-4543.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"552.5\" y=\"-4530.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (32)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722703144&#45;&gt;2422722702976 -->\r\n",
       "<g id=\"edge73\" class=\"edge\"><title>2422722703144&#45;&gt;2422722702976</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M552.5,-4522.84C552.5,-4515.01 552.5,-4505.54 552.5,-4497.28\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"556,-4497.04 552.5,-4487.04 549,-4497.04 556,-4497.04\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722703200 -->\r\n",
       "<g id=\"node75\" class=\"node\"><title>2422722703200</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"656.5,-4557 602.5,-4557 602.5,-4523 656.5,-4523 656.5,-4557\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"629.5\" y=\"-4543.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"629.5\" y=\"-4530.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (32)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722703200&#45;&gt;2422722702976 -->\r\n",
       "<g id=\"edge74\" class=\"edge\"><title>2422722703200&#45;&gt;2422722702976</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M609.269,-4522.84C597.907,-4513.77 583.781,-4502.48 572.462,-4493.44\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"574.445,-4490.55 564.447,-4487.04 570.076,-4496.02 574.445,-4490.55\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702864 -->\r\n",
       "<g id=\"node76\" class=\"node\"><title>2422722702864</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"697,-4430 616,-4430 616,-4396 697,-4396 697,-4430\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"656.5\" y=\"-4416.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"656.5\" y=\"-4403.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64, 32, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702864&#45;&gt;2422722702584 -->\r\n",
       "<g id=\"edge75\" class=\"edge\"><title>2422722702864&#45;&gt;2422722702584</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M644.177,-4395.88C636.387,-4385.7 626.307,-4372.52 618.257,-4361.99\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"620.921,-4359.71 612.066,-4353.89 615.361,-4363.96 620.921,-4359.71\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702640 -->\r\n",
       "<g id=\"node77\" class=\"node\"><title>2422722702640</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"749.5,-4360 685.5,-4360 685.5,-4326 749.5,-4326 749.5,-4360\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"717.5\" y=\"-4346.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"717.5\" y=\"-4333.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702640&#45;&gt;2422722702472 -->\r\n",
       "<g id=\"edge76\" class=\"edge\"><title>2422722702640&#45;&gt;2422722702472</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M717.5,-4325.84C717.5,-4318.01 717.5,-4308.54 717.5,-4300.28\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"721,-4300.04 717.5,-4290.04 714,-4300.04 721,-4300.04\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702696 -->\r\n",
       "<g id=\"node78\" class=\"node\"><title>2422722702696</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"821.5,-4360 767.5,-4360 767.5,-4326 821.5,-4326 821.5,-4360\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"794.5\" y=\"-4346.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"794.5\" y=\"-4333.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702696&#45;&gt;2422722702472 -->\r\n",
       "<g id=\"edge77\" class=\"edge\"><title>2422722702696&#45;&gt;2422722702472</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M774.269,-4325.84C762.907,-4316.77 748.781,-4305.48 737.462,-4296.44\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"739.445,-4293.55 729.447,-4290.04 735.076,-4299.02 739.445,-4293.55\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702360 -->\r\n",
       "<g id=\"node79\" class=\"node\"><title>2422722702360</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"862,-4233 781,-4233 781,-4199 862,-4199 862,-4233\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-4219.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">2.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-4206.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64, 64, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702360&#45;&gt;2422722702080 -->\r\n",
       "<g id=\"edge78\" class=\"edge\"><title>2422722702360&#45;&gt;2422722702080</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M809.177,-4198.88C801.387,-4188.7 791.307,-4175.52 783.257,-4164.99\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"785.921,-4162.71 777.066,-4156.89 780.361,-4166.96 785.921,-4162.71\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702136 -->\r\n",
       "<g id=\"node80\" class=\"node\"><title>2422722702136</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"914.5,-4163 850.5,-4163 850.5,-4129 914.5,-4129 914.5,-4163\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"882.5\" y=\"-4149.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">2.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"882.5\" y=\"-4136.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702136&#45;&gt;2422722701968 -->\r\n",
       "<g id=\"edge79\" class=\"edge\"><title>2422722702136&#45;&gt;2422722701968</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M882.5,-4128.84C882.5,-4121.01 882.5,-4111.54 882.5,-4103.28\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"886,-4103.04 882.5,-4093.04 879,-4103.04 886,-4103.04\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722702192 -->\r\n",
       "<g id=\"node81\" class=\"node\"><title>2422722702192</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"986.5,-4163 932.5,-4163 932.5,-4129 986.5,-4129 986.5,-4163\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"959.5\" y=\"-4149.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">2.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"959.5\" y=\"-4136.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722702192&#45;&gt;2422722701968 -->\r\n",
       "<g id=\"edge80\" class=\"edge\"><title>2422722702192&#45;&gt;2422722701968</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M939.269,-4128.84C927.907,-4119.77 913.781,-4108.48 902.462,-4099.44\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"904.445,-4096.55 894.447,-4093.04 900.076,-4102.02 904.445,-4096.55\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701744 -->\r\n",
       "<g id=\"node82\" class=\"node\"><title>2422722701744</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1204,-3979 1093,-3979 1093,-3945 1204,-3945 1204,-3979\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1148.5\" y=\"-3965.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.0.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"1148.5\" y=\"-3952.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64, 64, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701744&#45;&gt;2422722701520 -->\r\n",
       "<g id=\"edge81\" class=\"edge\"><title>2422722701744&#45;&gt;2422722701520</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1148.5,-3944.88C1148.5,-3935.31 1148.5,-3923.09 1148.5,-3912.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1152,-3912.89 1148.5,-3902.89 1145,-3912.89 1152,-3912.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701576 -->\r\n",
       "<g id=\"node83\" class=\"node\"><title>2422722701576</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"949,-3909 838,-3909 838,-3875 949,-3875 949,-3909\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"893.5\" y=\"-3895.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.0.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"893.5\" y=\"-3882.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701576&#45;&gt;2422722701408 -->\r\n",
       "<g id=\"edge82\" class=\"edge\"><title>2422722701576&#45;&gt;2422722701408</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M943.158,-3874.84C974.465,-3864.65 1014.32,-3851.69 1043.38,-3842.23\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1044.75,-3845.46 1053.18,-3839.04 1042.58,-3838.81 1044.75,-3845.46\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701632 -->\r\n",
       "<g id=\"node84\" class=\"node\"><title>2422722701632</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1067.5,-3909 967.5,-3909 967.5,-3875 1067.5,-3875 1067.5,-3909\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1017.5\" y=\"-3895.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.0.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"1017.5\" y=\"-3882.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701632&#45;&gt;2422722701408 -->\r\n",
       "<g id=\"edge83\" class=\"edge\"><title>2422722701632&#45;&gt;2422722701408</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1034.58,-3874.84C1043.98,-3865.94 1055.63,-3854.93 1065.08,-3845.98\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1067.56,-3848.46 1072.42,-3839.04 1062.75,-3843.37 1067.56,-3848.46\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701296 -->\r\n",
       "<g id=\"node85\" class=\"node\"><title>2422722701296</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1052,-3782 941,-3782 941,-3748 1052,-3748 1052,-3782\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"996.5\" y=\"-3768.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.0.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"996.5\" y=\"-3755.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64, 64, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701296&#45;&gt;2422722701016 -->\r\n",
       "<g id=\"edge84\" class=\"edge\"><title>2422722701296&#45;&gt;2422722701016</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1024.7,-3747.88C1044.43,-3736.61 1070.57,-3721.67 1089.74,-3710.72\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1091.59,-3713.7 1098.53,-3705.7 1088.11,-3707.62 1091.59,-3713.7\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701072 -->\r\n",
       "<g id=\"node86\" class=\"node\"><title>2422722701072</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"916,-3712 805,-3712 805,-3678 916,-3678 916,-3712\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"860.5\" y=\"-3698.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.0.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"860.5\" y=\"-3685.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701072&#45;&gt;2422722700848 -->\r\n",
       "<g id=\"edge85\" class=\"edge\"><title>2422722701072&#45;&gt;2422722700848</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M879.154,-3677.84C889.529,-3668.86 902.401,-3657.71 912.785,-3648.71\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"915.217,-3651.23 920.484,-3642.04 910.634,-3645.94 915.217,-3651.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722701128 -->\r\n",
       "<g id=\"node87\" class=\"node\"><title>2422722701128</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1034.5,-3712 934.5,-3712 934.5,-3678 1034.5,-3678 1034.5,-3712\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"984.5\" y=\"-3698.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.0.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"984.5\" y=\"-3685.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722701128&#45;&gt;2422722700848 -->\r\n",
       "<g id=\"edge86\" class=\"edge\"><title>2422722701128&#45;&gt;2422722700848</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M970.575,-3677.84C963.137,-3669.21 953.98,-3658.58 946.4,-3649.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"948.903,-3647.33 939.723,-3642.04 943.6,-3651.9 948.903,-3647.33\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700624 -->\r\n",
       "<g id=\"node88\" class=\"node\"><title>2422722700624</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1128,-3528 1017,-3528 1017,-3494 1128,-3494 1128,-3528\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1072.5\" y=\"-3514.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.1.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"1072.5\" y=\"-3501.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64, 64, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700624&#45;&gt;2422722700400 -->\r\n",
       "<g id=\"edge88\" class=\"edge\"><title>2422722700624&#45;&gt;2422722700400</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1072.5,-3493.88C1072.5,-3484.31 1072.5,-3472.09 1072.5,-3461.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1076,-3461.89 1072.5,-3451.89 1069,-3461.89 1076,-3461.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700456 -->\r\n",
       "<g id=\"node89\" class=\"node\"><title>2422722700456</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"873,-3458 762,-3458 762,-3424 873,-3424 873,-3458\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"817.5\" y=\"-3444.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.1.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"817.5\" y=\"-3431.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700456&#45;&gt;2422722700288 -->\r\n",
       "<g id=\"edge89\" class=\"edge\"><title>2422722700456&#45;&gt;2422722700288</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M858.75,-3423.84C884.302,-3413.83 916.709,-3401.14 940.73,-3391.73\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"942.106,-3394.95 950.141,-3388.04 939.553,-3388.43 942.106,-3394.95\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700512 -->\r\n",
       "<g id=\"node90\" class=\"node\"><title>2422722700512</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"991.5,-3458 891.5,-3458 891.5,-3424 991.5,-3424 991.5,-3458\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"941.5\" y=\"-3444.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.1.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"941.5\" y=\"-3431.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700512&#45;&gt;2422722700288 -->\r\n",
       "<g id=\"edge90\" class=\"edge\"><title>2422722700512&#45;&gt;2422722700288</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M950.17,-3423.84C954.61,-3415.57 960.035,-3405.46 964.634,-3396.89\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"967.736,-3398.51 969.38,-3388.04 961.568,-3395.2 967.736,-3398.51\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700176 -->\r\n",
       "<g id=\"node91\" class=\"node\"><title>2422722700176</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"976,-3331 865,-3331 865,-3297 976,-3297 976,-3331\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"920.5\" y=\"-3317.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.1.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"920.5\" y=\"-3304.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64, 64, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700176&#45;&gt;2422722699896 -->\r\n",
       "<g id=\"edge91\" class=\"edge\"><title>2422722700176&#45;&gt;2422722699896</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M948.701,-3296.88C968.429,-3285.61 994.574,-3270.67 1013.74,-3259.72\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1015.59,-3262.7 1022.53,-3254.7 1012.11,-3256.62 1015.59,-3262.7\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699952 -->\r\n",
       "<g id=\"node92\" class=\"node\"><title>2422722699952</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"840,-3261 729,-3261 729,-3227 840,-3227 840,-3261\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"784.5\" y=\"-3247.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.1.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"784.5\" y=\"-3234.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699952&#45;&gt;2422722699728 -->\r\n",
       "<g id=\"edge92\" class=\"edge\"><title>2422722699952&#45;&gt;2422722699728</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M803.154,-3226.84C813.529,-3217.86 826.401,-3206.71 836.785,-3197.71\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"839.217,-3200.23 844.484,-3191.04 834.634,-3194.94 839.217,-3200.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700008 -->\r\n",
       "<g id=\"node93\" class=\"node\"><title>2422722700008</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"958.5,-3261 858.5,-3261 858.5,-3227 958.5,-3227 958.5,-3261\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"908.5\" y=\"-3247.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">4.1.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"908.5\" y=\"-3234.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (64)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700008&#45;&gt;2422722699728 -->\r\n",
       "<g id=\"edge93\" class=\"edge\"><title>2422722700008&#45;&gt;2422722699728</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M894.575,-3226.84C887.137,-3218.21 877.98,-3207.58 870.4,-3198.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"872.903,-3196.33 863.723,-3191.04 867.6,-3200.9 872.903,-3196.33\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699504 -->\r\n",
       "<g id=\"node94\" class=\"node\"><title>2422722699504</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"995,-3077 884,-3077 884,-3043 995,-3043 995,-3077\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"939.5\" y=\"-3063.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"939.5\" y=\"-3050.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 64, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699504&#45;&gt;2422722690968 -->\r\n",
       "<g id=\"edge95\" class=\"edge\"><title>2422722699504&#45;&gt;2422722690968</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M939.5,-3042.88C939.5,-3033.31 939.5,-3021.09 939.5,-3010.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"943,-3010.89 939.5,-3000.89 936,-3010.89 943,-3010.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722691024 -->\r\n",
       "<g id=\"node95\" class=\"node\"><title>2422722691024</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"1131,-3007 1020,-3007 1020,-2973 1131,-2973 1131,-3007\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"1075.5\" y=\"-2993.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"1075.5\" y=\"-2980.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722691024&#45;&gt;2422722690856 -->\r\n",
       "<g id=\"edge96\" class=\"edge\"><title>2422722691024&#45;&gt;2422722690856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1043.27,-2972.88C1020.42,-2961.46 990.028,-2946.26 968.047,-2935.27\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"969.402,-2932.04 958.893,-2930.7 966.272,-2938.3 969.402,-2932.04\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699336 -->\r\n",
       "<g id=\"node96\" class=\"node\"><title>2422722699336</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"858.5,-3007 758.5,-3007 758.5,-2973 858.5,-2973 858.5,-3007\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"808.5\" y=\"-2993.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"808.5\" y=\"-2980.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722699336&#45;&gt;2422722690856 -->\r\n",
       "<g id=\"edge97\" class=\"edge\"><title>2422722699336&#45;&gt;2422722690856</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M839.545,-2972.88C861.46,-2961.51 890.568,-2946.4 911.717,-2935.42\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"913.557,-2938.41 920.82,-2930.7 910.332,-2932.2 913.557,-2938.41\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690744 -->\r\n",
       "<g id=\"node97\" class=\"node\"><title>2422722690744</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"876,-2867 765,-2867 765,-2833 876,-2833 876,-2867\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-2853.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-2840.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 128, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690744&#45;&gt;2422722690464 -->\r\n",
       "<g id=\"edge98\" class=\"edge\"><title>2422722690744&#45;&gt;2422722690464</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M842.777,-2832.88C857.935,-2821.92 877.89,-2807.48 892.902,-2796.62\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"895.045,-2799.39 901.096,-2790.7 890.942,-2793.72 895.045,-2799.39\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690520 -->\r\n",
       "<g id=\"node98\" class=\"node\"><title>2422722690520</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"715,-2797 604,-2797 604,-2763 715,-2763 715,-2797\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"659.5\" y=\"-2783.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"659.5\" y=\"-2770.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690520&#45;&gt;2422607646224 -->\r\n",
       "<g id=\"edge99\" class=\"edge\"><title>2422722690520&#45;&gt;2422607646224</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M678.154,-2762.84C688.529,-2753.86 701.401,-2742.71 711.785,-2733.71\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"714.217,-2736.23 719.484,-2727.04 709.634,-2730.94 714.217,-2736.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690576 -->\r\n",
       "<g id=\"node99\" class=\"node\"><title>2422722690576</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"833.5,-2797 733.5,-2797 733.5,-2763 833.5,-2763 833.5,-2797\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"783.5\" y=\"-2783.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"783.5\" y=\"-2770.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690576&#45;&gt;2422607646224 -->\r\n",
       "<g id=\"edge100\" class=\"edge\"><title>2422722690576&#45;&gt;2422607646224</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M769.575,-2762.84C762.137,-2754.21 752.98,-2743.58 745.4,-2734.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"747.903,-2732.33 738.723,-2727.04 742.6,-2736.9 747.903,-2732.33\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690352 -->\r\n",
       "<g id=\"node100\" class=\"node\"><title>2422722690352</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"649,-2860.5 502,-2860.5 502,-2839.5 649,-2839.5 649,-2860.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"575.5\" y=\"-2846.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690352&#45;&gt;2422607642808 -->\r\n",
       "<g id=\"edge101\" class=\"edge\"><title>2422722690352&#45;&gt;2422607642808</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M577.345,-2839.39C580.471,-2823.43 587.188,-2790.48 594.5,-2763 602.301,-2733.68 613.116,-2700.21 619.961,-2679.72\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.35,-2680.62 623.225,-2670.03 616.716,-2678.39 623.35,-2680.62\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690632 -->\r\n",
       "<g id=\"node101\" class=\"node\"><title>2422722690632</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"638,-2930.5 513,-2930.5 513,-2909.5 638,-2909.5 638,-2930.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"575.5\" y=\"-2916.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690632&#45;&gt;2422722690352 -->\r\n",
       "<g id=\"edge102\" class=\"edge\"><title>2422722690632&#45;&gt;2422722690352</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M575.5,-2909.26C575.5,-2899.26 575.5,-2883.42 575.5,-2870.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"579,-2870.72 575.5,-2860.72 572,-2870.72 579,-2870.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722699560&#45;&gt;2422722690632 -->\r\n",
       "<g id=\"edge103\" class=\"edge\"><title>2422722699560&#45;&gt;2422722690632</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M666.313,-2979.26C648.66,-2967.93 619.33,-2949.12 598.791,-2935.94\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"600.624,-2932.96 590.317,-2930.51 596.844,-2938.85 600.624,-2932.96\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722700568 -->\r\n",
       "<g id=\"node103\" class=\"node\"><title>2422722700568</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"604.5,-3007 498.5,-3007 498.5,-2973 604.5,-2973 604.5,-3007\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"551.5\" y=\"-2993.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.idconv.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"551.5\" y=\"-2980.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 64, 1, 1)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722700568&#45;&gt;2422722690632 -->\r\n",
       "<g id=\"edge105\" class=\"edge\"><title>2422722700568&#45;&gt;2422722690632</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M557.188,-2972.88C560.603,-2963.21 564.973,-2950.83 568.587,-2940.59\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"571.98,-2941.49 572.008,-2930.89 565.379,-2939.16 571.98,-2941.49\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690800 -->\r\n",
       "<g id=\"node104\" class=\"node\"><title>2422722690800</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"762.5,-2937 656.5,-2937 656.5,-2903 762.5,-2903 762.5,-2937\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"709.5\" y=\"-2923.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.idconv.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"709.5\" y=\"-2910.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690800&#45;&gt;2422722690352 -->\r\n",
       "<g id=\"edge106\" class=\"edge\"><title>2422722690800&#45;&gt;2422722690352</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M677.744,-2902.88C655.227,-2891.46 625.285,-2876.26 603.627,-2865.27\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"605.109,-2862.1 594.608,-2860.7 601.941,-2868.34 605.109,-2862.1\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664152 -->\r\n",
       "<g id=\"node105\" class=\"node\"><title>2422722664152</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"495,-2937 400,-2937 400,-2903 495,-2903 495,-2937\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"447.5\" y=\"-2923.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.0.idconv.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"447.5\" y=\"-2910.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664152&#45;&gt;2422722690352 -->\r\n",
       "<g id=\"edge107\" class=\"edge\"><title>2422722664152&#45;&gt;2422722690352</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M477.834,-2902.88C499.247,-2891.51 527.688,-2876.4 548.353,-2865.42\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"550.059,-2868.48 557.248,-2860.7 546.775,-2862.3 550.059,-2868.48\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607673832 -->\r\n",
       "<g id=\"node106\" class=\"node\"><title>2422607673832</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"927,-2613 816,-2613 816,-2579 927,-2579 927,-2613\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"871.5\" y=\"-2599.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.1.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"871.5\" y=\"-2586.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 128, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607673832&#45;&gt;2422607701496 -->\r\n",
       "<g id=\"edge108\" class=\"edge\"><title>2422607673832&#45;&gt;2422607701496</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M871.5,-2578.88C871.5,-2569.31 871.5,-2557.09 871.5,-2546.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"875,-2546.89 871.5,-2536.89 868,-2546.89 875,-2546.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607552240 -->\r\n",
       "<g id=\"node107\" class=\"node\"><title>2422607552240</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"672,-2543 561,-2543 561,-2509 672,-2509 672,-2543\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"616.5\" y=\"-2529.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.1.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"616.5\" y=\"-2516.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607552240&#45;&gt;2422607700768 -->\r\n",
       "<g id=\"edge109\" class=\"edge\"><title>2422607552240&#45;&gt;2422607700768</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M657.75,-2508.84C683.302,-2498.83 715.709,-2486.14 739.73,-2476.73\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"741.106,-2479.95 749.141,-2473.04 738.553,-2473.43 741.106,-2479.95\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607551736 -->\r\n",
       "<g id=\"node108\" class=\"node\"><title>2422607551736</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"790.5,-2543 690.5,-2543 690.5,-2509 790.5,-2509 790.5,-2543\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"740.5\" y=\"-2529.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.1.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"740.5\" y=\"-2516.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607551736&#45;&gt;2422607700768 -->\r\n",
       "<g id=\"edge110\" class=\"edge\"><title>2422607551736&#45;&gt;2422607700768</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M749.17,-2508.84C753.61,-2500.57 759.035,-2490.46 763.634,-2481.89\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"766.736,-2483.51 768.38,-2473.04 760.568,-2480.2 766.736,-2483.51\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422490288592 -->\r\n",
       "<g id=\"node109\" class=\"node\"><title>2422490288592</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"775,-2416 664,-2416 664,-2382 775,-2382 775,-2416\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"719.5\" y=\"-2402.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.1.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"719.5\" y=\"-2389.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128, 128, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422490288592&#45;&gt;2422722690128 -->\r\n",
       "<g id=\"edge111\" class=\"edge\"><title>2422490288592&#45;&gt;2422722690128</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M747.701,-2381.88C767.429,-2370.61 793.574,-2355.67 812.741,-2344.72\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"814.585,-2347.7 821.531,-2339.7 811.112,-2341.62 814.585,-2347.7\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690184 -->\r\n",
       "<g id=\"node110\" class=\"node\"><title>2422722690184</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"639,-2346 528,-2346 528,-2312 639,-2312 639,-2346\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"583.5\" y=\"-2332.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.1.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"583.5\" y=\"-2319.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690184&#45;&gt;2422722689960 -->\r\n",
       "<g id=\"edge112\" class=\"edge\"><title>2422722690184&#45;&gt;2422722689960</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M602.154,-2311.84C612.529,-2302.86 625.401,-2291.71 635.785,-2282.71\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"638.217,-2285.23 643.484,-2276.04 633.634,-2279.94 638.217,-2285.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722690240 -->\r\n",
       "<g id=\"node111\" class=\"node\"><title>2422722690240</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"757.5,-2346 657.5,-2346 657.5,-2312 757.5,-2312 757.5,-2346\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"707.5\" y=\"-2332.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">5.1.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"707.5\" y=\"-2319.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (128)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722690240&#45;&gt;2422722689960 -->\r\n",
       "<g id=\"edge113\" class=\"edge\"><title>2422722690240&#45;&gt;2422722689960</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M693.575,-2311.84C686.137,-2303.21 676.98,-2292.58 669.4,-2283.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"671.903,-2281.33 662.723,-2276.04 666.6,-2285.9 671.903,-2281.33\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689736 -->\r\n",
       "<g id=\"node112\" class=\"node\"><title>2422722689736</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"794,-2162 683,-2162 683,-2128 794,-2128 794,-2162\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"738.5\" y=\"-2148.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"738.5\" y=\"-2135.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256, 128, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689736&#45;&gt;2422722689456 -->\r\n",
       "<g id=\"edge115\" class=\"edge\"><title>2422722689736&#45;&gt;2422722689456</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M738.5,-2127.88C738.5,-2118.31 738.5,-2106.09 738.5,-2095.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"742,-2095.89 738.5,-2085.89 735,-2095.89 742,-2095.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689512 -->\r\n",
       "<g id=\"node113\" class=\"node\"><title>2422722689512</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"930,-2092 819,-2092 819,-2058 930,-2058 930,-2092\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"874.5\" y=\"-2078.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"874.5\" y=\"-2065.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689512&#45;&gt;2422722689344 -->\r\n",
       "<g id=\"edge116\" class=\"edge\"><title>2422722689512&#45;&gt;2422722689344</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M842.27,-2057.88C819.416,-2046.46 789.028,-2031.26 767.047,-2020.27\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"768.402,-2017.04 757.893,-2015.7 765.272,-2023.3 768.402,-2017.04\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689568 -->\r\n",
       "<g id=\"node114\" class=\"node\"><title>2422722689568</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"657.5,-2092 557.5,-2092 557.5,-2058 657.5,-2058 657.5,-2092\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"607.5\" y=\"-2078.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"607.5\" y=\"-2065.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689568&#45;&gt;2422722689344 -->\r\n",
       "<g id=\"edge117\" class=\"edge\"><title>2422722689568&#45;&gt;2422722689344</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M638.545,-2057.88C660.46,-2046.51 689.568,-2031.4 710.717,-2020.42\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"712.557,-2023.41 719.82,-2015.7 709.332,-2017.2 712.557,-2023.41\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689232 -->\r\n",
       "<g id=\"node115\" class=\"node\"><title>2422722689232</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"675,-1952 564,-1952 564,-1918 675,-1918 675,-1952\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"619.5\" y=\"-1938.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"619.5\" y=\"-1925.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256, 256, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689232&#45;&gt;2422722688952 -->\r\n",
       "<g id=\"edge118\" class=\"edge\"><title>2422722689232&#45;&gt;2422722688952</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M641.777,-1917.88C656.935,-1906.92 676.89,-1892.48 691.902,-1881.62\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"694.045,-1884.39 700.096,-1875.7 689.942,-1878.72 694.045,-1884.39\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689008 -->\r\n",
       "<g id=\"node116\" class=\"node\"><title>2422722689008</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"514,-1882 403,-1882 403,-1848 514,-1848 514,-1882\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"458.5\" y=\"-1868.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"458.5\" y=\"-1855.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689008&#45;&gt;2422722688784 -->\r\n",
       "<g id=\"edge119\" class=\"edge\"><title>2422722689008&#45;&gt;2422722688784</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M477.154,-1847.84C487.529,-1838.86 500.401,-1827.71 510.785,-1818.71\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"513.217,-1821.23 518.484,-1812.04 508.634,-1815.94 513.217,-1821.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689064 -->\r\n",
       "<g id=\"node117\" class=\"node\"><title>2422722689064</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"632.5,-1882 532.5,-1882 532.5,-1848 632.5,-1848 632.5,-1882\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"582.5\" y=\"-1868.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"582.5\" y=\"-1855.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689064&#45;&gt;2422722688784 -->\r\n",
       "<g id=\"edge120\" class=\"edge\"><title>2422722689064&#45;&gt;2422722688784</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M568.575,-1847.84C561.137,-1839.21 551.98,-1828.58 544.4,-1819.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"546.903,-1817.33 537.723,-1812.04 541.6,-1821.9 546.903,-1817.33\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688840 -->\r\n",
       "<g id=\"node118\" class=\"node\"><title>2422722688840</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"448,-1945.5 301,-1945.5 301,-1924.5 448,-1924.5 448,-1945.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-1931.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688840&#45;&gt;2422722688672 -->\r\n",
       "<g id=\"edge121\" class=\"edge\"><title>2422722688840&#45;&gt;2422722688672</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M376.345,-1924.39C379.471,-1908.43 386.188,-1875.48 393.5,-1848 401.301,-1818.68 412.116,-1785.21 418.961,-1764.72\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"422.35,-1765.62 422.225,-1755.03 415.716,-1763.39 422.35,-1765.62\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607673720 -->\r\n",
       "<g id=\"node119\" class=\"node\"><title>2422607673720</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"437,-2015.5 312,-2015.5 312,-1994.5 437,-1994.5 437,-2015.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"374.5\" y=\"-2001.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607673720&#45;&gt;2422722688840 -->\r\n",
       "<g id=\"edge122\" class=\"edge\"><title>2422607673720&#45;&gt;2422722688840</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M374.5,-1994.26C374.5,-1984.26 374.5,-1968.42 374.5,-1955.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"378,-1955.72 374.5,-1945.72 371,-1955.72 378,-1955.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689288&#45;&gt;2422607673720 -->\r\n",
       "<g id=\"edge123\" class=\"edge\"><title>2422722689288&#45;&gt;2422607673720</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M465.313,-2064.26C447.66,-2052.93 418.33,-2034.12 397.791,-2020.94\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"399.624,-2017.96 389.317,-2015.51 395.844,-2023.85 399.624,-2017.96\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722689400 -->\r\n",
       "<g id=\"node121\" class=\"node\"><title>2422722689400</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"403.5,-2092 297.5,-2092 297.5,-2058 403.5,-2058 403.5,-2092\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"350.5\" y=\"-2078.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.idconv.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"350.5\" y=\"-2065.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256, 128, 1, 1)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722689400&#45;&gt;2422607673720 -->\r\n",
       "<g id=\"edge125\" class=\"edge\"><title>2422722689400&#45;&gt;2422607673720</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M356.188,-2057.88C359.603,-2048.21 363.973,-2035.83 367.587,-2025.59\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370.98,-2026.49 371.008,-2015.89 364.379,-2024.16 370.98,-2026.49\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422490290832 -->\r\n",
       "<g id=\"node122\" class=\"node\"><title>2422490290832</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"561.5,-2022 455.5,-2022 455.5,-1988 561.5,-1988 561.5,-2022\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"508.5\" y=\"-2008.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.idconv.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"508.5\" y=\"-1995.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422490290832&#45;&gt;2422722688840 -->\r\n",
       "<g id=\"edge126\" class=\"edge\"><title>2422490290832&#45;&gt;2422722688840</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M476.744,-1987.88C454.227,-1976.46 424.285,-1961.26 402.627,-1950.27\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"404.109,-1947.1 393.608,-1945.7 400.941,-1953.34 404.109,-1947.1\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422910651080 -->\r\n",
       "<g id=\"node123\" class=\"node\"><title>2422910651080</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"294,-2022 199,-2022 199,-1988 294,-1988 294,-2022\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"246.5\" y=\"-2008.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.0.idconv.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"246.5\" y=\"-1995.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422910651080&#45;&gt;2422722688840 -->\r\n",
       "<g id=\"edge127\" class=\"edge\"><title>2422910651080&#45;&gt;2422722688840</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M276.834,-1987.88C298.247,-1976.51 326.688,-1961.4 347.353,-1950.42\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.059,-1953.48 356.248,-1945.7 345.775,-1947.3 349.059,-1953.48\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688560 -->\r\n",
       "<g id=\"node124\" class=\"node\"><title>2422722688560</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"727,-1698 616,-1698 616,-1664 727,-1664 727,-1698\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"671.5\" y=\"-1684.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.1.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"671.5\" y=\"-1671.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256, 256, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688560&#45;&gt;2422722688336 -->\r\n",
       "<g id=\"edge128\" class=\"edge\"><title>2422722688560&#45;&gt;2422722688336</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M671.5,-1663.88C671.5,-1654.31 671.5,-1642.09 671.5,-1631.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"675,-1631.89 671.5,-1621.89 668,-1631.89 675,-1631.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688392 -->\r\n",
       "<g id=\"node125\" class=\"node\"><title>2422722688392</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"472,-1628 361,-1628 361,-1594 472,-1594 472,-1628\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"416.5\" y=\"-1614.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.1.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"416.5\" y=\"-1601.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688392&#45;&gt;2422722688224 -->\r\n",
       "<g id=\"edge129\" class=\"edge\"><title>2422722688392&#45;&gt;2422722688224</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M466.158,-1593.84C497.465,-1583.65 537.32,-1570.69 566.379,-1561.23\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"567.75,-1564.46 576.176,-1558.04 565.584,-1557.81 567.75,-1564.46\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688448 -->\r\n",
       "<g id=\"node126\" class=\"node\"><title>2422722688448</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"590.5,-1628 490.5,-1628 490.5,-1594 590.5,-1594 590.5,-1628\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"540.5\" y=\"-1614.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.1.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"540.5\" y=\"-1601.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688448&#45;&gt;2422722688224 -->\r\n",
       "<g id=\"edge130\" class=\"edge\"><title>2422722688448&#45;&gt;2422722688224</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M557.578,-1593.84C566.981,-1584.94 578.627,-1573.93 588.084,-1564.98\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"590.557,-1567.46 595.415,-1558.04 585.746,-1562.37 590.557,-1567.46\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688112 -->\r\n",
       "<g id=\"node127\" class=\"node\"><title>2422722688112</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"575,-1501 464,-1501 464,-1467 575,-1467 575,-1501\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"519.5\" y=\"-1487.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.1.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"519.5\" y=\"-1474.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256, 256, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688112&#45;&gt;2422722687832 -->\r\n",
       "<g id=\"edge131\" class=\"edge\"><title>2422722688112&#45;&gt;2422722687832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M547.701,-1466.88C567.429,-1455.61 593.574,-1440.67 612.741,-1429.72\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"614.585,-1432.7 621.531,-1424.7 611.112,-1426.62 614.585,-1432.7\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687888 -->\r\n",
       "<g id=\"node128\" class=\"node\"><title>2422722687888</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"439,-1431 328,-1431 328,-1397 439,-1397 439,-1431\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"383.5\" y=\"-1417.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.1.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"383.5\" y=\"-1404.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687888&#45;&gt;2422722687664 -->\r\n",
       "<g id=\"edge132\" class=\"edge\"><title>2422722687888&#45;&gt;2422722687664</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M402.154,-1396.84C412.529,-1387.86 425.401,-1376.71 435.785,-1367.71\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"438.217,-1370.23 443.484,-1361.04 433.634,-1364.94 438.217,-1370.23\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687944 -->\r\n",
       "<g id=\"node129\" class=\"node\"><title>2422722687944</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"557.5,-1431 457.5,-1431 457.5,-1397 557.5,-1397 557.5,-1431\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"507.5\" y=\"-1417.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">6.1.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"507.5\" y=\"-1404.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (256)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687944&#45;&gt;2422722687664 -->\r\n",
       "<g id=\"edge133\" class=\"edge\"><title>2422722687944&#45;&gt;2422722687664</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M493.575,-1396.84C486.137,-1388.21 476.98,-1377.58 469.4,-1368.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"471.903,-1366.33 462.723,-1361.04 466.6,-1370.9 471.903,-1366.33\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687440 -->\r\n",
       "<g id=\"node130\" class=\"node\"><title>2422722687440</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"595,-1247 484,-1247 484,-1213 595,-1213 595,-1247\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"539.5\" y=\"-1233.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"539.5\" y=\"-1220.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512, 256, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687440&#45;&gt;2422722687160 -->\r\n",
       "<g id=\"edge135\" class=\"edge\"><title>2422722687440&#45;&gt;2422722687160</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M539.5,-1212.88C539.5,-1203.31 539.5,-1191.09 539.5,-1180.91\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"543,-1180.89 539.5,-1170.89 536,-1180.89 543,-1180.89\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687216 -->\r\n",
       "<g id=\"node131\" class=\"node\"><title>2422722687216</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"731,-1177 620,-1177 620,-1143 731,-1143 731,-1177\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"675.5\" y=\"-1163.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"675.5\" y=\"-1150.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687216&#45;&gt;2422722687048 -->\r\n",
       "<g id=\"edge136\" class=\"edge\"><title>2422722687216&#45;&gt;2422722687048</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M643.27,-1142.88C620.416,-1131.46 590.028,-1116.26 568.047,-1105.27\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"569.402,-1102.04 558.893,-1100.7 566.272,-1108.3 569.402,-1102.04\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687272 -->\r\n",
       "<g id=\"node132\" class=\"node\"><title>2422722687272</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"458.5,-1177 358.5,-1177 358.5,-1143 458.5,-1143 458.5,-1177\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"408.5\" y=\"-1163.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"408.5\" y=\"-1150.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687272&#45;&gt;2422722687048 -->\r\n",
       "<g id=\"edge137\" class=\"edge\"><title>2422722687272&#45;&gt;2422722687048</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M439.545,-1142.88C461.46,-1131.51 490.568,-1116.4 511.717,-1105.42\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"513.557,-1108.41 520.82,-1100.7 510.332,-1102.2 513.557,-1108.41\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666392 -->\r\n",
       "<g id=\"node133\" class=\"node\"><title>2422722666392</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"476,-1037 365,-1037 365,-1003 476,-1003 476,-1037\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"420.5\" y=\"-1023.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"420.5\" y=\"-1010.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512, 512, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666392&#45;&gt;2422722666112 -->\r\n",
       "<g id=\"edge138\" class=\"edge\"><title>2422722666392&#45;&gt;2422722666112</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M442.777,-1002.88C457.935,-991.919 477.89,-977.484 492.902,-966.624\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"495.045,-969.393 501.096,-960.696 490.942,-963.722 495.045,-969.393\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666168 -->\r\n",
       "<g id=\"node134\" class=\"node\"><title>2422722666168</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"315,-967 204,-967 204,-933 315,-933 315,-967\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"259.5\" y=\"-953.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"259.5\" y=\"-940.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666168&#45;&gt;2422722665944 -->\r\n",
       "<g id=\"edge139\" class=\"edge\"><title>2422722666168&#45;&gt;2422722665944</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M278.154,-932.842C288.529,-923.855 301.401,-912.705 311.785,-903.711\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"314.217,-906.235 319.484,-897.042 309.634,-900.944 314.217,-906.235\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666224 -->\r\n",
       "<g id=\"node135\" class=\"node\"><title>2422722666224</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"433.5,-967 333.5,-967 333.5,-933 433.5,-933 433.5,-967\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"383.5\" y=\"-953.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"383.5\" y=\"-940.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666224&#45;&gt;2422722665944 -->\r\n",
       "<g id=\"edge140\" class=\"edge\"><title>2422722666224&#45;&gt;2422722665944</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M369.575,-932.842C362.137,-924.211 352.98,-913.585 345.4,-904.789\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.903,-902.332 338.723,-897.042 342.6,-906.902 347.903,-902.332\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666000 -->\r\n",
       "<g id=\"node136\" class=\"node\"><title>2422722666000</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"249,-1030.5 102,-1030.5 102,-1009.5 249,-1009.5 249,-1030.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"175.5\" y=\"-1016.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">NativeBatchNormBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666000&#45;&gt;2422722665832 -->\r\n",
       "<g id=\"edge141\" class=\"edge\"><title>2422722666000&#45;&gt;2422722665832</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M177.345,-1009.39C180.471,-993.429 187.188,-960.484 194.5,-933 202.301,-903.68 213.116,-870.213 219.961,-849.721\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.35,-850.624 223.225,-840.03 216.716,-848.389 223.35,-850.624\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666280 -->\r\n",
       "<g id=\"node137\" class=\"node\"><title>2422722666280</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"238,-1100.5 113,-1100.5 113,-1079.5 238,-1079.5 238,-1100.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"175.5\" y=\"-1086.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">ThnnConv2DBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666280&#45;&gt;2422722666000 -->\r\n",
       "<g id=\"edge142\" class=\"edge\"><title>2422722666280&#45;&gt;2422722666000</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M175.5,-1079.26C175.5,-1069.26 175.5,-1053.42 175.5,-1040.78\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"179,-1040.72 175.5,-1030.72 172,-1040.72 179,-1040.72\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687104&#45;&gt;2422722666280 -->\r\n",
       "<g id=\"edge143\" class=\"edge\"><title>2422722687104&#45;&gt;2422722666280</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M266.313,-1149.26C248.66,-1137.93 219.33,-1119.12 198.791,-1105.94\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"200.624,-1102.96 190.317,-1100.51 196.844,-1108.85 200.624,-1102.96\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722688504 -->\r\n",
       "<g id=\"node139\" class=\"node\"><title>2422722688504</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"204.5,-1177 98.5,-1177 98.5,-1143 204.5,-1143 204.5,-1177\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-1163.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.idconv.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-1150.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512, 256, 1, 1)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722688504&#45;&gt;2422722666280 -->\r\n",
       "<g id=\"edge145\" class=\"edge\"><title>2422722688504&#45;&gt;2422722666280</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M157.188,-1142.88C160.603,-1133.21 164.973,-1120.83 168.587,-1110.59\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.98,-1111.49 172.008,-1100.89 165.379,-1109.16 171.98,-1111.49\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722666448 -->\r\n",
       "<g id=\"node140\" class=\"node\"><title>2422722666448</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"362.5,-1107 256.5,-1107 256.5,-1073 362.5,-1073 362.5,-1107\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"309.5\" y=\"-1093.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.idconv.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"309.5\" y=\"-1080.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722666448&#45;&gt;2422722666000 -->\r\n",
       "<g id=\"edge146\" class=\"edge\"><title>2422722666448&#45;&gt;2422722666000</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M277.744,-1072.88C255.227,-1061.46 225.285,-1046.26 203.627,-1035.27\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"205.109,-1032.1 194.608,-1030.7 201.941,-1038.34 205.109,-1032.1\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722687328 -->\r\n",
       "<g id=\"node141\" class=\"node\"><title>2422722687328</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"95,-1107 0,-1107 0,-1073 95,-1073 95,-1107\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"47.5\" y=\"-1093.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.0.idconv.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"47.5\" y=\"-1080.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722687328&#45;&gt;2422722666000 -->\r\n",
       "<g id=\"edge147\" class=\"edge\"><title>2422722687328&#45;&gt;2422722666000</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M77.8342,-1072.88C99.2469,-1061.51 127.688,-1046.4 148.353,-1035.42\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"150.059,-1038.48 157.248,-1030.7 146.775,-1032.3 150.059,-1038.48\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665720 -->\r\n",
       "<g id=\"node142\" class=\"node\"><title>2422722665720</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"528,-783 417,-783 417,-749 528,-749 528,-783\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"472.5\" y=\"-769.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.1.convs.0.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"472.5\" y=\"-756.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512, 512, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665720&#45;&gt;2422722665496 -->\r\n",
       "<g id=\"edge148\" class=\"edge\"><title>2422722665720&#45;&gt;2422722665496</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M472.5,-748.885C472.5,-739.309 472.5,-727.088 472.5,-716.912\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"476,-716.895 472.5,-706.895 469,-716.895 476,-716.895\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665552 -->\r\n",
       "<g id=\"node143\" class=\"node\"><title>2422722665552</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"273,-713 162,-713 162,-679 273,-679 273,-713\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"217.5\" y=\"-699.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.1.convs.0.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"217.5\" y=\"-686.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665552&#45;&gt;2422722665384 -->\r\n",
       "<g id=\"edge149\" class=\"edge\"><title>2422722665552&#45;&gt;2422722665384</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M267.158,-678.842C298.465,-668.654 338.32,-655.686 367.379,-646.23\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"368.75,-649.464 377.176,-643.042 366.584,-642.808 368.75,-649.464\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665608 -->\r\n",
       "<g id=\"node144\" class=\"node\"><title>2422722665608</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"391.5,-713 291.5,-713 291.5,-679 391.5,-679 391.5,-713\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"341.5\" y=\"-699.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.1.convs.0.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"341.5\" y=\"-686.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665608&#45;&gt;2422722665384 -->\r\n",
       "<g id=\"edge150\" class=\"edge\"><title>2422722665608&#45;&gt;2422722665384</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M358.578,-678.842C367.981,-669.944 379.627,-658.926 389.084,-649.978\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"391.557,-652.457 396.415,-643.042 386.746,-647.372 391.557,-652.457\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665272 -->\r\n",
       "<g id=\"node145\" class=\"node\"><title>2422722665272</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"376,-586 265,-586 265,-552 376,-552 376,-586\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"320.5\" y=\"-572.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.1.convs.1.0.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"320.5\" y=\"-559.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512, 512, 3, 3)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665272&#45;&gt;2422722664992 -->\r\n",
       "<g id=\"edge151\" class=\"edge\"><title>2422722665272&#45;&gt;2422722664992</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M348.701,-551.885C368.429,-540.612 394.574,-525.672 413.741,-514.719\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"415.585,-517.697 422.531,-509.696 412.112,-511.619 415.585,-517.697\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665048 -->\r\n",
       "<g id=\"node146\" class=\"node\"><title>2422722665048</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"240,-516 129,-516 129,-482 240,-482 240,-516\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"184.5\" y=\"-502.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.1.convs.1.1.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"184.5\" y=\"-489.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665048&#45;&gt;2422722664824 -->\r\n",
       "<g id=\"edge152\" class=\"edge\"><title>2422722665048&#45;&gt;2422722664824</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M203.154,-481.842C213.529,-472.855 226.401,-461.705 236.785,-452.711\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"239.217,-455.235 244.484,-446.042 234.634,-449.944 239.217,-455.235\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722665104 -->\r\n",
       "<g id=\"node147\" class=\"node\"><title>2422722665104</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"358.5,-516 258.5,-516 258.5,-482 358.5,-482 358.5,-516\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"308.5\" y=\"-502.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">7.1.convs.1.1.bias</text>\r\n",
       "<text text-anchor=\"middle\" x=\"308.5\" y=\"-489.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722665104&#45;&gt;2422722664824 -->\r\n",
       "<g id=\"edge153\" class=\"edge\"><title>2422722665104&#45;&gt;2422722664824</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M294.575,-481.842C287.137,-473.211 277.98,-462.585 270.4,-453.789\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"272.903,-451.332 263.723,-446.042 267.6,-455.902 272.903,-451.332\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422722664040 -->\r\n",
       "<g id=\"node148\" class=\"node\"><title>2422722664040</title>\r\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"285,-84.5 214,-84.5 214,-63.5 285,-63.5 285,-84.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"249.5\" y=\"-70.9\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">TBackward</text>\r\n",
       "</g>\r\n",
       "<!-- 2422722664040&#45;&gt;2422722663760 -->\r\n",
       "<g id=\"edge155\" class=\"edge\"><title>2422722664040&#45;&gt;2422722663760</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.188,-63.3906C218.438,-53.5067 193.688,-37.9749 175.366,-26.4773\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"177.036,-23.3929 166.705,-21.0419 173.315,-29.3221 177.036,-23.3929\"/>\r\n",
       "</g>\r\n",
       "<!-- 2422607703960 -->\r\n",
       "<g id=\"node149\" class=\"node\"><title>2422607703960</title>\r\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"281,-161 218,-161 218,-127 281,-127 281,-161\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"249.5\" y=\"-147.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\">10.weight</text>\r\n",
       "<text text-anchor=\"middle\" x=\"249.5\" y=\"-134.4\" font-family=\"Times New Roman,serif\" font-size=\"12.00\"> (10, 512)</text>\r\n",
       "</g>\r\n",
       "<!-- 2422607703960&#45;&gt;2422722664040 -->\r\n",
       "<g id=\"edge156\" class=\"edge\"><title>2422607703960&#45;&gt;2422722664040</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M249.5,-126.885C249.5,-117.309 249.5,-105.088 249.5,-94.9119\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"253,-94.8946 249.5,-84.8946 246,-94.8946 253,-94.8946\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x23415861518>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = learn.model.cpu()\n",
    "x = torch.randn([1, 3, 128, 128])\n",
    "graph = make_dot(model(x), params=dict(model.named_parameters()))\n",
    "graph.format = 'png'\n",
    "graph.render('bn')\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Train imagenette with the XResNet model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 Empirically determine a good learning rate for the XResNet-34 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "arch = partial(xresnet34, c_out=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(arch(), data, loss_func, lr=1, cb_funcs=cbfs, opt_func=opt_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'learning rate')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEOCAYAAACO+Hw9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxcdZX38c+p6up0OkkvIZ0Q0tkDBAhha5B9ExhQBAVEcFRUEJdxRGccN55HxXHBGRRZZkRGGdSHRXYQkUUBA7KEDgQICUv2pJOQTrqTTqe3Ws7zR1X1WpWkQ22p+r5fr7y6+tZyfzfdferU+Z37u+buiIhI6QjkewAiIpJbCvwiIiVGgV9EpMQo8IuIlBgFfhGREqPALyJSYrIW+M3sFjPbaGaL+m071MxeMLOFZtZoZkdla/8iIpJaNjP+W4EzB237D+Aqdz8U+G7iexERyaGsBX53nwe0DN4MVCVuVwPrsrV/ERFJrSzH+/sq8JiZXUP8TefYHO9fRKTk5TrwfxH4mrvfa2YXAr8BTkv1QDO7HLgcYNSoUUfMnj07d6MUESkCCxYs2OTudYO3WzbX6jGzacDD7j4n8f1WoMbd3cwM2OruVTt4CQAaGhq8sbExa+MUESlGZrbA3RsGb891O+c64KTE7VOBd3K8fxGRkpe1Uo+Z3QGcDIwzs7XA94DPAdeZWRnQRaKUIyIiuZO1wO/uF6e564hs7VNERHZOZ+6KiJQYBX4RkRKjwC8iUmIU+EVE8mjJ+jZyfQlcBX4RkTx5+91tnHXdMzSuas3pfhX4RUTyZEtHeMDXXFHgFxHJk0g0NuBrrijwi4jkSTjmA77migK/iEieRGPxTD8cUcYvIlISwtF4ph+JKfCLiJSESCLwJ98AckWBX0QkT5KZviZ3RURKRF+pRxm/iEhJ6J3cValHRKQ09Gb8KvWIiJSGZMBXH7+ISIlI1vaLJuM3s1vMbKOZLRq0/Z/N7C0ze8PM/iNb+xcRKXS9gb+IMv5bgTP7bzCzU4BzgbnufhBwTRb3LyJS0HpLPcWS8bv7PKBl0OYvAle7e3fiMRuztX8RkULXN7lbPBl/KvsBJ5jZi2b2NzM7Mt0DzexyM2s0s8bm5uYcDlFEJDd6T+Aq8iUbyoBa4Gjg34C7zMxSPdDdb3b3BndvqKury+UYRURyolSWbFgL3Odx84EYMC7HYxARKQhF19WTxgPAqQBmth9QDmzK8RhERApCvvr4y7L1wmZ2B3AyMM7M1gLfA24Bbkm0ePYAl3iurzIsIlIgwnnK+LMW+N394jR3fSJb+xQR2ZP0tXMWd41fREQS+iZ3i7vGLyIiCX2Tu8r4RURKQqn08YuISEK4RPr4RUQkITm5q4xfRKREqMYvIlJi1NUjIlJi+iZ3lfGLiJSEUlmWWUREEpIZv0o9IiIlIpnpq9QjIlIikgFfGb+ISIno7eNXjV9EpDT0Tu7qBC4RkdLQN7nr5PLSJAr8IiJ5Eu03qRvN4QRv1gK/md1iZhsTV9safN/XzczNTNfbFZGS1X9xtlx29mQz478VOHPwRjObDJwOrM7ivkVECl7/Sy725LCzJ2uB393nAS0p7roW+Aaga+2KSEkLx5yKUDwM57KzJ6c1fjM7B2hy91dzuV8RkUIUicYYGQr23s6VrF1sfTAzqwSuBM7YxcdfDlwOMGXKlCyOTEQk92IxJ+ZQEQoCYcJFUuMfbCYwHXjVzFYC9cDLZrZ3qge7+83u3uDuDXV1dTkcpohI9iUnc4s643f314Hxye8Twb/B3TflagwiIoUi2cM/sjwe+HN5+cVstnPeATwP7G9ma83s0mztS0RkT5MM9L0Zfw7P3s1axu/uF+/k/mnZ2reISKFLnrCVzPiLtqtHRETikjX9EWXJUk8R9PGLiEh64cEZf5F29YiISEIy4x+ZOIFLGb+ISJEbMrmrGr+ISHFLTu5WlOe+q0eBX0QkD5KlnYqyIurjFxGR9CJq5xQRKS19k7sq9YiIlITBk7sq9YiIFLkhk7tq5xQRKW7hWHJyV338IiIlITmZW1Src4qISHqa3BURKTHJtXoqNLkrIlIaooMuxKI+fhGRIpfM8MuDAcxU6hERKXrJDD8UDBAKBIqj1GNmt5jZRjNb1G/bf5rZm2b2mpndb2Y12dq/iEghS2b4ZUGjLGhF08d/K3DmoG1PAHPcfS7wNvDtLO5fRKRg9Wb8gQBlASuOC7G4+zygZdC2x909kvj2BaA+W/sXESlk/TP+UDBQMidwfRb4cx73LyKSN8mafjCQLPUUQca/I2Z2JRABbtvBYy43s0Yza2xubs7d4EREcqD/5G5ZINC7hEMu5Dzwm9klwNnAP7p72rc4d7/Z3RvcvaGuri53AxQRyYFILIZZPOMP5TjjL8vZngAzOxP4JnCSu3fkct8iIoUkEnNCgXjuXRYMFEcfv5ndATwP7G9ma83sUuBGYAzwhJktNLObsrV/EZFCFonGKAsaAGUBy2kff9Yyfne/OMXm32RrfyIie5Jw1AkG4oE/FAwUTR+/iIikEYnFCAWTpZ4i6eMXEZH0IlGnLJnxB0qnj19EpGRFYj4g4y+KtXpERCS9/pO7qvGLiJSAcKz/5K4yfhGRoheJxvr6+ANF0scvIiLpRaLe18dfCmv1iIiUukjMKUtM7oaCRb5Wj4iIxPv4k+2cZQFl/CIiRS/cr4+/LFgkl14UEZH0ItG+M3dDQdPkrohIsYvG+k3uBgIq9YiIFLt4qacv49eSDSIiRW7A5K4WaRMRKX4D+vgDAaIxZwcXJcwoBX4RkTwIxwZO7gI56+zJ5hW4bjGzjWa2qN+2sWb2hJm9k/ham639i4gUsuigdk4gZ5092cz4bwXOHLTtW8Bf3X1f4K+J70VESk6435m7yTeAPT7jd/d5QMugzecCv03c/i3w4WztX0SkkEWifZO7yZJPrpZm3qXAb2ZXmFmVxf3GzF42szN2Y38T3H09QOLr+N14DRGRPV7/yd1k4C+0jP+z7t4GnAHUAZ8Brs7aqAAzu9zMGs2ssbm5OZu7EhHJufCga+4COevl39XAb4mvHwD+191f7bdtON41s4kAia8b0z3Q3W929wZ3b6irq9uNXYmIFK5orN81dxOBP1e9/Lsa+BeY2ePEA/9jZjYG2J23poeASxK3LwEe3I3XEBHZo7l7/MzdYN+FWCB3Nf6yXXzcpcChwHJ37zCzscTLPWmZ2R3AycA4M1sLfI94eeguM7sUWA18dHcHLiKyp4omMvvBGX+uavy7GviPARa6+3Yz+wRwOHDdjp7g7henuev9wxifiEjRSZZ0+p+5G99eWDX+XwIdZnYI8A1gFfC7rI1KRKSIJQN/7zV3C/TM3YjHF5E4F7jO3a8DxmRvWCIixStZyx/czlloNf5tZvZt4JPACWYWBELZG5aISPFKZvb9L70IhdfV8zGgm3g//wZgEvCfWRuViEgRS9bye7t6ek/gKqAafyLY3wZUm9nZQJe7q8YvIrIbItHUXT25ugrXri7ZcCEwn3j75YXAi2Z2QTYHJiJSrHondwf38eeoq2dXa/xXAke6+0YAM6sD/gLck62BiYgUq6GTu4XZ1RNIBv2EzcN4roiI9DNkcjfH6/Hvasb/qJk9BtyR+P5jwCPZGZKISHHrndwN5Gc9/l0K/O7+b2Z2PnAc8cXZbnb3+7M6MhGRItWb8Q/p4y+gwA/g7vcC92ZxLCIiJSE6aHK3b3XOAij1mNk2INVbkAHu7lVZGZWISBHrndwdVOMviFKPu2tZBhGRDAvHBpd6CvNCLCIikiF9GX9+1uNX4BcRybGhk7uF2ccvIiIZMnhy18wIBqzg1uMXEZEM6evj77t0eVnACmutnkwzs6+Z2RtmtsjM7jCzinyMQ0QkH/rO3O0LwaFgoHhLPWY2CfgK0ODuc4AgcFGuxyEiki+D1+pJ3i72Uk8ZMNLMyoBKYF2exiEiknODr7kL8ey/aDN+d28CrgFWA+uBre7+eK7HISKSL8mMPzSg1GPF285pZrXEr907HdgHGGVmn0jxuMvNrNHMGpubm3M9TBGRrElm/MEhpZ4izfiB04AV7t7s7mHgPuDYwQ9y95vdvcHdG+rq6nI+SBGRbEmWdAZk/IFAUZ+5uxo42swqzcyA9wNL8jAOEZG8SDu5W8Q1/heJX7nrZeD1xBhuzvU4RETypXdyNzBwcrcgVufMFnf/HvC9fOxbRCTfIrEYZQEjXvSICwWteLt6RERKXSTqBPtl+xBfmrnY+/hFREpWOOq96/QkKeMXESlikVhswMQuxJdsKNo+fhGRUheJ+YB1eiCxSFsR9/GLiJS0SDTWuwZ/UlkwQE9EGb+ISFFKNbkbKvIzd0VESlo4NnRytyygGr+ISNGKJvr4+ytTV4+IyJ6rrSvMlo6etPeHo07Z4HbOHJ65q8AvIpJhV96/iC/d9nLa+yPR1Bl/rtbqycuSDSIixayptYPm9u6090dinrKPv5hX5xQRKWptXRG2bA+nvT8cjQ1YkhnUxy8iskdr6wyzrTuSNoOPpsj4y4KB4l2WWUSk2LV1xbP9LR2ps/6Uk7tBI6zJXRGRPU93JEpXOB7A03X2RFK1cwYCuMc/DWSbAr+ISAZt64r03m5Nk/FHop6yqwfIyQRvXgK/mdWY2T1m9qaZLTGzY/IxDhGRTGvr7Av2rWky/nA0lnJZZiAnE7z5aue8DnjU3S8ws3KgMk/jEBHJqLZ+GX+6Uk+qyd3kG0Eulm3IeeA3syrgRODTAO7eA6Q/xU1EZA8yMOPfweTu4HbORODPxbIN+Sj1zACagf81s1fM7NdmNioP4xARybhkRw+kL/WkmtwNBaz3vmzLR+AvAw4HfunuhwHbgW8NfpCZXW5mjWbW2NzcnOsxiojslrbOeKknGDBat6cJ/NHUffzJ+7ItH4F/LbDW3V9MfH8P8TeCAdz9ZndvcPeGurq6nA5QRGR3JTP+STUjd1DqST+5W5RdPe6+AVhjZvsnNr0fWJzrcYiIZENbZ5hQ0JhYXbHjyd0UffyQmxp/vrp6/hm4LdHRsxz4TJ7GISKSUW1dYaoqQtRWlrO0uT3lY8KxoWfu5rKPPy+B390XAg352LeISDa1dUaoGhmidlQ5W1alq/GnmNwd1MffFY6yuqWDKWMrqQgFMzpGnbkrIpJB8Yy/jNrKEFs6wrgPLN3EYk7MGTq5GxjYx79kfRtnXDuPZ9/ZlPExKvCLiGRQW2c4nvFXlhOJOdu6IwPuT2b0Q66521vqid+/trUTgPqxIzM+RgV+EZEMauuKUFURoqYyBDBkXf5kn/7QUk9gwP3JwD+pRoFfRKSgxTP+Mmory4GhJ3ElM/rgkK6eRI2/N+PvoKYyxJiKUMbHqMAvIpJBvV09o+IBe3DgT9bwh/bxJ9s5+zL+ybXZWcZMgV9EJEOSa/FXjQxRk8j4B1+MJVnjH3rm7sCunrWtHdTXZr7MAwr8IiIZk1yLP97Vk7rU0zu5O+Sau30Zv7uztrVTgV9EpNAlV+asGhmiemQIs6ErdCZLPUOXZe6r8W9q76E7EqNepR4RkcLW1pvxhwgGjKqK0JBlG9JN7vbv6lnb2gGgjF9EpND1ZfzxRRFqK0NDM/5Y6snd/n38vT38yvhFRApbcmXOqkQLZk1l+ZCMP9muOXQ9/r4zd3t7+JXxi4gUtuRa/Mne+3jGn2ZyN03GH4k5a1s7qK0MMXpEdpZTU+AXEcmQ3oy/t9RTTuvgM3fTTu72Lcu8prUza2UeUOAXEcmYts4wZQFjZGI1zVSlnp2fuRvLag8/KPCLiGRMW1d8gTazeBCvrQyxvSdKT6Rvjf10k7vJN4KeaIymLPbwgwK/iEjGtHVGqKroq8vXjEqevduX9aeb3DUzQkFj/daurPbwgwK/iEjGJDP+pNrK5Ho9fXX+dJO7ED97d+Wm7UD2evghj4HfzIJm9oqZPZyvMYiIZFJbZ7i3lRNgbGLZhpbt/TP+1JO7yW0rNycDf3Fm/FcAS/K4fxGRjGrrivR29AD9FmrrC/zhWOpSD8Q/BWxqjz82Wz38kKfAb2b1wAeBX+dj/yIi2TA44+9bmrlfqSeZ8QdSlXr6JoWz1cMP+cv4fwF8A8j+5eRFRHJkaI1/6Aqd6ZZlhr66/+Sx2SvzQB4Cv5mdDWx09wU7edzlZtZoZo3Nzc05Gp2IyO7pXYu/X1dPRShIRSiQsqsn5eRu4s0gmxO7kJ+M/zjgHDNbCdwJnGpm/2/wg9z9ZndvcPeGurq6XI9RRGRYetfiHznwUom1leWDunrihY7BJ3BBX6knmxO7kIfA7+7fdvd6d58GXAQ86e6fyPU4REQyqXdlzkHXyB189m7yzN3BF2KBvk8BxZjxi4gUnd61+EcOnJQdvDTzzto5ocgDv7s/7e5n53MMIiKZkC7jj5d6hje5W3SlHhGRYtS3MufgUk9owAXXIzsq9SS2Taop4oxfRKRYJNfiT5Xxb+noIZbI9COxGGYQSDW5GzTGjipnVBZ7+EGBX0QkIwavxZ9UUxki5n1dP+Gop8z2If4mMatudHYHCmT3bUVEpERs6xq4Fn9S/5O4qitDRKKxlPV9gB+cexDRxCeDbFLgFxHJgLbOyIC1+JP6lm3oYRqjiMQ85To9AHuNHpH1cYJKPSIiGdHWFR5w1m7S+DEVACxe3wbEa/ypztrNJQV+EZEMaOsMD+noAThonyrm1ldz09+WEY7GiEQ95Vm7uaTALyKSxqOL1vPn19fv0mPbuiJDOnogfmWtr562L2taOrn/5ab45G6eM37V+EVkj/Hc0k20dYU5c87ErO6nJxLjh39azO+eX0UoaDwyfjT7Thizw+e0dYaZUJW6Rn/K/uOZW1/NDU+9w9z6mrSTu7mijF9E9ghd4ShX/GEhX7ljIWtaOrK2n3fburj4f17gd8+v4pJjplJZXsaVDyzCfcfdNvEa/9CMH+JZ/xXvj2f9Ty7ZmHZyN1cU+EUkY5Y3t/eeqJRpdzWuoXlbN1F3rnn8rYy/fjTm3LNgLWff8CxL1rdxw8WHcdW5c/j2WbOZv6KFe19u2uHzk1096Zw6ezwHT6qmMxzNe6lHgX+QNze08bU/LOSy3zbSFY7u9PGb2rvZ3N6902ygkG3Y2pW1P1YpHfe9vJZTf/Y3fvromxl/7Z5IjF/9bTkNU2v5wkkzeHDhOl5buyUjr+3uPLH4Xc66bh5fv/tV9q6q4P4vHceHDtkHgAsbJnPE1Fp+/MgSWvtdO3fw+DrD0ZRdPUnJWj+kXpI5l0qmxu/urG7pYGL1SMrLhr7fLVjVwn89tYwn39xIZXmQjp4o37r3Na792KFD+nKTbntxFf/3gUXEHEaGguxTU8H+e4/h7Ln7cOrs8VT0O5FjS0cP0Zjvcp9uNOYsXtfG35dtIubOx4+a0nv9zqRN7d2sbulgcm0l40aXpx1nOp09UX766Jvc+txKDphYxXc+MJsT9tW1D2T4mrZ08r0H36CyPMiv5i3nsCk1Ga3DP/BKE01bOvnhR+bQMLWWO+ev4Ud/WsKdlx+NmRGOxrjhyaVsau/miyfN3OUrWC1q2spVf3yDl1a2Mn3cKP7r44dz1py9ByynEAgYP/zwHM6+4Vl++uibXH3+3CGvsy3NOj2DnTp7PIdOrmHMDt4gcqHoA/+m9m7uf7mJPzSuYenGdqaPG8WVHziA9x8wHjNj5abt/PiRJTy++F1qK0P8y+n78aljpvL751fxsyfe5oCJVXz+pJkDXtPdufaJt7n+yaWcvH8dJ+5bR9OWTppaO5m/opVHXt/A6BFlnDp7PB09ERava2Pd1i7M4PhZ4zj/8HrOOGgCATOWNbezrHk767Z0sqUjzNbOHpq3ddO4qnXAwk6/fGoZnzluGp89fjpL1m/jthdX8dgbG3rX9q4sDzJ1r1HM2aeKI6bWcsTUWiaPraStM8yWzjDbusLsNWoE+9TE3/heWd3Kv971Kss3bee8wycxf0ULn/zNfE7cr44r3r8vh06uGZKVtHdH2LC1EzACFl9JsL525LDfcPpb09JBe3eEAyZW7fZrSH7FYs6/3f0qMXce+vLx/Ovdr/L1u19jvwljmJFi+YG2rjCL17UxMhRkbn31Tn9/ItEY//30UuZMquLk/eri9fLT9uW7D77BX5dsZNb40Vxx5yu8unYrZQHj7sY1XHzUFL58yizGV1WkfM3N7d1c8/jb3PnSamory/nRR+ZwYcPktCWYAyZWcenx07l53nIqQkEuPX76gDeXLWlW5hzMzPjdpUfl/RO27QklioaGBm9sbBz28372+Fv88ullRGLO4VNqOP3AvblnwRqWNW/n+Fnj2G/CGH7/wkpCwQBfOnkmnz1+OpXl8fdCd+fLt7/CI4vWc8unj+SU/ccD8V/C79z/Onc1ruXChnp+/JGDKev3yxKNOc8v28xDrzbx1yUbGTuqnAP3qeKAiVV0dEe475Um1rZ2Ul4WIByN0f+/vzwYoLoyRG1liLn1NRw/axzHztyLlo4erv/rOzzy+gaCASMac6pHhrjgiHreN30s67Z0sqqlgxWbtvPqmi0D1v4eLGAwsXokG9q6mDBmBNd89BCOnTWO7kiU3z+/ihueXMrWzjDVI0McM2Mvjphay5rWDhpXtvLmhjYG/77O3nsMl50wg3MO2af3k1Qs5jRt6WTjti42tfewqb2byvIgp+w/vvdTy7auMDc+tZRbnl2BmXHPF45hbn1N2nG7Oy+vbmX6uNGMHVWe9nGSe7c8u4IfPLyYq887mIuOmkLTlk7Ovv4Zxo+p4P5/Opam1k6eW7aZF1ds5vWmraxp6ex97qzxo7mwoZ6PHFZP3ZjUn4YfXNjEFXcu5KZPHMGZc/YGIByN8Q/XzqMzHKWtM0xZMMDV5x3MIZNruOHJpdzduIayoPHpY6fzxZNmUl0ZD8idPVFufW4lv3x6KR09UT51zDSuOG1fqneSqQN09ET47oNv8MArTcTcOXPO3szeu4r5K1pYsKqVznCU3196VEF9ajazBe7eMGR7MQf+Bxc2sahpKxc2TO5txQpHY9z2wiqu/cs7tHWF+egR9Xz9jP1TZgYdPREu+OXzrGnp4Ihptaxu6WBtayc9kRhfOXUWXzt9v2Fnu7GYM39lC4+/8S7VI0PMHD+KWeNHM7m2ksry4A5fb8n6Nu5qXMOcfar54NyJA0pJSe7Oik3baVzVSvO2bqpHhqipDDF6RBmb2ntY3dLBmpYOaivL+erp+w7JULZ2hnn6rY38fekm/r50M01bOhlVHuSwKbUcPrWWmXWjeh/bur2HO+av4a13tzF+zAjeN2Mvlje3s6y5na5wbMjYggHjmBl7cfiUGm6fv4ZN7d2cf3g9Lyzf3JstpvrjX9PSwTfvfY3nlm0mFDROO2ACFxxRz0n71Q1405Ud64nEiLmn/L3ZXUs3buOD1z/L8bPG8etLGnp/f595p5lP3TKfirIgnYm5skk1Izl0cg0H7lPFgftU0dzWzR8a17BgVStlgfjP9aKjJnPCvnUEA4a7s6alk8t+9xIAj15x4oASzGNvbODzv1/AUdPH8ouPHco+/ZYyXrV5O9c+8TYPvrqOMSPK+OLJsxhTUcb1f32Hjdu6OWX/Or7zgQN22qKZyoatXdz63Epuf3EVbV0RZu89hvdNH8uxs8Zx+gETUq66mS8lGfh3pK0rTHtXZMAvSyprWzv4p9tfIRKNMWVsJZPHVnLUtLGcduCEjI6nUDVv66a2MpQ2wLo7897ZxK+fWc7y5u3MHD+a/caPZtb40exdXcG40SOoGzOCd9u6eHTRBh5dtIHlm7Zz+JQavvehgzhkcg2LmrZywU3PcfCkam677OjeTw7uzm0vruYnjywB4Gun78f6rV088EoTm7f3MLG6gs+dMIOLjprc+0lNBmrrCvPUmxt5fPG7PP3mRmpHlXP3F45hYvV7W+99/dZOfv3MCu6Yv5oRZQEe+9qJvUsTJN3+4mpeWtnC0TPGcuzMcWnr7ks3tnNX4xruWbCWlu09TKoZyczxo3l9bd+n1//6+OF8cO7QOYOlG7cxfdzotJOli9e18Z+PvclTbzUD0DC1lm+eNZsjp419L4cPxNtLu8Ox3k8ThahgAr+ZTQZ+B+wNxICb3f26HT0nG4Ff8sPd2dTeM2QyOvlx/uPvm8LpB0zgb28389RbG1m1uYPjZ43j6vMP7r0qUU8kxpNvbuSWv69g/ooWxo4q5zPHTuPCIyczIU1NF+ITef/ngUW829bF506YwcffN2W3s9/VmzuoGzOCkeWZy54zaf3WTn759DLufGkNPZEY40aXc8r+4/nzog1MqBrBXZ8/ZkijQU8kxqb2bpq3ddMZjtIwtXbIG35HT4SrHlrMfa+sJeZwziH78OVTZzEzA0sJ90RiPLH4Xe58aTXN27qZW1/N3Poajpha+57ngF5e3UpXOMoxM/Z6T3NSe5pCCvwTgYnu/rKZjQEWAB9298XpnqPAXxp+/MgSbp63HICKUIBjZ47jQ4dM5MOHTkr7x/rSyhb++6mlAzK6sw6eyPGzxrF3dQVVFWV0hqP84i/v8JtnV1BbWc6MulHMX9HChKoRfOGkmew7fgyBAAQtfhGMGXXpM8j1Wzu56qHFPPrGBkJB45D6Go6esRdnHDRhh3MU79W7bV2MHzNiyP9De3eEmxLzWONGl1M3ZgQvrWzhrpfWEnPn/MPrufDIeg6dXEswYLy4fDOfumU++04Yze2fO5pR5WU8umgD//PMchauGdge+aFD9uEXHzu09/8iEo3x+d8v4Km3NvLJo6dy2Qkzdrl7RvKjYAL/kAGYPQjc6O5PpHuMAn9piERj3PnSGqaMreSo6WOHlY0va27nkdfW88iiDSxJrIII8TeQUDDAtq4IFx81mW+deQDVlSGeX7aZnz/xFi+tbB3yWiNDQQ7ap4o5k6qZUTeKybWVTB47kqffaubnT7xNzJ3LT5hBT9R5YXl8wjIacy46cjLfOmv2kLbb96KjJ8IP/7SE219czSePnsoPzj2oN/iHozEu+20j895pJmjWey3XUNC44IjJfOnk1G2NT725kc/9rpHZE8ewtTPMmpZOpu1VyTmHTmJidQV1o0fwWtNWrpUENLgAAAmMSURBVP/rO1x05GR+ct7BAHzn/kXcMX81P/zwHD5x9NSMHaNkT0EGfjObBswD5rh726D7LgcuB5gyZcoRq1atyvn4ZM+0YtN2Xlu7heZt3bzb1sWWjjDnH1HP0TP2GvA4d+fNDdto6wwTdcc9PnH3etNWFjVt5Y11bb0Tk0nvnz2e759z0ICA2tYV5sYnl/KbZ1dQMzLE/zn7gB1+Shk8hqYtnbzzbjurWzqYulclh06uoaaynEVNW/nKna+wYtN2jpw2lvkrWvjMcdP47tkHAvDNe1/jrsa1XH3ewXzsyMls7QzHJ/QrQ0Pq7YP98dV1fPUPC5lbX83nT5zB6QfuPeRTzjWPvcWNTy3l0uOnU1sZ4prH3+ZLJ8/kG2fO3ulxSWEouMBvZqOBvwE/cvf7dvRYZfySD+5O87Zu1rR2sKalk3GjR3DcrPQ14sXr2vjO/a+zcM0WZo0fzWeOm8Z5h9UzsjzIO+9u44+vruPJtzayvTtKzJ2YOy3tPWzvGXqG+LS9Kmna0sleo0bw8wsP4ZiZe/HvDy/hlr+v4LLjpzO6ooxf/OUdvnLqLP7ljP136/jauyOM3sG1Xd2dq/64mFufWwnARw6bxM8vPKSkauR7uoIK/GYWAh4GHnP3n+/s8Qr8sqeIxZyHXl3Hr59dzqKmNmoqQ0wYU8Fb724jYHDktLGMr6ogmLjYdlVFiH0njGa/CWOYMraSZc3tLFyzhYWrtzB2VDnfPHM2tYnzFtyd7z/0Br99Pv7p9/zD67nmo3OzGohjMedHjyxh47ZufvbRQ1Ke9S6Fq2ACv8V/S38LtLj7V3flOQr8sqdxdxpXtfK/f19By/YezpozkbMO3nunJZhded3/eOwtNmzt4qfnz1Uglh0qpMB/PPAM8Drxdk6A77j7I+meo8AvIjJ86QJ/zs96cfdnARUJRUTyRJ8TRURKjAK/iEiJUeAXESkxCvwiIiVGgV9EpMQo8IuIlBgFfhGREpP31Tl3hZltBd7pt6ka2LqLt8cBm3Zz1/1fb7iPSbV98DYdx/DoOHQc6W7rOFKPZaq7D70WpLsX/D/iF2tJ+f3ObgONmdrvcB6TaruOQ8eh49Bx5PM4kv/2lFLPH3fw/a7cztR+h/OYVNt1HO+NjkPHsaPbu6uUjgPYQ0o974WZNXqKtSr2NDqOwqLjKCw6juHZUzL+9+LmfA8gQ3QchUXHUVh0HMNQ9Bm/iIgMVAoZv4iI9KPALyJSYhT4RURKTEkHfjMLmNmPzOwGM7sk3+PZXWZ2spk9Y2Y3mdnJ+R7Pe2Fmo8xsgZmdne+x7C4zOyDxs7jHzL6Y7/HsLjP7sJn9j5k9aGZn5Hs8u8vMZpjZb8zsnnyPZbgSfw+/Tfwc/jFTr7vHBn4zu8XMNprZokHbzzSzt8xsqZl9aycvcy4wCQgDa7M11h3J0HE40A5UsGcfB8A3gbuyM8qdy8RxuPsSd/8CcCGQlxbDDB3HA+7+OeDTwMeyONy0MnQcy9390uyOdNcN85jOA+5J/BzOydggdvcssXz/A04EDgcW9dsWBJYBM4By4FXgQOBg4OFB/8YD3wI+n3juPXvwcQQSz5sA3LYHH8dpwEXEA83Ze+pxJJ5zDvAc8PE9+TgSz/sZcHgRHEde/sbf4zF9Gzg08ZjbMzWGnF9zN1PcfZ6ZTRu0+ShgqbsvBzCzO4Fz3f0nwJDSgZmtBXoS30azN9r0MnEc/bQCI7Ixzp3J0M/jFGAU8V/4TjN7xN1jWR34IJn6ebj7Q8BDZvYn4PbsjTi1DP08DLga+LO7v5zdEaeW4b+PgjCcYyL+Cb4eWEgGKzR7bOBPYxKwpt/3a4H37eDx9wE3mNkJwLxsDmyYhnUcZnYe8A9ADXBjdoc2LMM6Dne/EsDMPg1synXQ34Hh/jxOJv4RfQTwSFZHNjzD/fv4Z+KfwqrNbJa735TNwQ3DcH8eewE/Ag4zs28n3iAKTbpjuh640cw+SGaWdQCKL/Bbim1pz1Bz9w6gYGp//Qz3OO4j/iZWaIZ1HL0PcL8180N5T4b783gaeDpbg3kPhnsc1xMPPIVmuMexGfhC9oaTESmPyd23A5/J9M722MndNNYCk/t9Xw+sy9NY3gsdR2HRcRSWYjmO/nJ6TMUW+F8C9jWz6WZWTnyi8KE8j2l36DgKi46jsBTLcfSX22PK9wz3e5gZvwNYT18r5qWJ7R8A3iY+Q35lvsep49Bx6Dh0HIV2TFqkTUSkxBRbqUdERHZCgV9EpMQo8IuIlBgFfhGREqPALyJSYhT4RURKjAK/7PHMrD0H+zhnF5eVzuQ+TzazY3O5TykNxbZWj8huM7Ogu6dcpdUTq21mYZ9l7h5Jc/fJxK+z8Fym9yulTRm/FBUz+zcze8nMXjOzq/ptfyBxZa83zOzyftvbzewHZvYicIyZrTSzq8zsZTN73cxmJx73aTO7MXH7VjO73syeM7PlZnZBYnvAzP47sY+HzeyR5H2Dxvi0mf3YzP4GXGFmHzKzF83sFTP7i5lNSCzb+wXga2a20MxOMLM6M7s3cXwvmdlx2fy/lOKljF+KhsUvD7gv8bXNjfh6+Ce6+zzgs+7eYmYjgZfM7F6Pr9o4ivgFMb6beA2ILwl9uJl9Cfg6cFmK3U0EjgdmE/8kcA/xpZinEb8gyHhgCXBLmuHWuPtJiX3WAke7u5vZZcA33P1fzewmoN3dr0k87nbgWnd/1symAI8BB+z2f5iULAV+KSZnJP69kvh+NPE3gnnAV8zsI4ntkxPbNxO/AM+9g14nucT1AuLBPJUHPH69gMVmNiGx7Xjg7sT2DWb21A7G+od+t+uBP5jZROJXX1qR5jmnAQcm3pwAqsxsjLtv28F+RIZQ4JdiYsBP3P1XAzbGL4xyGnCMu3eY2dPEr08M0JWirt+d+Bol/d9Id7/bNujrrtje7/YNwM/d/aHEWL+f5jkB4sfQOYz9iAyhGr8Uk8eAz5rZaAAzm2Rm44FqoDUR9GcDR2dp/88C5ydq/ROIT87uimqgKXH7kn7btwFj+n3/OPDl5DdmdujuD1VKmQK/FA13f5z49W2fN7PXidfdxwCPAmVm9hrw78ALWRrCvcSX2V0E/Ap4Edi6C8/7PnC3mT0DbOq3/Y/AR5KTu8BXgIbExPViCv+qUlKgtCyzSAaZ2Wh3b09c53U+cJy7b8j3uET6U41fJLMeNrMa4pO0/66gL4VIGb+ISIlRjV9EpMQo8IuIlBgFfhGREqPALyJSYhT4RURKjAK/iEiJ+f9MrkQ1PBcMoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# empirically determine a good learning rate\n",
    "learn.fit(1, cbs=[LR_Find(), Recorder()])\n",
    "learn.recorder.plot(skip_last = 3)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('learning rate')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Create learning rate schedules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "# input: a number on the interval [0,1], or a list of such numbers\n",
    "# output: extends the list with another number, such that all the numbers add to one\n",
    "def create_phases(phases):\n",
    "    phases = listify(phases)\n",
    "    return phases + [1-sum(phases)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.3, 0.7]\n",
      "[0.3, 0.2, 0.5]\n"
     ]
    }
   ],
   "source": [
    "# examples\n",
    "print(create_phases(0.3))\n",
    "print(create_phases([0.3,0.2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.1, 1.1]\n",
      "[0.7, 0.8, -0.5]\n",
      "[0.7, 0.8, 0.6, -1.1]\n"
     ]
    }
   ],
   "source": [
    "# caution: doesn't check that input is valid\n",
    "#     i.e. numbers in list should be positive and should have a sum that is less than 1, \n",
    "#      so can give negative phases!!!\n",
    "print(create_phases(-0.1))\n",
    "print(create_phases([0.7,0.8]))\n",
    "print(create_phases([0.7,0.8, 0.6]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create learning rate schedules\n",
    "lr = 1e-2\n",
    "pct_start = 0.5\n",
    "phases = create_phases(pct_start)\n",
    "sched_lr  = combine_scheds(phases, cos_1cycle_anneal(lr/10., lr, lr/1e5))\n",
    "sched_mom = combine_scheds(phases, cos_1cycle_anneal(0.95, 0.85, 0.95))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create callbacks for the learning schedules\n",
    "cbsched = [\n",
    "    ParamScheduler('lr', sched_lr),\n",
    "    ParamScheduler('mom', sched_mom)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 Train the XResNet-34 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a Learner object\n",
    "learn = Learner(arch(), data, loss_func, lr=lr, cb_funcs=cbfs, opt_func=opt_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.677569</td>\n",
       "      <td>0.501939</td>\n",
       "      <td>1.998155</td>\n",
       "      <td>0.448000</td>\n",
       "      <td>05:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.441444</td>\n",
       "      <td>0.615635</td>\n",
       "      <td>1.716789</td>\n",
       "      <td>0.510000</td>\n",
       "      <td>05:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.312632</td>\n",
       "      <td>0.666434</td>\n",
       "      <td>1.274288</td>\n",
       "      <td>0.698000</td>\n",
       "      <td>03:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.142816</td>\n",
       "      <td>0.736311</td>\n",
       "      <td>1.079520</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>01:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.972124</td>\n",
       "      <td>0.810299</td>\n",
       "      <td>0.879135</td>\n",
       "      <td>0.838000</td>\n",
       "      <td>01:41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# train the model\n",
    "learn.fit(5, cbs=cbsched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [Jump_to lesson 12 video](https://course.fast.ai/videos/?lesson=12&t=2711)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Putting it all together"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A flexible method to create a Learner object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def cnn_learner(arch, data, loss_func, opt_func, c_in=None, c_out=None,\n",
    "                lr=1e-2, cuda=True, norm=None, progress=True, mixup=0, xtra_cb=None, **kwargs):\n",
    "    cbfs = [partial(AvgStatsCallback,accuracy)]+listify(xtra_cb)\n",
    "    if progress: cbfs.append(ProgressCallback)\n",
    "    if cuda:     cbfs.append(CudaCallback)\n",
    "    if norm:     cbfs.append(partial(BatchTransformXCallback, norm))\n",
    "    if mixup:    cbfs.append(partial(MixUp, mixup))\n",
    "    arch_args = {}\n",
    "    if not c_in : c_in  = data.c_in\n",
    "    if not c_out: c_out = data.c_out\n",
    "    if c_in:  arch_args['c_in' ]=c_in\n",
    "    if c_out: arch_args['c_out']=c_out\n",
    "    return Learner(arch(**arch_args), data, loss_func, opt_func=opt_func, lr=lr, cb_funcs=cbfs, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XResNet-18 gets a slightly better result than XResNet-34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#learn = cnn_learner(xresnet34, data, loss_func, opt_func, norm=norm_imagenette)\n",
    "learn = cnn_learner(xresnet18, data, loss_func, opt_func, norm=norm_imagenette)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_accuracy</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>valid_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.688512</td>\n",
       "      <td>0.493408</td>\n",
       "      <td>1.595463</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.457410</td>\n",
       "      <td>0.600512</td>\n",
       "      <td>2.376162</td>\n",
       "      <td>0.396000</td>\n",
       "      <td>01:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.321719</td>\n",
       "      <td>0.662168</td>\n",
       "      <td>1.241997</td>\n",
       "      <td>0.698000</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.145524</td>\n",
       "      <td>0.739336</td>\n",
       "      <td>1.119666</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>01:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.980731</td>\n",
       "      <td>0.807585</td>\n",
       "      <td>0.910385</td>\n",
       "      <td>0.846000</td>\n",
       "      <td>01:32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(5, cbsched)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. A World-class Imagenet result with Fastai!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can see all this put together in the fastai [imagenet training script](https://github.com/fastai/fastai/blob/master/examples/train_imagenet.py). It's the same as what we've seen so far, except it also handles multi-GPU training. So how well does this work?\n",
    "\n",
    "We trained for 60 epochs, and got an error of 5.9%, compared to the official PyTorch resnet which gets 7.5% error in 90 epochs! Our xresnet 50 training even surpasses standard resnet 152, which trains for 50% more epochs and has 3x as many layers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python notebook2script.py 11_train_imagenette.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}