EarlyStopping_AlexNet.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DrJHSIM/PyTorch_Practice/blob/main/EarlyStopping_AlexNet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Building"
      ],
      "metadata": {
        "id": "wGHGrGaFFKYG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "hQG81-dJa5KT"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "from torchvision.transforms import ToTensor\n",
        "\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import torchvision.models as models\n",
        "from torchsummary import summary"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transform = transforms.Compose([\n",
        "    transforms.Resize(256),\n",
        "    transforms.CenterCrop(224),\n",
        "    transforms.RandomHorizontalFlip(p=1),\n",
        "    ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
        "])"
      ],
      "metadata": {
        "id": "wpfq6bAnD3H3"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4X2EVh9Wccw8",
        "outputId": "b33a2df5-d600-4923-91ee-a23bb1a3ed74"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ]
        }
      ],
      "source": [
        "# Download training data from open datasets.\n",
        "training_data = datasets.CIFAR10(\n",
        "    root=\"data\",\n",
        "    train=True,\n",
        "    download=True,\n",
        "    transform = transform)\n",
        "\n",
        "\n",
        "# Download test data from open datasets.\n",
        "test_data = datasets.CIFAR10(\n",
        "    root=\"data\",\n",
        "    train=False,\n",
        "    download=True,\n",
        "    transform = transform)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import random_split\n",
        "generator = torch.Generator().manual_seed(777)\n",
        "train_set, valid_set = random_split(training_data, [0.8, 0.2], generator = generator)\n",
        "\n",
        "len(training_data), len(train_set), len(valid_set)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "17MnW1_S6eSU",
        "outputId": "753bfd1e-85a5-4453-b96e-5e879bf2bd55"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 40000, 10000)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "v8m1Lwpvd3cA"
      },
      "outputs": [],
      "source": [
        "batch_size = 64\n",
        "\n",
        "# Create data loaders.\n",
        "train_dataloader = DataLoader(train_set, batch_size=batch_size)\n",
        "valid_dataloader = DataLoader(valid_set, batch_size = batch_size)\n",
        "test_dataloader = DataLoader(test_data, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LupnOzj1d5Vv",
        "outputId": "408cecbc-9473-4c92-c733-b14851b5a895"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda device\n"
          ]
        }
      ],
      "source": [
        "# Get cpu, gpu or mps device for training.\n",
        "device = (\n",
        "    \"cuda\"\n",
        "    if torch.cuda.is_available()\n",
        "    else \"cpu\"\n",
        ")\n",
        "print(f\"Using {device} device\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class AlexNet(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.cnns = nn.Sequential(\n",
        "            #conv1\n",
        "            nn.Conv2d(3, 96 , 11, stride = 4),\n",
        "            nn.ReLU(),\n",
        "            nn.LocalResponseNorm(5, alpha=0.0001, beta=0.75, k=2.0),\n",
        "            nn.MaxPool2d(kernel_size = 3, stride = 2),\n",
        "            #conv2\n",
        "            nn.Conv2d(96, 256, 5, stride = 1, padding = 2),\n",
        "            nn.ReLU(),\n",
        "            nn.LocalResponseNorm(5, alpha=0.0001, beta=0.75, k=2.0),\n",
        "            nn.MaxPool2d(kernel_size = 3, stride = 2),\n",
        "            #conv3\n",
        "            nn.Conv2d(256, 384, 3, stride = 1, padding = 1),\n",
        "            nn.ReLU(),\n",
        "            #conv4\n",
        "            nn.Conv2d(384, 384, 3, stride = 1, padding = 1),\n",
        "            nn.ReLU(),\n",
        "            #conv5\n",
        "            nn.Conv2d(384, 256, 3, stride = 1, padding = 1),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(kernel_size = 3, stride = 2),\n",
        "        )\n",
        "\n",
        "        self.classifier = nn.Sequential(\n",
        "            #fc1\n",
        "            nn.Linear(6400, 4096),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(),\n",
        "            #fc2\n",
        "            nn.Linear(4096, 4096),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(),\n",
        "            #fc3\n",
        "            nn.Linear(4096, 10)\n",
        "        )\n",
        "\n",
        "    def init_weights_bias(self):\n",
        "        for layer in self.cnns:\n",
        "            if isinstance(layer, nn.Conv2d):\n",
        "                nn.init.norm_(layer.weights, mean = 0, std =0.01)\n",
        "                nn.init.constant_(layer.bias, 0)\n",
        "    #conv 2, 4, 5는 bias 1로 초기화\n",
        "        nn.init.constant_(self.cnns[4].bias, 1)\n",
        "        nn.init.constant_(self.cnns[10].bias, 1)\n",
        "        nn.init.constant_(self.cnns[12].bias, 1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.cnns(x)\n",
        "        x = torch.flatten(x, 1)\n",
        "        x = self.classifier(x)\n",
        "        return x\n",
        "\n",
        "model = AlexNet().to(device)\n",
        "print(model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JdNMbWRWGGTO",
        "outputId": "4e2e8461-e3c5-41aa-ea16-5661f8f63081"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "AlexNet(\n",
            "  (cnns): Sequential(\n",
            "    (0): Conv2d(3, 96, kernel_size=(11, 11), stride=(4, 4))\n",
            "    (1): ReLU()\n",
            "    (2): LocalResponseNorm(5, alpha=0.0001, beta=0.75, k=2.0)\n",
            "    (3): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (4): Conv2d(96, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
            "    (5): ReLU()\n",
            "    (6): LocalResponseNorm(5, alpha=0.0001, beta=0.75, k=2.0)\n",
            "    (7): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (8): Conv2d(256, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (9): ReLU()\n",
            "    (10): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (11): ReLU()\n",
            "    (12): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (13): ReLU()\n",
            "    (14): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "  )\n",
            "  (classifier): Sequential(\n",
            "    (0): Linear(in_features=6400, out_features=4096, bias=True)\n",
            "    (1): ReLU()\n",
            "    (2): Dropout(p=0.5, inplace=False)\n",
            "    (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
            "    (4): ReLU()\n",
            "    (5): Dropout(p=0.5, inplace=False)\n",
            "    (6): Linear(in_features=4096, out_features=10, bias=True)\n",
            "  )\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "summary(model, (3, 224, 224))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kFWEAAs3PtMa",
        "outputId": "ceecb004-692e-4e42-954a-5c75bb607042"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 96, 54, 54]          34,944\n",
            "              ReLU-2           [-1, 96, 54, 54]               0\n",
            " LocalResponseNorm-3           [-1, 96, 54, 54]               0\n",
            "         MaxPool2d-4           [-1, 96, 26, 26]               0\n",
            "            Conv2d-5          [-1, 256, 26, 26]         614,656\n",
            "              ReLU-6          [-1, 256, 26, 26]               0\n",
            " LocalResponseNorm-7          [-1, 256, 26, 26]               0\n",
            "         MaxPool2d-8          [-1, 256, 12, 12]               0\n",
            "            Conv2d-9          [-1, 384, 12, 12]         885,120\n",
            "             ReLU-10          [-1, 384, 12, 12]               0\n",
            "           Conv2d-11          [-1, 384, 12, 12]       1,327,488\n",
            "             ReLU-12          [-1, 384, 12, 12]               0\n",
            "           Conv2d-13          [-1, 256, 12, 12]         884,992\n",
            "             ReLU-14          [-1, 256, 12, 12]               0\n",
            "        MaxPool2d-15            [-1, 256, 5, 5]               0\n",
            "           Linear-16                 [-1, 4096]      26,218,496\n",
            "             ReLU-17                 [-1, 4096]               0\n",
            "          Dropout-18                 [-1, 4096]               0\n",
            "           Linear-19                 [-1, 4096]      16,781,312\n",
            "             ReLU-20                 [-1, 4096]               0\n",
            "          Dropout-21                 [-1, 4096]               0\n",
            "           Linear-22                   [-1, 10]          40,970\n",
            "================================================================\n",
            "Total params: 46,787,978\n",
            "Trainable params: 46,787,978\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.57\n",
            "Forward/backward pass size (MB): 13.63\n",
            "Params size (MB): 178.48\n",
            "Estimated Total Size (MB): 192.69\n",
            "----------------------------------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "VezQ9rOpd6oy"
      },
      "outputs": [],
      "source": [
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum = 0.9, weight_decay = 0.0005)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "YsCpzmGJd8ke"
      },
      "outputs": [],
      "source": [
        "def train(dataloader, model, loss_fn, optimizer):\n",
        "    size = len(dataloader.dataset)\n",
        "    model.train()\n",
        "    for batch, (X, y) in enumerate(dataloader):\n",
        "        X, y = X.to(device), y.to(device)\n",
        "\n",
        "        # Compute prediction error\n",
        "        pred = model(X)\n",
        "        loss = loss_fn(pred, y)\n",
        "\n",
        "        # Backpropagation\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        if batch % 100 == 0:\n",
        "            loss, current = loss.item(), (batch + 1) * len(X)\n",
        "            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def valid(dataloader, model, loss_fn):\n",
        "    num_batches = len(dataloader)\n",
        "    size = len(dataloader.dataset)\n",
        "    model.eval()\n",
        "    valid_loss, Acurracy = 0, 0\n",
        "    with torch.no_grad():\n",
        "        for X, y in dataloader:\n",
        "            X, y = X.to(device), y.to(device)\n",
        "            pred = model(X)\n",
        "            valid_loss += loss_fn(pred, y).item()\n",
        "            Acurracy += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
        "    valid_loss /= num_batches\n",
        "    Acurracy /= size\n",
        "    print(f\"Valid Error : \\n Accuracy : {(Acurracy*100):>0.1f}%, epoch_valid_loss: {valid_loss:>8f} \\n\")\n",
        "    return valid_loss # earlystopping시 적용"
      ],
      "metadata": {
        "id": "M9VU5hCQ0U0O"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "uhO55RXtd93_"
      },
      "outputs": [],
      "source": [
        "def test(dataloader, model, loss_fn):\n",
        "    size = len(dataloader.dataset)\n",
        "    num_batches = len(dataloader)\n",
        "    model.eval()\n",
        "    test_loss, correct = 0, 0\n",
        "    with torch.no_grad():\n",
        "        for X, y in dataloader:\n",
        "            X, y = X.to(device), y.to(device)\n",
        "            pred = model(X)\n",
        "            test_loss += loss_fn(pred, y).item()\n",
        "            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
        "    test_loss /= num_batches\n",
        "    correct /= size\n",
        "    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 추가 실험"
      ],
      "metadata": {
        "id": "jJiMoDz5DuE9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### EarlyStopping 적용"
      ],
      "metadata": {
        "id": "0v5nnf56AU3L"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 30\n",
        "min_loss = float('inf')  # 검증데이터셋의 최소 loss값\n",
        "es_con = 5               # 학습 종료 조건\n",
        "es_counts = 0            # 검증 데이터 loss가 상승하는 epochs의 수\n",
        "\n",
        "for t in range(epochs):\n",
        "    print(f\"Epoch {t+1}\\n-------------------------------\")\n",
        "    train(train_dataloader, model, loss_fn, optimizer)\n",
        "    valid_loss = valid(valid_dataloader, model, loss_fn)\n",
        "    if valid_loss > min_loss:\n",
        "        es_counts += 1\n",
        "    else:\n",
        "        min_loss = valid_loss\n",
        "        es_counts = 0\n",
        "    if es_counts == es_con:\n",
        "        print(\"EarlyStopping\")\n",
        "        break\n",
        "    print(min_loss, es_con, es_counts)\n",
        "print(\"Done!\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4pXgKJU6Mlld",
        "outputId": "3d3a6ed7-d3ce-4a8f-c034-4331747be7f3"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1\n",
            "-------------------------------\n",
            "loss: 2.302305  [   64/40000]\n",
            "loss: 2.302297  [ 6464/40000]\n",
            "loss: 2.299439  [12864/40000]\n",
            "loss: 2.047245  [19264/40000]\n",
            "loss: 1.955898  [25664/40000]\n",
            "loss: 1.912456  [32064/40000]\n",
            "loss: 1.890727  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 30.3%, epoch_valid_loss: 1.838696 \n",
            "\n",
            "1.838695923234247 5 0\n",
            "Epoch 2\n",
            "-------------------------------\n",
            "loss: 1.884799  [   64/40000]\n",
            "loss: 1.896578  [ 6464/40000]\n",
            "loss: 1.545689  [12864/40000]\n",
            "loss: 1.398454  [19264/40000]\n",
            "loss: 1.582658  [25664/40000]\n",
            "loss: 1.283064  [32064/40000]\n",
            "loss: 1.613618  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 46.4%, epoch_valid_loss: 1.460210 \n",
            "\n",
            "1.4602095169626224 5 0\n",
            "Epoch 3\n",
            "-------------------------------\n",
            "loss: 1.312006  [   64/40000]\n",
            "loss: 1.543977  [ 6464/40000]\n",
            "loss: 1.166419  [12864/40000]\n",
            "loss: 1.124926  [19264/40000]\n",
            "loss: 1.198185  [25664/40000]\n",
            "loss: 1.099540  [32064/40000]\n",
            "loss: 1.410838  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 57.5%, epoch_valid_loss: 1.200006 \n",
            "\n",
            "1.2000060313066858 5 0\n",
            "Epoch 4\n",
            "-------------------------------\n",
            "loss: 0.994041  [   64/40000]\n",
            "loss: 1.392035  [ 6464/40000]\n",
            "loss: 0.920679  [12864/40000]\n",
            "loss: 0.825938  [19264/40000]\n",
            "loss: 0.804796  [25664/40000]\n",
            "loss: 0.814476  [32064/40000]\n",
            "loss: 0.992193  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 65.5%, epoch_valid_loss: 0.968792 \n",
            "\n",
            "0.9687922771569271 5 0\n",
            "Epoch 5\n",
            "-------------------------------\n",
            "loss: 0.714298  [   64/40000]\n",
            "loss: 1.035820  [ 6464/40000]\n",
            "loss: 0.718746  [12864/40000]\n",
            "loss: 0.825663  [19264/40000]\n",
            "loss: 0.708404  [25664/40000]\n",
            "loss: 0.651470  [32064/40000]\n",
            "loss: 0.915451  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 68.1%, epoch_valid_loss: 0.950107 \n",
            "\n",
            "0.9501065349882576 5 0\n",
            "Epoch 6\n",
            "-------------------------------\n",
            "loss: 0.640604  [   64/40000]\n",
            "loss: 1.047049  [ 6464/40000]\n",
            "loss: 0.583689  [12864/40000]\n",
            "loss: 0.674492  [19264/40000]\n",
            "loss: 0.626894  [25664/40000]\n",
            "loss: 0.532636  [32064/40000]\n",
            "loss: 0.735334  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 68.5%, epoch_valid_loss: 0.974694 \n",
            "\n",
            "0.9501065349882576 5 1\n",
            "Epoch 7\n",
            "-------------------------------\n",
            "loss: 0.535642  [   64/40000]\n",
            "loss: 0.841015  [ 6464/40000]\n",
            "loss: 0.514333  [12864/40000]\n",
            "loss: 0.660697  [19264/40000]\n",
            "loss: 0.478025  [25664/40000]\n",
            "loss: 0.472807  [32064/40000]\n",
            "loss: 0.735547  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 70.3%, epoch_valid_loss: 0.943700 \n",
            "\n",
            "0.9437000789460103 5 0\n",
            "Epoch 8\n",
            "-------------------------------\n",
            "loss: 0.398117  [   64/40000]\n",
            "loss: 0.674779  [ 6464/40000]\n",
            "loss: 0.427274  [12864/40000]\n",
            "loss: 0.553886  [19264/40000]\n",
            "loss: 0.562801  [25664/40000]\n",
            "loss: 0.496704  [32064/40000]\n",
            "loss: 0.431255  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 70.4%, epoch_valid_loss: 1.038077 \n",
            "\n",
            "0.9437000789460103 5 1\n",
            "Epoch 9\n",
            "-------------------------------\n",
            "loss: 0.315427  [   64/40000]\n",
            "loss: 0.532928  [ 6464/40000]\n",
            "loss: 0.357917  [12864/40000]\n",
            "loss: 0.565360  [19264/40000]\n",
            "loss: 0.526531  [25664/40000]\n",
            "loss: 0.375487  [32064/40000]\n",
            "loss: 0.504832  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 71.0%, epoch_valid_loss: 1.009143 \n",
            "\n",
            "0.9437000789460103 5 2\n",
            "Epoch 10\n",
            "-------------------------------\n",
            "loss: 0.287633  [   64/40000]\n",
            "loss: 0.380175  [ 6464/40000]\n",
            "loss: 0.425283  [12864/40000]\n",
            "loss: 0.485552  [19264/40000]\n",
            "loss: 0.310895  [25664/40000]\n",
            "loss: 0.244445  [32064/40000]\n",
            "loss: 0.502676  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 73.8%, epoch_valid_loss: 0.894828 \n",
            "\n",
            "0.8948281878119062 5 0\n",
            "Epoch 11\n",
            "-------------------------------\n",
            "loss: 0.205963  [   64/40000]\n",
            "loss: 0.402495  [ 6464/40000]\n",
            "loss: 0.209322  [12864/40000]\n",
            "loss: 0.474313  [19264/40000]\n",
            "loss: 0.374400  [25664/40000]\n",
            "loss: 0.217439  [32064/40000]\n",
            "loss: 0.347065  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 76.7%, epoch_valid_loss: 0.829233 \n",
            "\n",
            "0.8292326061588944 5 0\n",
            "Epoch 12\n",
            "-------------------------------\n",
            "loss: 0.154455  [   64/40000]\n",
            "loss: 0.375109  [ 6464/40000]\n",
            "loss: 0.262818  [12864/40000]\n",
            "loss: 0.341136  [19264/40000]\n",
            "loss: 0.243232  [25664/40000]\n",
            "loss: 0.197785  [32064/40000]\n",
            "loss: 0.459589  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 76.3%, epoch_valid_loss: 0.846244 \n",
            "\n",
            "0.8292326061588944 5 1\n",
            "Epoch 13\n",
            "-------------------------------\n",
            "loss: 0.178166  [   64/40000]\n",
            "loss: 0.321362  [ 6464/40000]\n",
            "loss: 0.196588  [12864/40000]\n",
            "loss: 0.200705  [19264/40000]\n",
            "loss: 0.423252  [25664/40000]\n",
            "loss: 0.119337  [32064/40000]\n",
            "loss: 0.241655  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 76.8%, epoch_valid_loss: 0.877290 \n",
            "\n",
            "0.8292326061588944 5 2\n",
            "Epoch 14\n",
            "-------------------------------\n",
            "loss: 0.141734  [   64/40000]\n",
            "loss: 0.361257  [ 6464/40000]\n",
            "loss: 0.086383  [12864/40000]\n",
            "loss: 0.165300  [19264/40000]\n",
            "loss: 0.166160  [25664/40000]\n",
            "loss: 0.147319  [32064/40000]\n",
            "loss: 0.177290  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 76.1%, epoch_valid_loss: 0.930453 \n",
            "\n",
            "0.8292326061588944 5 3\n",
            "Epoch 15\n",
            "-------------------------------\n",
            "loss: 0.205394  [   64/40000]\n",
            "loss: 0.288037  [ 6464/40000]\n",
            "loss: 0.249709  [12864/40000]\n",
            "loss: 0.340554  [19264/40000]\n",
            "loss: 0.126236  [25664/40000]\n",
            "loss: 0.066642  [32064/40000]\n",
            "loss: 0.086415  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 75.9%, epoch_valid_loss: 0.984997 \n",
            "\n",
            "0.8292326061588944 5 4\n",
            "Epoch 16\n",
            "-------------------------------\n",
            "loss: 0.208106  [   64/40000]\n",
            "loss: 0.174194  [ 6464/40000]\n",
            "loss: 0.045875  [12864/40000]\n",
            "loss: 0.299735  [19264/40000]\n",
            "loss: 0.086460  [25664/40000]\n",
            "loss: 0.073328  [32064/40000]\n",
            "loss: 0.167345  [38464/40000]\n",
            "Valid Error : \n",
            " Accuracy : 75.7%, epoch_valid_loss: 0.995768 \n",
            "\n",
            "EarlyStopping\n",
            "Done!\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": [],
      "toc_visible": true,
      "mount_file_id": "1mEb1JNFM5UobjyPpycj0UnhdH3dJPAHN",
      "authorship_tag": "ABX9TyPXDt6GWHY1QYdmnW6hFaLb",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}