functional_api_v1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "functional_api_v1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNkSqDYPlPVkK4jwYczviW9",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bipinKrishnan/torchkeras/blob/master/functional_api_v1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sTRyumaWcSwb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install pkbar"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q-oEWtsIX6Xz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "from torch import optim\n",
        "from torch.autograd import Variable\n",
        "from torchsummary import summary as summary_\n",
        "import pkbar\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VCugQ8DJae0g",
        "colab_type": "text"
      },
      "source": [
        "# TorchKeras"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_h4x_jTPpIfL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Input(shape):\n",
        "  Input.shape = shape\n",
        "  return Input.shape\n",
        "\n",
        "def get_conv_output(shape, inputs):\n",
        "  bs = 1\n",
        "  data = Variable(torch.rand(bs, *shape))\n",
        "  output_feat = inputs(data)\n",
        "\n",
        "  return output_feat.size(1)\n",
        "\n",
        "def same_pad(h_in, kernal, stride, dilation):\n",
        "  return (stride*(h_in-1)-h_in+(dilation*(kernal-1))+1) / 2.0"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-JJKHynxYHIj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Dense(nn.Module):\n",
        "  def __init__(self, outputs, activation):\n",
        "    super().__init__()\n",
        "    self.outputs = outputs\n",
        "    self.activation = activation\n",
        "\n",
        "  def __call__(self, inputs):\n",
        "    self.inputs_size = 1\n",
        "    \n",
        "    if type(inputs) == tuple:\n",
        "      for i in range(len(inputs)):\n",
        "        self.inputs_size *= inputs[i]\n",
        "      \n",
        "      self.layers = nn.Sequential(\n",
        "        nn.Linear(self.inputs_size, self.outputs),\n",
        "        self.activation\n",
        "    )\n",
        "\n",
        "      return self.layers\n",
        "\n",
        "    elif isinstance(inputs[-2], nn.Linear):\n",
        "      self.inputs = inputs\n",
        "      self.layers = list(self.inputs)\n",
        "      self.layers.extend([nn.Linear(self.layers[-2].out_features, self.outputs), self.activation])\n",
        "\n",
        "      self.layers = nn.Sequential(*self.layers)\n",
        "\n",
        "      return self.layers\n",
        "\n",
        "    else:\n",
        "      self.inputs = inputs\n",
        "      self.layers = list(self.inputs)\n",
        "      self.layers.extend([nn.Linear(get_conv_output(Input.shape, self.inputs), self.outputs), self.activation])\n",
        "\n",
        "      self.layers = nn.Sequential(*self.layers)\n",
        "\n",
        "      return self.layers\n",
        "\n",
        "\n",
        "class FlattenedLayer(nn.Module):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "    pass\n",
        "\n",
        "  def forward(self, input):\n",
        "      self.inputs = input.view(input.size(0), -1)\n",
        "      return self.inputs\n",
        "\n",
        "\n",
        "class Flatten():\n",
        "  def __init__(self):\n",
        "    pass\n",
        "\n",
        "  def __call__(self, inputs):\n",
        "    self.inputs = inputs\n",
        "    self.layers = list(self.inputs)\n",
        "    self.layers.extend([FlattenedLayer()])\n",
        "    self.layers = nn.Sequential(*self.layers)\n",
        "\n",
        "    return self.layers"
      ],
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IALWgA3QwhtN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Conv2d(nn.Module):\n",
        "  def __init__(self, filters, kernel_size, strides, padding, dilation, activation):\n",
        "    super().__init__()\n",
        "    self.filters = filters\n",
        "    self.kernel = kernel_size\n",
        "    self.strides = strides\n",
        "    self.padding = padding\n",
        "    self.dilation = dilation\n",
        "    self.activation = activation\n",
        "\n",
        "  def __call__(self, inputs):\n",
        "\n",
        "    if type(inputs) == tuple:\n",
        "      self.inputs_size = inputs\n",
        "\n",
        "      if self.padding == 'same':\n",
        "        self.padding = int(same_pad(self.inputs_size[-2], self.kernel, self.strides, self.dilation))\n",
        "      else:\n",
        "        self.padding = self.padding\n",
        "\n",
        "      self.layers = nn.Sequential(\n",
        "        nn.Conv2d(self.inputs_size[-3],\n",
        "                  self.filters, \n",
        "                  self.kernel, \n",
        "                  self.strides, \n",
        "                  self.padding,\n",
        "                  self.dilation),\n",
        "        self.activation\n",
        "    )\n",
        "\n",
        "      return self.layers\n",
        "\n",
        "    else:\n",
        "      if self.padding == 'same':\n",
        "        self.padding = int(same_pad(get_conv_output(Input.shape, inputs), self.kernel, self.strides, self.dilation))\n",
        "      else:\n",
        "        self.padding = self.padding\n",
        "\n",
        "      self.inputs = inputs\n",
        "      self.layers = list(self.inputs)\n",
        "      self.layers.extend(\n",
        "             [nn.Conv2d(self.layers[-2].out_channels, \n",
        "                    self.filters, \n",
        "                    self.kernel, \n",
        "                    self.strides, \n",
        "                    self.padding,\n",
        "                    self.dilation),\n",
        "             self.activation]\n",
        "          )\n",
        "      self.layers = nn.Sequential(*self.layers)\n",
        "\n",
        "      return self.layers\n"
      ],
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XZ81UK7tegjA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model():\n",
        "  def __init__(self, inputs, outputs, device):\n",
        "    self.input_size = inputs\n",
        "    self.device = device\n",
        "    self.model = outputs.to(self.device)\n",
        "\n",
        "  def parameters(self):\n",
        "    return self.model.parameters()\n",
        "\n",
        "  def compile(self, optimizer, loss):\n",
        "    self.opt = optimizer\n",
        "    self.criterion = loss\n",
        "\n",
        "  def summary(self):\n",
        "    summary_(self.model, self.input_size, device=self.device)\n",
        "    print(\"Device Type:\", self.device)\n",
        "\n",
        "  def fit(self, data_x, data_y, epochs):\n",
        "    self.model.train()\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "      print(\"Epoch {}/{}\".format(epoch+1, epochs))\n",
        "      progress = pkbar.Kbar(target=len(data_x), width=25)\n",
        "      \n",
        "      for i, (data, target) in enumerate(zip(data_x, data_y)):\n",
        "        self.opt.zero_grad()\n",
        "\n",
        "        train_out = self.model(data.to(self.device))\n",
        "        loss = self.criterion(train_out, target.to(self.device))\n",
        "        loss.backward()\n",
        "\n",
        "        self.opt.step()\n",
        "\n",
        "        progress.update(i, values=[(\"loss: \", loss.item())])\n",
        "\n",
        "      progress.add(1)\n",
        "\n",
        "  def evaluate(self, test_x, test_y):\n",
        "    self.model.eval()\n",
        "    correct, loss = 0.0, 0.0\n",
        "\n",
        "    progress = pkbar.Kbar(target=len(test_x), width=25)\n",
        "\n",
        "    for i, (data, target) in enumerate(zip(test_x, test_y)):\n",
        "      out = self.model(data.to(self.device))\n",
        "      loss += self.criterion(out, target.to(self.device))\n",
        "\n",
        "      correct += ((torch.max(out, 1)[1]) == target.to(self.device)).sum()\n",
        "\n",
        "      progress.update(i, values=[(\"loss\", loss.item()/len(test_x)), (\"acc\", (correct/len(test_x)).item())])\n",
        "    progress.add(1)\n",
        "\n",
        "\n",
        "  def fit_generator(self, generator, epochs):\n",
        "    self.model.train()\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "      print(\"Epoch {}/{}\".format(epoch+1, epochs))\n",
        "      progress = pkbar.Kbar(target=len(generator), width=25)\n",
        "\n",
        "      for i, (data, target) in enumerate(generator):\n",
        "        self.opt.zero_grad()\n",
        "\n",
        "        train_out = self.model(data.to(self.device))\n",
        "        loss = self.criterion(train_out.squeeze(), target.to(self.device))\n",
        "        loss.backward()\n",
        "\n",
        "        self.opt.step()\n",
        "\n",
        "        progress.update(i, values=[(\"loss: \", loss.item())])\n",
        "\n",
        "      progress.add(1)\n",
        "      \n",
        "\n",
        "  def evaluate_generator(self, generator):\n",
        "    self.model.eval()\n",
        "    correct, loss = 0.0, 0.0\n",
        "\n",
        "    progress = pkbar.Kbar(target=len(generator), width=25)\n",
        "\n",
        "    for i, (data, target) in enumerate(generator):\n",
        "      out = self.model(data.to(self.device))\n",
        "      loss += self.criterion(out.squeeze(), target.to(self.device))\n",
        "\n",
        "      correct += (torch.max(out.squeeze(), 1)[1] == target.to(self.device)).sum()\n",
        "\n",
        "      progress.update(i, values=[(\"test_acc\", (correct/len(generator)).item()), (\"test_loss\", loss.item()/len(generator))])\n",
        "\n",
        "    progress.add(1)\n",
        "\n",
        "  def predict_generator(self, generator):\n",
        "    self.model.train()\n",
        "    out = []\n",
        "    for i, (data, labels) in enumerate(generator):\n",
        "      out.append(self.model(data.to(self.device)))\n",
        "\n",
        "    return out\n",
        "      "
      ],
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N21QvHSGaZ3L",
        "colab_type": "text"
      },
      "source": [
        "# Test data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Otnl-l-iZJJO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "inputs = Input((1, 1024))\n",
        "x = Dense(54, nn.ReLU())(inputs)\n",
        "y = Dense(34, nn.ReLU())(x)\n",
        "z = Dense(43, activation=nn.ReLU())(y)\n",
        "a = Dense(10, activation=nn.ReLU())(z)\n",
        "b = Dense(1, activation=nn.Sigmoid())(a)"
      ],
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKHqRcKYeLPI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Model(inputs, b, 'cpu')\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.001), nn.BCELoss())"
      ],
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ajMFyfaGyrGJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        },
        "outputId": "5cb9a25e-9a84-46ff-c5ef-4512b162681f"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1                [-1, 1, 54]          55,350\n",
            "              ReLU-2                [-1, 1, 54]               0\n",
            "            Linear-3                [-1, 1, 34]           1,870\n",
            "              ReLU-4                [-1, 1, 34]               0\n",
            "            Linear-5                [-1, 1, 43]           1,505\n",
            "              ReLU-6                [-1, 1, 43]               0\n",
            "            Linear-7                [-1, 1, 10]             440\n",
            "              ReLU-8                [-1, 1, 10]               0\n",
            "            Linear-9                 [-1, 1, 1]              11\n",
            "          Sigmoid-10                 [-1, 1, 1]               0\n",
            "================================================================\n",
            "Total params: 59,176\n",
            "Trainable params: 59,176\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.00\n",
            "Params size (MB): 0.23\n",
            "Estimated Total Size (MB): 0.23\n",
            "----------------------------------------------------------------\n",
            "Device Type: cpu\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99Pyms_5rYtP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.manual_seed(42)\n",
        "\n",
        "#input should be of the dimension - (batch_size, 1, n_rows*n_columns)\n",
        "x = torch.rand((10, 1, 1024), dtype=torch.float)\n",
        "y = torch.tensor((torch.rand(10, 1) < 0.5), dtype=torch.float)\n",
        "#y = torch.tensor([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], dtype=torch.int64).reshape(-1, 1)"
      ],
      "execution_count": 99,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HSCwl4yqsPhw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "99e67062-225e-46af-b935-3606aec92064"
      },
      "source": [
        "model.fit(x, y, 4)"
      ],
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/4\n",
            "10/10 [=========================] - 0s 2ms/step - loss: : 0.6709\n",
            "Epoch 2/4\n",
            "10/10 [=========================] - 0s 2ms/step - loss: : 0.5571\n",
            "Epoch 3/4\n",
            "10/10 [=========================] - 0s 2ms/step - loss: : 0.5080\n",
            "Epoch 4/4\n",
            "10/10 [=========================] - 0s 2ms/step - loss: : 0.4903\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3oRV0zycttMR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fb755721-4473-4ef9-a381-f196ada24a45"
      },
      "source": [
        "model.evaluate(x, y)"
      ],
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10/10 [=========================] - 0s 1ms/step - loss: 0.2493 - acc: 0.1111    \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qtA78EwuB_QB",
        "colab_type": "text"
      },
      "source": [
        "# CIFAR100 -Dense layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGCt0wTGCAjI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4jc8Zb5CE2Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bs = 128\n",
        "\n",
        "transform = transforms.Compose([transforms.Grayscale(num_output_channels=1),\n",
        "                                transforms.Resize((1, 28*28)),\n",
        "                                transforms.ToTensor()\n",
        "])\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR100(root='./data', train=True, download=True, transform=transform)\n",
        "testset = torchvision.datasets.CIFAR100(root='./data', train=False, download=True, transform=transform)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFRZvUpvUQ1n",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "aa553362-d5d7-4105-a398-e6ccee4326bd"
      },
      "source": [
        "#images for feeding fit_generator dense network \n",
        "#must be resized to (1, 1, height*width*channel)\n",
        "trainset[0][0].shape"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 1, 784])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9hYwVNEsCJ7o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trainloader = DataLoader(trainset, batch_size=bs, shuffle=True)\n",
        "testloader = DataLoader(testset, batch_size=bs)"
      ],
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vhQI6UtDE1DZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        },
        "outputId": "96be7846-01ae-4d38-dfd2-d632410c8c96"
      },
      "source": [
        "input = Input((1, 784))\n",
        "x = Dense(1000, nn.ReLU())(input)\n",
        "y = Dense(500, nn.ReLU())(x)\n",
        "z = Dense(700, activation=nn.ReLU())(y)\n",
        "a = Dense(200, activation=nn.ReLU())(z)\n",
        "b = Dense(100, activation=nn.ReLU())(a)\n",
        "\n",
        "model = Model(input, b, device)\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.001), nn.CrossEntropyLoss())\n",
        "model.summary()"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1              [-1, 1, 1000]         785,000\n",
            "              ReLU-2              [-1, 1, 1000]               0\n",
            "            Linear-3               [-1, 1, 500]         500,500\n",
            "              ReLU-4               [-1, 1, 500]               0\n",
            "            Linear-5               [-1, 1, 700]         350,700\n",
            "              ReLU-6               [-1, 1, 700]               0\n",
            "            Linear-7               [-1, 1, 200]         140,200\n",
            "              ReLU-8               [-1, 1, 200]               0\n",
            "            Linear-9               [-1, 1, 100]          20,100\n",
            "             ReLU-10               [-1, 1, 100]               0\n",
            "================================================================\n",
            "Total params: 1,796,500\n",
            "Trainable params: 1,796,500\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.04\n",
            "Params size (MB): 6.85\n",
            "Estimated Total Size (MB): 6.89\n",
            "----------------------------------------------------------------\n",
            "Device Type: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q2ahcZ9IDeEd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "2400a0d3-eb9b-4571-aaae-e74d78ac50b1"
      },
      "source": [
        "model.fit_generator(trainloader, 3)"
      ],
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "391/391 [=========================] - 13s 34ms/step - loss: : 4.5967\n",
            "Epoch 2/3\n",
            "391/391 [=========================] - 13s 34ms/step - loss: : 4.5844\n",
            "Epoch 3/3\n",
            "391/391 [=========================] - 13s 34ms/step - loss: : 4.5749\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIP9jwX1bsP2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "65429b62-cf3e-47fd-9448-8c67582a0097"
      },
      "source": [
        "model.evaluate_generator(testloader)"
      ],
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "79/79 [=========================] - 3s 32ms/step - test_acc: 2.0894 - test_loss: 2.3429\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A64KPIvKxkTs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out = model.predict_generator(testloader)"
      ],
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hud23hIZxyQ8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.max(out[0][0], 1)[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lXLnuHX4nxiT",
        "colab_type": "text"
      },
      "source": [
        "# CIFAR100 -Conv2d"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGO2iw25nxNd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bs = 128\n",
        "\n",
        "transform = transforms.ToTensor()\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR100(root='./data', train=True, download=True, transform=transform)\n",
        "testset = torchvision.datasets.CIFAR100(root='./data', train=False, download=True, transform=transform)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtHxqh0qn8az",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "207b4283-b61e-4d02-ac09-00b0778f1838"
      },
      "source": [
        "trainset[0][0].shape"
      ],
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 32, 32])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SyZEIipnoFz9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trainloader = DataLoader(trainset, batch_size=bs, shuffle=True)\n",
        "testloader = DataLoader(testset, batch_size=bs)"
      ],
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pOi7FTboHRy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 416
        },
        "outputId": "49cd815c-aebc-4672-a809-76dc4f853ab1"
      },
      "source": [
        "input = Input((3, 32, 32))\n",
        "x = Conv2d(3, 3, 1, 'same', 1, nn.ReLU())(input)\n",
        "y = Conv2d(5, 3, 1, 0, 1, nn.ReLU())(x)\n",
        "z = Conv2d(6, 3, 1, 'same', 1, nn.ReLU())(y)\n",
        "a = Flatten()(z)\n",
        "b = Dense(100, activation=nn.ReLU())(a)\n",
        "\n",
        "model = Model(input, b, device)\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.001), nn.CrossEntropyLoss())\n",
        "model.summary()"
      ],
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1            [-1, 3, 32, 32]              84\n",
            "              ReLU-2            [-1, 3, 32, 32]               0\n",
            "            Conv2d-3            [-1, 5, 30, 30]             140\n",
            "              ReLU-4            [-1, 5, 30, 30]               0\n",
            "            Conv2d-5            [-1, 6, 30, 30]             276\n",
            "              ReLU-6            [-1, 6, 30, 30]               0\n",
            "    FlattenedLayer-7                 [-1, 5400]               0\n",
            "            Linear-8                  [-1, 100]         540,100\n",
            "              ReLU-9                  [-1, 100]               0\n",
            "================================================================\n",
            "Total params: 540,600\n",
            "Trainable params: 540,600\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.01\n",
            "Forward/backward pass size (MB): 0.24\n",
            "Params size (MB): 2.06\n",
            "Estimated Total Size (MB): 2.31\n",
            "----------------------------------------------------------------\n",
            "Device Type: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gd3SirTpo6Z1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "15e87270-87ad-4adb-c04f-d2c5aa72ec52"
      },
      "source": [
        "model.fit_generator(trainloader, 3)"
      ],
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "391/391 [=========================] - 7s 17ms/step - loss: : 4.5724\n",
            "Epoch 2/3\n",
            "391/391 [=========================] - 7s 18ms/step - loss: : 4.5473\n",
            "Epoch 3/3\n",
            "391/391 [=========================] - 7s 18ms/step - loss: : 4.5396\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p2R5sVz1pHAS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d9555926-3622-4e68-ba8c-79b7354a930e"
      },
      "source": [
        "model.evaluate_generator(testloader)"
      ],
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "79/79 [=========================] - 1s 14ms/step - test_acc: 2.3575 - test_loss: 2.3294\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JCnrHa1uyUG2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "0c07e9aa-62fc-4ca2-c16b-05b298bcf32a"
      },
      "source": [
        "out = model.predict_generator(testloader)\n",
        "torch.max(out[0], 1)[1]"
      ],
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([61, 61,  0,  0,  0,  0,  0, 90, 61, 40,  0, 86, 90, 61, 40, 40,  0, 86,\n",
              "        61,  0, 61, 61, 90, 90,  0,  0, 61, 61,  0, 86, 24, 61, 86,  0,  0,  0,\n",
              "        86, 90, 86, 90,  0, 24, 61, 61, 86,  0,  0, 86, 90, 90, 40, 40, 86, 86,\n",
              "         0,  0,  0, 90,  0, 61, 86,  0,  0,  0,  0,  0, 40, 24,  0,  0, 61, 90,\n",
              "        61,  0,  0, 90,  0,  0,  0, 61, 90, 61, 40,  0,  0, 90, 61, 90, 61,  0,\n",
              "        90,  9, 90,  0, 90,  0, 86, 86, 90, 86,  0, 90, 90, 86, 90, 61,  0, 90,\n",
              "        24, 90, 61, 61,  0,  0,  9, 90, 61, 86,  0, 40, 90,  0,  0,  0,  0,  0,\n",
              "        90, 90], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "7PqD8aKda0GR"
      },
      "source": [
        "# CIFAR10- Dense layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RxbpAFqEa0Gv",
        "colab": {}
      },
      "source": [
        "bs = 64\n",
        "\n",
        "transform = transforms.Compose([transforms.Grayscale(num_output_channels=1),\n",
        "                                transforms.Resize((1, 28*28)),\n",
        "                                transforms.ToTensor()\n",
        "])\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
        "testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZnANxHHFa0HF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7daa6acf-5808-43e9-bd4b-a25ba24d193e"
      },
      "source": [
        "trainset[0][0].shape"
      ],
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 1, 784])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "B_ywq52Ba0HR",
        "colab": {}
      },
      "source": [
        "trainloader = DataLoader(trainset, batch_size=bs, shuffle=True)\n",
        "testloader = DataLoader(testset, batch_size=bs)"
      ],
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "jSO7gtHea0Hb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        },
        "outputId": "86063ff8-7a98-467d-fce6-93643accf783"
      },
      "source": [
        "input = Input(shape=(1, 784))\n",
        "x = Dense(1024, nn.ReLU())(input)\n",
        "y = Dense(800, nn.ReLU())(x)\n",
        "z = Dense(300, activation=nn.ReLU())(y)\n",
        "a = Dense(200, activation=nn.ReLU())(z)\n",
        "b = Dense(10, activation=nn.ReLU())(a)\n",
        "\n",
        "model = Model(input, b, 'cuda')\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.001), nn.CrossEntropyLoss())\n",
        "model.summary()"
      ],
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1              [-1, 1, 1024]         803,840\n",
            "              ReLU-2              [-1, 1, 1024]               0\n",
            "            Linear-3               [-1, 1, 800]         820,000\n",
            "              ReLU-4               [-1, 1, 800]               0\n",
            "            Linear-5               [-1, 1, 300]         240,300\n",
            "              ReLU-6               [-1, 1, 300]               0\n",
            "            Linear-7               [-1, 1, 200]          60,200\n",
            "              ReLU-8               [-1, 1, 200]               0\n",
            "            Linear-9                [-1, 1, 10]           2,010\n",
            "             ReLU-10                [-1, 1, 10]               0\n",
            "================================================================\n",
            "Total params: 1,926,350\n",
            "Trainable params: 1,926,350\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.04\n",
            "Params size (MB): 7.35\n",
            "Estimated Total Size (MB): 7.39\n",
            "----------------------------------------------------------------\n",
            "Device Type: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fIfJtFala0Hm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "105f2084-2ce2-4324-bfb1-728064da3597"
      },
      "source": [
        "model.fit_generator(trainloader, 3)"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "782/782 [=========================] - 15s 19ms/step - loss: : 2.3024\n",
            "Epoch 2/3\n",
            "782/782 [=========================] - 15s 19ms/step - loss: : 2.3026\n",
            "Epoch 3/3\n",
            "782/782 [=========================] - 15s 19ms/step - loss: : 2.3026\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Yo1h0ywGa0Hx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bb28947e-9b11-4234-d834-8f13b380613c"
      },
      "source": [
        "model.evaluate_generator(testloader)"
      ],
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "157/157 [=========================] - 3s 17ms/step - test_acc: 3.2235 - test_loss: 1.1660\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hguj7CRsy1uL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out = model.predict_generator(testloader)\n",
        "torch.max(out[0][0], 1)[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "oy1zHKBmqO0m"
      },
      "source": [
        "# CIFAR10 -Conv2d"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "A6RU5p21qO00",
        "colab": {}
      },
      "source": [
        "bs = 64\n",
        "\n",
        "transform = transforms.ToTensor()\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
        "testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7n9mt120qO1N",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d20f7003-32ff-42f8-813c-81603e5acb71"
      },
      "source": [
        "trainset[0][0].shape"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 32, 32])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ORhwioJ_qO1e",
        "colab": {}
      },
      "source": [
        "trainloader = DataLoader(trainset, batch_size=bs, shuffle=True)\n",
        "testloader = DataLoader(testset, batch_size=bs)"
      ],
      "execution_count": 127,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ph8a0VdpqO1s",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 416
        },
        "outputId": "b0dc881a-39e6-4973-ddb6-86824826a1c5"
      },
      "source": [
        "input = Input(shape=(3, 32, 32))\n",
        "x = Conv2d(7, 3, 1, 'same', 1, nn.ReLU())(input)\n",
        "y = Conv2d(10, 3, 1, 'same', 1, nn.ReLU())(x)\n",
        "z = Conv2d(6, 3, 1, 'same', 1, nn.ReLU())(y)\n",
        "a = Flatten()(z)\n",
        "b = Dense(10, activation=nn.ReLU())(a)\n",
        "\n",
        "model = Model(input, b, 'cuda')\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.001), nn.CrossEntropyLoss())\n",
        "model.summary()"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1            [-1, 7, 32, 32]             196\n",
            "              ReLU-2            [-1, 7, 32, 32]               0\n",
            "            Conv2d-3           [-1, 10, 32, 32]             640\n",
            "              ReLU-4           [-1, 10, 32, 32]               0\n",
            "            Conv2d-5            [-1, 6, 32, 32]             546\n",
            "              ReLU-6            [-1, 6, 32, 32]               0\n",
            "    FlattenedLayer-7                 [-1, 6144]               0\n",
            "            Linear-8                   [-1, 10]          61,450\n",
            "              ReLU-9                   [-1, 10]               0\n",
            "================================================================\n",
            "Total params: 62,832\n",
            "Trainable params: 62,832\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.01\n",
            "Forward/backward pass size (MB): 0.41\n",
            "Params size (MB): 0.24\n",
            "Estimated Total Size (MB): 0.66\n",
            "----------------------------------------------------------------\n",
            "Device Type: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eBykn4s6qO14",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "57dde2fc-615b-4a93-b4e7-773aef0c644c"
      },
      "source": [
        "model.fit_generator(trainloader, 3)"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "782/782 [=========================] - 8s 10ms/step - loss: : 2.3028\n",
            "Epoch 2/3\n",
            "782/782 [=========================] - 8s 10ms/step - loss: : 2.2852\n",
            "Epoch 3/3\n",
            "782/782 [=========================] - 8s 10ms/step - loss: : 2.2474\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RdF15rBJqO2E",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7df734de-978e-45c3-b88d-1a93decec27b"
      },
      "source": [
        "model.evaluate_generator(testloader)"
      ],
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "157/157 [=========================] - 1s 8ms/step - test_acc: 6.6300 - test_loss: 1.1336\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y9rfTSEtzOsI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "cb8fae37-bdbc-40a9-d1af-197135058126"
      },
      "source": [
        "out = model.predict_generator(testloader)\n",
        "torch.max(out[0], 1)[1]"
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3, 0, 0, 0, 4, 0, 0, 3, 4, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0,\n",
              "        4, 4, 4, 0, 0, 4, 3, 4, 4, 3, 0, 0, 4, 0, 0, 3, 0, 3, 3, 4, 0, 0, 3, 0,\n",
              "        4, 4, 0, 0, 3, 3, 0, 0, 0, 3, 3, 3, 4, 3, 3, 0], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 131
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wlI2RCv5bI0g"
      },
      "source": [
        "# MNIST- Dense layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fN8MtErvbI1E",
        "colab": {}
      },
      "source": [
        "bs = 32\n",
        "\n",
        "transform = transforms.Compose([\n",
        "                                transforms.Resize((1, 28*28)),\n",
        "                                transforms.ToTensor()\n",
        "])\n",
        "\n",
        "trainset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)\n",
        "testset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)"
      ],
      "execution_count": 132,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Sw437nMwbI1V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c3ea824b-c7ad-49f4-dcb0-281a01e2bb52"
      },
      "source": [
        "trainset[0][0].shape"
      ],
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 1, 784])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zslLOsG3bI1j",
        "colab": {}
      },
      "source": [
        "trainloader = DataLoader(trainset, batch_size=bs, shuffle=True)\n",
        "testloader = DataLoader(testset, batch_size=bs)"
      ],
      "execution_count": 134,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "p91ByqQ2bI1v",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        },
        "outputId": "be11176c-9170-414e-9c5a-215bd396dab9"
      },
      "source": [
        "input = Input(shape=(1, 784))\n",
        "x = Dense(700, nn.ReLU())(input)\n",
        "y = Dense(200, nn.ReLU())(x)\n",
        "z = Dense(300, activation=nn.ReLU())(y)\n",
        "a = Dense(100, activation=nn.ReLU())(z)\n",
        "b = Dense(10, activation=nn.ReLU())(a)\n",
        "\n",
        "model = Model(input, b, 'cuda')\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.001), nn.CrossEntropyLoss())\n",
        "model.summary()"
      ],
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1               [-1, 1, 700]         549,500\n",
            "              ReLU-2               [-1, 1, 700]               0\n",
            "            Linear-3               [-1, 1, 200]         140,200\n",
            "              ReLU-4               [-1, 1, 200]               0\n",
            "            Linear-5               [-1, 1, 300]          60,300\n",
            "              ReLU-6               [-1, 1, 300]               0\n",
            "            Linear-7               [-1, 1, 100]          30,100\n",
            "              ReLU-8               [-1, 1, 100]               0\n",
            "            Linear-9                [-1, 1, 10]           1,010\n",
            "             ReLU-10                [-1, 1, 10]               0\n",
            "================================================================\n",
            "Total params: 781,110\n",
            "Trainable params: 781,110\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.02\n",
            "Params size (MB): 2.98\n",
            "Estimated Total Size (MB): 3.00\n",
            "----------------------------------------------------------------\n",
            "Device Type: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "G9dQdMOybI16",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "44d3f783-662c-445b-d2ff-bd26c0230ac4"
      },
      "source": [
        "model.fit_generator(trainloader, 3)"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "1875/1875 [=========================] - 18s 10ms/step - loss: : 2.1482\n",
            "Epoch 2/3\n",
            "1875/1875 [=========================] - 18s 10ms/step - loss: : 2.1327\n",
            "Epoch 3/3\n",
            "1875/1875 [=========================] - 18s 10ms/step - loss: : 2.1260\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "w4vo6b2YbI2E",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a280e544-0193-42b1-a391-d57446071ee5"
      },
      "source": [
        "model.evaluate_generator(testloader)"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "313/313 [=========================] - 2s 8ms/step - test_acc: 3.2877 - test_loss: 1.0559\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRfi44ZezzYj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out = model.predict_generator(testloader)\n",
        "torch.max(out[0][0], 1)[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hRBTwshsrjrz"
      },
      "source": [
        "# MNIST -Conv2d"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "jJ_fr5k6rjr6",
        "colab": {}
      },
      "source": [
        "bs = 32\n",
        "\n",
        "transform = transforms.Compose([\n",
        "                                transforms.ToTensor(),\n",
        "                                transforms.Normalize(0.5, 1)\n",
        "                                ])\n",
        "\n",
        "trainset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)\n",
        "testset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)"
      ],
      "execution_count": 152,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "thk7zRJvrjsK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ead1027e-3024-4d94-d922-21d471f7bacd"
      },
      "source": [
        "#images for feeding fit_generator dense network \n",
        "#must be of the form (1, 1, height*width*channel)\n",
        "trainset[0][0].shape"
      ],
      "execution_count": 153,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 28, 28])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gx0sJBazrjsd",
        "colab": {}
      },
      "source": [
        "trainloader = DataLoader(trainset, batch_size=bs, shuffle=True)\n",
        "testloader = DataLoader(testset, batch_size=bs)"
      ],
      "execution_count": 154,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "m8_dpt0irjsq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        },
        "outputId": "cb87dbd3-78d7-4412-e00e-ec48d33ef823"
      },
      "source": [
        "input = Input(shape=(1, 28, 28))\n",
        "x = Conv2d(25, 3, 1, 3, 1, nn.ReLU())(input)\n",
        "y = Conv2d(50, 5, 1, 'same',1 , nn.ReLU())(x)\n",
        "y = Conv2d(20, 3, 1, 'same', 1, nn.ReLU())(y)\n",
        "z = Flatten()(y)\n",
        "a = Dense(100, activation=nn.ReLU())(z)\n",
        "b = Dense(10, activation=nn.ReLU())(a)\n",
        "\n",
        "model = Model(input, b, 'cuda')\n",
        "model.compile(optim.Adam(model.parameters(), lr=0.0001), nn.CrossEntropyLoss())\n",
        "model.summary()"
      ],
      "execution_count": 185,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 25, 32, 32]             250\n",
            "              ReLU-2           [-1, 25, 32, 32]               0\n",
            "            Conv2d-3           [-1, 50, 32, 32]          31,300\n",
            "              ReLU-4           [-1, 50, 32, 32]               0\n",
            "            Conv2d-5           [-1, 20, 32, 32]           9,020\n",
            "              ReLU-6           [-1, 20, 32, 32]               0\n",
            "    FlattenedLayer-7                [-1, 20480]               0\n",
            "            Linear-8                  [-1, 100]       2,048,100\n",
            "              ReLU-9                  [-1, 100]               0\n",
            "           Linear-10                   [-1, 10]           1,010\n",
            "             ReLU-11                   [-1, 10]               0\n",
            "================================================================\n",
            "Total params: 2,089,680\n",
            "Trainable params: 2,089,680\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 1.64\n",
            "Params size (MB): 7.97\n",
            "Estimated Total Size (MB): 9.62\n",
            "----------------------------------------------------------------\n",
            "Device Type: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kxJm65blrjs1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "1105fe68-fea6-4a32-93f1-485bf6970680"
      },
      "source": [
        "model.fit_generator(trainloader, 3)"
      ],
      "execution_count": 186,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "1875/1875 [=========================] - 19s 10ms/step - loss: : 0.4994\n",
            "Epoch 2/3\n",
            "1875/1875 [=========================] - 19s 10ms/step - loss: : 0.3068\n",
            "Epoch 3/3\n",
            "1875/1875 [=========================] - 19s 10ms/step - loss: : 0.2813\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "G1DHE6hWrjtD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "29b5ff35-5049-46b1-8a9f-896af821d43a"
      },
      "source": [
        "model.evaluate_generator(testloader)"
      ],
      "execution_count": 187,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "313/313 [=========================] - 2s 7ms/step - test_acc: 14.2766 - test_loss: 0.1430\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hx1DCH1NvV1J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = model.predict_generator(testloader)"
      ],
      "execution_count": 188,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q4uSA_f3oXRG",
        "colab_type": "text"
      },
      "source": [
        "Model prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QhLaRtBFFkoz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "65c6ccee-9aa6-4844-fa2a-db8d59525a68"
      },
      "source": [
        "torch.max(x[0], 1)[1]"
      ],
      "execution_count": 189,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([7, 2, 1, 0, 4, 1, 4, 0, 5, 0, 0, 6, 0, 0, 1, 5, 0, 7, 8, 4, 7, 6, 6, 5,\n",
              "        4, 0, 7, 4, 0, 1, 3, 1], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 189
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "11xQdCrYoZow",
        "colab_type": "text"
      },
      "source": [
        "Original target data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EsS7YY5IyqZ8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 573
        },
        "outputId": "a0273df1-6dae-4408-f219-f6415fcdf522"
      },
      "source": [
        "for i in range(32):\n",
        "  print(testloader.dataset[i][1])"
      ],
      "execution_count": 190,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7\n",
            "2\n",
            "1\n",
            "0\n",
            "4\n",
            "1\n",
            "4\n",
            "9\n",
            "5\n",
            "9\n",
            "0\n",
            "6\n",
            "9\n",
            "0\n",
            "1\n",
            "5\n",
            "9\n",
            "7\n",
            "3\n",
            "4\n",
            "9\n",
            "6\n",
            "6\n",
            "5\n",
            "4\n",
            "0\n",
            "7\n",
            "4\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}