2_vanilla_neural_net_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch tutorial 60 minutes blitz : 2) Vanilla Neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Autograd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you set its attribute .requires_grad as True, it starts to track all operations on it. When you finish your computation you can call .backward() and have all the gradients computed automatically. The gradient for this tensor will be accumulated into .grad attribute.\n",
    "\n",
    "To stop a tensor from tracking history, you can call .detach() to detach it from the computation history, and to prevent future computation from being tracked.\n",
    "\n",
    "To prevent tracking history (and using memory), you can also wrap the code block in with torch.no_grad():. This can be particularly helpful when evaluating a model because the model may have trainable parameters with requires_grad=True, but for which we don’t need the gradients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.rand(5,3, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1804, 0.9977, 0.1434],\n",
       "        [0.2356, 0.7562, 0.2739],\n",
       "        [0.6528, 0.2323, 0.5545],\n",
       "        [0.6372, 0.7173, 0.7901],\n",
       "        [0.9934, 0.4070, 0.0940]], requires_grad=True)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=x+2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2.1804, 2.9977, 2.1434],\n",
       "        [2.2356, 2.7562, 2.2739],\n",
       "        [2.6528, 2.2323, 2.5545],\n",
       "        [2.6372, 2.7173, 2.7901],\n",
       "        [2.9934, 2.4070, 2.0940]], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AddBackward0 at 0x7f84a130c588>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.grad_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "z=y**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4.7539, 8.9863, 4.5940],\n",
       "        [4.9981, 7.5968, 5.1707],\n",
       "        [7.0374, 4.9833, 6.5254],\n",
       "        [6.9549, 7.3836, 7.7846],\n",
       "        [8.9605, 5.7939, 4.3850]], grad_fn=<PowBackward0>)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "m=z.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.3939, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.3939, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "## m = mean(z), z = y**2, y=x+2, x= rand tensor 5,3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### gives dm/dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.2907, 0.3997, 0.2858],\n",
      "        [0.2981, 0.3675, 0.3032],\n",
      "        [0.3537, 0.2976, 0.3406],\n",
      "        [0.3516, 0.3623, 0.3720],\n",
      "        [0.3991, 0.3209, 0.2792]])\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "## can only done on scalar outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "grad can be implicitly created only for scalar outputs",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-ab75bb780f4c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    193\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m         \"\"\"\n\u001b[0;32m--> 195\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0mgrad_tensors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m     \u001b[0mgrad_tensors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_make_grads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mretain_graph\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m         \u001b[0mretain_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36m_make_grads\u001b[0;34m(outputs, grads)\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"grad can be implicitly created only for scalar outputs\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m                 \u001b[0mnew_grads\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory_format\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreserve_format\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: grad can be implicitly created only for scalar outputs"
     ]
    }
   ],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vanilla NN on MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms, datasets\n",
    "import torchvision   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to MNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6b4bd9f31bd46d2b987976f24b6e7f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST/raw/train-images-idx3-ubyte.gz to MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to MNIST/raw/train-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "547baf03fed4414b88b5e447aca30e23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST/raw/train-labels-idx1-ubyte.gz to MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to MNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9670304d0564b09beec9dd5034322cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST/raw/t10k-images-idx3-ubyte.gz to MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "633deb60490d437a84e10540871bf5ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST/raw/t10k-labels-idx1-ubyte.gz to MNIST/raw\n",
      "Processing...\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "train = datasets.MNIST(\"\",train=True,download=True, transform=transforms.Compose([transforms.ToTensor()]))\n",
    "test = datasets.MNIST(\"\",train=False,download=True, transform=transforms.Compose([transforms.ToTensor()]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = torch.utils.data.DataLoader(train,batch_size=10,shuffle=True)\n",
    "testset = torch.utils.data.DataLoader(test,batch_size=10,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for data in trainset:\n",
    "#     print(data)\n",
    "#     print(data[0].shape)  # 10 images of 28*28 size\n",
    "#     print(data[1].shape)  # true labels of 10 images\n",
    "#     print(data[0][0].shape)  #1 image of 28*28 with depth 1 i,e, grayscale\n",
    "    \n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#extract 1st image and label and visualize\n",
    "x,y = data[0][0], data[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_x = x.view(28,28)    ## plot imshow requires 2d array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAN8UlEQVR4nO3df6zV9X3H8ddLBKxUMyjCGBJFZe3IOrG5pa7SlsbOWbYUusStpHN2IaOJxanp2hmbFJKtGVt1xnVdN5xMNNbOBI0uM10JGm3TxXCxTKHYCQ4QYWBhrbq0/Hzvj3tYrnrP51zO93t+XN/PR3Jzzvm+z/f7fXPC637POZ/v934cEQLw9ndGrxsA0B2EHUiCsANJEHYgCcIOJHFmN3c2wRPjLE3q5i6BVH6u/9XROOKRapXCbvtqSXdKGifpHyNiden5Z2mSPuArq+wSQMHTsbFpre238bbHSfq6pI9Lmitpqe257W4PQGdV+cw+X9KOiHgxIo5K+pakxfW0BaBuVcI+U9JLwx7vbSx7A9vLbQ/aHjymIxV2B6CKKmEf6UuAt5x7GxFrImIgIgbGa2KF3QGookrY90qaNezx+ZL2VWsHQKdUCfsmSXNsz7Y9QdKnJD1aT1sA6tb20FtEHLe9QtK/aWjobW1EbKutMwC1qjTOHhGPSXqspl4AdBCnywJJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BEpVlcMfadcfbZxfqLX7q0WD867Xix/l+/dVfT2rE4UVy3lZUHLyvWt3xsatPaiUOHK+17LKoUdtu7JL0m6YSk4xExUEdTAOpXx5H9oxHx4xq2A6CD+MwOJFE17CHpO7Y3214+0hNsL7c9aHvwmI5U3B2AdlV9G39FROyzPU3SBtvPR8RTw58QEWskrZGkcz0lKu4PQJsqHdkjYl/j9qCkhyXNr6MpAPVrO+y2J9k+59R9SVdJ2lpXYwDqVeVt/HRJD9s+tZ1vRsS3a+kKtYkr5hXr5/zlS8X6s7P/ptL+j0Xz48lJnay07ZXTNhfrl/7xjU1rF6z8fqV9j0Vthz0iXpRUPuMCQN9g6A1IgrADSRB2IAnCDiRB2IEkuMT1bWDn7Zc3rf3DkuaXmErSgrN+Xnc76FMc2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZx4DSOLokbf69O5rWzvaE4rrVLjJtbe6Ty5rWtn6kfA4A6sWRHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJx9DFj0kfKfTC6NpZ8hF9f96cmjxfr1uz9RrP/3bRcX63O2N58a+YzHqx1r9hz/WbE+80mmGxuOIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+xjwzF+8r1h/90eb13/tvbuK6x6688Ji/eyHni7W36FDxbrefUnTUtUpmz/27ZuL9V9+fFOl7b/dtDyy215r+6DtrcOWTbG9wfYLjdvJnW0TQFWjeRt/j6Sr37TsFkkbI2KOpI2NxwD6WMuwR8RTkt58zuNiSesa99dJWlJzXwBq1u4XdNMjYr8kNW6nNXui7eW2B20PHhPnKgO90vFv4yNiTUQMRMTAeE3s9O4ANNFu2A/YniFJjduD9bUEoBPaDfujkq5r3L9O0iP1tAOgU1qOs9t+QNJCSVNt75W0UtJqSQ/aXiZpj6RrOtlkdpPWl8e656xvXitf8S2drQOn39Bp2LFqUse2fdGDnf6r928vLcMeEUublK6suRcAHcTpskAShB1IgrADSRB2IAnCDiTBJa6oZN8XP1isb/vw15rWWg2czf3nG4r1OU8OFuvRYvvZcGQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ0fRuHPPLdZ/6Tf3FOulKaNbTRc9+1/K9Th+vFjHG3FkB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGGdH0e5/mlWsP/Oee4r1/ymMpS/8+heK68584vvFOk4PR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9uQOLfv1Yv3hgdtabGFCsXr97k80rc1czTh6N7U8sttea/ug7a3Dlq2y/bLtLY2fRZ1tE0BVo3kbf4+kq0dYfkdEzGv8PFZvWwDq1jLsEfGUpMNd6AVAB1X5gm6F7Wcbb/MnN3uS7eW2B20PHtORCrsDUEW7Yf+GpIslzZO0X9LtzZ4YEWsiYiAiBsZrYpu7A1BVW2GPiAMRcSIiTkq6S9L8etsCULe2wm57xrCHn5S0tdlzAfSHluPsth+QtFDSVNt7Ja2UtND2PA1Ngb1L0mc72CMq2Hn75cX649d8tVifPq780evLB99frL++mFnS+0XLsEfE0hEW392BXgB0EKfLAkkQdiAJwg4kQdiBJAg7kASXuI4BraZNPvFw8/qP3vN3xXVP6h1t9XTKn0/bXKwvWLSiae0X7vv3SvvG6eHIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM4+BlSZNvlki9/nJ3WynZZGbf1Xml9Cu2RCecrmd93NOHydOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs/eBTk+bXHLzvg8V65+Z+t1i/dIWu54+rvn18t/8cvnf9en4k2J9ylrG4U8HR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSMIR3ZtS91xPiQ/4yq7tr1+Mu2R2sf73j99XrLeaNrnkt5//nWL9zCU/KdZ33fzeYv2RZeUpn2efeVbTWqtr6Q+cOFKs/8Hzv1+sT7xqV7H+dvR0bNSrcdgj1Voe2W3Psv2E7e22t9m+sbF8iu0Ntl9o3E6uu3EA9RnN2/jjkj4fEb8i6XJJn7M9V9ItkjZGxBxJGxuPAfSplmGPiP0R8Uzj/muStkuaKWmxpHWNp62TtKRTTQKo7rS+oLN9oaTLJD0taXpE7JeGfiFImtZkneW2B20PHlP5MxiAzhl12G2/U9J6STdFxKujXS8i1kTEQEQMjFf7XzQBqGZUYbc9XkNBvz8iHmosPmB7RqM+Q9LBzrQIoA4th95sW0OfyQ9HxE3Dln9V0qGIWG37FklTIuKLpW1lHXp7/5YTxfrK87ZU2v4f7m7+mr7ywfLQWlU777+sWN+w4GtNa+efWW266D3Hf1asX3/BgkrbH4tKQ2+juZ79CknXSnrO9qn/lbdKWi3pQdvLJO2RdE0dzQLojJZhj4jvSRrxN4WkfIdpYIzidFkgCcIOJEHYgSQIO5AEYQeS4E9J1+An15b/FPTK8/62WG91qecPjpR/J+/9ypymtYnaVFy3qos//YNiffmHbmhae/nGY8V1F16wo1j/4ary5bed/rePNRzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlr8NNLml0UWI8b/mxFsT7lX/t36uIzvtt8HH5WeTZo7WyxbcbRTw9HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2Lmj1980X3fuFYv2iB8rXjJevhgeGcGQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSRGMz/7LEn3SvpFDQ3promIO22vkvRHkl5pPPXWiHistK2s87MD3VJ1fvbjkj4fEc/YPkfSZtsbGrU7IuK2uhoF0DmjmZ99v6T9jfuv2d4uaWanGwNQr9P6zG77QkmXSXq6sWiF7Wdtr7U9uck6y20P2h48piOVmgXQvlGH3fY7Ja2XdFNEvCrpG5IuljRPQ0f+20daLyLWRMRARAyM18QaWgbQjlGF3fZ4DQX9/oh4SJIi4kBEnIiIk5LukjS/c20CqKpl2G1b0t2StkfEXw9bPmPY0z4paWv97QGoy2i+jb9C0rWSnrO9pbHsVklLbc+TFJJ2SfpsRzoEUIvRfBv/PUkjjdsVx9QB9BfOoAOSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTR8k9J17oz+xVJu4ctmirpx11r4PT0a2/92pdEb+2qs7cLIuK8kQpdDftbdm4PRsRAzxoo6Nfe+rUvid7a1a3eeBsPJEHYgSR6HfY1Pd5/Sb/21q99SfTWrq701tPP7AC6p9dHdgBdQtiBJHoSdttX2/6R7R22b+lFD83Y3mX7OdtbbA/2uJe1tg/a3jps2RTbG2y/0LgdcY69HvW2yvbLjddui+1FPeptlu0nbG+3vc32jY3lPX3tCn115XXr+md22+Mk/aek35C0V9ImSUsj4oddbaQJ27skDUREz0/AsP1hSa9LujcifrWx7K8kHY6I1Y1flJMj4k/7pLdVkl7v9TTejdmKZgyfZlzSEkmfUQ9fu0Jfv6suvG69OLLPl7QjIl6MiKOSviVpcQ/66HsR8ZSkw29avFjSusb9dRr6z9J1TXrrCxGxPyKeadx/TdKpacZ7+toV+uqKXoR9pqSXhj3eq/6a7z0kfcf2ZtvLe93MCKZHxH5p6D+PpGk97ufNWk7j3U1vmma8b167dqY/r6oXYR9pKql+Gv+7IiLeJ+njkj7XeLuK0RnVNN7dMsI0432h3enPq+pF2PdKmjXs8fmS9vWgjxFFxL7G7UFJD6v/pqI+cGoG3cbtwR738//6aRrvkaYZVx+8dr2c/rwXYd8kaY7t2bYnSPqUpEd70Mdb2J7U+OJEtidJukr9NxX1o5Kua9y/TtIjPezlDfplGu9m04yrx69dz6c/j4iu/0hapKFv5HdK+lIvemjS10WS/qPxs63XvUl6QENv645p6B3RMknvkrRR0guN2yl91Nt9kp6T9KyGgjWjR70t0NBHw2clbWn8LOr1a1foqyuvG6fLAklwBh2QBGEHkiDsQBKEHUiCsANJEHYgCcIOJPF/Zl8y88E/WSgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(img_x)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#checking for balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "total = 0\n",
    "counter = {0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0}\n",
    "for data in trainset:\n",
    "    xs,ys = data\n",
    "    for y in ys:\n",
    "        counter[int(y)]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 5923,\n",
       " 1: 6742,\n",
       " 2: 5958,\n",
       " 3: 6131,\n",
       " 4: 5842,\n",
       " 5: 5421,\n",
       " 6: 5918,\n",
       " 7: 6265,\n",
       " 8: 5851,\n",
       " 9: 5949}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        '''\n",
    "        Note -\n",
    "        # Weights are initialized ~ UNIFORM(-SQRT(K),SQRT(K)), K = 1/n_input_features\n",
    "        # PARAMETERS i.e, weights in Linear layers have require_grad=True by default \n",
    "        & autograd detects it while performing backward() and gives derivative of output \n",
    "        wrt parameters\n",
    "        '''\n",
    "        self.layer1 = nn.Linear(28*28, 128)\n",
    "        self.layer2 = nn.Linear(128, 128)\n",
    "        self.layer3 = nn.Linear(128, 128)\n",
    "        self.layer4 = nn.Linear(128, 10)\n",
    "    def forward(self,x):\n",
    "        x = F.relu(self.layer1(x))\n",
    "        x = F.relu(self.layer2(x))\n",
    "        x = F.relu(self.layer3(x))\n",
    "        x = self.layer4(x)\n",
    "        \n",
    "        return F.log_softmax(x,dim=1)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (layer1): Linear(in_features=784, out_features=128, bias=True)\n",
      "  (layer2): Linear(in_features=128, out_features=128, bias=True)\n",
      "  (layer3): Linear(in_features=128, out_features=128, bias=True)\n",
      "  (layer4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = Net()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.1242, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0038, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1925, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0002, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "#net.parameters() - everything adjustable in model\n",
    "optimizer = optim.Adam(net.parameters(),lr=0.001)\n",
    "#epochs = a pass through whole data\n",
    "EPOCHS = 4\n",
    "\n",
    "for epochs in range(EPOCHS):\n",
    "    for data in trainset:\n",
    "        # data is a batch of featuresets and labels and batch size=10\n",
    "        X,y  = data\n",
    "        net.zero_grad()   #zero the gradient after batch is over\n",
    "        output = net(X.view(-1,28*28))\n",
    "        loss = F.nll_loss(output,y)           #negative likelihood loss\n",
    "        #print(\"Loss=\"+str(loss))\n",
    "        loss.backward()   # computes d(loss)/dparameter\n",
    "        optimizer.step()\n",
    "    print(loss)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### viewing prameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer1.weight tensor([[ 0.0247,  0.0268,  0.0049,  ...,  0.0031,  0.0227,  0.0066],\n",
      "        [ 0.0111,  0.0065, -0.0315,  ...,  0.0212,  0.0343,  0.0003],\n",
      "        [-0.0280,  0.0346,  0.0357,  ..., -0.0149, -0.0163,  0.0147],\n",
      "        ...,\n",
      "        [-0.0353, -0.0031, -0.0350,  ..., -0.0260, -0.0068,  0.0101],\n",
      "        [ 0.0320, -0.0109, -0.0129,  ...,  0.0316, -0.0105,  0.0025],\n",
      "        [-0.0283,  0.0298, -0.0016,  ...,  0.0161,  0.0176,  0.0174]])\n",
      "layer1.bias tensor([-0.0221, -0.0953,  0.1431,  0.2258, -0.0369,  0.0642,  0.0212,  0.0158,\n",
      "         0.0917,  0.0168,  0.2361, -0.0072,  0.0079,  0.0132, -0.0800,  0.1626,\n",
      "        -0.0595, -0.0116, -0.0254,  0.0912,  0.0447, -0.0099,  0.0180,  0.1490,\n",
      "        -0.0302, -0.0389, -0.0182, -0.0994, -0.0059, -0.0383, -0.0542,  0.1763,\n",
      "         0.1112, -0.0625, -0.1228, -0.1412,  0.1001,  0.0652, -0.0652, -0.0758,\n",
      "        -0.1176,  0.1359,  0.0722, -0.0880, -0.0438, -0.0264,  0.0468,  0.0100,\n",
      "        -0.0388,  0.1565, -0.0920, -0.0118,  0.0116,  0.0300, -0.0770,  0.0096,\n",
      "        -0.1812, -0.0878,  0.0891,  0.0171,  0.0160,  0.0131,  0.0389,  0.1827,\n",
      "        -0.0189,  0.1016,  0.1132, -0.0117, -0.2142,  0.0098,  0.0179, -0.1412,\n",
      "        -0.0704, -0.1399, -0.0593,  0.0338,  0.1723, -0.1584,  0.0195, -0.0327,\n",
      "         0.0025,  0.0723, -0.0295,  0.0366,  0.1071,  0.1322, -0.0408, -0.0843,\n",
      "        -0.0233, -0.1316,  0.0522,  0.0853,  0.0042,  0.0077,  0.0610, -0.0083,\n",
      "        -0.0665,  0.1004, -0.0653,  0.0461, -0.0092, -0.0142, -0.0607, -0.1646,\n",
      "         0.1233,  0.0255, -0.1104,  0.0057, -0.0682, -0.0090,  0.0981, -0.1103,\n",
      "        -0.0341,  0.0015, -0.0699,  0.0031, -0.0234, -0.0090, -0.0862,  0.0750,\n",
      "         0.0476,  0.1161,  0.0851, -0.0359, -0.0513, -0.0627, -0.0372, -0.0397])\n",
      "layer2.weight tensor([[-0.0324,  0.2255,  0.3178,  ..., -0.0524,  0.1040, -0.0155],\n",
      "        [-0.2394,  0.1500,  0.1458,  ..., -0.1041,  0.0581, -0.2326],\n",
      "        [ 0.0570,  0.2413, -0.0437,  ...,  0.1826,  0.0733, -0.4017],\n",
      "        ...,\n",
      "        [-0.3274,  0.2014,  0.1814,  ..., -0.3179,  0.2687,  0.3354],\n",
      "        [-0.1480, -0.0065,  0.3190,  ..., -0.1220, -0.0085,  0.0198],\n",
      "        [-0.2326,  0.0703,  0.0369,  ..., -0.0034,  0.0773, -0.4800]])\n",
      "layer2.bias tensor([ 8.9962e-02, -2.5540e-01, -2.7836e-02,  1.1238e-01, -1.1460e-01,\n",
      "        -1.2444e-02, -1.2327e-02,  8.6152e-03, -2.2073e-01,  1.7650e-01,\n",
      "         5.0469e-02,  1.0253e-01, -1.7659e-01,  2.4876e-01, -4.2093e-02,\n",
      "        -4.3210e-02, -1.1371e-02, -1.5678e-01, -3.1405e-01, -1.6858e-01,\n",
      "        -5.2066e-02, -7.0773e-02, -9.1892e-03, -4.7348e-02,  1.6834e-01,\n",
      "         4.0009e-02, -4.2312e-02,  7.2387e-02, -3.5867e-01, -3.5238e-02,\n",
      "         3.1000e-01,  9.1420e-02, -1.0708e-01, -5.3423e-02, -1.0725e-01,\n",
      "        -1.1951e-01,  4.0418e-02, -2.1378e-01, -1.6457e-01,  2.4475e-02,\n",
      "        -3.5714e-02, -1.1441e-01,  4.1630e-01, -7.3671e-02, -6.7439e-02,\n",
      "        -4.4144e-02,  1.0678e-01,  2.1023e-01, -1.3208e-01,  9.5689e-02,\n",
      "         1.0454e-01,  1.3670e-01, -1.1289e-01,  1.2022e-01, -1.6954e-01,\n",
      "         1.4278e-01, -1.2813e-01, -5.6414e-02, -2.6510e-01, -2.1511e-01,\n",
      "         3.4464e-01, -1.8296e-01, -3.0616e-01,  2.2375e-02,  7.7372e-02,\n",
      "        -1.2667e-01,  5.2136e-02,  1.2608e-01, -1.0423e-01,  2.1909e-04,\n",
      "        -1.6492e-01,  7.3583e-02,  6.3754e-02,  4.6723e-02,  1.3104e-01,\n",
      "        -1.6746e-01, -9.5686e-02, -1.3271e-01,  3.0792e-01, -4.3396e-01,\n",
      "         1.6902e-01, -3.1935e-01, -8.4667e-02, -3.4232e-02, -8.8839e-02,\n",
      "        -1.2866e-01, -1.8464e-01, -1.7631e-01, -1.2018e-01, -1.2693e-02,\n",
      "        -3.7851e-02,  5.4052e-02,  1.4647e-01,  2.5960e-01,  1.0828e-01,\n",
      "        -3.7048e-01, -5.3104e-02, -8.0175e-02, -7.4132e-02,  1.8955e-01,\n",
      "        -2.4120e-01, -1.8731e-02,  2.7324e-02, -3.8469e-01, -1.3512e-01,\n",
      "         2.5039e-02, -2.3353e-02,  1.3162e-02,  7.2810e-02, -8.9747e-02,\n",
      "        -8.3281e-03, -4.7281e-02,  2.2931e-01,  4.7900e-02, -5.4528e-02,\n",
      "         1.3370e-01, -6.9460e-03, -2.8263e-01, -5.1095e-02, -2.3200e-01,\n",
      "         5.4728e-02,  6.8317e-02, -1.7856e-01, -1.6832e-01, -1.9808e-02,\n",
      "         8.0334e-02, -2.9944e-01, -4.6267e-02])\n",
      "layer3.weight tensor([[ 0.0810, -0.0578, -0.1078,  ...,  0.0208,  0.1808, -0.0153],\n",
      "        [-0.0210,  0.0248, -0.0471,  ..., -0.0039,  0.2125,  0.0149],\n",
      "        [ 0.0125,  0.1231, -0.0330,  ...,  0.1482, -0.0166,  0.0377],\n",
      "        ...,\n",
      "        [-0.3380,  0.1128, -0.2414,  ..., -0.0963,  0.0571, -0.0261],\n",
      "        [-0.0422,  0.1372, -0.0378,  ..., -0.0775,  0.0112,  0.0241],\n",
      "        [ 0.0424, -0.0122,  0.1167,  ..., -0.1880,  0.1697,  0.0052]])\n",
      "layer3.bias tensor([-0.1645, -0.1400,  0.2087,  0.2349,  0.0008, -0.1998,  0.1970, -0.0839,\n",
      "         0.0577,  0.0414,  0.0062, -0.1212, -0.1022,  0.2831, -0.0673,  0.0219,\n",
      "         0.0168,  0.1838,  0.0878,  0.4087,  0.0075, -0.0464, -0.0614, -0.0724,\n",
      "        -0.0214,  0.0342, -0.1322, -0.1447, -0.1013, -0.0739, -0.0374, -0.2290,\n",
      "        -0.0458,  0.1689, -0.2247,  0.0118,  0.2784,  0.1598, -0.0414,  0.0716,\n",
      "        -0.0266, -0.0048,  0.2341, -0.2237,  0.0251,  0.0934, -0.0146, -0.1451,\n",
      "         0.1384, -0.0743,  0.0658, -0.1409, -0.1126, -0.0312,  0.1490,  0.1016,\n",
      "        -0.1478, -0.0732, -0.0519,  0.2037, -0.0912, -0.1071,  0.1243, -0.1321,\n",
      "         0.2765,  0.4099, -0.0965, -0.1157,  0.2813, -0.0228, -0.1404,  0.0468,\n",
      "        -0.0576, -0.1372, -0.0373, -0.1702,  0.0934, -0.0641,  0.0464,  0.0281,\n",
      "        -0.0475,  0.2726, -0.0382,  0.0705, -0.0369, -0.0028, -0.0753,  0.0243,\n",
      "        -0.0833,  0.0208, -0.1188, -0.0114, -0.0083,  0.3765,  0.0578,  0.0468,\n",
      "        -0.1023,  0.0918,  0.0676, -0.0613, -0.1860, -0.0022,  0.0346, -0.0565,\n",
      "        -0.0339, -0.2261,  0.1830,  0.3080,  0.3313, -0.0035, -0.1177, -0.3421,\n",
      "        -0.1596, -0.0716, -0.0056,  0.0593,  0.1934,  0.2483, -0.1370, -0.1133,\n",
      "         0.1882, -0.1517, -0.0483, -0.0889,  0.4078, -0.1241,  0.1118,  0.0867])\n",
      "layer4.weight tensor([[-2.5987e-02, -2.7091e-01,  3.6907e-02,  ..., -8.8366e-03,\n",
      "         -7.6024e-02, -3.1551e-01],\n",
      "        [ 8.9010e-02, -3.5451e-03, -4.7837e-01,  ..., -2.6092e-01,\n",
      "         -2.9350e-03, -3.6564e-01],\n",
      "        [-5.1477e-02, -1.5490e-02,  6.5694e-02,  ..., -2.4826e-01,\n",
      "         -2.7357e-01, -5.2380e-02],\n",
      "        ...,\n",
      "        [-1.1257e-01, -1.2851e-01, -3.5072e-01,  ..., -1.2489e-01,\n",
      "         -1.9138e-01,  1.1374e-01],\n",
      "        [-4.9816e-02, -1.6467e-01,  3.8437e-02,  ..., -2.0833e-01,\n",
      "          5.9020e-02,  3.7908e-05],\n",
      "        [-1.3746e-01, -1.0214e-01,  6.5796e-02,  ..., -4.5534e-02,\n",
      "         -1.8492e-03,  6.5497e-02]])\n",
      "layer4.bias tensor([-0.0907, -0.2505,  0.0789, -0.0570, -0.0201,  0.0409, -0.2191, -0.1143,\n",
      "         0.4819,  0.1082])\n"
     ]
    }
   ],
   "source": [
    "for name, param in net.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name, param.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.9728\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "with torch.no_grad():   #do not calculate gradients , just give me forward pass and do the following\n",
    "    for data in testset:\n",
    "        X,y = data\n",
    "        output = net(X.view(-1,28*28))\n",
    "        for idx, i in enumerate(output):\n",
    "            if torch.argmax(i) ==y[idx]:\n",
    "                correct+=1\n",
    "            total+=1\n",
    "print(\"Accuracy = \"+str(correct/total))\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convenience :  nn.Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=128, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=128, out_features=128, bias=True)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=128, out_features=128, bias=True)\n",
      "  (5): ReLU()\n",
      "  (6): Linear(in_features=128, out_features=10, bias=True)\n",
      "  (7): LogSoftmax()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Defining architecture of our network\n",
    "input_size = 28*28\n",
    "hidden = [128,128,128]\n",
    "output = 10\n",
    "\n",
    "# 784 -> 128 -> 128 -> 128 -> 10\n",
    "\n",
    "model = nn.Sequential(nn.Linear(input_size,hidden[0]), \n",
    "                      nn.ReLU(),\n",
    "                      nn.Linear(hidden[0],hidden[1]),\n",
    "                     nn.ReLU(),\n",
    "                      nn.Linear(hidden[1],hidden[2]),\n",
    "                     nn.ReLU(),\n",
    "                      nn.Linear(hidden[2],output),\n",
    "                      nn.LogSoftmax(dim=1)\n",
    "                     )\n",
    "\n",
    "print(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0046, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0040, grad_fn=<NllLossBackward>)\n",
      "tensor(0.1974, grad_fn=<NllLossBackward>)\n",
      "tensor(0.0039, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "#net.parameters() - everything adjustable in model\n",
    "optimizer = optim.Adam(model.parameters(),lr=0.001)\n",
    "#epochs = a pass through whole data\n",
    "EPOCHS = 4\n",
    "\n",
    "for epochs in range(EPOCHS):\n",
    "    for data in trainset:\n",
    "        # data is a batch of featuresets and labels and batch size=10\n",
    "        X,y  = data\n",
    "        model.zero_grad()   #zero the gradient after batch is over\n",
    "        output = model(X.view(-1,28*28))\n",
    "        loss = F.nll_loss(output,y)           #negative likelihood loss\n",
    "        #print(\"Loss=\"+str(loss))\n",
    "        loss.backward()   # computes d(loss)/dparameter\n",
    "        optimizer.step()\n",
    "    print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.9671\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "with torch.no_grad():   #do not calculate gradients , just give me forward pass and do the following\n",
    "    for data in testset:\n",
    "        X,y = data\n",
    "        output = model(X.view(-1,28*28))\n",
    "        for idx, i in enumerate(output):\n",
    "            if torch.argmax(i) ==y[idx]:\n",
    "                correct+=1\n",
    "            total+=1\n",
    "print(\"Accuracy = \"+str(correct/total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}