02_train_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision.all import *\n",
    "from moving_mnist.models.conv_rnn import *\n",
    "from moving_mnist.models.transformer import *\n",
    "from moving_mnist.data import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quadro RTX 8000\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():\n",
    "    torch.cuda.set_device(0)\n",
    "    print(torch.cuda.get_device_name())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Transformer Model:\n",
    "> Based on DETR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We wil predict:\n",
    "- `n_in`: 5 images\n",
    "- `n_out`: 5 images  \n",
    "- `n_obj`: up to 3 objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Path('/home/tcapelle/SteadySun/moving_mnist')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Path.cwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path.cwd()/'data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = MovingMNIST(DATA_PATH, n_in=5, n_out=5, n_obj=[1,2], th=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tl = TfmdLists(range(300), ImageTupleTransform(ds, cl_type=TensorMask))\n",
    "valid_tl = TfmdLists(range(100), ImageTupleTransform(ds, cl_type=TensorMask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = DataLoaders.from_dsets(train_tl, valid_tl, bs=1,\n",
    "                             after_batch=[Normalize.from_stats(imagenet_stats[0][0], \n",
    "                                                               imagenet_stats[1][0])]).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_func = StackLoss(CrossEntropyLossFlat(axis=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Left: Input, Right: Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dls.show_batch() #does not work for bs=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`StackUnstack` takes cares of stacking the list of images into a fat tensor, and unstacking them at the end, we will need to modify our loss function to take a list of tensors as input and target."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = StackUnstack(DETR(resnet18, n_in=1, n_out=2, n=int(5*(64/32)**2), nheads=1, num_encoder_layers=1, num_decoder_layers=1))\n",
    "model = StackUnstack(TransformerTS(n_in=1, n_out=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StackUnstack(\n",
       "  (module): TransformerTS(\n",
       "    (backbone): TimeDistributed(Encoder(\n",
       "      (body): Sequential(\n",
       "        (0): Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "        (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "        (4): Sequential(\n",
       "          (0): BasicBlock(\n",
       "            (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (1): BasicBlock(\n",
       "            (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (2): BasicBlock(\n",
       "            (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (5): Sequential(\n",
       "          (0): BasicBlock(\n",
       "            (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (downsample): Sequential(\n",
       "              (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "              (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            )\n",
       "          )\n",
       "          (1): BasicBlock(\n",
       "            (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (2): BasicBlock(\n",
       "            (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (3): BasicBlock(\n",
       "            (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (6): Sequential(\n",
       "          (0): BasicBlock(\n",
       "            (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (downsample): Sequential(\n",
       "              (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "              (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            )\n",
       "          )\n",
       "          (1): BasicBlock(\n",
       "            (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (2): BasicBlock(\n",
       "            (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (3): BasicBlock(\n",
       "            (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (4): BasicBlock(\n",
       "            (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (5): BasicBlock(\n",
       "            (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (7): Sequential(\n",
       "          (0): BasicBlock(\n",
       "            (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (downsample): Sequential(\n",
       "              (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "              (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            )\n",
       "          )\n",
       "          (1): BasicBlock(\n",
       "            (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "          (2): BasicBlock(\n",
       "            (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace=True)\n",
       "            (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    ))\n",
       "    (conv): TimeDistributed(Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1)))\n",
       "    (transformer): Transformer(\n",
       "      (layers_encoding): ModuleList(\n",
       "        (0): Encoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (1): Encoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (2): Encoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (3): Encoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (layers_decoding): ModuleList(\n",
       "        (0): Decoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_encoderDecoderAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (1): Decoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_encoderDecoderAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (2): Decoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_encoderDecoderAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (3): Decoder(\n",
       "          (_selfAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_encoderDecoderAttention): MultiHeadAttention(\n",
       "            (_W_q): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_k): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_v): Linear(in_features=256, out_features=64, bias=True)\n",
       "            (_W_o): Linear(in_features=64, out_features=256, bias=True)\n",
       "          )\n",
       "          (_feedForward): PositionwiseFeedForward(\n",
       "            (_linear1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "            (_linear2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          )\n",
       "          (_layerNorm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_layerNorm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (_dopout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (_embedding): Linear(in_features=256, out_features=256, bias=True)\n",
       "      (_linear): Linear(in_features=256, out_features=256, bias=True)\n",
       "    )\n",
       "    (decoder): TimeDistributed(Sequential(\n",
       "      (0): UpsampleBLock(in=256, out=128, blur=False, residual=False, act=ReLU(), attn=False, norm=None)\n",
       "      (1): UpsampleBLock(in=128, out=128, blur=False, residual=False, act=ReLU(), attn=False, norm=None)\n",
       "      (2): UpsampleBLock(in=128, out=64, blur=False, residual=False, act=ReLU(), attn=False, norm=None)\n",
       "      (3): UpsampleBLock(in=64, out=32, blur=False, residual=False, act=ReLU(), attn=False, norm=None)\n",
       "      (4): UpsampleBLock(in=32, out=16, blur=False, residual=False, act=ReLU(), attn=False, norm=None)\n",
       "      (5): Conv2d(16, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    ))\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As the `ImageSeq` is a `tuple` of images, we will need to stack them to compute loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls, model, loss_func=loss_func, splitter=partial(tf_split, stacked=True), cbs=[]).to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have a weird bug that if I use `nn.LeakyReLU` after doing `learn.lr_find()` the model does not train (the loss get stucked)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=0.017378008365631102, lr_steep=0.14454397559165955)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3ic1Zn38e+t3uUm924M2BgXLGzAQGApoQUDgQCmJlmMl0B2s9lsYJM3CUkIKZuQ0GJMCUlooVdjINmAITR3bOOCMMaW5SIXFauX+/1jxo4QY1uSNXpmRr/PdenyPOVo7sMg3TrnOcXcHRERkdaSgg5ARERikxKEiIhEpAQhIiIRKUGIiEhEShAiIhKREoSIiESUEnQAnalPnz4+fPjwoMMQEYkbixYt2u7uBZGuJVSCGD58OAsXLgw6DBGRuGFmn+7rmrqYREQkIiUIERGJSAlCREQiUoIQEZGIlCBERCQiJQgREYkooYa5iogkmsraBpZtLKdPbhoD8jPJy0jBzLrkvZUgRERi1LKNZXzjkcUU76rZey47LZljR/Xh95cfRWpydDuBlCBERGKMu/Pg2+v52dxV9M3NYPblR9HY7Gwuq+WjbZU8vrCYP73zKV8/fkRU41CCEBGJIbvrGvmvx5cxb+UWTh3Tl/+9aAI9stL2Xnd3tlTU8dvX1nLuhIEU5KZHLRY9pBYRiSF/fHs981Zu4XtnjeHeKws/kxwAzIwffmkstY1N/HLe6qjGogQhIhJDFq7fyaH9crjmxJH7fBg9qiCHr00bwROLilmyYVfUYlGCEBGJEe7O0o1lTBrS84D33nDKaPrmpvOj51fS3OxRiUcJQkQkRny6o5pd1Q1MHNrjgPfmpKdw01mHs6y4nCcXFUclHiUIEZEYsWRjqLtoUhsSBMB5EwcxeVhPfjFvNTX1TZ0ej0YxiYjEiCUbyshOS2Z039w23W9m/PS8ceysqiczLbnT41GCEBGJEUs2lDFhSA+Sk9o+U3rMgLyoxaMuJhGRGFDb0MSqzRVMHNK27qWuENUEYWZnmNkaMysysxsjXP+OmS0Nf60wsyYz69WWsiIiiWTFpnIam51JQw88gqmrRC1BmFkycBdwJjAWuNTMxra8x91/5e4T3X0icBPwhrvvbEtZEZFEsmRDGUC3aUFMAYrcfZ271wOPAdP3c/+lwKMdLCsiEteWbNzFkF6ZUV06o72imSAGARtbHBeHz32OmWUBZwBPtbesiEgiWLKhbRPkulI0E0Skx/D7mu73JeAf7r6zvWXNbKaZLTSzhaWlpR0IU0QkWFvKa9lcXhtT3UsQ3QRRDAxpcTwYKNnHvZfwz+6ldpV19znuXujuhQUFBQcRrohIMJa2c4JcV4lmglgAjDazEWaWRigJPN/6JjPLB74APNfesiIiiWDJhjLSkpMYOzB6cxo6ImoT5dy90cyuB14BkoEH3H2lmc0KX58dvvV84FV3rzpQ2WjFKiISpCUbyjhiUB7pKZ0/G/pgRHUmtbvPBea2Oje71fGDwINtKSsikmgam5r5YFMZl04ZGnQon6OZ1CIiAVq9pZLahuaYmiC3hxKEiEiAVpaUAzBhcH7AkXyeEoSISIB2VjUAxNQEuT2UIEREAlRZ20BykpGZGlsPqEEJQkQkUJW1jeRmpOxz/+kgKUGIiASosraB3IzY3JpHCUJEJECVtY3kpqcGHUZEShAiIgHa08UUi5QgREQCVFnXSG6GWhAiItJKZW0DeWpBiIhIa+piEhGRz3F3dquLSUREWquub6Kp2dWCEBGRz6qsbQRQC0JERD6rsja0DpNaECIi8hkVe1sQShAiItLCP1sQ3bCLyczOMLM1ZlZkZjfu456TzGypma00szdanF9vZsvD1xZGM04RkSDseQYRq/MgohaVmSUDdwGnAcXAAjN73t0/bHFPD+Bu4Ax332BmfVt9m5PdfXu0YhQRCVJ3fkg9BShy93XuXg88Bkxvdc8M4Gl33wDg7tuiGI+ISEzpzg+pBwEbWxwXh8+1dCjQ08xeN7NFZnZli2sOvBo+PzOKcYqIBKKytpHkJCMrLfY2C4IodjEBkXa/8AjvPxk4BcgE3jGzd919LTDN3UvC3U6vmdlqd5//uTcJJY+ZAEOHDu3UCoiIRFNlbQM56bG5WRBEtwVRDAxpcTwYKIlwzzx3rwo/a5gPTABw95Lwv9uAZwh1WX2Ou89x90J3LywoKOjkKoiIRE8sr8ME0U0QC4DRZjbCzNKAS4DnW93zHHCCmaWYWRYwFVhlZtlmlgtgZtnA6cCKKMYqItLlKmpjdx0miGIXk7s3mtn1wCtAMvCAu680s1nh67PdfZWZzQM+AJqB+9x9hZmNBJ4JN7tSgEfcfV60YhURCUIsbzcK0X0GgbvPBea2Oje71fGvgF+1OreOcFeTiEiiqqxtZEB+RtBh7JNmUouIBKSyLrZbEEoQIiIBqYzxZxBKECIiAXD3bj2KSURE9qGmYc9mQWpBiIhIC5UxvtQ3KEGIiAQi1tdhAiUIEZFAVOxd6ltdTCIi0oK6mEREJKJY300OlCBERAKhFoSIiESkh9QiIhJRZW0jZpCdpgQhIiItVNY2kpOeQlJSbG4WBEoQIiKBqKhtiOkhrqAEISISiFhfhwmUIEREAhHrmwWBEoSISCBifalviHKCMLMzzGyNmRWZ2Y37uOckM1tqZivN7I32lBURiVe762K/iylq0ZlZMnAXcBpQDCwws+fd/cMW9/QA7gbOcPcNZta3rWVFROJZd38GMQUocvd17l4PPAZMb3XPDOBpd98A4O7b2lFWRCQuhTYLaujWXUyDgI0tjovD51o6FOhpZq+b2SIzu7IdZQEws5lmttDMFpaWlnZS6CIi0VPX2ExDk8d8CyKa0UWa/eER3n8ycAqQCbxjZu+2sWzopPscYA5AYWFhxHtERGJJxZ5lNtK7b4IoBoa0OB4MlES4Z7u7VwFVZjYfmNDGsp2qsamZlSUVvP/JTpZuLGNUQTbTJw1iVEHO5+6tqG0gIyWZtJTOa4C5O7uqG9hcXsOW8lpKK+soraxjW2Udu6rrqW9spr6pmYamZtwhPzM19JWVSrIZWypq2VIe+qqsayQrLZmstBSy0pJJNqO6oZHquiaq65swgx5ZafTKTqVnVhpZacm4fzYDG2AGSWbkZqTQPz+TAfkZ9M/PYEB+Bn1zM0iO4RmgIrHsnwv1xXYXUzQTxAJgtJmNADYBlxB65tDSc8CdZpYCpAFTgduA1W0o2ylqG5qY+edFLFq/k6r6JgAG5mfw8orN3P5/RRw5KJ9zxg+gpqGJFZsqWFlSzubyWgD65KTRLy+Dgtx0ahuaKK9ppLy6nsraRprdSTIDg+QkIz8z9Mu4V3YauRkp1NQ3UVXfyO7aRsprGthSUUttQ/Pn4svPTKV3dhppKUmkpSSRmhxKSkXbdlNW00B5TQNNzU6/3HT652cwZmAeuekp1DSEkkF1fWP4egaZvZPJTkuhyZ2y6np2VtVTUlZBTThphJKC4e444A7N7pTXNFDX+NnYkpOM/nmhZDGsdzaj+mZzSEEOo/rmMKhHJhmpydH4uEQSQjys5ApRTBDu3mhm1wOvAMnAA+6+0sxmha/PdvdVZjYP+ABoBu5z9xUAkcpGI86M1GRSkowLjhrMlBG9mDKiF/3yMthaUcsLy0p4dukmbn15NWYwqiCHqSN6cfiAPGobmtga/qu9dHcdWakpDOqRydgBeeRlppBktvcXbFNz6Jfsrup6tlbUUrQt9Bd+dnoKPbLSGNIri9PG9qN/fiYDw3+l983LoE9OGukpB/5F29zsUV3PxcNJYnN5LZvLa0L/ltVSUl7Dpl01vFVUylOLiz9TJjstmT656fTOTuPQfrkcNbQnRw3rwcg+OTG99oxIV4iHvSAAzD1xuu0LCwt94cKFnf59S8pqyM9MJTvG+wuDVFnbwMelVXy8bTdbK2vZXlnPjqo6tlXUsbKkvMX2iimMGZDH4f1zOax/HocPyGXcwPxO7a4TiXVzl2/muocX8/K/n8CYAXmBxmJmi9y9MNI1/cZrg4E9MoMOIeblZqQycUgPJg7p8blrzc3Ouu1VLN6wiyUbyli9pYInFxXv7dLLTkvm2FF9+MJhBZx0aAFDemV1dfgiXSoe9oIAJQjpAklJxiF9czikbw5fKQyNPWhudjaV1bCypJy3irbz+ppS/rpqKwBDe2Ux7ZA+TDukN8eO7E3vnPQgwxfpdHpILbIfSUnGkF5ZDOmVxRnjBuAeamW8saaUtz/ezovLSnj0/Q2YwaQhPTj9iP588Yj+jOiTHXToIgdtT5drTox3W8d2dNJtmBmjCnIYVZDD144fQWNTM8s3lfPmR9t57cOt/Pzl1fz85dUc2i+HK44ZxkWFQzRSSuJWZW0DOekpMT9UXAlCYlJKchKThvZk0tCefPOU0Wwqq+G1lVt4ZmkJ/++5lfz2rx/x1WnDueKY4eRnxXYzXaS1eFiHCZQgJE4M6pHJ1dNGcNVxw3nvk53MfuNj/vfVtdz9+seccUR/zp04kOMP6UNKskZDSeyLh70gQAlC4oyZcczI3hwzsjerNlfwx7fXM3f5Zp5esone2Wl8acJAvnnKaHplpwUdqsg+xcNeEKANgySOjRmQx8+/PJ4F3z+VOVdM5phRvXn4vU857Tdv8PLyzUGHJ7JP8dLFpAQhcS89JZnTj+jPXTOO4oUbjmdAjwz+7eHF3PDoEnZW1QcdnsjnxMNS36AEIQnm8P55PHPdNL592qHMW7GZ0297gzfWahl4iS1qQYgEJDU5iRtOGc1z3zieXtlpXPXA+9z68ioamj6/GKJIEJQgRAI2dmAez33jeGZMHco9b6zjotnvsHFnddBhSTdX29BEfVMzeepiEglWZloyPzv/SO6acRQfl+7mrNvf5JWVW4IOS7qxeFnqG5QgpJs4e/wA5n7zBEb0yebaPy/i1pdX0aguJwlAvCzUB0oQ0o0M6ZXFE7OO5bJwl9Nl973HtsraoMOSbmZvCyJdXUwiMSU9JZlbzj+S33xlAsuKy/jSHW+xrnR30GFJN6IuJpEYd8FRg3nmumk0Njkz7n2PT3dUBR2SdBPxspscRDlBmNkZZrbGzIrM7MYI108ys3IzWxr++kGLa+vNbHn4fOdvEyfd3pgBeTz0r1OpbWxixr3vUbxLI5wk+raHJ2/Gw3IwUUsQZpYM3AWcCYwFLjWzsRFufdPdJ4a/ftzq2snh8xG3wxM5WGMG5PHQ16dSUdvAjHvfY0u5nklIdG3YUUVaShJ9c2N/I6w2JQgzyzazpPDrQ83sXDM7UPtoClDk7uvcvR54DJh+cOGKdL5xg/L509emsLOqnhn3vUtZtZbnkOj5dEc1Q3tlkRTje0FA21sQ84EMMxsE/A34KvDgAcoMAja2OC4On2vtWDNbZmYvm9kRLc478KqZLTKzmft6EzObaWYLzWxhaamWVJCOmTS0J/dfVcjGndV845HFmnUtUbNhZzXD4mTf9bYmCHP3auAC4A53P59Qt9F+y0Q4562OFwPD3H0CcAfwbItr09z9KEJdVN8wsxMjvYm7z3H3QncvLCgoaEtdRCKaOrI3Pzv/SP5RtIObX1gZdDiSgNydDTurGdo7wRKEmR0LXAa8FD53oDFaxcCQFseDgZKWN7h7hbvvDr+eC6SaWZ/wcUn4323AM4S6rESi6qLCIVz7hZE89O4G/vTO+qDDkQSzfXc91fVNCdeC+A/gJuAZd19pZiOBvx+gzAJgtJmNMLM04BLg+ZY3mFl/M7Pw6ynheHaEn3nkhs9nA6cDK9paKZGD8d9fPJxTx/Tl5hc+ZL5WgpVOtGFnaDh1QrUg3P0Ndz/X3X8Rfli93d2/eYAyjcD1wCvAKuDxcHKZZWazwrddCKwws2XA7cAl7u5AP+Ct8Pn3gZfcfV6HaijSTslJxm8vmcTovjlc/8hiLfAnnebTHaH/l4b2yg44krZp6yimR8wsL/zX/IfAGjP7zoHKuftcdz/U3Ue5+y3hc7PdfXb49Z3ufoS7T3D3Y9z97fD5deFzE8LXb+l4FUXaLyc9hTlXFOIO1z+6hPpGPbSWg/fpjmrMYEivzKBDaZO2djGNdfcK4DxgLjAUuCJqUYnEgKG9s/jFheNZtrGMX85bHXQ4kgA27KxmQF4G6SnJQYfSJm1NEKnheQ/nAc+5ewOfH5EkknDOOnIAVxwzjPve+oS/frg16HAkzn26oypunj9A2xPEPcB6IBuYb2bDgIpoBSUSS7539hjGDsjj208sY1NZTdDhSBzbsLOGYXHy/AHa/pD6dncf5O5necinwMlRjk0kJmSkJnPXZUfR2NTM9Y8spq6xKeiQJA5V1TWyfXdd4rUgzCzfzH6zZ8aymf2aUGtCpFsY0SebX100gSUbyrjp6eWEBtuJtN2GnXtGMCVYggAeACqBr4S/KoA/RCsokVh01pED+Naph/L04k3MfmNd0OFInNkzxHVYHLUg2rpjxSh3/3KL45vNbGk0AhKJZd885RCKSnfzy1dWM6ogm9OP6B90SBIn9kySS7hnEECNmR2/58DMpgF6WifdjpnxqwvHM35QPv/xl6V8WKKxGtI2n+6oJj8zlfys2N8oaI+2JohZwF3hTXzWA3cC10YtKpEYlpGazL1XFpKXkcqshxZRXd8YdEgSBzbsrI6r7iVo+yimZeEVV8cD4919EvAvUY1MJIb1zcvgd5dMZMPOan796tqgw5E4sGFndVw9oIZ27igXXn11T5v6P6MQj0jcmDqyN5dNHcof/vEJSzbsCjociWGNTc1s2lWTmC2IfYj97ZBEouzGMw+nX14GNz61XOs1yT6VlNXS2OyJ3YJoRQPBpdvLzUjlp+eNY83WSn7/+sdBhyMx6tM9y3zH0QgmOECCMLNKM6uI8FUJDOyiGEVi2ilj+vGlCQO58+8fsXZrZdDhSAyKxzkQcIAE4e657p4X4SvX3ds6h0Ik4f3wS2PJSU/hq39YwBMLN9KoPa2lhY07q0lLSaJ/XkbQobTLwXQxiUhYn5x07r2ykJ7ZqXznyQ849Tdv8NSiYiUKAUItiCE9M0lKiq9Ht1FNEGZ2hpmtMbMiM7sxwvWTzKzczJaGv37Q1rIisaZweC9euP547r2ykOz0FL79xDIuv/89PbwWPt1ZzbDe8fX8AaKYIMwsGbgLOBMYC1xqZmMj3Pqmu08Mf/24nWVFYoqZcdrYfrx4w/Hccv443l23k5+8+GHQYUmA3J0NO6ribgQTRLcFMQUoCm8fWg88BkzvgrIigTMzLps6jJknjuTP737K4ws2Bh2SBGRHVT1V9U1KEK0MAlr+VBSHz7V2rJktM7OXzeyIdpYViWn//cXDOP6QPnz/2RWaTNdNxesIJohugoj0NKb13InFwLDwMh53AM+2o2zoRrOZe/apKC0t7XCwItGQkpzEHZdOol9+OrMeWsS2ytqgQ5IuVrwrlCCGqAXxGcXAkBbHg4GSljeEl+7YHX49l9De133aUrbF95jj7oXuXlhQUNCZ8Yt0ip7ZadxzeSEVNY38+6NLaW7WHNPuZGtF6I+CAfnxNcQVopsgFgCjzWyEmaUBlwDPt7zBzPqbmYVfTwnHs6MtZUXiydiBedx87hG8s24Hf3xnfdDhSBfaXF5LTnoKuRnxs8z3HlFLEO7eCFwPvAKsAh5395VmNsvMZoVvuxBYYWbLgNuBS8J7XkcsG61YRbrCRYWDOfmwAn4xbzXrSncHHY50kS3ltfTLSw86jA6xRNpbt7Cw0BcuXBh0GCL7tLWiltNvm8/IgmyenHUcyXE2cUra77y7/kFOegoP/evUoEOJyMwWuXthpGuaSS3ShfrlZfDj6UewZEMZc+ZrX+vuYGtFLf3j8PkDKEGIdLlzJwzkzHH9ue21tazZosX9ElljUzPbKuvibg2mPZQgRLqYmfHT88aRl5nCNX9ayOZybe+eqLbvrqep2dWCEJG26x1e3G9nVT0z7n1v71BISSxb4niIKyhBiARm0tCe/PFrR7OtopYZ975LaWVd0CFJJ9sSbh2qBSEi7TZ5WC/+8NUplJSFksT23UoSiWRzeagFoWcQItIhU0b04oGrj2bjrmpueGRJh2daN2mGdszZUl5LWnISvbLTgg6lQ5QgRGLAsaN684NzQjOtH3l/Q7vLf1hSwcQfv8oNjy6hvLohChFKR2wJD3ENLxgRd5QgRGLEpVOGMO2Q3tw6d9XeBd7aorymgX97eBHJScbLyzdzxu/m84+i7VGMVNpqc3lt3HYvgRKESMwwM35+wXgcuOnp5bRllYPmZufbjy9l064a7r+qkKevO47MtGQuu+89fvLih9Q1NkU/cNmnLeXxO0kOlCBEYsqQXlnceObhvPnRdh5feOBNhn7/xsf8ddU2vnf2GCYP68X4wT146YYTuOrYYdz/1ifc/IJ2swuKu7OlojZuh7iCEoRIzLl86jCmjujFT19ctd9JdG99tJ1fv7qGcycM5Orjhu89n5mWzM3Tx3HtF0byyHsbeOmDzV0QtbS2q7qB+sZmtSBEpPMkJRm/vHA8Dc3NXP3AAkrKPp8kFqzfyfWPLmZUQQ63XnBkxIeg/3X6YUwc0oMbn/6AjTvb/kxDOsee5K5nECLSqYb1zub+q46mpKyGC+5+m9VbKvZee+z9Dcy49116ZqVx31WFZKenRPweqeHd7HC44dElNDQ1d1X4Quj5A8TvJDlQghCJWdMO6cPjs44F4KLfv8P8taX86PmV3Pj0co4d1Ydnr5vGsN7Z+/0eQ3pl8fMvj2fpxjL+99U1XRG2hP1zmY3MgCPpOCUIkRg2ZkAeT193HAN6ZHDlA+/z4NvrueaEETxwVSH5WW3boezs8QOYMXUo97yxTsNfu9CW8lqSDPrkxOckOVCCEIl5A3tk8sSs47ho8mB+e/FEvnf2WFKS2/ej+4NzxjKiTzY3Pb2cmnoNfe0Km8tr6Zub0e7PKpZENXIzO8PM1phZkZnduJ/7jjazJjO7sMW59Wa23MyWmpm2iZNuLT8zlV9dNIHzJg3qUPmM1GRuOX8cG3ZW89u/re3k6CSSeJ8DAVFMEGaWDNwFnAmMBS41s7H7uO8XhPafbu1kd5+4r+3wRKTtjhvVh4sLh3Dfm5+wYlN50OEkvHifAwHRbUFMAYrcfZ271wOPAdMj3HcD8BSwLYqxiAjwP2eNoWdWGjc9vZxGjWqKqi3ltfSL4yGuEN0EMQhoORW0OHxuLzMbBJwPzI5Q3oFXzWyRmc2MWpQi3Uh+Vio/OncsyzeV8+Db64MOJ2FV1jawu65RLYj9iLR8YevFZX4LfNfdIz01m+buRxHqovqGmZ0Y8U3MZprZQjNbWFpaenARi3QDZx85gFMO78uvX13L+u1VQYeTkBJhDgREN0EUA0NaHA8GSlrdUwg8ZmbrgQuBu83sPAB3Lwn/uw14hlCX1ee4+xx3L3T3woKCgs6tgUgCMjN+ct440lOTmPnnheyuaww6pISzZw5EPM+ihugmiAXAaDMbYWZpwCXA8y1vcPcR7j7c3YcDTwLXufuzZpZtZrkAZpYNnA6siGKsIt3KwB6Z3HnpURRt2823H1/a4U2KJLI9O8nF8yQ5iGKCcPdG4HpCo5NWAY+7+0ozm2Vmsw5QvB/wlpktA94HXnL3edGKVaQ7On50H/7nrDG8snIrd/69KOhwEsqeLqa+eekBR3JwIi/i0kncfS4wt9W5SA+kcferW7xeB0yIZmwiAl8/fgQrSyr4zWtrGTMgj9PG9gs6pISwpaKW3tlpZKQmBx3KQYnfKX4ictDMjFsvOJIjB+Xzrb8sZe3WyqBDSgiJMMQVlCBEur2M1GTuuWIymWnJfPUPC9hWWRt0SHFvc3n8T5IDJQgRIfTQ+oGrjmZnVT3X/HGh1ms6SFvKa+J+iCsoQYhI2JGD87n90kl8sKmcf39sCU0a2dQhtQ1N7KpuiPshrqAEISItnDa2Hz84ZyyvfriVn81dFXQ4cWlrRWJMkoMoj2ISkfjz1Wkj+HRHNfe/9QmH9c/lK4VDDlxI9tqcILOoQS0IEYng+2ePYdohvfn+sytYXqyVX9tj067QXtSDesT3JDlQghCRCFKSk7j9kkkU5KQz66FF7KyqDzqkuLGpLJQgBipBiEii6p2Tzu8vP4rS3XV881E9tG6rTbtq6JOTHveT5EAJQkT2Y/zgHvx0+jjeKtrOr19dE3Q4cWFTWQ2DesZ/6wGUIETkAL5y9BAunTKUu1//mEfe2xB0ODGveFc1g5UgRKS7uPncIzj5sAK+9+xynlpUHHQ4Mau52Skpq2VwAjx/ACUIEWmDtJQkfn/5ZI4b1ZvvPLmMFz9ovbWLAGzfXUd9U7O6mESke8lITebeKwuZPKwn//HYUl5duSXokGJOcVniDHEFJQgRaYestBQeuPpojhiUzzceWcwzS9Td1NLeORBqQYhId5SbkcqfvjaFycN68q2/LON3f/0Idw2BhX/OgVALQkS6rfzMVP70talccNQgbvvrWv7riQ+ob2wOOqzAFe+qJj8zldyM1KBD6RRRTRBmdoaZrTGzIjO7cT/3HW1mTWZ2YXvLikgw0lKS+PVFE/jWqYfy1OJirnrgfXbXNQYdVqA27apJmNYDRDFBmFkycBdwJjAWuNTMxu7jvl8Q2ru6XWVFJFhmxr+fOprbLp7A++t3cuX971FR2xB0WIFJpElyEN0WxBSgyN3XuXs98BgwPcJ9NwBPAds6UFZEYsD5kwZz14yjWL6pnMvve4+y6u63dpO7s2lXTcJMkoPoJohBwMYWx8Xhc3uZ2SDgfGB2e8u2+B4zzWyhmS0sLS096KBFpGPOGNef2ZdPZvXmSmbc+163W+CvvKaBqvomdTG1kUU413qow2+B77p76/0N21I2dNJ9jrsXunthQUFBB8IUkc5yyph+3HdVIR+X7uYr97zDJ9urgg6pyxSHh7iqBdE2xUDLnUYGA62nXxYCj5nZeuBC4G4zO6+NZUUkBp14aAEPfnUKO3bXce4dbzFvxeagQ+oSxXv3gcgKOJLOE80EsQAYbWYjzCwNuAR4vuUN7j7C3Ye7+3DgSeA6d3+2LWVFJHYdO6o3L3uS6/gAAA1fSURBVH7zBEb2zWHWQ4u55aUPaWhK7GGwe+dAqAVxYO7eCFxPaHTSKuBxd19pZrPMbFZHykYrVhHpfIN6ZPL4tcdw5bHDuPfNT7jsvvfYvrsu6LCiZtOuGrLSkumZlRhzIAAskWZAFhYW+sKFC4MOQ0RaeW7pJr771Af0zk5nzpWTOWJgftAhdbpr/7yQdaVVvPafXwg6lHYxs0XuXhjpmmZSi0jUTZ84iCdnHYe78+Xfv52Qq8Em2hwIUIIQkS4yblA+z11/POMG5nP9I0v49atrEmoNp+IEm0UNShAi0oUKctN5+JqpXFw4hDv+r4jvPvUBjQnw8LqqrpGy6oaEa0GkBB2AiHQv6SnJ/PzLR9I/P4Pf/e0jdlY1cOeMSWSkJgcdWoftGcE0uGfiDHEFtSBEJABmxrdOO5SfTD+Cv63eyhX3v0d5dfyu4bR3Hwh1MYmIdI4rjh3OHZdOYunGMi699924XQ22uCzxZlGDEoSIBOyc8QOZc2Uha7ZWcsMji+PymcSmXTWkJSdRkJMedCidSglCRAJ38mF9ufncI/j7mlJ++tKqoMNpt+Jd1QzskUFSUqRl5OKXHlKLSEy4/JhhrN9exX1vfcLw3llcPW1E0CG1WSLOgQC1IEQkhtx01hhOG9uPH7/4IfNWbAk6nDZLtJ3k9lCCEJGYkZxk/O6SiRwxMJ9ZDy1i5p8WUrRtd9Bh7VddYxPbKusSahXXPZQgRCSmZKWl8Jdrj+Hbpx3K2x/v4PTb3uCmpz9g/faqmJx5vWxjOZBYq7juoWcQIhJzstJSuOGU0cyYOpQ7/q+Ih9/7lEff30i/vHQKh/fi6GE9+eK4/gzID+6XctG2Sn73tyJe/KCEnPQUpgzvFVgs0aLVXEUk5hXvqubva0pZ8MlOFqzfyebyWvIzU5l9+WSOHdW7y+Kob2zm7Y+389TiTbz4QQmZqclcddxwrjlhJL2y07osjs60v9VclSBEJO58tLWSf3t4Meu3V3HL+eO4+OihUX2/+WtLeW5pCa99uIWK2kZy0lO47JihzDxhJL3jfO7D/hKEuphEJO6M7pfL09cdxzceXsx3n1rOutIq/vuMw0mOwjyEOfM/5mdzV5OXkcKpY/tx1rgBHD+6T1yvHdVWShAiEpfyMlL5w9VHc/MLH3LP/HV8XLqb2y6eSG5G5+3o9uj7G/jZ3NWcPX4At31lImkp3WtcT1Rra2ZnmNkaMysysxsjXJ9uZh+Y2VIzW2hmx7e4tt7Mlu+5Fs04RSQ+pSQn8ZPzxu2dhX3+3W/zyfaqTvneLywr4X+eWc5JhxV0y+QAUUwQZpYM3AWcCYwFLjWzsa1u+xswwd0nAl8D7mt1/WR3n7iv/jEREYCrjhvOn78+hR2765h+51u8sba0w9+rudn564db+dZflnL0sF78/rLJ3TI5QHS7mKYARe6+DsDMHgOmAx/uucHdW86AyQYS54m5iHSp40b14fnrj+eaPy3kq394n++ecTgzTxyJ2f6fSyzZsIsH317PJ9urKK2so7SyjsZm58hB+dx/dSGZaYn/rGFfopkgBgEbWxwXA1Nb32Rm5wO3An2Bs1tccuBVM3PgHnefE+lNzGwmMBNg6NDojmQQkdg2pFcWT193HN954gNufXk1HxSX84sLx5OT/tlfde7OOx/v4K7Xi/hH0Q7yM1OZOKQHh/XLpSA3nf75GUyfMKhTn2fEo2gmiEhp+3MtBHd/BnjGzE4EfgKcGr40zd1LzKwv8JqZrXb3+RHKzwHmQGiYa6dFLyJxKSsthTtnTGL8/Hx+MW81a7dWMvuKyYzsk83qLZX8bdVW5q3cwopNFfTNTef7Z4/h0ilDyU7XmJ3WovlfpBgY0uJ4MFCyr5vdfb6ZjTKzPu6+3d1Lwue3mdkzhLqsPpcgRERaMzOu/cIoxg3K5/pHFjP9zn+Qn5m6d2vQCYPz+el547hw8uBuMVy1o6KZIBYAo81sBLAJuASY0fIGMzsE+Njd3cyOAtKAHWaWDSS5e2X49enAj6MYq4gkoGmH9OGFG47nh8+txAxu+JdD+JfD+9I3LyPo0OJC1BKEuzea2fXAK0Ay8IC7rzSzWeHrs4EvA1eaWQNQA1wcThb9CHU77YnxEXefF61YRSRxDe6Zxf1XHx10GHFJS22IiHRj+1tqo3sO7hURkQNSghARkYiUIEREJCIlCBERiUgJQkREIlKCEBGRiJQgREQkooSaB2Fm5cBHLU7lA+VtfN0H2N7Bt275/dp7vfW1th53dR32d0+k8/urR6TXLc91tB5B16Hl61j9LNpSp1ivQ+tj/Wzv34HqMMzdCyJecfeE+QLm7Ov4QK+BhZ31vu25vr+Y93fc1XXY3z2Rzrf3s2h1rkP1CLoO8fBZtKVOsV6HWPosEuVne19fidbF9MJ+jtvyurPetz3X9xfz/o67ug77uyfS+fZ+FolQh7bGcCDRrEdb6hTrdWh9rJ/t/evw90ioLqaDYWYLPc53rkuEOkBi1EN1iB2JUI+g6pBoLYiDEXFDojiTCHWAxKiH6hA7EqEegdRBLQgREYlILQgREYlICUJERCJSghARkYiUINrAzE4ws9lmdp+ZvR10PB1hZklmdouZ3WFmVwUdT0eY2Ulm9mb4szgp6HgOhpllm9kiMzsn6Fg6wszGhD+HJ83s34KOp6PM7Dwzu9fMnjOz04OOpyPMbKSZ3W9mT3b29074BGFmD5jZNjNb0er8GWa2xsyKzOzG/X0Pd3/T3WcBLwJ/jGa8kXRGHYDpwCCgASiOVqz70kl1cGA3kEEAdYBOqwfAd4HHoxPl/nXSz8Sq8M/EV4BAhpB2Uj2edfdrgKuBi6MYbkSdVId17v71qATY0Rl28fIFnAgcBaxocS4Z+BgYCaQBy4CxwJGEkkDLr74tyj0O5MVjHYAbgWvDZZ+M0zokhcv1Ax6O1/+fgFOBSwj9UjonHusQLnMu8DYwI14/ixblfg0cFed16PSf6xQSnLvPN7PhrU5PAYrcfR2AmT0GTHf3W4GITX4zGwqUu3tFFMONqDPqYGbFQH34sCl60UbWWZ9D2C4gPRpxHkgnfRYnA9mEfuhrzGyuuzdHNfAWOuuzcPfngefN7CXgkehFHFknfRYG/Bx42d0XRzfiz+vkn4tOl/AJYh8GARtbHBcDUw9Q5uvAH6IWUfu1tw5PA3eY2QnA/GgG1g7tqoOZXQB8EegB3Bnd0NqlXfVw9+8BmNnVwPauTA770d7P4iTgAkKJem5UI2uf9v5c3ECoRZdvZoe4++xoBtdG7f0segO3AJPM7KZwIukU3TVBWIRz+50x6O4/jFIsHdWuOrh7NaEkF0vaW4enCSW6WNPu/58A3P3Bzg+lw9r7WbwOvB6tYA5Ce+txO3B79MLpkPbWYQcwKxqBJPxD6n0oBoa0OB4MlAQUS0epDrEjEeqRCHWAxKhHzNShuyaIBcBoMxthZmmEHhg+H3BM7aU6xI5EqEci1AESox6xU4cgRh908SiBR4HN/HN459fD588C1hIaLfC9oONUHWK/DolSj0SoQ6LUI9broMX6REQkou7axSQiIgegBCEiIhEpQYiISERKECIiEpEShIiIRKQEISIiESlBSEIzs91d/H6dsl9IeO+LcjNbYmarzex/21DmPDMb2xnvLwJKECLtYmb7Xb/M3Y/rxLd7090nAZOAc8xs2gHuP4/QCrEinaK7LtYn3ZiZjQLuAgqAauAad19tZl8Cvk9oDf4dwGXuvtXMfgQMBIYD281sLTCU0Hr9Q4HfemjRN8xst7vnhFc7/RGwHRgHLAIud3c3s7OA34SvLQZGuvs+l3F29xozW0polU/M7BpgZjjOIuAKYCKh/Rm+YGbfB74cLv65eh7EfzrpZtSCkO5oDnCDu08G/gu4O3z+LeCY8F/tjwH/3aLMZEJr8s8IHx9OaOnxKcAPzSw1wvtMAv6D0F/1I4FpZpYB3AOc6e7HE/rlvV9m1hMYzT+XaX/a3Y929wnAKkLLM7xNaL2e77j7RHf/eD/1FGkTtSCkWzGzHOA44InQXjHAPzcfGgz8xcwGEPrr/JMWRZ9395oWxy+5ex1QZ2bbCO1y13ob1PfdvTj8vksJtUB2A+vcfc/3fpRQayCSE8zsA+Aw4OfuviV8fpyZ/ZTQvhg5wCvtrKdImyhBSHeTBJS5+8QI1+4AfuPuz7foItqjqtW9dS1eNxH5ZynSPZHW+t+XN939HDM7FHjLzJ5x96XAg8B57r4svOnQSRHK7q+eIm2iLibpVjy0ZewnZnYRhLacNLMJ4cv5wKbw66uiFMJqYGSLbSYvPlABd18L3Ap8N3wqF9gc7ta6rMWtleFrB6qnSJsoQUiiyzKz4hZf/0nol+rXzWwZsBKYHr73R4S6ZN4k9AC504W7qa4D5pnZW8BWoLwNRWcDJ5rZCOD/Ae8BrxFKOHs8BnwnPDR2FPuup0ibaLlvkS5mZjnuvttCDwfuAj5y99uCjkukNbUgRLreNeGH1isJdWvdE3A8IhGpBSEiIhGpBSEiIhEpQYiISERKECIiEpEShIiIRKQEISIiESlBiIhIRP8faLEccxaY6YcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.129871</td>\n",
       "      <td>0.129139</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.125856</td>\n",
       "      <td>0.139628</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.130224</td>\n",
       "      <td>0.139459</td>\n",
       "      <td>00:29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.123834</td>\n",
       "      <td>0.124233</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.128979</td>\n",
       "      <td>0.119502</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.116724</td>\n",
       "      <td>0.128846</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.113866</td>\n",
       "      <td>0.114182</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.115080</td>\n",
       "      <td>0.123263</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.119888</td>\n",
       "      <td>0.122002</td>\n",
       "      <td>00:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.112150</td>\n",
       "      <td>0.119879</td>\n",
       "      <td>00:29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, 5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p,t = learn.get_preds()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, the results is a list of 5 tensors with 100 samples each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, torch.Size([100, 2, 64, 64]))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(p), p[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_res(t, idx, argmax=False):\n",
    "    if argmax:\n",
    "        im_seq = ImageSeq.create([t[i][idx].argmax(0).unsqueeze(0) for i in range(5)], TensorMask)\n",
    "    else:\n",
    "        im_seq = ImageSeq.create([t[i][idx] for i in range(5)])\n",
    "    im_seq.show(figsize=(8,4));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAABnCAYAAACNd3IfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMsElEQVR4nO3df0xV9R/H8de9gAIWJIxyUWA6ylZZGy2h6IfTWQlaWq0yZlbOMCf9sI20tbXamlut/mCzHI7ETSeVFLKSXLpBY22o4TLYyESIxB8bAXIxft1zvn/wvQzioke518M99/nY2IIPnL19dzgvzo/P57hM0xQAALg4t90FAAAQCghMAAAsIDABALCAwAQAwAICEwAACwhMAAAsiLzEOHNOAADhxuXvi5xhAgBgAYEJAIAFBCYAABYQmAAAWEBgAgBgAYEJAIAFBCYAABYQmAAAWEBgAgBgAYEJAIAFBOZ/tLW16ejRo3aXAQCYZMIyMA3DkNfr9fvx5ZdfasWKFRocHJTX65VhGHaXCwCYBFymedH11R25+HpRUZHee+89v2Pd3d3q7e1VUlKSJGnVqlXavHnz1SwPAGAvv4uvX+ptJSHr2LFjqqys9Du2f/9+DQ4O6rPPPrvoNj788EOdP38+GOUBAEKMYwLT4/FoYGBg+POysjK9//77kqSbb75Z06ZNG/X9WVlZys3Nlcs19g8JwzDU1dWl4uLioNYcLgYGBtTT06P4+Hi//QaAUOCYwMzLy9OuXbuGPx95qbmyslK33377mJ8Z7+Dd2dmppKQkGYbh9+dweerq6vTwww+rvb1dsbGxdpcDAFfEMfcw29raxr18OnPmTEVHR1ve1j///KPExER9/fXXysrK0owZMwJVpmM1NTWpoKDA71hra6tqa2uVnZ2t6Oho3XTTTZe8HA4ANvJ7NuWYwAyUs2fP6siRI8rOztaff/6p2bNn213SpNHT06PGxka/YzU1NXr99df17LPPaurUqeNuo7a2Vi6XS/X19cEqM6w0NTUpIiJCqampdpcCOEl4PfRzJQzDUHFxsTZt2qTIyMiwv99mmuaoS9t//PGH0tPTJUlut1tu9+hZSdHR0dq2bduY+8U+hmHo448/1o4dO4JXtANdbGrThg0bNH36dG3btk3S0G2GcN9vgWAhMP/P6/Xq/vvvV0NDg9LS0nTkyJGwv99WWlo66jLrhQsXhv978+bNysvLG/X9Lpdr3J55vV498MADqq+vV0pKSnAKdqj8/HxVVFT4HTtz5owiIiJ04MABSVJhYaGWLl16NcsDwgaBOUJLS4s8Ho+ioqJ07bXX2l2O7dLT07Vx40a/YwsWLLjsHrW0tCgnJ0fPPPNMIMpzlKqqKv3+++9+xw4cOKD4+Hi99tpr4/681+tVQUGBurq6glUiEPYITEmDg4PyeDzyer2aPn26Zs2aZXdJk0JaWprS0tImvB1ffw3D0OLFi7V8+fIAVBfafP3wKSws1J49eyQNTYOKiIgY9f2LFy8ec0bv4+vvBx98ELyCw0hfX58GBgZ0zTXX2F0KJhkCU0PTHjIzM+X1erVjxw7l5ubaXZKjjOwvhu4Np6en68SJE8Nf8/UmIiJCx44dG3P2frH7kkePHlVGRgb9DZCioiJt2bJF9fX13A/GKASmhg5gIw9Y/JIEzqeffqry8nK5XC5VV1frzjvvtLsk27lcLpWXl6u3t9fv2LRp08Y8UHUxvgez6K911dXVKioq8jt25MgRtbS0aOXKlZKkBx98UGvWrLma5WGSCvvAbG1t1eHDh+VyufTEE09wOTbAampqVF1draioKKWnp4f9g1Q+c+bMCch2fPuv2+2mv//R2dmpU6dO+R0rKyvTN998o6VLl475A3nu3LmaO3euBgYG9NNPPykqKorAhCQCU/n5+fruu+8UFRWlnTt3csAJoJFTUiIjw35XCzjTNPXGG2+orKzsshbmcKr/zin/6quv9Oqrr0ryP91m5syZ2r1797hXlEzT5AG1APP9PwrVq3hh+XovBN/58+c1Z84c7du3T88//7xOnz6tmJgYu8tyjJH9fe6553TmzJmw7+8777yj2267bfhj5BPe+/fvV3t7+6iPurq6cQ/cvv7+8MMPV6t8xzMMQ4sWLVJ5ebndpVyxsP2zv7e3V6WlpWpoaNBdd92l1atXKyoqyu6yHMMwDB0/flymaSo2Nlbx8fF2l+Qo9Hesxx57TDfccIPfsbvvvlvXXXed5W35+rtu3To99dRTgSrR8UzT1N69e9XR0TFmzOv16tChQyotLVVnZ6ckadmyZSG174Z1YL700ksyTVOvvPKK8vPz7S7JMfr7+4fX9U1ISND06dNtrshZ+vv71d3dLWmov5cTBE42f/58zZ8/f8LbGdnfdevWBex+s5P8+++/flegMk1T69evV0dHh9952rGxsaqqqtLBgwd17tw5ZWRkEJgIb1988YXefvttmaap6upq3vgSYFu3btWGDRvob5AUFRXpzTffHHNPFENM09S9996r48eP+x0fGBhQYWGh1q5dO+42urq6lJSUFKwSgyYsA7O6ulpbt26VaZr6/PPP9eijj9pdkqMMDg4Ov5s0MjLysqZI4NLob3CYpqmNGzfqxx9/VGJior799luWcdTQ1bi33npLg4ODkob61NLSooGBASUkJKiiomLMQhu33HLLmK/5/Pzzz9q6detF10ierMIyMA8fPqyysjLl5ORoyZIlSk5OtrskR/D9IrW0tCg6OloLFy4cdyF2XD76G1ymaWrXrl1qbW1VWlqa5s2bF7JPcwba2bNn1d/fP/y579L39ddfr8zMzMvqU11dnfbs2aOcnJyQ23/DMjBN01RKSor27t3LL0QAmaaprKwsnTp1Srfeeiv9DbCR/U1LS6O/AcYlWP+io6OHl20MBMMwlJycHJL7b1gFpmEYys7O1qFDh5SYmGh3OQAmicbGRi1fvlxtbW366KOPtHbt2pA7mE92hmEoJydHhw4dCqkHfUYKq8A0TVO//fabMjIy/K7wgSvX0dGhyspKeTweZWdn098A6+zs1L59++TxePT444/rySefpL8BdOHCBTU0NEiSkpKSePI4wHz77y+//KLMzMyQfQVdWAWmz9NPP61Vq1bZXYajnDx5UitWrFBCQoJefPFFVkgJMF9/paH99+WXX7a5Iufo6+tTT0+PpKFpOqF2Xy0UNDc3D++/y5cv1+rVq22u6MqEZWAiOCIjI3XixAnFxcXZXQpg2Zo1a7R79272X1wSgYkJKy4uVkVFhSSmOQTD9u3bVV5eLrfbrbKyMs2bN8/ukhylv79f/f39ioyMZP8NgpKSEu3du1dut1t79uxRRkaG3SVdsbAKTJfLpUceeUSpqal2l+IolZWVqq6u1sKFC8ede4Ur5+vvokWLtGjRorBfMzZQvF6v/v77b7W3tyshIUH33Xcf+28QVFZWqqysTBEREVq4cGFIv5g7rALT7XZr586ddpfhSEuWLNH27dvtLsOxcnJyVFJSYncZjtLT06NZs2bJMAytXLmS/gaZEx5S49oDMEl5PB5lZmbq+++/t7sU4LL19PQM779Lly5Va2tryL8+MazOMIFQYhiGamtr9cILL2jZsmV2l+Mof/31lw4ePCjTNJWbm0t/g8Dr9aq2tlaGYSguLk4zZsywu6QJIzAxYbGxsTyKH0QFBQW644477C7DUSoqKrR+/XrFxcXp3Xff5Y0kAeb1etXX1ydp6PgQ6meWPgQmJqyoqMgR9ycQXlJSUtTY2Mh7cIOgpKRE+fn5MgxDVVVVuueee+wuKSAITEwYBxyECtM09cknn6iiokJut1tTpkzhj70g6O3tHV4MIjo6WpGRzogaZ/wrAAeKiIhQZmamYy5nTQamaWrLli3q7+/XQw89ZHc5jnTu3DmdPn1aUVFRSk9Pd9T+67rECv0s3w/AMQzD0OzZs1VQUKC8vDy7y3Ec0zQ1a9YsNTc3a+bMmWpqagrVM3i/RTOtBAAQUJs2bVJNTU2ohuW4uCQLAJgwj8ej2tpadXd3Kzk5WTfeeKPdJQUcgQkgbLhcLsXGxmrKlCl2l+I4zc3NWrBggWJiYhzbXwITQNhwuVz69ddfWTM2SNxut06ePKnExES7SwkKAhNAWJk6dardJThaTEyMY6aR/BcP/QAAYAGBCQCABczDBABMWG9vr9ra2pSamuqEe8R+58MQmAAAjMbCBQAAXCkCEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALCAwAQAwAICEwAACwhMAAAsIDABALAg8hLjrqtSBQAAkxxnmAAAWEBgAgBgAYEJAIAFBCYAABYQmAAAWEBgAgBgwf8AdPZYDAxB/OoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAABnCAYAAACNd3IfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAABxElEQVR4nO3XsQ3CQBBFQYwohH5oiMwioyH6oZOlAWS9CBtpJrxLfva0y8ycAIBt570HAMA/EEwACAQTAALBBIBAMAEgEEwACC5bn9f7a/3VEAA4gvfz9vj27sIEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAAsEEgEAwASAQTAAIBBMAgmVm9t4AAIfnwgSAQDABIBBMAAgEEwACwQSAQDABIPgA1csKyc+RHVoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "k = random.randint(0,99)\n",
    "show_res(t,k)\n",
    "show_res(p,k, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.5.1'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}