DQN_Batch_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "require 'os'\n",
    "require 'nn'\n",
    "require 'rnn'\n",
    "require 'optim'\n",
    "require 'cunn'\n",
    "require 'cunnx'\n",
    "require 'cutorch'\n",
    "require 'parallel'\n",
    "\n",
    "dl = require 'dataload'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "...Utils file loaded\t\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "...Utils file loaded\t\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dofile(\"Code/utils.lua\")\n",
    "dofile(\"Code/utilsNNbatch.lua\")\n",
    "dofile(\"Code/Utils/load_cnn.lua\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8\t\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.getnumthreads()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.setnumthreads(torch.getnumthreads())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "outputpath = '/home/francisco/GitHub/DQN-Event-Summarization/data/training_ss/'\n",
    "datafile = \"cnn_data_ss.dat\"\n",
    "\n",
    "data = torch.load(outputpath .. datafile)\n",
    "\n",
    "queries = data[1]\n",
    "trueSummaries = data[2]\n",
    "sentences= data[3]\n",
    "\n",
    "n = 1000\n",
    "\n",
    "queries = queries[{{1, n}}]\n",
    "trueSummaries = trueSummaries[{{1, n}}]\n",
    "\n",
    "tmp = {}\n",
    "for j=1,#sentences do \n",
    "    tmp[j] = sentences[j][{{1, n}}]\n",
    "end\n",
    "\n",
    "sentences = tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size = 20001\n",
    "learning_rate = 1e-5\n",
    "embDim = 50\n",
    "gamma = 0.\n",
    "batch_size = 25\n",
    "mem_multiplier = 1\n",
    "cuts = 4\n",
    "endexplorerate = 0.8\n",
    "base_explore_rate = 0.1\n",
    "nepochs = 100\n",
    "epsilon = 1.\n",
    "print_perf = false\n",
    "adapt = false\n",
    "adapt_lambda = 0.25\n",
    "usecuda = false\n",
    "seedval = 420"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "...running on CPU\t\n"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manualSeed(seedval)\n",
    "if usecuda then\n",
    "    Tensor = torch.CudaTensor\n",
    "    LongTensor = torch.CudaLongTensor   \n",
    "    ByteTensor = torch.CudaByteTensor\n",
    "    maskLayer = nn.MaskedSelect():cuda()\n",
    "    print(\"...running on GPU\")\n",
    "else\n",
    "    Tensor = torch.Tensor\n",
    "    LongTensor = torch.LongTensor\n",
    "    ByteTensor = torch.ByteTensor\n",
    "    maskLayer = nn.MaskedSelect()\n",
    "    print(\"...running on CPU\")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SKIP = 1\n",
    "SELECT = 2\n",
    "\n",
    "k = sentences[1]:size(2)\n",
    "n = queries:size(1)\n",
    "q = queries:size(2)\n",
    "n_s = #sentences\n",
    "\n",
    "optimParams = { learningRate = learning_rate }\n",
    "delta = cuts / nepochs\n",
    "end_baserate = torch.round(nepochs * endexplorerate )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qTokens = {}\n",
    "for i=1, n do\n",
    "    qTokens[i] = Tokenize(unpackZeros(trueSummaries[1]:totable()))\n",
    "    -- Tokenize({trueSummaries[i]:totable()})\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Running bag-of-words model to learn f1\t\n"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- qTokens = {}\n",
    "-- for i=1, n do\n",
    "--     qTokens[i] = Tokenize({trueSummaries[i]:totable()}, false)\n",
    "-- end\n",
    "\n",
    "-- Building the model\n",
    "model = buildModel('bow', vocab_size, embDim, 'f1', adapt, usecuda)\n",
    "params, gradParams = model:getParameters()\n",
    "\n",
    "if adapt then \n",
    "    criterion = nn.ParallelCriterion():add(nn.MSECriterion()):add(nn.BCECriterion())\n",
    "    criterion[\"weights\"] = {1, adapt_lambda}\n",
    "else \n",
    "    criterion = nn.MSECriterion()\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "j = 1\n",
    "epoch = 1\n",
    "i = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Running model with 1000 queries and 122 sentences\t\n"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(string.format(\"Running model with %i queries and %i sentences\", n, n_s)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mem_multiplier = 0.02\n",
    "qValues = {}\n",
    "qActions = {}\n",
    "qPreds = {}\n",
    "rewards = {}\n",
    "lossfull = {}\n",
    "rouguef1 = {}\n",
    "rougue_scores = {}\n",
    "\n",
    "totalPredsummary = LongTensor(n, n_s * k):fill(0)\n",
    "\n",
    "memfull = false\n",
    "curr_memsize = 0\n",
    "memsize = torch.round(n * n_s * mem_multiplier)\n",
    "queryMemory = Tensor(memsize, q):fill(0)\n",
    "qActionMemory = Tensor(memsize, 2):fill(0)\n",
    "predSummaryMemory = Tensor(memsize, n_s * k):fill(0)\n",
    "sentenceMemory = Tensor(memsize, k):fill(0)\n",
    "sentencetp1Memory  = Tensor(memsize, k):fill(0)\n",
    "predSummarytp1Memory = Tensor(memsize, n_s * k):fill(0)\n",
    "qPredsMemory = Tensor(memsize, 2):fill(0)\n",
    "qValuesMemory = Tensor(memsize, 1):fill(0)\n",
    "rewardMemory = Tensor(memsize, 1):fill(0)\n",
    "totalPreds = Tensor(n, 2):fill(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "--- Initializing thingss\n",
    "for i = 1, n_s do\n",
    "    qPreds[i] = Tensor(n, 2):fill(0) \n",
    "    qValues[i] = Tensor(n, 1):fill(0)\n",
    "    qActions[i] = Tensor(n, 2):fill(0)\n",
    "    rewards[i] = Tensor(n, 1):fill(0)\n",
    "    rougue_scores[i] = Tensor(n, 1):fill(0)\n",
    "    if adapt then\n",
    "        regPreds[i] = Tensor(n, 1):fill(0)\n",
    "    end        \n",
    "end "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if usecuda then\n",
    "    criterion = criterion:cuda()\n",
    "    model = model:cuda()\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "totalPreds:fill(0)\n",
    "start_row = 1\n",
    "end_row = batch_size\n",
    "c = 1             \n",
    "for start_row=1, n, batch_size do                 \n",
    "    end_row = c * batch_size\n",
    "\n",
    "    if end_row > n then \n",
    "        end_row = n\n",
    "    end\n",
    "    totalPreds[{{start_row, end_row}}]:copy(\n",
    "        model:forward({\n",
    "            queries[{{start_row, end_row}}], \n",
    "            sentences[i][{{start_row, end_row}}], \n",
    "            totalPredsummary[{{start_row, end_row}}]\n",
    "        })\n",
    "    )\n",
    "    c = c + 1\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if adapt then \n",
    "    qPreds[i]:copy(totalPreds[1])\n",
    "    regPreds[i]:copy(totalPreds[2])\n",
    "else\n",
    "    qPreds[i]:copy(totalPreds)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if torch.uniform(0, 1) <= epsilon then\n",
    "    -- randomly choosing actions\n",
    "    xrand = torch.rand(qPreds[i]:size())\n",
    "    qActions[i]:select(2, SELECT):copy(torch.ge(xrand:select(2, SELECT), xrand:select(2, SKIP)))\n",
    "    qActions[i]:select(2, SKIP):copy(torch.ge(xrand:select(2, SKIP), xrand:select(2, SELECT)))\n",
    "    qValues[i]:copy( maskLayer:forward({totalPreds, qActions[i]:byte()}) )\n",
    "else \n",
    "    qMax, qindx = torch.max(qPreds[i], 2)  -- Pulling the best actions\n",
    "    -- Here's the fast way to select the optimal action for each query\n",
    "    qActions[i]:copy(\n",
    "        qActions[i]:scatter(2, qindx, torch.ones(qPreds[i]:size())):clone()\n",
    "    )\n",
    "    qValues[i]:copy(\n",
    "        qMax\n",
    "    )\n",
    "end\n",
    "\n",
    "-- This is where we begin to store the data in our memory \n",
    "    -- notice that we store the reward after this part\n",
    "start_row = curr_memsize + 1\n",
    "if memsize < (start_row + n) then \n",
    "    start_row = memsize - n + 1\n",
    "    end_row = start_row + n - 1\n",
    "    curr_memsize = 0\n",
    "    if (end_row + n) >= memsize then \n",
    "        memfull = true\n",
    "    end \n",
    "else \n",
    "    end_row = start_row + n - 1\n",
    "    curr_memsize = end_row\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "-- Now that we've stored our memory, we can build the summary to evaluate our action\n",
    "predsummary = buildPredsummaryFast(qActions[i], sentences[i], SELECT)\n",
    "totalPredsummary = buildTotalSummaryFast(predsummary, totalPredsummary, usecuda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if i < n_s then\n",
    "    sentencetp1Memory[{{start_row, end_row}}]:copy(sentences[i + 1])\n",
    "    predSummarytp1Memory[{{start_row, end_row}}]:copy(totalPredsummary)\n",
    "else \n",
    "    sentencetp1Memory[{{start_row, end_row}}]:copy(Tensor(sentences[i]:size()):fill(0) )\n",
    "    predSummarytp1Memory[{{start_row, end_row}}]:copy(Tensor(totalPredsummary:size()):fill(0) )\n",
    "end \n",
    "\n",
    "qActionMemory[{{start_row, end_row}}]:copy(qActions[i])\n",
    "qPredsMemory[{{start_row, end_row}}]:copy(qPreds[i])\n",
    "qValuesMemory[{{start_row, end_row}}]:copy(qValues[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = totalPredsummary[j]:totable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : 1\n",
       "  2 : 2\n",
       "  3 : 3\n",
       "  4 : 4\n",
       "  5 : 5\n",
       "  6 : 6\n",
       "  7 : 7\n",
       "}\n"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unpackZeros(totalPredsummary[1]:totable())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28\t\n"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(x):sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out = 1\n",
    "for j = 1, n do\n",
    "    recall, prec, f1 = rougeScores( Tokenize(totalPredsummary[j]:totable()),\n",
    "                                    qTokens[j]\n",
    "        )\n",
    "    rougue_scores[i][j]:fill(f1)\n",
    "    out = j\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if i == n_s then \n",
    "    rouguef1[epoch] = rougue_scores[i]:mean()\n",
    "end \n",
    "\n",
    "if i == 1 then\n",
    "    -- Calculating change in rougue f1\n",
    "    rewards[i]:copy(rougue_scores[i])\n",
    "else \n",
    "    rewards[i]:copy(rougue_scores[i] - rougue_scores[i-1])\n",
    "end\n",
    "\n",
    "rewardMemory[{{start_row, end_row}}]:copy(rewards[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i=1, n_s do\n",
    "    totalPreds:fill(0)\n",
    "    start_row = 1\n",
    "    end_row = batch_size\n",
    "    c = 1             \n",
    "    for start_row=1, n, batch_size do                 \n",
    "        end_row = c * batch_size\n",
    "\n",
    "        if end_row > n then \n",
    "            end_row = n\n",
    "        end\n",
    "        totalPreds[{{start_row, end_row}}]:copy(\n",
    "            model:forward({\n",
    "                queries[{{start_row, end_row}}], \n",
    "                sentences[i][{{start_row, end_row}}], \n",
    "                totalPredsummary[{{start_row, end_row}}]\n",
    "            })\n",
    "        )\n",
    "        c = c + 1\n",
    "    end\n",
    "\n",
    "    if adapt then \n",
    "        qPreds[i]:copy(totalPreds[1])\n",
    "        regPreds[i]:copy(totalPreds[2])\n",
    "    else\n",
    "        qPreds[i]:copy(totalPreds)\n",
    "    end\n",
    "\n",
    "    if torch.uniform(0, 1) <= epsilon then\n",
    "        -- randomly choosing actions\n",
    "        xrand = torch.rand(qPreds[i]:size())\n",
    "        qActions[i]:select(2, SELECT):copy(torch.ge(xrand:select(2, SELECT), xrand:select(2, SKIP)))\n",
    "        qActions[i]:select(2, SKIP):copy(torch.ge(xrand:select(2, SKIP), xrand:select(2, SELECT)))\n",
    "        qValues[i]:copy( maskLayer:forward({totalPreds, qActions[i]:byte()}) )\n",
    "    else \n",
    "        qMax, qindx = torch.max(qPreds[i], 2)  -- Pulling the best actions\n",
    "        -- Here's the fast way to select the optimal action for each query\n",
    "        qActions[i]:copy(\n",
    "            qActions[i]:scatter(2, qindx, torch.ones(qPreds[i]:size())):clone()\n",
    "        )\n",
    "        qValues[i]:copy(\n",
    "            qMax\n",
    "        )\n",
    "    end\n",
    "\n",
    "    -- This is where we begin to store the data in our memory \n",
    "        -- notice that we store the reward after this part\n",
    "    start_row = curr_memsize + 1\n",
    "    if memsize < (start_row + n) then \n",
    "        start_row = memsize - n + 1\n",
    "        end_row = start_row + n - 1\n",
    "        curr_memsize = 0\n",
    "        if (end_row + n) >= memsize then \n",
    "            memfull = true\n",
    "        end \n",
    "    else \n",
    "        end_row = start_row + n - 1\n",
    "        curr_memsize = end_row\n",
    "    end\n",
    "\n",
    "    -- Update memory sequentially until it's full then restart updating it\n",
    "    queryMemory[{{start_row, end_row}}]:copy(queries)\n",
    "    sentenceMemory[{{start_row, end_row}}]:copy(sentences[i])\n",
    "    predSummaryMemory[{{start_row, end_row}}]:copy(totalPredsummary)\n",
    "\n",
    "    -- Now that we've stored our memory, we can build the summary to evaluate our action\n",
    "    predsummary = buildPredsummaryFast(qActions[i], sentences[i], SELECT)\n",
    "    totalPredsummary = buildTotalSummaryFast(predsummary, totalPredsummary, usecuda)\n",
    "\n",
    "    if i < n_s then\n",
    "        sentencetp1Memory[{{start_row, end_row}}]:copy(sentences[i + 1])\n",
    "        predSummarytp1Memory[{{start_row, end_row}}]:copy(totalPredsummary)\n",
    "    else \n",
    "        sentencetp1Memory[{{start_row, end_row}}]:copy(Tensor(sentences[i]:size()):fill(0) )\n",
    "        predSummarytp1Memory[{{start_row, end_row}}]:copy(Tensor(totalPredsummary:size()):fill(0) )\n",
    "    end \n",
    "\n",
    "    qActionMemory[{{start_row, end_row}}]:copy(qActions[i])\n",
    "    qPredsMemory[{{start_row, end_row}}]:copy(qPreds[i])\n",
    "    qValuesMemory[{{start_row, end_row}}]:copy(qValues[i])\n",
    "\n",
    "    if adapt then\n",
    "        regMemory[{{start_row, end_row}}]:copy(regPreds[i])\n",
    "    end\n",
    "\n",
    "    for j = 1, n do\n",
    "        recall, prec, f1 = rougeScores( Tokenize(totalPredsummary[j]:totable()),\n",
    "                                        qTokens[j]\n",
    "            )\n",
    "        rougue_scores[i][j]:fill(f1)\n",
    "    end\n",
    "\n",
    "    if i == n_s then \n",
    "        rouguef1[epoch] = rougue_scores[i]:mean()\n",
    "    end \n",
    "\n",
    "    if i == 1 then\n",
    "        -- Calculating change in rougue f1\n",
    "        rewards[i]:copy(rougue_scores[i])\n",
    "    else \n",
    "        rewards[i]:copy(rougue_scores[i] - rougue_scores[i-1])\n",
    "    end\n",
    "    rewardMemory[{{start_row, end_row}}]:copy(rewards[i])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"training...\")\n",
    "\n",
    "nClock = os.clock()\n",
    "for epoch=1, nepochs do\n",
    "    --- Reset things at the start of each epoch\n",
    "    for i=1, n_s do\n",
    "        qPreds[i]:fill(0)\n",
    "        qValues[i]:fill(0)\n",
    "        qActions[i]:fill(0)\n",
    "        rewards[i]:fill(0)\n",
    "        rougue_scores[i]:fill(0)\n",
    "        totalPredsummary:fill(0)\n",
    "        if adapt then\n",
    "            regMemory[i]:fill(0)\n",
    "        end        \n",
    "    end\n",
    "\n",
    "    for i=1, n_s do\n",
    "        totalPreds:fill(0)\n",
    "        start_row = 1\n",
    "        end_row = batch_size\n",
    "        c = 1             \n",
    "        for start_row=1, n, batch_size do                 \n",
    "            end_row = c * batch_size\n",
    "\n",
    "            if end_row > n then \n",
    "                end_row = n\n",
    "            end\n",
    "            totalPreds[{{start_row, end_row}}]:copy(\n",
    "                model:forward({\n",
    "                    queries[{{start_row, end_row}}], \n",
    "                    sentences[i][{{start_row, end_row}}], \n",
    "                    totalPredsummary[{{start_row, end_row}}]\n",
    "                })\n",
    "            )\n",
    "            c = c + 1\n",
    "        end\n",
    "\n",
    "        if adapt then \n",
    "            qPreds[i]:copy(totalPreds[1])\n",
    "            regPreds[i]:copy(totalPreds[2])\n",
    "        else\n",
    "            qPreds[i]:copy(totalPreds)\n",
    "        end\n",
    "\n",
    "        if torch.uniform(0, 1) <= epsilon then\n",
    "            -- randomly choosing actions\n",
    "            xrand = torch.rand(qPreds[i]:size())\n",
    "            qActions[i]:select(2, SELECT):copy(torch.ge(xrand:select(2, SELECT), xrand:select(2, SKIP)))\n",
    "            qActions[i]:select(2, SKIP):copy(torch.ge(xrand:select(2, SKIP), xrand:select(2, SELECT)))\n",
    "            qValues[i]:copy( maskLayer:forward({totalPreds, qActions[i]:byte()}) )\n",
    "        else \n",
    "            qMax, qindx = torch.max(qPreds[i], 2)  -- Pulling the best actions\n",
    "            -- Here's the fast way to select the optimal action for each query\n",
    "            qActions[i]:copy(\n",
    "                qActions[i]:scatter(2, qindx, torch.ones(qPreds[i]:size())):clone()\n",
    "            )\n",
    "            qValues[i]:copy(\n",
    "                qMax\n",
    "            )\n",
    "        end\n",
    "\n",
    "        -- This is where we begin to store the data in our memory \n",
    "            -- notice that we store the reward after this part\n",
    "        start_row = curr_memsize + 1\n",
    "        if memsize < (start_row + n) then \n",
    "            start_row = memsize - n + 1\n",
    "            end_row = start_row + n - 1\n",
    "            curr_memsize = 0\n",
    "            if (end_row + n) >= memsize then \n",
    "                memfull = true\n",
    "            end \n",
    "        else \n",
    "            end_row = start_row + n - 1\n",
    "            curr_memsize = end_row\n",
    "        end\n",
    "\n",
    "        -- Update memory sequentially until it's full then restart updating it\n",
    "        queryMemory[{{start_row, end_row}}]:copy(queries)\n",
    "        sentenceMemory[{{start_row, end_row}}]:copy(sentences[i])\n",
    "        predSummaryMemory[{{start_row, end_row}}]:copy(totalPredsummary)\n",
    "\n",
    "        -- Now that we've stored our memory, we can build the summary to evaluate our action\n",
    "        predsummary = buildPredsummaryFast(qActions[i], sentences[i], SELECT)\n",
    "        totalPredsummary = buildTotalSummaryFast(predsummary, totalPredsummary, usecuda)\n",
    "\n",
    "        if i < n_s then\n",
    "            sentencetp1Memory[{{start_row, end_row}}]:copy(sentences[i + 1])\n",
    "            predSummarytp1Memory[{{start_row, end_row}}]:copy(totalPredsummary)\n",
    "        else \n",
    "            sentencetp1Memory[{{start_row, end_row}}]:copy(Tensor(sentences[i]:size()):fill(0) )\n",
    "            predSummarytp1Memory[{{start_row, end_row}}]:copy(Tensor(totalPredsummary:size()):fill(0) )\n",
    "        end \n",
    "\n",
    "        qActionMemory[{{start_row, end_row}}]:copy(qActions[i])\n",
    "        qPredsMemory[{{start_row, end_row}}]:copy(qPreds[i])\n",
    "        qValuesMemory[{{start_row, end_row}}]:copy(qValues[i])\n",
    "\n",
    "        if adapt then\n",
    "            regMemory[{{start_row, end_row}}]:copy(regPreds[i])\n",
    "        end\n",
    "\n",
    "        for j = 1, n do\n",
    "            recall, prec, f1 = rougeScores( Tokenize(totalPredsummary[j]:totable()),\n",
    "                                            qTokens[j]\n",
    "                )\n",
    "            rougue_scores[i][j]:fill(f1)\n",
    "        end\n",
    "\n",
    "        if i == n_s then \n",
    "            rouguef1[epoch] = rougue_scores[i]:mean()\n",
    "        end \n",
    "\n",
    "        if i == 1 then\n",
    "            -- Calculating change in rougue f1\n",
    "            rewards[i]:copy(rougue_scores[i])\n",
    "        else \n",
    "            rewards[i]:copy(rougue_scores[i] - rougue_scores[i-1])\n",
    "        end\n",
    "        rewardMemory[{{start_row, end_row}}]:copy(rewards[i])\n",
    "    end\n",
    "\n",
    "    if memfull then \n",
    "        memrows = memsize\n",
    "    else \n",
    "        memrows = curr_memsize\n",
    "    end\n",
    "\n",
    "    if usecuda then \n",
    "        dataloader = dl.TensorLoader({\n",
    "                        queryMemory[{{1, memrows}}]:cuda(), \n",
    "                        sentenceMemory[{{1, memrows}}]:cuda(), \n",
    "                        predSummaryMemory[{{1, memrows}}]:cuda(),\n",
    "                        qPredsMemory[{{1, memrows}}]:cuda(), \n",
    "                        ByteTensor(memrows, 2):copy(qActionMemory[{{1, memrows}}]), \n",
    "                        qValuesMemory[{{1, memrows}}]:cuda(),\n",
    "                        sentencetp1Memory[{{1, memrows}}]:cuda(),\n",
    "                        predSummarytp1Memory[{{1, memrows}}]:cuda()               \n",
    "                            }, \n",
    "                    rewardMemory[{{1, memrows}}]:cuda()\n",
    "                )\n",
    "        if adapt then            \n",
    "            table.insert(dataloader['inputs'], regMemory[{{1, memrows}}]:cuda() )\n",
    "        end\n",
    "    else \n",
    "        dataloader = dl.TensorLoader({\n",
    "                    queryMemory[{{1, memrows}}], \n",
    "                    sentenceMemory[{{1, memrows}}], \n",
    "                    predSummaryMemory[{{1, memrows}}], \n",
    "                    qPredsMemory[{{1, memrows}}], \n",
    "                    ByteTensor(memrows, 2):copy(qActionMemory[{{1, memrows}}]), \n",
    "                    qValuesMemory[{{1, memrows}}],\n",
    "                    sentencetp1Memory[{{1, memrows}}],\n",
    "                    predSummarytp1Memory[{{1, memrows}}]                    \n",
    "                    }, \n",
    "                rewardMemory[{{1, memrows}}]\n",
    "            )\n",
    "        if adapt then\n",
    "            table.insert(dataloader['inputs'], regMemory[{{1, memrows}}] )\n",
    "        end\n",
    "    end\n",
    "    loss = {}\n",
    "    c = 1\n",
    "    for k, xin, reward in dataloader:sampleiter(batch_size, memsize) do\n",
    "        local function feval(params)\n",
    "            gradParams:zero()\n",
    "            if adapt then\n",
    "                local predtp1 = model:forward({xin[1], xin[7], xin[8]})\n",
    "                local predQOnActions = maskLayer:forward({xin[4], xin[5]}) \n",
    "                local ones = torch.ones(reward:size(1)):resize(reward:size(1))\n",
    "                if usecuda then\n",
    "                    ones = ones:cuda()\n",
    "                end\n",
    "                lossf = criterion:forward({predQOnActions, xin[7]}, {reward, ones})\n",
    "                local gradOutput = criterion:backward({predQOnActions, xin[6]}, {reward, ones})\n",
    "                local gradMaskLayer = maskLayer:backward({xin[4], xin[5]}, gradOutput[1])\n",
    "                model:backward({xin[1], xin[2], xin[3]}, {gradMaskLayer[1], gradOutput[2]})\n",
    "            else\n",
    "                model:forget()\n",
    "                local predtp1 = model:forward({xin[1], xin[7], xin[8]})\n",
    "                local predtp1max, _ = torch.max(predtp1, 2)\n",
    "                model:forget()\n",
    "                local predt = model:forward({xin[1], xin[2], xin[3]})\n",
    "                local y_j = reward + (gamma * predtp1max) \n",
    "                local predQOnActions = maskLayer:forward({predt, xin[5]}) \n",
    "                lossf = criterion:forward(predQOnActions, y_j )\n",
    "                local gradOutput = criterion:backward(predQOnActions, y_j)\n",
    "                local gradMaskLayer = maskLayer:backward({predt, xin[5]}, gradOutput)\n",
    "                model:backward({xin[1], xin[2], xin[3]}, gradMaskLayer[1])\n",
    "\n",
    "            end \n",
    "            return lossf, gradParams\n",
    "        end\n",
    "        --- optim.rmsprop returns \\theta, f(\\theta):= loss function\n",
    "         _, lossv  = optim.rmsprop(feval, params, optimParams)\n",
    "        loss[c] = lossv[1]\n",
    "        c = c + 1\n",
    "    end\n",
    "\n",
    "    lossfull[epoch] = Tensor(loss):sum() / #lossv\n",
    "    if print_perf then\n",
    "        print(\n",
    "            string.format('epoch = %i; rougue = %.6f; epsilon = %.6f; loss = %.6f' , \n",
    "                epoch, rouguef1[epoch], epsilon, lossfull[epoch])\n",
    "            )\n",
    "    end\n",
    "\n",
    "    if (epsilon - delta) <= base_explore_rate then\n",
    "        epsilon = base_explore_rate\n",
    "        if epoch > end_baserate then \n",
    "            base_explore_rate = 0.\n",
    "        end\n",
    "    else \n",
    "        epsilon = epsilon - delta\n",
    "    end\n",
    "\n",
    "end\n",
    "print(string.format(\"Elapsed time: %.5f\" % (os.clock()-nClock) ))\n",
    "print(\n",
    "    string.format('First rougue = %.6f; Last rougue = %.6f',\n",
    "        rouguef1[1], rouguef1[nepochs]) \n",
    "    )\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "iTorch",
   "language": "lua",
   "name": "itorch"
  },
  "language_info": {
   "name": "lua",
   "version": "5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}