DeepLeaningFromScratch/chapter05.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 誤差逆伝播法\n",
    "\n",
    "❌ gradientの計算ができていない"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 単純なレイヤの実装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "type MulLayer\n",
    "    x::Number\n",
    "    y::Number\n",
    "    forward::Function\n",
    "    backward::Function\n",
    "    \n",
    "    function MulLayer()\n",
    "        instance = new()\n",
    "\n",
    "        instance.forward = function(x, y) \n",
    "            instance.x = x\n",
    "            instance.y = y\n",
    "            x * y\n",
    "        end\n",
    "        \n",
    "        instance.backward = function(dout)\n",
    "            dx = dout * instance.y\n",
    "            dy = dout * instance.x\n",
    "            dx, dy\n",
    "        end\n",
    "        instance\n",
    "    end    \n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "220.00000000000003\n"
     ]
    }
   ],
   "source": [
    "apple = 100\n",
    "apple_num = 2\n",
    "tax = 1.1\n",
    "\n",
    "# layer\n",
    "mul_apple_layer = MulLayer()\n",
    "mul_tax_layer = MulLayer()\n",
    "\n",
    "# forward\n",
    "apple_price = mul_apple_layer.forward(apple, apple_num)\n",
    "price = mul_tax_layer.forward(apple_price, tax)\n",
    "\n",
    "println(price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2 110.00000000000001 200\n"
     ]
    }
   ],
   "source": [
    "# backward\n",
    "dprice = 1\n",
    "dapple_price, dtax = mul_tax_layer.backward(dprice)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)\n",
    "\n",
    "println(\"$dapple $dapple_num $dtax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "type AddLayer\n",
    "    forward::Function\n",
    "    backward::Function\n",
    "    \n",
    "    function AddLayer()\n",
    "        forward = function(x, y)\n",
    "            x+y\n",
    "        end\n",
    "        \n",
    "        backward = function(dout)\n",
    "            dx = dout * 1\n",
    "            dy = dout * 1\n",
    "            dx, dy\n",
    "        end\n",
    "        \n",
    "        new(forward, backward)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "715.0000000000001\n",
      "110.00000000000001 2.2 3.3000000000000003 165.0 650\n"
     ]
    }
   ],
   "source": [
    "apple = 100\n",
    "apple_num = 2\n",
    "orange = 150\n",
    "orange_num = 3\n",
    "tax = 1.1\n",
    "\n",
    "# layer\n",
    "mul_apple_layer = MulLayer()\n",
    "mul_orange_layer = MulLayer()\n",
    "add_apple_orange_layer = AddLayer()\n",
    "mul_tax_layer = MulLayer()\n",
    "\n",
    "# forward\n",
    "apple_price = mul_apple_layer.forward(apple, apple_num)\n",
    "orange_price = mul_orange_layer.forward(orange, orange_num)\n",
    "all_price = add_apple_orange_layer.forward(apple_price, orange_price)\n",
    "price = mul_tax_layer.forward(all_price, tax)\n",
    "\n",
    "# backward\n",
    "dprice = 1\n",
    "dall_price, dtax= mul_tax_layer.backward(dprice)\n",
    "dapple_price, dorange_price = add_apple_orange_layer.backward(dall_price)\n",
    "dorange, dorange_num = mul_orange_layer.backward(dorange_price)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)\n",
    "\n",
    "println(price)\n",
    "println(\"$dapple_num $dapple $dorange $dorange_num $dtax\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 活性化関数レイヤの実装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "type Relu\n",
    "    mask::Array{Float64}\n",
    "    forward::Function\n",
    "    backward::Function\n",
    "    \n",
    "    function Relu()\n",
    "        instance = new()\n",
    "        instance.forward = function(x)\n",
    "            instance.mask = (x .<=  0)[:]\n",
    "            out = copy(x)\n",
    "            out[instance.mask] = 0\n",
    "            out\n",
    "        end\n",
    "\n",
    "        instance.backward = function(dout)\n",
    "            dout[instance.mask] = 0\n",
    "            dx = dout\n",
    "            dx\n",
    "        end\n",
    "        instance\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.0 -0.5; -2.0 3.0]\n",
      "Bool[false true; true false]\n"
     ]
    }
   ],
   "source": [
    "x = [1.0 -0.5; -2.0 3.0]\n",
    "println(x)\n",
    "mask = x .<= 0\n",
    "println(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "type Sigmoid\n",
    "    forward::Function\n",
    "    backward::Function\n",
    "    \n",
    "    function Sigmoid()\n",
    "        instance = new()\n",
    "        instance.forward = function(x)\n",
    "            out = 1 ./ (1 .+ exp.(-x))\n",
    "            out\n",
    "        end\n",
    "        \n",
    "        instance.backward = function(dout)\n",
    "            dx = dout .* (1.0 .- instance.out) .* instance.out\n",
    "            dx\n",
    "        end\n",
    "        instance\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Affine/Softmaxレイヤの実装"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Affineレイヤ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size(X) = (2,)\n",
      "size(W) = (2, 3)\n",
      "size(B) = (3,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3-element Array{Float64,1}:\n",
       " 0.415242\n",
       " 0.452299\n",
       " 0.973252"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = rand(2)\n",
    "W = rand(2, 3)\n",
    "B = rand(3)\n",
    "\n",
    "@show size(X)\n",
    "@show size(W)\n",
    "@show size(B)\n",
    "\n",
    "Y = W'*X + B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### バッチ版Affineレイヤ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_dot_W = [0 0 0; 10 10 10]\n",
      "X_dot_W .+ B = [1 2 3; 11 12 13]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2×3 Array{Int64,2}:\n",
       "  1   2   3\n",
       " 11  12  13"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_dot_W = [0 0 0; 10 10 10]\n",
    "B = [1 2 3]\n",
    "@show X_dot_W\n",
    "@show X_dot_W .+ B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×3 Array{Int64,2}:\n",
       " 1  2  3\n",
       " 4  5  6"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dY = [1 2 3; 4 5 6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1×3 Array{Int64,2}:\n",
       " 5  7  9"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dB = sum(dY,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "type Affine\n",
    "    W::Array{Float64}\n",
    "    b::Array{Float64}\n",
    "    x::Array{Float64}\n",
    "    dW::Array{Float64}\n",
    "    db::Array{Float64}\n",
    "    forward::Function\n",
    "    backward::Function\n",
    "\n",
    "    function Affine(W, b)\n",
    "        instance = new()\n",
    "        instance.W, instance.b = W, b\n",
    "        \n",
    "        instance.forward = function(x)\n",
    "            instance.x = x\n",
    "            out = instance.W' * x .+ instance.b\n",
    "            out\n",
    "        end\n",
    "        \n",
    "        instance.backward = function(dout)\n",
    "            dx = instance.W' * dout\n",
    "            instance.dW = dout * instance.x'\n",
    "            instance.db = sum(dout, 1)\n",
    "            dx\n",
    "        end\n",
    "        instance\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Softmax-with-Lossレイヤ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "type SoftmaxWithLoss\n",
    "    loss::Float64\n",
    "    y::Float64\n",
    "    t::Array{Float64}\n",
    "    forward::Function\n",
    "    backward::Function\n",
    "    \n",
    "    function SoftmaxWithLoss()\n",
    "        instance = new()\n",
    "        instance.forward = function(x, t)\n",
    "            instance.t = t\n",
    "            instance.y = softmax(x)\n",
    "            instance.loss = cross_entropy_error(instance.y, instance.t)\n",
    "        end\n",
    "        \n",
    "        instance.backward = function(dout)\n",
    "            batch_size = size(instance.t)[1]\n",
    "            dx = (instance.y - instance.t) / batch_size\n",
    "            dx\n",
    "        end\n",
    "        instance\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 誤差逆伝播法の実装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cross_entropy_error (generic function with 1 method)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function sigmoid(x)\n",
    "    1 ./ (1 .+ exp.(-x))\n",
    "end\n",
    "\n",
    "function softmax(a)\n",
    "    exp_a = exp.(a)\n",
    "    sum_exp_a = sum(exp_a)\n",
    "    y = exp_a / sum_exp_a\n",
    "    y\n",
    "end\n",
    "\n",
    "function cross_entropy_error(y, t) # one-hot vector\n",
    "    if ndims(y) == 0\n",
    "        t = reshape(t, 1, length(t))\n",
    "        y = reshpae(y, 1, length(y))\n",
    "    end\n",
    "    batch_size = size(y)[end]\n",
    "    -sum(t.*log.(y))/batch_size\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataStructures\n",
    "\n",
    "type TwoLayerNet\n",
    "    params::Dict{String, Array{Float64}}\n",
    "    predict::Function\n",
    "    loss::Function\n",
    "    accuracy::Function\n",
    "    numerical_gradient::Function\n",
    "    gradient::Function\n",
    "    layers::OrderedDict\n",
    "    lastLayer::SoftmaxWithLoss\n",
    "    \n",
    "    function TwoLayerNet(input_size, hidden_size, output_size; weight_init_std=0.01)\n",
    "        params = Dict()\n",
    "        params[\"W1\"] = weight_init_std * randn(input_size, hidden_size)\n",
    "        params[\"b1\"] = zeros(hidden_size)\n",
    "        params[\"W2\"] = weight_init_std * randn(hidden_size, output_size)\n",
    "        params[\"b2\"] = zeros(output_size)\n",
    "        \n",
    "        layers = OrderedDict()\n",
    "        layers[\"Affine1\"] = Affine(params[\"W1\"], params[\"b1\"])\n",
    "        layers[\"Relu1\"] = Relu()\n",
    "        layers[\"Affine2\"] = Affine(params[\"W2\"], params[\"b2\"])\n",
    "        lastLayer = SoftmaxWithLoss()\n",
    "        \n",
    "        predict = function(x)\n",
    "            for layer in values(layers)\n",
    "                x = layer.forward(x[:])\n",
    "            end\n",
    "            x\n",
    "        end\n",
    "        \n",
    "        loss = function(x, t)\n",
    "            y = predict(x)\n",
    "            cross_entropy_error(y, t)\n",
    "        end\n",
    "        \n",
    "        accuracy = function(x, t)\n",
    "            y = predict(x)\n",
    "            tmp = findmax(y, 1)[2]\n",
    "            y = [tmp[j] - (i-1)*output_size for i in 1:length(tmp)]\n",
    "            t = indmax(t)\n",
    "            accuracy = sum(t[i:i+batch_size-1] .== p-1) / float(size(x)[1])\n",
    "        end\n",
    "        \n",
    "        numerical_gradient = function(x, t)\n",
    "            loss_W = W -> loss(x, t)\n",
    "            grads = Dict()\n",
    "            grads[\"W1\"] = numerical_gradient(loss_W, params[\"W1\"])\n",
    "            grads[\"b1\"] = numerical_gradient(loss_W, params[\"b1\"])\n",
    "            grads[\"W2\"] = numerical_gradient(loss_W, params[\"W2\"])\n",
    "            grads[\"b2\"] = numerical_gradient(loss_W, params[\"b2\"])\n",
    "            grads\n",
    "        end\n",
    "        \n",
    "        gradient = function(x, t)\n",
    "            # forward\n",
    "            loss(x, t)\n",
    "            \n",
    "            # backward\n",
    "            dout = 1\n",
    "            dout = lastLayer.backward(dout)\n",
    "            \n",
    "            layers = values(instance.layers)\n",
    "            layers = reverse(layers)\n",
    "            for layer in layers\n",
    "                dout = layer.backward(dout)\n",
    "            end\n",
    "            \n",
    "            grads = Dict()\n",
    "            grads[\"W1\"] = layers[\"Affine1\"].dW\n",
    "            grads[\"b1\"]  = layers[\"Affine1\"].db\n",
    "            grads[\"W2\"] = layers[\"Affine2\"].dW\n",
    "            grads[\"b2\"]  = layers[\"Affine2\"].db\n",
    "            grads\n",
    "        end\n",
    "        \n",
    "        new(params, predict, loss, accuracy, numerical_gradient, gradient, layers, lastLayer)\n",
    "    end    \n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using MLDatasets\n",
    "\n",
    "x_train, t_train = MNIST.traindata()\n",
    "x_test, t_test = MNIST.testdata()\n",
    "\n",
    "network = TwoLayerNet(784, 50, 10)\n",
    "x_batch = x_train[:, :, 3]\n",
    "t_batch = t_train[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mStackOverflowError:\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mStackOverflowError:\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1m(::##42#50{Dict{Any,Any},##39#47{##38#46}})\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Function, ::Array{Float64,2}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[16]:47\u001b[22m\u001b[22m",
      " [2] \u001b[1m(::##42#50{Dict{Any,Any},##39#47{##38#46}})\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Function, ::Array{Float64,2}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[16]:49\u001b[22m\u001b[22m (repeats 11141 times)",
      " [3] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "grad_numerical = network.numerical_gradient(x_batch, t_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mUndefVarError: params not defined\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mUndefVarError: params not defined\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "Affine(params[\"W1\"], params[\"b1\"]).forward(x_batch[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mBoundsError: attempt to access 784-element Array{Float64,1} at index [[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]]\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mBoundsError: attempt to access 784-element Array{Float64,1} at index [[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]]\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1mthrow_boundserror\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}, ::Tuple{Array{Float64,1}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:433\u001b[22m\u001b[22m",
      " [2] \u001b[1mcheckbounds\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:362\u001b[22m\u001b[22m [inlined]",
      " [3] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m./multidimensional.jl:487\u001b[22m\u001b[22m [inlined]",
      " [4] \u001b[1m_setindex!\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::IndexLinear, ::Array{Float64,1}, ::Int64, ::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./multidimensional.jl:484\u001b[22m\u001b[22m",
      " [5] \u001b[1m(::##53#56{Relu})\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[22]:11\u001b[22m\u001b[22m",
      " [6] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "Relu().forward(x_batch[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mBoundsError: attempt to access 50-element Array{Float64,1} at index [[1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0]]\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mBoundsError: attempt to access 50-element Array{Float64,1} at index [[1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0]]\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1mthrow_boundserror\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}, ::Tuple{Array{Float64,1}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:433\u001b[22m\u001b[22m",
      " [2] \u001b[1mcheckbounds\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:362\u001b[22m\u001b[22m [inlined]",
      " [3] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m./multidimensional.jl:487\u001b[22m\u001b[22m [inlined]",
      " [4] \u001b[1m_setindex!\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::IndexLinear, ::Array{Float64,1}, ::Int64, ::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./multidimensional.jl:484\u001b[22m\u001b[22m",
      " [5] \u001b[1m(::##9#12{Relu})\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[6]:11\u001b[22m\u001b[22m",
      " [6] \u001b[1m(::##38#46)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[16]:28\u001b[22m\u001b[22m",
      " [7] \u001b[1m#39\u001b[22m\u001b[22m at \u001b[1m./In[16]:34\u001b[22m\u001b[22m [inlined]",
      " [8] \u001b[1m(::##44#52{SoftmaxWithLoss,##39#47{##38#46}})\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}, ::Int64\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[16]:58\u001b[22m\u001b[22m",
      " [9] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "grad_backprop = network.gradient(x_batch[:], t_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数値微分できなかったので比較できていない\n",
    "``` julia\n",
    "for key in keys(grad_numerical)\n",
    "    diff = average(abs(grad_backprop[key] - grad_numerical[key]))\n",
    "    println(\"$key:$diff\")\n",
    "end\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.0",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}