Bird_ID_404_species_CNN_TL_clean.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bird_ID_404_species_CNN_TL_clean.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z-7UORN7Ymoo",
        "colab_type": "text"
      },
      "source": [
        "Mount the drive to the instance."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9X2hja-OpXM9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fjU2n68zYxU3",
        "colab_type": "text"
      },
      "source": [
        "Import various modules.\n",
        "\n",
        "I used Tnesorflow and its Keras library as the backend. The Keras module is for preprocessing images. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QIR81L-ArDn3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q tensorflow-gpu==2.0.0-alpha0\n",
        "import cv2\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.patches as patches\n",
        "from __future__ import absolute_import, division, print_function\n",
        "import os\n",
        "import tensorflow as tf\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.utils import to_categorical\n",
        "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n",
        "from keras.preprocessing import image\n",
        "keras = tf.keras\n",
        "datapath = '/content/gdrive/My Drive/Bird_ID_project/nabirds'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GVPN6iW0IvZ5",
        "colab_type": "text"
      },
      "source": [
        "Reading in the pandas dataframe. It actually doesn't provide much beside the unique species class names."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K99e_KEnkN4p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_frame = pd.read_csv(datapath+'/train.csv')\n",
        "val_frame = pd.read_csv(datapath+'/val.csv')\n",
        "test_frame = pd.read_csv(datapath+'/test.csv')\n",
        "# train_frame\n",
        "Bird_list=train_frame.class_name_sp.unique()\n",
        "Bird_id=train_frame.class_id_sp.unique()\n",
        "# print(Bird_list, Bird_id)\n",
        "Birds = dict(zip(Bird_id, Bird_list))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R9pJGyS8b83K",
        "colab_type": "text"
      },
      "source": [
        "Unzip the tar.gz files **to the Google Colab instance**. This drastically increased the training speed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cYGuyyUINUQ3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!tar -xzf \"gdrive/My Drive/Bird_ID_project/nabirds/data.tar.gz\"\n",
        "!tar -xzf \"gdrive/My Drive/Bird_ID_project/nabirds/Darren_data.tar.gz\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GWP0UQZdBDk1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mv /content/Darren_test/0289 /content/Darren_test/0867\n",
        "!mv /content/Darren_test/0095 /content/Darren_test/0553\n",
        "!rm -rf /content/Darren_test/NIL"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0vRSm4DVI9Jd",
        "colab_type": "text"
      },
      "source": [
        "Using Keras image generator to reduce RAM footprint (instead of using a large numpy array)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BxCuJxxxl2ge",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_datagen = image.ImageDataGenerator(\n",
        "        rescale=1./255,\n",
        "#         width_shift_range=0.1,\n",
        "#         height_shift_range=0.1,\n",
        "#         zoom_range=0.1,\n",
        "        fill_mode='constant',\n",
        "        horizontal_flip=True,\n",
        "        dtype=np.float32)\n",
        "\n",
        "val_datagen = image.ImageDataGenerator(\n",
        "        rescale=1./255, \n",
        "        dtype=np.float32)\n",
        "\n",
        "test_datagen = image.ImageDataGenerator(\n",
        "        rescale=1./255,\n",
        "        dtype=np.float32)\n",
        "\n",
        "train_generator = train_datagen.flow_from_directory(\n",
        "        directory='/content/data/train/',\n",
        "        #directory=datapath + '/data/train/',\n",
        "        #classes=list(Bird_list),\n",
        "        target_size=(224, 224),\n",
        "        batch_size=32,\n",
        "        class_mode='categorical')\n",
        "\n",
        "validation_generator = val_datagen.flow_from_directory(\n",
        "        #directory=datapath + '/data/val/',\n",
        "        directory='/content/data/val/',\n",
        "        #classes=list(Bird_list),\n",
        "        target_size=(224, 224),\n",
        "        batch_size=32,\n",
        "        class_mode='categorical')\n",
        "\n",
        "test_generator = test_datagen.flow_from_directory(\n",
        "        #directory=datapath + '/data/test/',\n",
        "        directory='/content/data/test/',\n",
        "        shuffle=False,\n",
        "        #classes=list(Bird_list),\n",
        "        target_size=(224, 224),\n",
        "        batch_size=32,\n",
        "        class_mode='categorical')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HlsRpRXDMicF",
        "colab_type": "text"
      },
      "source": [
        "Research has shown that starting from scratch will not work. So we use MobileNet V2 as our feature extractor. This is a relatively small network, which will help in the future when we want to run near real time inferences. We also add two layers: One global average and one prediction. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytj6wsAdM7Un",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "IMG_SHAPE = (224,224,3)\n",
        "base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,\n",
        "                                               include_top=False, \n",
        "                                               weights='imagenet')\n",
        "\n",
        "base_model.trainable = False\n",
        "\n",
        "global_average_layer = tf.keras.layers.GlobalAveragePooling2D()\n",
        "prediction_layer = keras.layers.Dense(404,activation='softmax')\n",
        "base_learning_rate = 0.0001\n",
        "model = tf.keras.Sequential([\n",
        "    base_model,\n",
        "    global_average_layer,\n",
        "    prediction_layer\n",
        "])\n",
        "model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=base_learning_rate), \n",
        "              loss='categorical_crossentropy', \n",
        "              metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bGi69hxZRSvy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fit_model(model, batch_size=32, epochs=10):    \n",
        "    history = model.fit_generator(\n",
        "        generator=train_generator,\n",
        "        steps_per_epoch=(len(train_frame) // batch_size),\n",
        "        epochs=epochs,\n",
        "        validation_data=validation_generator,\n",
        "        callbacks=None\n",
        "        )\n",
        "    score = model.evaluate_generator(train_generator, verbose=1)\n",
        "    probs = model.predict_generator(test_generator, verbose=1)\n",
        "    return model, score, probs, history\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dt80nu7nQlzm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fit_model, score, probs, history = fit_model(model, batch_size=32, epochs=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YdfI5Fzw6CRx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def top3(probs, GT):\n",
        "    t3 = np.argsort(probs)[-3:]\n",
        "    #print(t3)\n",
        "    if GT in t3:\n",
        "        return 1\n",
        "    else:\n",
        "        return 0\n",
        "    \n",
        "def top5(probs, GT):\n",
        "    t5 = np.argsort(probs)[-5:]\n",
        "    if GT in t5:\n",
        "        return 1\n",
        "    else:\n",
        "        return 0\n",
        "    \n",
        "def top3_idx(probs):\n",
        "    return np.flip(np.argsort(probs)[-3:],0), np.flip(probs[np.argsort(probs)[-3:]],0)\n",
        "    #print(t3)\n",
        "\n",
        "def top5_idx(probs):\n",
        "    return np.flip(np.argsort(probs)[-5:])\n",
        "    #print(t3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Kb1Q3eeP7DI",
        "colab_type": "text"
      },
      "source": [
        "After 10 epochs we achieve 44, 64, and 72 % top 1, 3, and 5 hit rate. Not bad! But it can be better. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2iQYi1kdSLtF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct_prediction = 0\n",
        "correct_top3 = 0\n",
        "correct_top5 = 0\n",
        "for ii in range(len(probs)):\n",
        "    P_this = np.argmax(probs[ii])\n",
        "    GT_this = test_generator.labels[ii]\n",
        "    if P_this == GT_this:\n",
        "        correct_prediction += 1\n",
        "    correct_top3 += top3(probs[ii],GT_this)\n",
        "    correct_top5 += top5(probs[ii],GT_this)\n",
        "    if ii < 10:\n",
        "        print(\"Prediction: {} ({})\".format(P_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(P_this)])]))\n",
        "        print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n",
        "        print(\"Prediction: {} \".format(P_this))\n",
        "        print(\"Actual: {} \".format(GT_this))\n",
        "        #plt.figure()\n",
        "        #plt.imshow(X_test[ii,:,:,:])\n",
        "        #plt.show()\n",
        "print(correct_prediction, correct_prediction/len(probs))\n",
        "print(correct_top3, correct_top3/len(probs))\n",
        "print(correct_top5, correct_top5/len(probs))\n",
        "print(len(probs))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ou1-W3SQYOK",
        "colab_type": "text"
      },
      "source": [
        "We see overfitting after second epoch, which seems to be a common thing when training models for bird IDs. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7tJdIW8F6mIv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "acc = history.history['accuracy']\n",
        "val_acc = history.history['val_accuracy']\n",
        "\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.subplot(2, 1, 1)\n",
        "plt.plot(acc, label='Training Accuracy')\n",
        "plt.plot(val_acc, label='Validation Accuracy')\n",
        "plt.legend(loc='lower right')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.ylim([min(plt.ylim()),1])\n",
        "plt.title('Training Accuracy')\n",
        "\n",
        "plt.subplot(2, 1, 2)\n",
        "plt.plot(loss, label='Training Loss')\n",
        "plt.plot(val_loss, label='Validation Loss')\n",
        "plt.legend(loc='upper right')\n",
        "plt.ylabel('Cross Entropy')\n",
        "# plt.ylim([0,1.0])\n",
        "plt.title('Training Loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FK_OysQzQ4_S",
        "colab_type": "text"
      },
      "source": [
        "We start to fine tune the model by allowing the last 55 layers to be trained."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BffxJucn6wlf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_model.trainable = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WCA4VLaUYr7P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's take a look to see how many layers are in the base model\n",
        "print(\"Number of layers in the base model: \", len(base_model.layers))\n",
        "\n",
        "# Fine tune from this layer onwards\n",
        "fine_tune_at = 100\n",
        "\n",
        "# Freeze all the layers before the `fine_tune_at` layer\n",
        "for layer in base_model.layers[:fine_tune_at]:\n",
        "    layer.trainable =  False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fbNXMyuEZd2h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer = tf.keras.optimizers.RMSprop(lr=base_learning_rate/10),\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmYtpM-GZit-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "initial_epochs = 10\n",
        "fine_tune_epochs = 10\n",
        "total_epochs =  initial_epochs + fine_tune_epochs\n",
        "\n",
        "\n",
        "def fit_model_FT(model, batch_size=32, epochs=10):    \n",
        "    history = model.fit_generator(\n",
        "        generator=train_generator,\n",
        "        steps_per_epoch=(len(train_frame) // batch_size),\n",
        "        epochs=total_epochs,\n",
        "        initial_epoch=initial_epochs,\n",
        "        validation_data=validation_generator,\n",
        "        callbacks=None\n",
        "        )\n",
        "    score = model.evaluate_generator(train_generator, verbose=1)\n",
        "    probs = model.predict_generator(test_generator, verbose=1)\n",
        "    return model, score, probs, history"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ng5tOm_VZ2LZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fit_model, score, probs, history = fit_model_FT(model, batch_size=32, epochs=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ee6U1HZLWnK8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "acc = history.history['accuracy']\n",
        "val_acc = history.history['val_accuracy']\n",
        "\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.subplot(2, 1, 1)\n",
        "plt.plot(acc, label='Training Accuracy')\n",
        "plt.plot(val_acc, label='Validation Accuracy')\n",
        "plt.legend(loc='lower right')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.ylim([min(plt.ylim()),1])\n",
        "plt.title('Training Accuracy')\n",
        "\n",
        "plt.subplot(2, 1, 2)\n",
        "plt.plot(loss, label='Training Loss')\n",
        "plt.plot(val_loss, label='Validation Loss')\n",
        "plt.legend(loc='upper right')\n",
        "plt.ylabel('Cross Entropy')\n",
        "# plt.ylim([0,1.0])\n",
        "plt.title('Training Loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bFU-cJgNRNr3",
        "colab_type": "text"
      },
      "source": [
        "After 10 fine tuning epochs the hit rates increased to 63 (top), 82 (top3), and 88 % (top5)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJWas9HFIm8p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct_prediction = 0\n",
        "correct_top3 = 0\n",
        "correct_top5 = 0\n",
        "for ii in range(len(probs)):\n",
        "    P_this = np.argmax(probs[ii])\n",
        "    GT_this = test_generator.labels[ii]\n",
        "    if P_this == GT_this:\n",
        "        correct_prediction += 1\n",
        "    correct_top3 += top3(probs[ii],GT_this)\n",
        "    correct_top5 += top5(probs[ii],GT_this)\n",
        "    if ii % 100 == 20:\n",
        "        t3, p3 = top3_idx(probs[ii])\n",
        "        print(\"Prediction: {}, {}, or {} ({} ({:.1f} %), {} ({:.1f} %), or {} ({:.1f} %))\".format(t3[0], t3[1], t3[2] ,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[0])])], p3[0] * 100,\n",
        "                                                                                      Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[1])])], p3[1] * 100,\n",
        "                                                                                      Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[2])])], p3[2] * 100))\n",
        "        print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n",
        "\n",
        "    \n",
        "        plt.figure()\n",
        "        image = plt.imread('data/test/'+test_generator.filenames[ii])\n",
        "        plt.imshow(image)\n",
        "        plt.show()\n",
        "print(correct_prediction, correct_prediction/len(probs))\n",
        "print(correct_top3, correct_top3/len(probs))\n",
        "print(correct_top5, correct_top5/len(probs))\n",
        "print(len(probs))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5BfjDfYfSaGs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Prediction = []\n",
        "Prediction3 = []\n",
        "Correct_prediction3 = []\n",
        "for ii in range(len(probs)):\n",
        "    Prediction.append(np.argmax(probs[ii]))\n",
        "    Prediction3.append(top3_idx(probs[ii])[0])\n",
        "    Correct_prediction3.append(np.asscalar(np.in1d(test_generator.labels[ii],Prediction3[ii])))\n",
        "\n",
        "    \n",
        "Correct_predicted = []\n",
        "Correct_predicted3 = []\n",
        "Species_length = []\n",
        "for ii in range(len(np.unique(test_generator.labels))):\n",
        "    Species_length.append(sum((test_generator.labels == ii)))\n",
        "    Correct_predicted.append((sum((test_generator.labels == ii) & (Prediction == test_generator.labels)))/sum(test_generator.labels == ii))\n",
        "    Correct_predicted3.append(sum((test_generator.labels == ii) & (Correct_prediction3))/sum(test_generator.labels == ii))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y090L25JReE_",
        "colab_type": "text"
      },
      "source": [
        "For some species the model performed terribly. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hYumabaNVOMA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for ii in range(len(np.unique(test_generator.labels))):\n",
        "#     if sum((test_generator.labels == ii)) < 8:\n",
        "    if Correct_predicted3[ii] < 0.4:\n",
        "        print('{}: {:.2f}, {:.2f}, {}, {}'.format(ii, Correct_predicted[ii], Correct_predicted3[ii], sum((train_generator.labels == ii)), Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(ii)])]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eWALVyB-PwNz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "initial_epochs = 20\n",
        "fine_tune_epochs = 10\n",
        "total_epochs =  initial_epochs + fine_tune_epochs\n",
        "\n",
        "\n",
        "def fit_model_FT2(model, batch_size=32, epochs=10):    \n",
        "    history = model.fit_generator(\n",
        "        generator=train_generator,\n",
        "        steps_per_epoch=(len(train_frame) // batch_size),\n",
        "        epochs=total_epochs,\n",
        "        initial_epoch=initial_epochs,\n",
        "        validation_data=validation_generator,\n",
        "        callbacks=None\n",
        "        )\n",
        "    score = model.evaluate_generator(train_generator, verbose=1)\n",
        "    probs = model.predict_generator(test_generator, verbose=1)\n",
        "    return model, score, probs, history"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sqEGCIJmP3aC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#fit_model, score, probs, history = fit_model_FT(model, X_train, X_test, Y_train, Y_test, batch_size=32, epochs=15)\n",
        "fit_model, score, probs, history = fit_model_FT2(model, batch_size=32, epochs=10)\n",
        "model.save(datapath + '/model3_30.h5')\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DyFi0YJH5iv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "probs = model.predict_generator(test_generator, verbose=1)\n",
        "np.savetxt(datapath + '/probs30.txt', probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fB66sNewSaue",
        "colab_type": "text"
      },
      "source": [
        "After another 10 fine tuning epochs (total = 30 epochs) we have 66, 84, 89 % top 1, 3, 5 hit rate. At this point it may be more helpful to refine the probability based on the location and time of year that the picture is taken. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bjudiCZmKEe3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct_prediction = 0\n",
        "correct_top3 = 0\n",
        "correct_top5 = 0\n",
        "for ii in range(len(probs)):\n",
        "    P_this = np.argmax(probs[ii])\n",
        "    GT_this = test_generator.labels[ii]\n",
        "    if P_this == GT_this:\n",
        "        correct_prediction += 1\n",
        "    correct_top3 += top3(probs[ii],GT_this)\n",
        "    correct_top5 += top5(probs[ii],GT_this)\n",
        "    if ii % 100 == 26:\n",
        "#     if test_generator.labels[ii] == 10: # Gadwall\n",
        "        #print(\"Prediction: {} ({})\".format(P_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(top3_idx(probs[ii]))])]))\n",
        "        t3, p3 = top3_idx(probs[ii])\n",
        "        print(\"Prediction: {}, {}, or {} ({} ({:.1f} %), {} ({:.1f} %), or {} ({:.1f} %))\".format(t3[0], t3[1], t3[2] ,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[0])])], p3[0] * 100,\n",
        "                                                                                      Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[1])])], p3[1] * 100,\n",
        "                                                                                      Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[2])])], p3[2] * 100))\n",
        "        print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n",
        "#         print(\"Prediction: {} \".format(P_this))\n",
        "#         print(\"Actual: {} \".format(GT_this))\n",
        "#         print('data/test/'+test_generator.filenames[ii])\n",
        "    \n",
        "        plt.figure()\n",
        "        image = plt.imread('data/test/'+test_generator.filenames[ii])\n",
        "        plt.imshow(image)\n",
        "        plt.show()\n",
        "print(correct_prediction, correct_prediction/len(probs))\n",
        "print(correct_top3, correct_top3/len(probs))\n",
        "print(correct_top5, correct_top5/len(probs))\n",
        "print(len(probs))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RlxJGKb8TNYY",
        "colab_type": "text"
      },
      "source": [
        "Using my photos to test."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TRgQdjw7APzO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Darren_test_generator = test_datagen.flow_from_directory(\n",
        "        #directory=datapath + '/data/test/',\n",
        "        directory='/content/Darren_test/',\n",
        "        shuffle=False,\n",
        "        #classes=list(Bird_list),\n",
        "        target_size=(224, 224),\n",
        "        batch_size=32,\n",
        "        class_mode='categorical')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FbuTUb_6AaGy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Darren_probs = model.predict_generator(Darren_test_generator, verbose=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf_PABndAb7s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct_prediction = 0\n",
        "correct_top3 = 0\n",
        "correct_top5 = 0\n",
        "for ii in range(len(Darren_probs)):\n",
        "# for ii in [0]:\n",
        "    P_this = np.argmax(Darren_probs[ii])\n",
        "    GT_this = test_generator.class_indices[list(Darren_test_generator.class_indices.keys())[Darren_test_generator.labels[ii]]]\n",
        "    if P_this == GT_this:\n",
        "        correct_prediction += 1\n",
        "    correct_top3 += top3(Darren_probs[ii],GT_this)\n",
        "    correct_top5 += top5(Darren_probs[ii],GT_this)\n",
        "#     if ii % 100 == 26:\n",
        "#     if test_generator.labels[ii] == 10: # Gadwall\n",
        "        #print(\"Prediction: {} ({})\".format(P_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(top3_idx(probs[ii]))])]))\n",
        "    t3, p3 = top3_idx(Darren_probs[ii])\n",
        "    print(\"Prediction: {}, {}, or {} ({} ({:.1f} %), {} ({:.1f} %), or {} ({:.1f} %))\".format(\n",
        "        t3[0], t3[1], t3[2] ,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[0])])], p3[0] * 100,\n",
        "                             Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[1])])], p3[1] * 100,\n",
        "                             Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[2])])], p3[2] * 100))\n",
        "    print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n",
        "#         print(\"Prediction: {} \".format(P_this))\n",
        "#         print(\"Actual: {} \".format(GT_this))\n",
        "#         print('data/test/'+test_generator.filenames[ii])\n",
        "    \n",
        "    plt.figure()\n",
        "    image = plt.imread('Darren_test/'+Darren_test_generator.filenames[ii])\n",
        "    plt.imshow(image)\n",
        "    plt.show()\n",
        "print(correct_prediction, correct_prediction/len(Darren_probs))\n",
        "print(correct_top3, correct_top3/len(Darren_probs))\n",
        "print(correct_top5, correct_top5/len(Darren_probs))\n",
        "print(len(Darren_probs))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yg-w5GA6TIZy",
        "colab_type": "text"
      },
      "source": [
        "Below are scratch codes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_6UTR_-hBIs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Birds\n",
        "class_indices_inv_map = {v: k for k, v in test_generator.class_indices.items()} \n",
        "\n",
        "import pickle\n",
        "\n",
        "# write python dict to a file\n",
        "output = open(datapath+'/class_indices_inv_map.pkl', 'wb')\n",
        "pickle.dump(class_indices_inv_map, output)\n",
        "output.close()\n",
        "output = open(datapath+'/Birds.pkl', 'wb')\n",
        "pickle.dump(Birds, output)\n",
        "output.close()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}