notebooks/classifier_scikitlearn_LogisticRegression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adversarial-Robustness-Toolbox for scikit-learn LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from art.estimators.classification import SklearnClassifier\n",
    "from art.attacks.evasion import ProjectedGradientDescent\n",
    "from art.utils import load_mnist\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and transform MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test), min_, max_ = load_mnist()\n",
    "\n",
    "n_samples_train = x_train.shape[0]\n",
    "n_features_train = x_train.shape[1] * x_train.shape[2] * x_train.shape[3]\n",
    "n_samples_test = x_test.shape[0]\n",
    "n_features_test = x_test.shape[1] * x_test.shape[2] * x_test.shape[3]\n",
    "\n",
    "x_train = x_train.reshape(n_samples_train, n_features_train)\n",
    "x_test = x_test.reshape(n_samples_test, n_features_test)\n",
    "\n",
    "y_train = np.argmax(y_train, axis=1)\n",
    "y_test = np.argmax(y_test, axis=1)\n",
    "\n",
    "n_samples_max = 100\n",
    "x_train = x_train[0:n_samples_max]\n",
    "y_train = y_train[0:n_samples_max]\n",
    "x_test = x_test[0:n_samples_max]\n",
    "y_test = y_test[0:n_samples_max]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train LogisticRegression classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(penalty='l2', dual=False, tol=0.0001, C=1.0, fit_intercept=True, intercept_scaling=1, \n",
    "                           class_weight='balanced', random_state=None, solver='lbfgs', max_iter=100, \n",
    "                           multi_class='ovr', verbose=0, warm_start=False, n_jobs=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight='balanced', dual=False,\n",
       "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                   max_iter=100, multi_class='ovr', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X=x_train, y=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and apply ProjectedGradientDescent Attack with ART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "art_classifier = SklearnClassifier(model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pgd = ProjectedGradientDescent(estimator=art_classifier, norm=np.inf, eps=.3, eps_step=0.1, max_iter=20, \n",
    "                               targeted=False, num_random_init=0, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "x_train_adv = pgd.generate(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_adv = pgd.generate(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate LogisticRegression classifier on benign and adversarial samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Training Score: 1.0000\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_train, y_train)\n",
    "print(\"Benign Training Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAO90lEQVR4nO3dbYxc5XnG8euKvdg1mMRbx45DHXCMU2igMemKFxkBFQp1o0qAKkKtKHJoWtMEJ6F1JahVFVqRyq2AlFKKZIqLkYAEAhR/oEksCwFRYYvtEjBxgARcarxdY1ZgIMTYu3c/7Ljdkt1ndndeznjv/09azcy5Z+bcPravfc6cZ85xRAhAXh+ougEA1SIEgOQIASA5QgBIjhAAkiMEgOQqCQHby20/b/sntq+uoocS27tsP2v7adtbO6CfDbb32t4xYlm37c22X6zdzumw/q61/WptGz5t+7MV9rfQ9iO2d9p+zvbXa8s7YhsW+mvLNnS75wnYnibpBUmfkbRb0lOSVkTEj9raSIHtXZJ6ImJf1b1Iku1zJL0t6c6IOKW27G8lDUTEulqQzomIqzqov2slvR0R11fR00i2F0haEBHbbc+WtE3SRZK+qA7YhoX+Pqc2bMMqRgKnS/pJRLwUEe9J+pakCyvo44gREY9JGnjf4gslbazd36jhfzSVGKO/jhERfRGxvXb/LUk7JR2nDtmGhf7aoooQOE7Sf414vFtt/AOPU0j6vu1ttldV3cwY5kdEnzT8j0jSvIr7Gc1q28/Udhcq210ZyfYJkk6T1KsO3Ibv609qwzasIgQ8yrJOm7u8LCI+Lem3JV1RG+5iYm6VtFjSUkl9km6oth3J9jGS7pd0ZUTsr7qf9xulv7ZswypCYLekhSMe/4qkPRX0MaaI2FO73SvpQQ3vwnSa/tq+5OF9yr0V9/P/RER/RAxGxJCk21TxNrTdpeH/YHdFxAO1xR2zDUfrr13bsIoQeErSEtuLbB8l6fckbaqgj1HZPrr24YxsHy3pAkk7yq+qxCZJK2v3V0p6qMJefsHh/1w1F6vCbWjbkm6XtDMibhxR6ohtOFZ/7dqGbT86IEm1Qx1/J2mapA0R8Y22NzEG2x/X8G9/SZou6e6q+7N9j6TzJM2V1C/pGkn/IuleSR+T9IqkSyKikg/nxujvPA0PY0PSLkmXH97/rqC/syU9LulZSUO1xWs1vN9d+TYs9LdCbdiGlYQAgM7BjEEgOUIASI4QAJIjBIDkCAEguUpDoIOn5Eqiv0Z1cn+d3JvU3v6qHgl09F+E6K9RndxfJ/cmtbG/qkMAQMUamixke7mkmzQ88++fImJd6flHeUbM1NH/+/igDqhLMya9/lajv8Z0cn+d3JvU/P5+rnf0XhwY7ct7kw+ByZwc5Fh3xxk+f1LrAzB5vbFF+2Ng1BBoZHeAk4MAU0AjIXAknBwEQB3TG3jtuE4OUjvUsUqSZmpWA6sD0AqNjATGdXKQiFgfET0R0dPJH8QAWTUSAh19chAA4zPp3YGIOGR7taTv6f9ODvJc0zoD0BaNfCagiHhY0sNN6gVABZgxCCRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJNXRpchxZPL381z3tw3Nbuv7n//SEYn1w1lCxfvzivcX6rK+4WP/vG48q1rf3fLtY3zf4TrF+xn1rivUT/+TJYr0qDYWA7V2S3pI0KOlQRPQ0oykA7dOMkcBvRsS+JrwPgArwmQCQXKMhEJK+b3ub7VXNaAhAezW6O7AsIvbYnidps+0fR8RjI59QC4dVkjRTsxpcHYBma2gkEBF7ard7JT0o6fRRnrM+InoioqdLMxpZHYAWmHQI2D7a9uzD9yVdIGlHsxoD0B6N7A7Ml/Sg7cPvc3dEfLcpXU1R005eUqzHjK5ifc+5HyrW3z2zfBy7+4Pl+uOfKh8nr9q//mx2sf43/7C8WO899e5i/eWD7xbr6/o/U6x/9PEo1jvVpEMgIl6S9Kkm9gKgAhwiBJIjBIDkCAEgOUIASI4QAJIjBIDkOJ9AEw2e9+li/cY7binWP9FV/r77VHcwBov1v7j5i8X69HfKx+nPum91sT771UPF+ox95XkEs7b2FuudipEAkBwhACRHCADJEQJAcoQAkBwhACRHCADJMU+giWY8v6dY3/bzhcX6J7r6m9lO063pO7NYf+nt8nUL7lj8nWL9zaHycf75f/9vxXqrHZlnC6iPkQCQHCEAJEcIAMkRAkByhACQHCEAJEcIAMk5on1HP491d5zh89u2vk4zcNlZxfr+5eXrAkx75phi/YdfuXnCPY103b5fL9afOrc8D2DwjTeL9TirfIb6XV8rlrVoxQ/LT8CYemOL9seAR6sxEgCSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDnmCXSQaXN/uVgffH2gWH/57vJx/ufO2VCsn/7XXy3W591S7ff5MXkNzROwvcH2Xts7Rizrtr3Z9ou12znNbBhA+4xnd+AOScvft+xqSVsiYomkLbXHAI5AdUMgIh6T9P5x6IWSNtbub5R0UZP7AtAmk/1gcH5E9ElS7XZe81oC0E4tP9Go7VWSVknSTM1q9eoATNBkRwL9thdIUu1271hPjIj1EdETET1dmjHJ1QFolcmGwCZJK2v3V0p6qDntAGi3ursDtu+RdJ6kubZ3S7pG0jpJ99r+kqRXJF3SyiazGNz3ekOvP7j/qIZe/8nP/6hYf+3WaeU3GBpsaP2oRt0QiIgVY5SY9QNMAUwbBpIjBIDkCAEgOUIASI4QAJIjBIDkWj5tGO1z8lUvFOuXnVo+qvvPx28p1s+95Ipiffa3nyzW0ZkYCQDJEQJAcoQAkBwhACRHCADJEQJAcoQAkBzzBKaQwTfeLNZf//LJxform94t1q++7s5i/c8+d3GxHv/xwWJ94TeeKNbVxmtkZMJIAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5BxtPPZ6rLvjDHOm8k418PtnFet3XXN9sb5o+syG1v/JO1cX60tu6yvWD720q6H1T2W9sUX7Y8Cj1RgJAMkRAkByhACQHCEAJEcIAMkRAkByhACQHPMEMG6xbGmxfuy63cX6PR//XkPrP+mRPyjWf/Uvy+dTGHzxpYbWfyRraJ6A7Q2299reMWLZtbZftf107eezzWwYQPuMZ3fgDknLR1n+zYhYWvt5uLltAWiXuiEQEY9JGmhDLwAq0MgHg6ttP1PbXZjTtI4AtNVkQ+BWSYslLZXUJ+mGsZ5oe5Xtrba3HtSBSa4OQKtMKgQioj8iBiNiSNJtkk4vPHd9RPRERE+XZky2TwAtMqkQsL1gxMOLJe0Y67kAOlvdeQK275F0nqS5kvolXVN7vFRSSNol6fKIKH/ZW8wTmOqmzZ9XrO+59MRivfeqm4r1D9T5nfX5ly8o1t88+/VifSorzROoe/GRiFgxyuLbG+4KQEdg2jCQHCEAJEcIAMkRAkByhACQHCEAJMf5BNAx7t39RLE+y0cV6z+L94r13/nqleX3f7C3WD+Scd0BAGMiBIDkCAEgOUIASI4QAJIjBIDkCAEgubpfJQYOGzq7fN2Bn14ys1g/ZemuYr3ePIB6bh44rfz+D21t6P2nKkYCQHKEAJAcIQAkRwgAyRECQHKEAJAcIQAkxzyBRNxzSrH+wtfKx+lvW7axWD9nZvn7/I06EAeL9ScHFpXfYKjupTFSYiQAJEcIAMkRAkByhACQHCEAJEcIAMkRAkByzBM4gkxfdHyx/tPLPlqsX3vpt4r13z1m34R7aqa1/T3F+qM3nVmsz9lYvm4BRld3JGB7oe1HbO+0/Zztr9eWd9vebPvF2u2c1rcLoNnGsztwSNKaiDhZ0pmSrrD9a5KulrQlIpZI2lJ7DOAIUzcEIqIvIrbX7r8laaek4yRdKOnwPNKNki5qVZMAWmdCHwzaPkHSaZJ6Jc2PiD5pOCgkzWt2cwBab9whYPsYSfdLujIi9k/gdatsb7W99aAOTKZHAC00rhCw3aXhALgrIh6oLe63vaBWXyBp72ivjYj1EdETET1dmtGMngE00XiODljS7ZJ2RsSNI0qbJK2s3V8p6aHmtweg1cYzT2CZpC9Ietb207VlayWtk3Sv7S9JekXSJa1pceqYfsLHivU3f2NBsX7pX323WP+jDz1QrLfamr7ycfwn/rE8D6D7jn8v1ucMMQ+gFeqGQET8QJLHKJ/f3HYAtBvThoHkCAEgOUIASI4QAJIjBIDkCAEgOc4nMAHTF3ykWB/YcHSx/uVFjxbrK2b3T7inZlr96tnF+vZblxbrc7+zo1jvfovj/J2IkQCQHCEAJEcIAMkRAkByhACQHCEAJEcIAMmlmifw3m+Vv8/+3h8PFOtrT3y4WL/gl96ZcE/N1D/4brF+zqY1xfpJf/7jYr37jfJx/qFiFZ2KkQCQHCEAJEcIAMkRAkByhACQHCEAJEcIAMmlmiew66Jy5r1w6n0tXf8tbywu1m969IJi3YNjnfl92EnXvVysL+nvLdYHi1VMVYwEgOQIASA5QgBIjhAAkiMEgOQIASA5QgBIzhFRfoK9UNKdkj6i4a+Mr4+Im2xfK+kPJb1We+raiCh+4f5Yd8cZ5mrmQLv1xhbtj4FRJ5qMZ7LQIUlrImK77dmSttneXKt9MyKub1ajANqvbghERJ+kvtr9t2zvlHRcqxsD0B4T+kzA9gmSTpN0eP7patvP2N5ge06TewPQBuMOAdvHSLpf0pURsV/SrZIWS1qq4ZHCDWO8bpXtrba3HtSBJrQMoJnGFQK2uzQcAHdFxAOSFBH9ETEYEUOSbpN0+mivjYj1EdETET1dmtGsvgE0Sd0QsG1Jt0vaGRE3jli+YMTTLpZUviQtgI40nqMDyyR9QdKztp+uLVsraYXtpZJC0i5Jl7ekQwAtNZ6jAz+QNNrxxfJJ+AEcEZgxCCRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcoQAkBwhACRHCADJEQJAcnWvO9DUldmvSfrPEYvmStrXtgYmjv4a08n9dXJvUvP7Oz4iPjxaoa0h8Asrt7dGRE9lDdRBf43p5P46uTepvf2xOwAkRwgAyVUdAusrXn899NeYTu6vk3uT2thfpZ8JAKhe1SMBABUjBIDkCAEgOUIASI4QAJL7H4v8SYP7urYSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_train[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Training Predicted Label: 5\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_train[0:1, :])[0]\n",
    "print(\"Benign Training Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Training Score: 0.2400\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_train_adv, y_train)\n",
    "print(\"Adversarial Training Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQqElEQVR4nO3dfZBV9X3H8c83skIEVB6EUnzEkIkmnaBZHyKdjJWpJTbxYVoy5Q9LZ5yuk8aONmmrYzqjnTEt6URT/4kdjEywNXZ01Eqn1IeiU2RMiIulgmJBkRBk3VUR8RF34ds/9pJuye7vt3vPPfec9ft+zTC7e7737vlydvlw7v39zu+YuwtAXJ+ougEA1SIEgOAIASA4QgAIjhAAgiMEgOAqCQEzW2xm/2NmL5nZDVX0kGJmO81ss5ltMrPuGvSz0sz6zGzLkG3TzexxM9ve+DitZv3dbGavNo7hJjO7pML+TjKzJ81sq5k9b2bXNrbX4hgm+mvLMbR2zxMws6MkbZP025J2S3pG0lJ3f6GtjSSY2U5Jne7+RtW9SJKZfUnSu5LudvfPNbb9naS97r68EaTT3P36GvV3s6R33f17VfQ0lJnNkTTH3Z81s6mSNkq6XNIfqQbHMNHf19SGY1jFmcC5kl5y9x3u/pGkf5Z0WQV9jBvuvk7S3iM2XyZpVePzVRr8panECP3Vhrv3uPuzjc/fkbRV0lzV5Bgm+muLKkJgrqRfDPl6t9r4Fx4ll/SYmW00s66qmxnBbHfvkQZ/iSTNqrif4VxjZs81Xi5U9nJlKDM7VdJZkjaohsfwiP6kNhzDKkLAhtlWt7nLC939bElflvSNxukuxuYOSadLWiCpR9Kt1bYjmdkUSQ9Ius7d91fdz5GG6a8tx7CKENgt6aQhX58oaU8FfYzI3fc0PvZJekiDL2HqprfxWvLwa8q+ivv5f9y9190PuvshSXeq4mNoZh0a/Ad2j7s/2Nhcm2M4XH/tOoZVhMAzkuab2WlmdrSkP5C0uoI+hmVmkxtvzsjMJku6WNKW9LMqsVrSssbnyyQ9XGEvv+LwP66GK1ThMTQzk3SXpK3uftuQUi2O4Uj9tesYtn10QJIaQx1/L+koSSvd/Tttb2IEZjZPg//7S9IEST+uuj8zu1fShZJmSuqVdJOkf5F0n6STJe2StMTdK3lzboT+LtTgaaxL2inp6sOvvyvo7zclPSVps6RDjc03avB1d+XHMNHfUrXhGFYSAgDqgxmDQHCEABAcIQAERwgAwRECQHCVhkCNp+RKor+i6txfnXuT2ttf1WcCtf5BiP6KqnN/de5NamN/VYcAgIoVmixkZosl3a7BmX8/dPflqccfbRN9kib/8ut+HVCHJja9/7LRXzF17q/OvUmt7+9DvaeP/MBwF+81HwLNLA5yrE3382xRU/sD0LwNvlb7fe+wIVDk5QCLgwAfA0VCYDwsDgIgY0KB545qcZDGUEeXJE3SMQV2B6AMRc4ERrU4iLuvcPdOd++s8xsxQFRFQqDWi4MAGJ2mXw64+4CZXSPpUf3f4iDPt6wzAG1R5D0BufsaSWta1AuACjBjEAiOEACCIwSA4AgBIDhCAAiOEACCKzRECAw1cNEXKt3/hCc2Vrr/8YozASA4QgAIjhAAgiMEgOAIASA4QgAIjhAAgmOeAH5p35VfLPT8Ka9+1KJOmpObp8A8guFxJgAERwgAwRECQHCEABAcIQAERwgAwRECQHDMEwhkwrxTk/XcOH9unL3q9QRycv29O/foUvd//D/+pNTv3yzOBIDgCAEgOEIACI4QAIIjBIDgCAEgOEIACI55AjVS9vX8/RMsWX/zjPQ4+ZSZ54+5p6Emde1J1mdMei9Zf/Ynn04/f3N6//vmp+tLvro+Wd/bPzlZ3/aXZ6Z3UFOFQsDMdkp6R9JBSQPu3tmKpgC0TyvOBH7L3d9owfcBUAHeEwCCKxoCLukxM9toZl2taAhAexV9ObDQ3feY2SxJj5vZi+6+bugDGuHQJUmTdEzB3QFotUJnAu6+p/GxT9JDks4d5jEr3L3T3Ts7NLHI7gCUoOkQMLPJZjb18OeSLpa0pVWNAWiPIi8HZkt6yMwOf58fu/sjLemqSbnr5d9YOCdZz13vXfa69rn9v3btBcn6e3PTZ1pTvtSXrC+e81Ky/t3Zm5L1ovr9YLL+77PTx+e7v7E4Wb989o5kfdcH05L1p7am5ymc8Vz6+6f/dtVpOgTcfYekz7ewFwAVYIgQCI4QAIIjBIDgCAEgOEIACI4QAIL7WK0nMLBjZ/oBmXkCuev5c9fr5xy3fkayftrkNzPf4elktfvNk5P1zhm7kvXcPIDrexck67fMSs+T+Ku+9DyLgUPp/5Me/O+zk3V7/6hkffM307/uh17+ebL+6QPdyXpd5wHkcCYABEcIAMERAkBwhAAQHCEABEcIAMERAkBwH695Apnr/XPj/Ln70+fqH2au93903g+S9UUvXJqs58b5HznjoWS9w9Lj6DkfHOxI1q/Y/pVk/eTJbyXr+/snJevz/smTdWkgWT34wrbM82PiTAAIjhAAgiMEgOAIASA4QgAIjhAAgiMEgOBqNU8gdz1/bl3+3Lr/uXkERc3cfCBZX3xy+s7tu+85Plnf89ZxY+5pqKL3DVj/w3T/07al52G8/ERPsv7tHT9N1r/zxL5kHc3hTAAIjhAAgiMEgOAIASA4QgAIjhAAgiMEgOBqNU+gqNw8gNx6ADlF7zvw1tJzkvVj1lmyPv3F9DyESbf1j7mnob6w8WvJem4eQNF5Gn9xU+a+DxcVO/45uf4/rrJnAma20sz6zGzLkG3TzexxM9ve+Dit3DYBlGU0Lwd+JGnxEdtukLTW3edLWtv4GsA4lA0Bd18nae8Rmy+TtKrx+SpJl7e4LwBt0uwbg7PdvUeSGh9nta4lAO1U+huDZtYlqUuSJumYsncHYIyaPRPoNbM5ktT42DfSA919hbt3untnhyY2uTsAZWk2BFZLWtb4fJmkh1vTDoB2y74cMLN7JV0oaaaZ7ZZ0k6Tlku4zs6sk7ZK0pBXN5NYLKKroOH9O0XkIR+/PrKufKb/83gnJ+qLX0/c12LsnvV7BzCeeSTeQkTs+Zf98cspeb6Ku8xCyIeDuS0coLWpxLwAqwLRhIDhCAAiOEACCIwSA4AgBIDhCAAjuY7WeQFFlj2OXPQ6+95L0RIJX/+zEZH3ewt2F9p8bZ696HkDRn29dx/mL4kwACI4QAIIjBIDgCAEgOEIACI4QAIIjBIDgmCcwBnW/Hv7gvreT9VNuejpZ73v4M8n6X2/flqzfcPcFybo+vz9Znrp6avr5mfUUcqr++dQVZwJAcIQAEBwhAARHCADBEQJAcIQAEBwhAATHPIEhcuPIRe8rkFP19eofftSRrF86+f10/es/SNav712QrL/zp5OS9afuPztZn9V9IFkvquz7EuR+v8q6LwdnAkBwhAAQHCEABEcIAMERAkBwhAAQHCEABMc8gTEo+3r03Dh02fMIfv2O9Dj1/Je/nqxv/8M7kvVbZqX777CjkvVHuv4rWf/zT1yVrM/+WXoeQdnzQOoqeyZgZivNrM/MtgzZdrOZvWpmmxp/Lim3TQBlGc3LgR9JWjzM9u+7+4LGnzWtbQtAu2RDwN3XSdrbhl4AVKDIG4PXmNlzjZcL01rWEYC2ajYE7pB0uqQFknok3TrSA82sy8y6zay7X+Ve4AFg7JoKAXfvdfeD7n5I0p2Szk08doW7d7p7Z4cmNtsngJI0FQJmNmfIl1dI2jLSYwHUW3aegJndK+lCSTPNbLekmyRdaGYLNLgS/E5JV5fY46jlxtHLvh68bEX7L3q9+vS5X0zWF/ztnyTrb5+Vnmfx4u+k5xks+uTBZP29UwaSdf0sXc7NAyn6+1X1ehEjyYaAuy8dZvNdJfQCoAJMGwaCIwSA4AgBIDhCAAiOEACCIwSA4EKtJzDe5xGUfb37vivT8wByZjyfnhY+4/n08z+8OD3O/zevp/v76jnp9QYefaMzWT/xyfQ8gdzxKXu9ibJwJgAERwgAwRECQHCEABAcIQAERwgAwRECQHCh5gnklH29d9Fx+LIVvZ5+++3nJ+s7lvxDsv7Y+8cm6zmPvnxGsj73P8sdx8/O48j8/HPrOZSFMwEgOEIACI4QAIIjBIDgCAEgOEIACI4QAIJjnkAb5caBq76vQM83L0jWZ3/7pGR9yfSfJuvX9y5I1nMOHEr/uvb3fjJZ7/iPDcl62etJVDUPIIczASA4QgAIjhAAgiMEgOAIASA4QgAIjhAAgmOeQI0UXc9g4uXnJutvr/lUsv7lOU8X2n9Rh9yS9X99Kn3fgPnXpecplD0PILceQ249idquJ2BmJ5nZk2a21cyeN7NrG9unm9njZra98XFa+e0CaLXRvBwYkPQtdz9D0vmSvmFmZ0q6QdJad58vaW3jawDjTDYE3L3H3Z9tfP6OpK2S5kq6TNKqxsNWSbq8rCYBlGdMbwya2amSzpK0QdJsd++RBoNC0qxWNwegfKMOATObIukBSde5+/4xPK/LzLrNrLtf6RtWAmi/UYWAmXVoMADucfcHG5t7zWxOoz5HUt9wz3X3Fe7e6e6dHZrYip4BtNBoRgdM0l2Strr7bUNKqyUta3y+TNLDrW8PQNlGM09goaQrJW02s02NbTdKWi7pPjO7StIuSUvKaTGOo044IVn/xYp0fdbU15L1dWeuTtYXvXBpst45Y1eyfv9zZyfr2t+RLJ/68EC6fqg/Wc/NA8jeF6Cgsr9/WbIh4O7rJY00i2NRa9sB0G5MGwaCIwSA4AgBIDhCAAiOEACCIwSA4FhPoIVeWZ6+Xvy4z72ZrF80d1uyvmb242Puaah+P5isf+rY15P1+586L1mftiX9f8rxL6Wvt69abj2AovMAuO8AgFoiBIDgCAEgOEIACI4QAIIjBIDgCAEgOOYJDPHadRck6+92fpCs//5nqx0H3j3wbrJ+zSu/l6zvundesn7atmrH+XPj9Llx/sLj9Jn7BoxXnAkAwRECQHCEABAcIQAERwgAwRECQHCEABBcqHkCB373nGR95ld2p+sF9/9vr3w2Wf/MCb3J+rz1V6eff8sryfqbF6fnAaTvCpAfp8+Nw+fuC5BT1+vxxzvOBIDgCAEgOEIACI4QAIIjBIDgCAEgOEIACM7cPf0As5Mk3S3p1yQdkrTC3W83s5sl/bGkw4vV3+jua1Lf61ib7ufZ+L2bedFx7rJNeGJjsr6v5Ovhc9fz5+T6R/M2+Frt9702XG00k4UGJH3L3Z81s6mSNprZ4btgfN/dv9eqRgG0XzYE3L1HUk/j83fMbKukuWU3BqA9xvSegJmdKuksSRsam64xs+fMbKWZTWtxbwDaYNQhYGZTJD0g6Tp33y/pDkmnS1qgwTOFW0d4XpeZdZtZd78OtKBlAK00qhAwsw4NBsA97v6gJLl7r7sfdPdDku6UdO5wz3X3Fe7e6e6dHZrYqr4BtEg2BMzMJN0laau73zZk+5whD7tC0pbWtwegbKMZHVgo6UpJm81sU2PbjZKWmtkCSS5pp6T0da4Aamk0owPrJQ03vpicEzAejfd5AFUrel8AVIMZg0BwhAAQHCEABEcIAMERAkBwhAAQHCEABBfqvgM5uXHuspW9rj7r9mM4nAkAwRECQHCEABAcIQAERwgAwRECQHCEABBc9r4DLd2Z2euSfj5k00xJb7StgbGjv2Lq3F+de5Na398p7n7CcIW2hsCv7Nys2907K2sgg/6KqXN/de5Nam9/vBwAgiMEgOCqDoEVFe8/h/6KqXN/de5NamN/lb4nAKB6VZ8JAKgYIQAERwgAwRECQHCEABDc/wI2UNnGRLE8DQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_train_adv[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Training Predicted Label: 7\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_train_adv[0:1, :])[0]\n",
    "print(\"Adversarial Training Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Test Score: 0.7000\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_test, y_test)\n",
    "print(\"Benign Test Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAODklEQVR4nO3df4xc5XXG8eeJvazjtWnsOHZcY3BDSBSSBlNtIJHbyhElJYmQQQltLNVypTSLWpCgitoiSxGW2qYU8aO0aZFMceNEhoTGUFDiprGstBSVOtiWAYNpTalLHW+9gNPaBPDP0z/2mm7J7ju7Oz/urM/3I61m5p479x5fzz773pl37zoiBCCvt9XdAIB6EQJAcoQAkBwhACRHCADJEQJAcrWEgO0rbP+L7edt31RHDyW299l+2vYu29u7oJ/1tods7x6xbK7tLbb3Vrdzuqy/tbZ/WB3DXbY/VWN/i21/3/Ye28/YvqFa3hXHsNBfR46hOz1PwPY0Sf8q6XJJ+yU9IWllRDzb0UYKbO+T1B8RL9fdiyTZ/kVJr0r6WkR8qFp2q6RDEXFLFaRzIuL3uqi/tZJejYjb6uhpJNsLJS2MiJ22Z0vaIekqSb+uLjiGhf5+RR04hnWMBC6R9HxEvBARxyR9Q9KKGvqYMiLiUUmH3rJ4haQN1f0NGn7R1GKM/rpGRAxGxM7q/hFJeyQtUpccw0J/HVFHCCyS9J8jHu9XB//B4xSSvmd7h+2BupsZw4KIGJSGX0SS5tfcz2iut/1UdbpQ2+nKSLaXSLpY0jZ14TF8S39SB45hHSHgUZZ129zlZRHxc5I+Kem6ariLiblb0vmSlkoalHR7ve1ItmdJ2iTpxog4XHc/bzVKfx05hnWEwH5Ji0c8PkfSgRr6GFNEHKhuhyQ9pOFTmG5zsDqXPH1OOVRzP/9PRByMiJMRcUrSPar5GNru0fA32MaIeLBa3DXHcLT+OnUM6wiBJyRdYPtnbJ8l6XOSHqmhj1HZ7qvenJHtPkmfkLS7/KxaPCJpdXV/taSHa+zlJ5z+5qpcrRqPoW1LulfSnoi4Y0SpK47hWP116hh2/NMBSao+6vgTSdMkrY+IP+x4E2Ow/R4N//SXpOmS7qu7P9v3S1ouaZ6kg5JulvQ3kh6QdK6kFyVdExG1vDk3Rn/LNTyMDUn7JF17+vy7hv5+XtI/Snpa0qlq8RoNn3fXfgwL/a1UB45hLSEAoHswYxBIjhAAkiMEgOQIASA5QgBIrtYQ6OIpuZLor1nd3F839yZ1tr+6RwJd/R8h+mtWN/fXzb1JHeyv7hAAULOmJgvZvkLSXRqe+feXEXFLaf2z3Bsz1Pfm4+M6qh71Tnr/7UZ/zenm/rq5N6n1/b2hH+tYHB3tl/cmHwKTuTjI2Z4bl/qySe0PwORti606HIdGDYFmTge4OAhwBmgmBKbCxUEANDC9ieeO6+Ig1UcdA5I0QzOb2B2AdmhmJDCui4NExLqI6I+I/m5+IwbIqpkQ6OqLgwAYn0mfDkTECdvXS/o7/d/FQZ5pWWcAOqKZ9wQUEZslbW5RLwBqwIxBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSm97Mk23vk3RE0klJJyKivxVNAeicpkKg8vGIeLkF2wFQA04HgOSaDYGQ9D3bO2wPtKIhAJ3V7OnAsog4YHu+pC22n4uIR0euUIXDgCTN0Mwmdweg1ZoaCUTEgep2SNJDki4ZZZ11EdEfEf096m1mdwDaYNIhYLvP9uzT9yV9QtLuVjUGoDOaOR1YIOkh26e3c19EfLclXQHomEmHQES8IOmiFvYCoAZ8RAgkRwgAyRECQHKEAJAcIQAkRwgAybXitwjTeOULHyvWz131fLH+3NCCYv3Y0Z5ifdH95frM/a8W66d2PVusIydGAkByhACQHCEAJEcIAMkRAkByhACQHCEAJMc8gQn43d+5r1j/TN+Pyhs4v8kGlpfL+068Vqzf9dLHm2xgavvB0HnFet/tP1WsT9+6o5XtdA1GAkByhACQHCEAJEcIAMkRAkByhACQHCEAJOeI6NjOzvbcuNSXdWx/rfbjz15arL/84XKmztlTPtY/+oCL9bM+/N/F+q0ferBYv/ztrxfr33ltVrH+6Znl6xU06/U4VqxvO9pXrC+fcbyp/b/3O9cW6+8beKKp7ddpW2zV4Tg06guMkQCQHCEAJEcIAMkRAkByhACQHCEAJEcIAMlxPYEJ6PvWtgb15rZ/dnNP15+9e3mx/gfLlpT3/w/lv5tw6/L3TrCjiZn++qlive+pwWL9nY9uKtZ/9qwGf7dhX7l+pmo4ErC93vaQ7d0jls21vcX23up2TnvbBNAu4zkd+KqkK96y7CZJWyPiAklbq8cApqCGIRARj0o69JbFKyRtqO5vkHRVi/sC0CGTfWNwQUQMSlJ1O791LQHopLa/MWh7QNKAJM3QzHbvDsAETXYkcND2QkmqbofGWjEi1kVEf0T096h3krsD0C6TDYFHJK2u7q+W9HBr2gHQaQ1PB2zfr+Er3s+zvV/SzZJukfSA7c9LelHSNe1sEuNz4r8OFut9m8r1kw223/etVybYUWsd/I2PFesfPKv8cr7t0PuL9SV/9UKxfqJYnboahkBErByjNHWvDgLgTUwbBpIjBIDkCAEgOUIASI4QAJIjBIDkuJ4Ausb08xYX619Z85VivcfTivW/vuuXivV3Dj5erJ+pGAkAyRECQHKEAJAcIQAkRwgAyRECQHKEAJAc8wTQNZ777UXF+kd6Xaw/c+z1Yn3us69NuKcMGAkAyRECQHKEAJAcIQAkRwgAyRECQHKEAJAc8wTQMUc//ZFifedn72ywhfJfsPrNG24o1t/+Tz9osP2cGAkAyRECQHKEAJAcIQAkRwgAyRECQHKEAJAc8wTQMS9+svwzZ5bL8wBW/vvlxfrM7z5ZrEexmlfDkYDt9baHbO8esWyt7R/a3lV9faq9bQJol/GcDnxV0hWjLL8zIpZWX5tb2xaATmkYAhHxqKRDHegFQA2aeWPwettPVacLc1rWEYCOmmwI3C3pfElLJQ1Kun2sFW0P2N5ue/txHZ3k7gC0y6RCICIORsTJiDgl6R5JlxTWXRcR/RHR39Pgt8AAdN6kQsD2whEPr5a0e6x1AXS3hvMEbN8vabmkebb3S7pZ0nLbSzX80es+Sde2sUdMEW+bPbtYX/ULjxXrh0+9UawPffk9xXrv0SeKdYyuYQhExMpRFt/bhl4A1IBpw0ByhACQHCEAJEcIAMkRAkByhACQHNcTQMvsXfvBYv3b8/6iWF+x9zPFeu9m5gG0AyMBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSY54Axu1/fu2jxfpTv/qnxfq/nTherL/6x+cU670aLNYxOYwEgOQIASA5QgBIjhAAkiMEgOQIASA5QgBIjnkCeNP0RT9drN/4pW8W670uv5w+9+SqYv1df8v1AurASABIjhAAkiMEgOQIASA5QgBIjhAAkiMEgOSYJ5CIp5f/uy/69v5i/ZpZrxTrG4/ML9YXfKn8M+dUsYp2aTgSsL3Y9vdt77H9jO0bquVzbW+xvbe6ndP+dgG02nhOB05I+mJEfEDSRyVdZ/tCSTdJ2hoRF0jaWj0GMMU0DIGIGIyIndX9I5L2SFokaYWkDdVqGyRd1a4mAbTPhN4YtL1E0sWStklaEBGD0nBQSCqfEALoSuMOAduzJG2SdGNEHJ7A8wZsb7e9/biOTqZHAG00rhCw3aPhANgYEQ9Wiw/aXljVF0oaGu25EbEuIvojor9Hva3oGUALjefTAUu6V9KeiLhjROkRSaur+6slPdz69gC023jmCSyTtErS07Z3VcvWSLpF0gO2Py/pRUnXtKdFtMxF7y+Wf3/+15va/J9/ufwSeMeTjze1fbRHwxCIiMckeYzyZa1tB0CnMW0YSI4QAJIjBIDkCAEgOUIASI4QAJLjegJnkGkXvq9YH/hGc/O5Llx/XbG+5Ov/3NT2UQ9GAkByhACQHCEAJEcIAMkRAkByhACQHCEAJMc8gTPIc79Vvur7lTPHfVW4UZ3z98fKK0Q0tX3Ug5EAkBwhACRHCADJEQJAcoQAkBwhACRHCADJMU9gCnnjykuK9a1X3t5gCzNb1wzOGIwEgOQIASA5QgBIjhAAkiMEgOQIASA5QgBIruE8AduLJX1N0rslnZK0LiLusr1W0hckvVStuiYiNrerUUgHlk0r1s+d3tw8gI1H5hfrPYfL1xPgagJT03gmC52Q9MWI2Gl7tqQdtrdUtTsj4rb2tQeg3RqGQEQMShqs7h+xvUfSonY3BqAzJvSegO0lki6WtK1adL3tp2yvt12+thWArjTuELA9S9ImSTdGxGFJd0s6X9JSDY8URp24bnvA9nbb24/raAtaBtBK4woB2z0aDoCNEfGgJEXEwYg4GRGnJN0jadTfbomIdRHRHxH9PeptVd8AWqRhCNi2pHsl7YmIO0YsXzhitasl7W59ewDabTyfDiyTtErS07Z3VcvWSFppe6mGPxnaJ+natnQIoK3G8+nAY5I8Sok5AVPMH71yYbH++C8vKdZj8OkWdoNuwYxBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSc3Twb8qf7blxqS/r2P4ADNsWW3U4Do0234eRAJAdIQAkRwgAyRECQHKEAJAcIQAkRwgAyXV0noDtlyT9x4hF8yS93LEGJo7+mtPN/XVzb1Lr+zsvIt41WqGjIfATO7e3R0R/bQ00QH/N6eb+urk3qbP9cToAJEcIAMnVHQLrat5/I/TXnG7ur5t7kzrYX63vCQCoX90jAQA1IwSA5AgBIDlCAEiOEACS+1/8tsxjstIf5QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Test Predicted Label: 7\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_test[0:1, :])[0]\n",
    "print(\"Benign Test Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Test Score: 0.1000\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_test_adv, y_test)\n",
    "print(\"Adversarial Test Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPy0lEQVR4nO3df6xX9X3H8dcbuUDlh0AQREoLUlxplxXdjXSjabREw0wWtZlb+cPRrOn1j5rWrNlqSBb5Z4nbqq5ZNhccrLRaNxNU7GY2GetCTTsmUCIw6kSHFr29F3vbgZUhP9774x7cLd77OZf7+Z4fl/fzkZD7/Z739/s9b8+Fl+d8z+d8jrm7AMQ1oekGADSLEACCIwSA4AgBIDhCAAiOEACCayQEzGy1mb1oZofM7J4mekgxs8Nmts/M9prZrhb0s8nM+s1s/5Bls81sm5m9VPyc1bL+1pvZ68U23GtmNzfY30Iz+46ZHTSzA2b2pWJ5K7Zhor9atqHVPU7AzC6R9F+SbpR0RNLzkta4+3/W2kiCmR2W1O3ubzbdiySZ2SclvSXpG+7+y8WyP5U04O73FUE6y92/0qL+1kt6y92/2kRPQ5nZfEnz3X2PmU2XtFvSrZI+qxZsw0R/v60atmETewLXSTrk7q+4+zuS/k7SLQ30MW64+w5JA+ctvkXS5uLxZg3+pWnECP21hrv3uvue4vFxSQclLVBLtmGiv1o0EQILJP1oyPMjqvE/eJRc0rNmttvMeppuZgTz3L1XGvxLJGluw/0M5y4ze6E4XGjscGUoM1sk6RpJO9XCbXhef1IN27CJELBhlrVt7PJKd79W0m9I+kKxu4sL85CkJZKWS+qVdH+z7UhmNk3SFkl3u/uxpvs53zD91bINmwiBI5IWDnn+fklvNNDHiNz9jeJnv6QnNXgI0zZ9xbHkuWPK/ob7+QXu3ufuZ9z9rKSH1fA2NLMuDf4De9TdnygWt2YbDtdfXduwiRB4XtJSM1tsZpMkfUbS0w30MSwzm1p8OSMzmyrpJkn70+9qxNOS1haP10ra2mAv73HuH1fhNjW4Dc3MJG2UdNDdHxhSasU2HKm/urZh7WcHJKk41fHnki6RtMnd/7j2JkZgZldp8P/+kjRR0rea7s/MHpN0vaQ5kvok3SvpKUmPS/qApNck3e7ujXw5N0J/12twN9YlHZZ057nj7wb6+4Sk70raJ+lssXidBo+7G9+Gif7WqIZt2EgIAGgPRgwCwRECQHCEABAcIQAERwgAwTUaAi0ekiuJ/nK1ub829ybV21/TewKt/kWI/nK1ub829ybV2F/TIQCgYVmDhcxstaSvaXDk39+4+32p10+yyT5FU999fkon1aXJY15/1egvT5v7a3NvUuf7+1/9XO/4yeEu3ht7CIxlcpAZNttX2KoxrQ/A2O307TrmA8OGQM7hAJODABeBnBAYD5ODACgxMeO9o5ocpDjV0SNJU3RpxuoAVCFnT2BUk4O4+wZ373b37jZ/EQNElRMCrZ4cBMDojPlwwN1Pm9ldkv5Z/z85yIGOdQagFjnfCcjdn5H0TId6AdAARgwCwRECQHCEABAcIQAERwgAwRECQHBZpwgRy8/u+LWmW0ia9vo7yfrEf91dUyfjC3sCQHCEABAcIQAERwgAwRECQHCEABAcIQAExzgBXDTeWjApWZ9ZUx/jDXsCQHCEABAcIQAERwgAwRECQHCEABAcIQAExzgBvOv0p3616RYqlTsfQtl8BWXaOp8BewJAcIQAEBwhAARHCADBEQJAcIQAEBwhAATHOIEhmp5Xf+Y3v5/1/tzz/GXX4zet7Dx9Wf8X63n+XFkhYGaHJR2XdEbSaXfv7kRTAOrTiT2BG9z9zQ58DoAG8J0AEFxuCLikZ81st5n1dKIhAPXKPRxY6e5vmNlcSdvM7IfuvmPoC4pw6JGkKbo0c3UAOi1rT8Dd3yh+9kt6UtJ1w7xmg7t3u3t3lybnrA5ABcYcAmY21cymn3ss6SZJ+zvVGIB65BwOzJP0pJmd+5xvufs/daSripSNAxjv97cv6y93HEHuefqmPx/DG3MIuPsrkj7WwV4ANIBThEBwhAAQHCEABEcIAMERAkBwhAAQ3EU1n0DZefDc6/UnXJoe9vzq7y9P1i/t9WT95GxL1q/8s+8l61Vr/Hr9hud7uFixJwAERwgAwRECQHCEABAcIQAERwgAwRECQHAX1TiBqq/3f/GvliXrSxb+qNL1n129MFmf977jyfojizZmrf+qbb+XrA/0Zs4cdVPeOIAz808m6//4qb9I1n/ns1/MWv94xZ4AEBwhAARHCADBEQJAcIQAEBwhAARHCADBmXv6GvdOmmGzfYWtquzzc+fVzx1nYNd8NFn3HxxI1k/c+p4bOP2Co8vTwzpOLT2RrB+64W+T9U/uuy1Z/8Tcl5P1XM/1L0nWe39yWbJ++7I9Wet/7PkVyfrc76a3f5vvW7HTt+uYDww7YQV7AkBwhAAQHCEABEcIAMERAkBwhAAQHCEABBdqPoHccQRl7889D/y+p/4jWf/AU1kfr5svvzG9/qP/nazvnXdFsn6mrz9Zv2Te3PT6+9LrvypZlX64Y16y/uHpfcn6oi3pMTMTTuXdV6GtSvcEzGyTmfWb2f4hy2ab2TYze6n4OavaNgFUZTSHA1+XtPq8ZfdI2u7uSyVtL54DGIdKQ8Ddd0gaOG/xLZI2F483S7q1w30BqMlYvxic5+69klT8TB/sAWityr8YNLMeST2SNEXpG3oCqN9Y9wT6zGy+JBU/R/xa2N03uHu3u3d3KXM2WgAdN9YQeFrS2uLxWklbO9MOgLqVHg6Y2WOSrpc0x8yOSLpX0n2SHjezz0l6TdLtVTY5Wj/LvX99yfvLrhdvuzNHj+a9v2QcQNk4itMlnz+xbJzBR38pWV86/cVk/clDH0vWJ314UrI+ZSBz7o2Sv18zv/n9vM8fo9IQcPc1I5Sqmx0EQG0YNgwERwgAwRECQHCEABAcIQAERwgAwV1U8wnkauo8bafkzneQO9/CWwvS59nLzCypT/jLY8n63oH3J+uL16fHefzk2vruwdEm7AkAwRECQHCEABAcIQAERwgAwRECQHCEABAc4wRaJHs+hDKZ8yVkjwMoGYfx07Xp/i47cyRZ//Gx6cn6yTsuS9ZnHUiWL1rsCQDBEQJAcIQAEBwhAARHCADBEQJAcIQAEBzjBIYoO09fdp4793r83PP0ue/PHQdQZsLUqcn6pDV9yfq2Zd9O1hd/+/PJ+tzMcQBl27dsvoYyufNBjBV7AkBwhAAQHCEABEcIAMERAkBwhAAQHCEABMc4gQtQ9fX+Zeehq77eP1dZf0c/8yvJ+qyu9HwBi7f2JOuLtqbvGzDhnWrP849XpXsCZrbJzPrNbP+QZevN7HUz21v8ubnaNgFUZTSHA1+XtHqY5Q+6+/LizzOdbQtAXUpDwN13SBqooRcADcj5YvAuM3uhOFyY1bGOANRqrCHwkKQlkpZL6pV0/0gvNLMeM9tlZrtO6eQYVwegKmMKAXfvc/cz7n5W0sOSrku8doO7d7t7d5cmj7VPABUZUwiY2fwhT2+TtH+k1wJoN3NPn1s1s8ckXS9pjqQ+SfcWz5dLckmHJd3p7r1lK5ths32FrcpqOEfl8/pXrOpxArnXy7/Zk96+u9c/lKyvfOHTyfqMT/84WT/79tvJett//2XzVeTY6dt1zAdsuFrpYCF3XzPM4o3ZXQFoBYYNA8ERAkBwhAAQHCEABEcIAMERAkBwoeYTKDsPm3seOfc8b9m8803PF1Cm6+30mJMys6acSNZPfHxZ1ueXKRsnUabtv5+RsCcABEcIAMERAkBwhAAQHCEABEcIAMERAkBwocYJlKnyeu7RaPt55omLP5isT/nd9PX+ZV75l8XJ+vyKp6fLve/AzA71UTf2BIDgCAEgOEIACI4QAIIjBIDgCAEgOEIACI5xAjUqmy8gV+718Ccu70rWtz+yNVk/42eT9Y3/c2WyPv3VvPkIcsdZlP1+cscRlM1X0dQ4FfYEgOAIASA4QgAIjhAAgiMEgOAIASA4QgAIjnECLZJ7nr+MTxz29vTv+t6Df531+Y8en5usP77sivQH3JEul40DKNt+Ze/PHQdQpun5KkZSuidgZgvN7DtmdtDMDpjZl4rls81sm5m9VPycVX27ADptNIcDpyV92d2XSfq4pC+Y2Uck3SNpu7svlbS9eA5gnCkNAXfvdfc9xePjkg5KWiDpFkmbi5dtlnRrVU0CqM4FfTFoZoskXSNpp6R57t4rDQaFpPQBIYBWGnUImNk0SVsk3e3uxy7gfT1mtsvMdp2qeKJIABduVCFgZl0aDIBH3f2JYnGfmc0v6vMl9Q/3Xnff4O7d7t7dpcmd6BlAB43m7IBJ2ijpoLs/MKT0tKS1xeO1ktLXmQJopdGME1ipwTO4+8xsb7FsnaT7JD1uZp+T9Jqk26tpsT1y5wMoOw9d9XwDP/1Q+jx52XwAf9S/PFn/+3/79WT9Q/r3ZD13nETZ9h2v9wWoWmkIuPtzkkYaZbKqs+0AqBvDhoHgCAEgOEIACI4QAIIjBIDgCAEgOOYTuAC55/mrntf+ko9cnaz/4JGNyfq6vvQ4gC3/sDJZn/NSspytdD6Birdv2e+vrL9xO58AgIsbIQAERwgAwRECQHCEABAcIQAERwgAwTFOYBwpu7/9b33l2UrX3/Xz9H0Lph3Jmz6u7Dx709re31ixJwAERwgAwRECQHCEABAcIQAERwgAwRECQHCME+igqq9X7195Jln/g9kvZ62/zII/SV8Pf/qGa9P1zPsq5N6XoOr5HMYr9gSA4AgBIDhCAAiOEACCIwSA4AgBIDhCAAiudJyAmS2U9A1JV0g6K2mDu3/NzNZL+ryko8VL17n7M1U12gZVn2cue/+Vf3hV1uffePA3k/VDh+cl61f7rmQ997+/7L4BTWvrfQNyjWaw0GlJX3b3PWY2XdJuM9tW1B50969W1x6AqpWGgLv3SuotHh83s4OSFlTdGIB6XNB3Ama2SNI1knYWi+4ysxfMbJOZzepwbwBqMOoQMLNpkrZIutvdj0l6SNISScs1uKdw/wjv6zGzXWa265Ty5qAD0HmjCgEz69JgADzq7k9Ikrv3ufsZdz8r6WFJ1w33Xnff4O7d7t7dpcmd6htAh5SGgJmZpI2SDrr7A0OWzx/ystsk7e98ewCqNpqzAysl3SFpn5ntLZatk7TGzJZLckmHJd1ZSYcAKjWaswPPSRpuwvmLbkxA7vXubffygSuT9au/uDNZb7uo8wHkYsQgEBwhAARHCADBEQJAcIQAEBwhAARHCADBmbvXtrIZNttX2Kra1gdg0E7frmM+MNx4H/YEgOgIASA4QgAIjhAAgiMEgOAIASA4QgAIrtZxAmZ2VNKrQxbNkfRmbQ1cOPrL0+b+2tyb1Pn+Pujulw9XqDUE3rNys13u3t1YAyXoL0+b+2tzb1K9/XE4AARHCADBNR0CGxpefxn6y9Pm/trcm1Rjf41+JwCgeU3vCQBoGCEABEcIAMERAkBwhAAQ3P8Biyeq37yblYMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test_adv[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Test Predicted Label: 9\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_test_adv[0:1, :])[0]\n",
    "print(\"Adversarial Test Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Investigate dependence on attack budget eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3QU9f3/8ec7mxsQwh1BLiIVFRAViCBCtVZ/rVewCBYoVrwU8YK17a/92q/6q7U9tdraWhBFFLXeCl4q4l1bxXKXoCiCogEsIlSi3EESAu/fH7vUNeSygUxmk3k9ztmT3dmZ3ZcjyWtnZucz5u6IiEh0ZYQdQEREwqUiEBGJOBWBiEjEqQhERCJORSAiEnEqAhGRiAusCMzsfjPbYGbvVfK8mdkEMysys3fNrE9QWUREpHJBbhE8CJxRxfNnAt0St7HA3QFmERGRSgRWBO7+L2BjFbMMAR7yuAVAczNrH1QeERGpWGaI790B+CTp8drEtPXlZzSzscS3GmjSpEnfo48+uk4Ciog0FIsXL/7c3dtU9FyYRWAVTKtwvAt3nwJMASgoKPDCwsIgc4mINDhm9u/KngvzW0NrgU5JjzsC60LKIiISWWEWwUzgh4lvD50IbHH3/XYLiYhIsALbNWRmfwO+BbQ2s7XAr4AsAHefDLwAnAUUATuBi4PKIiIilQusCNx9ZDXPO3BVUO8vIiKp0ZnFIiIRpyIQEYk4FYGISMSpCEREIk5FICIScSoCEZGIUxGIiEScikBEJOJUBCIiEaciEBGJOBWBiEjEqQhERCJORSAiEnEqAhGRiFMRiIhEnIpARCTiVAQiIhGnIhARiTgVgYhIxKkIREQiTkUgIhJxKgIRkYhTEYiIRJyKQEQk4lQEIiIRpyIQEYk4FYGISMSpCEREIk5FICIScSoCEZGIUxGIiEScikBEJOJUBCIiEaciEBGJOBWBiEjEBVoEZnaGma0wsyIzu66C5zub2etm9raZvWtmZwWZR0RE9hdYEZhZDJgEnAn0AEaaWY9ys90APO7uvYERwF1B5RERkYoFuUXQDyhy91XuXgpMA4aUm8eB/MT9ZsC6APOIiEgFgiyCDsAnSY/XJqYluwkYbWZrgReA8RW9kJmNNbNCMyssLi4OIquISGQFWQRWwTQv93gk8KC7dwTOAh42s/0yufsUdy9w94I2bdoEEFVEJLqCLIK1QKekxx3Zf9fPpcDjAO4+H8gFWgeYSUREygmyCBYB3czscDPLJn4weGa5edYApwGYWXfiRaB9PyIidSiwInD3MuBq4GXgfeLfDlpmZjeb2eDEbD8DfmRm7wB/A8a4e/ndRyIiEqDMIF/c3V8gfhA4edr/S7q/HBgYZAYREamaziwWEYk4FYGISMSpCEREIk5FICIScSoCEZGIUxGIiEScikBEJOJUBCIiEaciEBGJOBWBiEjEqQhERCJORSAiEnEqAhGRiFMRiIhEnIpARCTiVAQiIhGnIhARiTgVgYhIxEWqCMr27A07gohI2olMEbz03noG3zmXjz/fEXYUEZG0EpkiyM2K8enmLzn3zjm8suw/YccREUkbkSmCbx3VlufGD6JLqyaMfXgxv3/xA+0qEhEhQkUA0KllY54YN4BR/Tsz+Y2VjJ66kOJtJWHHEhEJVaSKAOK7iH73vV7cPvw4lnyymbMnzGbRxxvDjiUiEprIFcE+5/ftyIyrBtIkJ5MRUxZw3+xVuHvYsURE6lxkiwDg6Hb5PHP1QE7v3pbfPv8+Vz76Ftt27Q47lohInaq2CMxsoZldbmb5dRGoruXnZjF5dF+uP6s7ryz/jCF3zmXFf7aFHUtEpM6kskVwEdAVWGJmj5jZaQFnqnNmxo9O7spjl/VnW0kZ502ay9Nvrw07lohInai2CNz9A3f/H6Ab8BTwkJmtNrMbzax54AnrUP+urXj+mkH06tiMn0x/hxtmLKWkbE/YsUREApXSMQIz6wH8HrgFeAYYDZQCrwUXLRxtm+by2GX9ufyUrjyyYA0XTJ7P2k07w44lIhKYlI4RAHcBS4He7n6lu89191uBBrn/JDOWwS/P7M49F/ZlVfEOzpk4h1krNoQdS0QkEKlsEVzo7t9y94fc/cvkJ9x9cEC50sJ3e7bj2fGDaJefy8UPLuLPr37Inr36iqmINCwpFUHysQAza2Fmvw4wU1rp0roJT185kKG9O/KXf37EmAfeZOOO0rBjiYjUmlSK4Bx337zvgbtvAs4NLlL6aZQd44/Dj+WWob1YuHoj50yYzZJPNle/oIhIPZBKEcTMLHvfAzPLBbKrmP+/zOwMM1thZkVmdl0l81xgZsvNbJmZPZZa7LpnZozs15mnxp1ERoYxfPI8Hp7/sc5GFpF6L5UimAa8amYXmdkPgZeBR6tbyMxiwCTgTKAHMDLx7aPkeboBvwQGuntP4Noa5q9zvTo247nxg/hmtzbc+Mwyrp2+hJ2lZWHHEhE5YKmcR/A74I9Ab6AvcJu735LCa/cDitx9lbuXEi+UIeXm+REwKbG7CXevF1/Nad44m/t+WMDPv3sUz76zjvMmzWVl8fawY4mIHJCUziNw92fd/Vp3/7G7P5/ia3cAPkl6vDYxLdmRwJFmNtfMFpjZGRW9kJmNNbNCMyssLi5O8e2DlZFhXHXqETx0SX8+317K4IlzeGHp+rBjiYjUWCrnEZyQ+CO9xcx2mVmJmW1N4bWtgmnld6hnEj9j+VvASOC+is5Wdvcp7l7g7gVt2rRJ4a3rzqBurXn+mkEc1a4pVz76Fjc/u5zduuCNiNQjqWwR3EV8vKFVQFPgauCOFJZbC3RKetwRWFfBPM+4+253Xw2sIF4M9Ur7Zo2YNnYAY07qwv1zVzNyygL+s2VX2LFERFKSShFkuPsKIDPxB/te4PQUllsEdDOzwxPfOhoBzCw3zwzgVAAza018V9GqlNOnkezMDG4a3JOJI3uzfP1Wzpk4m3lFn4cdS0SkWqkUwY7EH/J3zOx3ZjYeyKtuIXcvI7718DLwPvC4uy8zs5vNbN8ZyS8DX5jZcuB14Ofu/sUB/ZekiXOPO5SZVw+keeNsRk9dyKTXi9irs5FFJI1Zdd+DN7OuxHfp5AI/A5oBd7r7h8HH219BQYEXFhaG8dY1sqOkjOv+vpRn31nH6d3bcvvw42nWOCvsWCISUWa22N0LKnquyi2CxLkAv3L3Xe6+2d1vdPdrwiqB+qRJTiYTRhzPTef24I0Piznnztm89+mWsGOJiOynyiJw9z1AezPTR9kDYGaMGXg40y8fQNkeZ+jd85i+aE3YsUREviYzhXlWAbPN7Blgx76J7j4hsFQNTJ/OLXhu/CB+PG0J//PUUhb/exM3DzmG3KxY2NFERFI6WFwMvAo0Btok3aQGWuXl8NdL+nHNt4/g8cK1DL1rHv/+Ykf1C4qIBKzag8Xppr4cLK7K6x9s4NrpS9jrzp8uOJ7/0+OQsCOJSAN3wAeLEwu/amavlL/VfszoOPXotjw3fhBdWjXhRw8VcutLH1Cms5FFJCSpHCO4Iel+LnA+UBJMnOjo1LIxT4wbwM3PLefuWStZsmYzE0b2pk3TnLCjiUjEHNCuITN7w91PCSBPtRrCrqHynlq8lutnLCU/N4tJP+jDCV1ahh1JRBqYg901lJ90a25mpwHtaz1lhJ3ftyNPXzmQxtkxRkxZwH2zV+mCNyJSZ1LZNbSM+KihBpQBq4lfR0BqUff2+cwcP4ifP/EOv33+fd5as4lbzz+Wprk6hUNEglVtEbh7p+rmkdqRn5vF5NF9uXf2Km59aQUfrJ/L3aP7clS7pmFHE5EGLJVdQ+OSrxFgZi3MbGywsaLLzBh78jd47LL+bCsp47xJc5nx9qdhxxKRBiyVE8rGufvmfQ8Sl5W8IrhIAtC/ayueHz+IXh2bce30JdwwYyklZXvCjiUiDVAqRfC1cRDMLAPQjus60DY/l8cu68/lJ3flkQVruGDyfNZu2hl2LBFpYFIpglfN7G9mdoqZnQw8Cvwj4FySkBnL4JdndWfy6L6sKt7BORPnMGvFhrBjiUgDkkoR/ByYC/yE+PUI5gD/N8hQsr8zjmnHzPGDaJefy8UPLuKOf3yoC96ISK1I5cI0uUCpu+9NPM4Ast09lIvyNsQTymriy9I9XD9jKX9/61NOPrINd3z/eFo2yQ47loikuYM6oYz4JSSbJD1uArxWG8Gk5hplx7h9+HH87nu9WLDyC86dOIcln2yufkERkUqkUgSN3H3bvgeJ+42DiyTVMTNG9e/MU1echBkMnzyPh+d/rLORReSApFIEO83suH0PzOx4IJTdQvJ1vTo247nxgxh0RGtufGYZP5m+hJ2lZWHHEpF6JpUhJn4CPG1m/0487gyMCi6S1ETzxtlMvegE7ppVxO2vfsjy9Vu5e3RfvtEmL+xoIlJPpDT6qJnlAN2Jjze0DNiTuJ5xnYv6weKqzP6omB9PW0Jp2V5uG3YsZ/XS2IAiEnewB4tx9xJ3XwI0BSYAGvMgDX2zWxueGz+IbofkceWjb/Gb55azWxe8EZFqpDLWUF8zu93MPgZeBBYBxwQdTA7Moc0bMX3sAMac1IWpc1YzcsoC/rNFh3REpHKVFoGZ/drMPgD+BHwEnABscPep7v55XQWUmsvOzOCmwT2ZMLI3y9dv5ZyJs5m3Uv/LRKRiVW0RXA1sAP4M3O/uxcSvSyD1xODjDmXm1QNp1iiL0fct5K5ZRTobWUT2U1URtAP+AFwArDKzB4BGiTOLpZ44om1Tnrl6EGf1as9tL61g7MOL2fLl7rBjiUgaqfSPurvvdvdn3X0UcCTwMvAm8KmZPVRXAeXg5eVkMnFkb246twezVmzg3IlzWLZuS9ixRCRNpPqtoZ3uPs3dhwA9gDeCjSW1zcwYM/Bwpl8+gNKyvQy9ax6PL/ok7FgikgZqvJvH3Te5+9Qgwkjw+h7WguevGcQJXVryi6fe5RdPvsOu3brgjUiUaX9/BLXKy+Gvl/Rj/LeP4PHCtQy9ax5rvtAFb0SiKpXzCPYbhqKiaVK/xDKMn33nKO4fU8Cnm7/k7Imz+cfyz8KOJSIhSGWL4M0Up0k99O2jD+G58YM4rFVjLnuokNte+oAynY0sEimVfrI3s7ZAe+JfGe1FfJwhgHw0DHWD0qllY54cdxK/fnY5d81aydtrNjNhZG/aNM0JO5qI1IGqdvGcDVwCdAQm8VURbANuDDiX1LHcrBi3DO1F38NacP3TSzln4mwmjepDQZeWYUcTkYBVdR7BA+7+TeBSdz/Z3b+ZuJ3l7k+k8uJmdoaZrTCzIjO7ror5hpmZm1mFI+NJ3RnWtyMzrhpIo6wYI6YsYMbbGl9QpKFL5RhBWzPLBzCzyWb2ppmdVt1CZhYjviVxJvFzD0aaWY8K5msKXAMsrFFyCUz39vnMHB//ium105dw/5zVYUcSkQClUgRj3X2rmX2H+G6iK4DbUliuH1Dk7qvcvRSYBgypYL7fJF5PQ2SmkfzcLB64+ATO6NmOm59bzh9e/kCXwhRpoFIpgn2//WcCD7j74hSX6wAkn7q6NjHtv8ysN9DJ3Z+r6oXMbKyZFZpZYXFxcQpvLbUhNyvGpB/0YWS/zkx6fSW//PtSfaNIpAFK5XyAd8zsBeLjDV1vZnmkNgqpVTDtv8slBq/7MzCmuhdy9ynAFIhfoSyF95ZaEsswfve9Y2idl83E14rYtLOUv4zoTW5WLOxoIlJLUvlkfzFwE9DP3XcCucClKSy3FuiU9LgjsC7pcVPiF7iZlbjozYnATB0wTj9m8ZPPfnVuD15e9hljHniTbbs0gqlIQ1FtESSuTdyV+LEBgEapLEf8SmbdzOxwM8sGRgAzk153i7u3dvcu7t4FWAAMdnddkDhNXTzwcP4y4ngKP97EiCkLKN5WEnYkEakFqQwxcSdwKjA6MWkHMLm65dy9jPjFbV4G3gced/dlZnazmQ0+8MgSpiHHd+C+iwpYVbyDYZM1RpFIQ2DVfRPEzN5y9z5m9ra7905Me8fdj6uThOUUFBR4YaE2GsL29ppNXPzgIrJiGTx0ST+6t88PO5KIVMHMFrt7hbveU9nFsztxYNcTL9YK0FdHIq535xY8OW4AmRnGBffM583VG8OOJCIHqKqL1+/7RtEk4CmgjZn9GpgD3FoH2STNHdG2KU9ecRJtm+Zw4dSFvKrRS0Xqpaq2CN4EcPeHgBuAPwKbgOHuPq0Oskk90KF5I54YdxJHt89n3COLebxQVz0TqW+qOo/gv+cBuPsyYFnwcaQ+atkkm8cu68+4RxbziyffZeOOUsad8o2wY4lIiqoqgjZm9tPKnnT3PwWQR+qpJjmZTL3oBH76+BJ+/+IHbNxRynVnHE1GRkXnFYpIOqmqCGJAHhWfISyyn+zMDCaM6E2rJtlM+dcqvtheyu/P70VWTFdEFUlnVRXBene/uc6SSIOQkWHcNLgnrfJy+NOrH7J5Zyl3jupDo2wNSSGSrqr6qKYtATkgZsY1p3Xjt+cdw2srNnDh1IVs2akhKUTSVVVFUO01B0SqMvrEw5g0qg/vrt3CBffM57OtGmlcJB1VdYUynSEkB+2sXu154OITWLtpJ0Pvmseq4u1hRxKRcnQUTwI38IjWTBs7gF279zB88nyWrt0SdiQRSaIikDrRq2Mznhg3gNysGCOmzGde0edhRxKRBBWB1JmubfL4+5Un0bFFY8Y8sIgXlq4PO5KIoCKQOnZIfi6PXz6AYzs246rH3uKRBf8OO5JI5KkIpM41a5zFw5f259Sj2nLDjPeY8M+PqG44dBEJjopAQtEoO8Y9F/ZlaJ8O/OnVD7lp5jL27lUZiIQhlYvXiwQiK5bBH4cdR6sm2dw7ezUbd+7m9uHHkZ2pzycidUlFIKHKyDCuP7sHrfJy+P2LH7B5ZymTR/elSY7+aYrUFX30krQw7pRvcNv5xzK36HNG3beQjTtKw44kEhkqAkkbF5zQicmj+/L++q0MnzyPTzd/GXYkkUhQEUha+U7Pdjx8ST82bC1h2N3zKNqwLexIIg2eikDSTv+urZh++QB273GGTZ7P22s2hR1JpEFTEUha6nFoPn+/4iSaNcpi1L0LeePD4rAjiTRYKgJJW51bNeaJcQM4vHUTLn1wEc8s+TTsSCINkopA0lrbprlMu/xE+h7Wgh9PW8IDc1eHHUmkwVERSNrLz83ir5f04zs9DuHXzy7n9ldWaEgKkVqkIpB6ITcrxl0/6MOIEzox8bUi/vfp99ijISlEaoVO35R6IzOWwS1De9EqL5tJr69k045S7hhxPLlZsbCjidRr2iKQesXM+Pl3j+bGc3rw0rL/cPEDi9i2a3fYsUTqNRWB1EuXDjqcO75/PIs+3siIKQso3lYSdiSRektFIPXWeb07cO9FBaws3s7wyfP4ZOPOsCOJ1EsqAqnXTj2qLY9ediKbdu5m6N3zeH/91rAjidQ7KgKp9/oe1oInxg0gZsYF98znzdUbw44kUq+oCKRBOPKQpjx5xQDa5OVw4dSF/GP5Z2FHEqk3Ai0CMzvDzFaYWZGZXVfB8z81s+Vm9q6Z/dPMDgsyjzRsHVvEh6Q4ql1TLn9kMU8UfhJ2JJF6IbAiMLMYMAk4E+gBjDSzHuVmexsocPdjgSeB24LKI9HQKi+Hx350IgO6tuLnT77LPW+sDDuSSNoLcougH1Dk7qvcvRSYBgxJnsHdX3f3fV/1WAB0DDCPREReTiZTxxRw9rHtueXFD/jdC+9rSAqRKgR5ZnEHIHnbfC3Qv4r5LwVerOgJMxsLjAXo3LlzbeWTBiwnM8aEEb1p2TibKf9axRfbS7n1/F5kxnRYTKS8IIvAKphW4ccyMxsNFACnVPS8u08BpgAUFBToo52kJJZh3DykJ63ysrnjHx+xeWcpd47qQ6NsDUkhkizIj0drgU5JjzsC68rPZGanA9cDg91dp4dKrTIzrj39SH4zpCevrdjAD+9fyJadGpJCJFmQRbAI6GZmh5tZNjACmJk8g5n1Bu4hXgIbAswiEXfhgC5MHNmbJZ9s5vtT5vPZ1l1hRxJJG4EVgbuXAVcDLwPvA4+7+zIzu9nMBidm+wOQBzxhZkvMbGYlLydy0M459lAeGNOPNRt3cv7d81j9+Y6wI4mkBatv36YoKCjwwsLCsGNIPfbOJ5u5+MFFZBg8eHE/junQLOxIIoEzs8XuXlDRc/oKhUTOcZ2a88S4AeRkxhgxZQHzVn4ediSRUKkIJJK+0SaPp644iUOb5zLm/kW89N76sCOJhEZFIJHVrlkuj18+gGM65HPlo2/x2MI1YUcSCYWKQCKteeNsHr3sRE45sg3/+/RS7nztI52FLJGjIpDIa5QdY8oPC/he7w788ZUP+fWzy9m7V2Ug0aGL14sAWbEMbh9+HC2bZDN1zmo27ijlj8OPIztTn5Wk4VMRiCRkZBg3nN2dVnnZ3PbSCjZ/uZvJo/vQOFu/JtKw6eOOSBIz48pvHcHvh/ZizkfFjLp3IZt2lIYdSyRQKgKRCozo15m7R/dl+fqtDL9nPus2fxl2JJHAqAhEKvHdnu146JJ+fLZlF+ffPY+iDdvCjiQSCBWBSBVO7NqKaZefyO49zrDJ83l7zaawI4nUOhWBSDV6HtqMp64YQH5uFqPuXcgbHxaHHUmkVqkIRFJwWKsmPDluAF1aN+Gyvy7imSWfhh1JpNaoCERS1DY/l2ljT6R35xZcO30JD85dHXYkkVqhIhCpgWaNsnjokn6c3v0Qbnp2OX96ZYWGpJB6T0UgUkO5WTHu/kEfLijoyITXirh+xnvs0ZAUUo/plEmRA5AZy+DW84+lVV4Od89ayaYdpdwx4nhyMmNhRxOpMRWByAEyM/7njKNp1SSb3z7/Pl/c9yanHt2WvNxM8nMzaZqbSV5OVuJnJvm5WeTlZhLLsLCji3yNikDkIF32za60bJLNjTPe482PN1Y7f+PsGE1zM2mam0VeTmbifiZN95VG4rn4tK/u5yXNl5uVgZkKRWqHikCkFgzt05Hv9e7Art172bZrN1t3lbG9pIxtu3azbVcZ23eVsXXX7sS0+PR997fuKmPd5i/j85WUsbN0T7Xvl5lhX5VDokD2L5csbZ1ISlQEIrXEzGiUHaNRdoy2+Qf+OmV79iYVxtdLY1u5ctl3f1tJGZ9u3sX2km3/XS6VA9jaOhFQEYikncxYBs0bZ9O8cfYBv4a7V7h1sn3Xvq2QirdOttXh1kleTia5WRlkZ8ZvOZkxcjIzyI5lkKEtlTqlIhBpgILaOkne3VXR1sn2kviurgPZOkmWFTOyY18VxFdlkSiOWAY5WbH4zwqfyyA7Fvv69MT9nMxyrxvLqGC+aJWSikBEKlXbWyfbkrdCdpVRUraX0rK9lJTtid/fs+9x/Oe+50oTz5XsTvws28uWL3cn5tnz1fxJy9fWuR1RKCUVgYgE6mtbJ3X4vnv2+teKpCS5YPbspWT3noqLp6rnUiil5Odqu5R+c94xXHjiYbXyWslUBCLSIMUyviqgsNWklJLLp3wpHduhWSD5VAQiIgFLp1KqiMYaEhGJOBWBiEjEqQhERCJORSAiEnEqAhGRiFMRiIhEnIpARCTiVAQiIhEXaBGY2RlmtsLMiszsugqezzGz6YnnF5pZlyDziIjI/gIrAjOLAZOAM4EewEgz61FutkuBTe5+BPBn4Nag8oiISMWC3CLoBxS5+yp3LwWmAUPKzTME+Gvi/pPAaaYrXIiI1KkgxxrqAHyS9Hgt0L+yedy9zMy2AK2Az5NnMrOxwNjEw+1mtuIAM7Uu/9ppQrlqRrlqLl2zKVfNHEyuSoctDbIIKvpkX34s1lTmwd2nAFMOOpBZobsXHOzr1Dblqhnlqrl0zaZcNRNUriB3Da0FOiU97gisq2weM8sEmgEbA8wkIiLlBFkEi4BuZna4mWUDI4CZ5eaZCVyUuD8MeM3da+cKDiIikpLAdg0l9vlfDbwMxID73X2Zmd0MFLr7TGAq8LCZFRHfEhgRVJ6Eg969FBDlqhnlqrl0zaZcNRNILtMHcBGRaNOZxSIiEaciEBGJuAZTBCkMZ3Gymb1lZmVmNqzccxeZ2UeJ20Xllw0x1x4zW5K4lT/QHnSun5rZcjN718z+aWaHJT0X5vqqKleY62ucmS1NvPec5LPozeyXieVWmNl30yGXmXUxsy+T1tfkusyVNN8wM3MzK0iaFtr6qixX2OvLzMaYWXHS+1+W9NzB/z66e72/ET8YvRLoCmQD7wA9ys3TBTgWeAgYljS9JbAq8bNF4n6LsHMlntse4vo6FWicuH8FMD1N1leFudJgfeUn3R8MvJS43yMxfw5weOJ1YmmQqwvwXljrKzFfU+BfwAKgIB3WVxW5Ql1fwBjgzgqWrZXfx4ayRVDtcBbu/rG7vwvsLbfsd4FX3X2ju28CXgXOSINcQUol1+vuvjPxcAHx80Ag/PVVWa4gpZJra9LDJnx1YuQQYJq7l7j7aqAo8Xph5wpSKsPLAPwGuA3YlTQt1PVVRa4gpZqrIrXy+9hQiqCi4Sw61MGyQb92rpkVmtkCMzuvljIdSK5LgRcPcNm6ygUhry8zu8rMVhL/I3JNTZYNIRfA4Wb2tpm9YWbfrKVMKeUys95AJ3d/rqbLhpQLQlxfCecndok+aWb7TtatlfUV5BATdSmloSoCWDbo1+7s7uvMrCvwmpktdfeVdZnLzEYDBcApNV22jnNByOvL3ScBk8xsFHAD8ZMlQ19fleRaT3x9fWFmfYEZZtaz3BZEILnMLIP4aMNjarrsQTqYXKGtr4Rngb+5e4mZjSM+WOe3U1y2Wg1liyCV4SyCWDbQ13b3dYmfq4BZQO+6zGVmpwPXA4PdvaQmy4aQK/T1lWQasG+LJPT1VVGuxK6XLxL3FxPfR31kHeVqChwDzDKzj4ETgZmJA7Nhrq9Kc4W8vnD3L5L+rd8L9E112ZQEcfCjrm/Et2xWET+4tO9gS89K5n2Q/Q8WryZ+oKVF4n7LNMjVAshJ3G8NfEQFB7aCykX8j+hKoFu56aGurypyhb2+uiXdP5f42fMAPfn6wc9V1N7Bz4PJ1WZfDuIHKT8N4999Yv5ZfHVQNtT1VUWuUNcX0D7p/m6hCqcAAAH2SURBVPeABYn7tfL7eND/EelyA84CPkz8kbg+Me1m4p8aAU4g3p47gC+AZUnLXkL8oFQRcHE65AJOApYm/lEsBS6t41z/AD4DliRuM9NkfVWYKw3W11+AZYlMryf/IhPfelkJrADOTIdcwPmJ6e8AbwHn1mWucvPOIvEHN+z1VVmusNcXcEvS+78OHJ207EH/PmqICRGRiGsoxwhEROQAqQhERCJORSAiEnEqAhGRiFMRiIhEnIpARCTiVAQiIhGnIhCpATMbbWZvJsaEv8fMYma23cxut/h1Jf5pZm0S815jX107YVrY2UUqoyIQSZGZdQe+Dwx09+OBPcAPiA/v/Ja79wHeAH6VWOQ6oLe7HwuMCyGySEoayuijInXhNOKDfS0yM4BGwAbi15KYnpjnEeDvifvvAo+a2QxgRt1GFUmdtghEUmfAX939+MTtKHe/qYL59o3bcjYwiXh5LDYzffCStKQiEEndP4FhZtYWwMxaWvyayRnAvutNjwLmJMa27+TurwO/AJoDeSFkFqmWPqGIpMjdl5vZDcAriT/0u4GriI8c29PMFgNbiB9HiAGPmFkz4lsSf3b3zSFFF6mSRh8VOUhmtt3d9Wlf6i3tGhIRiThtEYiIRJy2CEREIk5FICIScSoCEZGIUxGIiEScikBEJOL+P2Gi0wttRzusAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "eps_list = [0.1, 0.2, 0.3, 0.4, 0.5]\n",
    "score_list = list()\n",
    "\n",
    "for eps in eps_list:\n",
    "    pgd = ProjectedGradientDescent(estimator=art_classifier, norm=np.inf, eps=eps, eps_step=0.05, max_iter=20, \n",
    "                                   targeted=False, num_random_init=0, batch_size=128)\n",
    "    x_test_adv = pgd.generate(x_test)\n",
    "    score = model.score(x_test_adv, y_test)\n",
    "    score_list.append(score)\n",
    "\n",
    "plt.plot(eps_list, score_list)\n",
    "plt.xlabel('eps')\n",
    "plt.ylabel('Test Accuracy')\n",
    "plt.ylim((0, 1));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Targeted PGD attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "pgd = ProjectedGradientDescent(estimator=art_classifier, norm=np.inf, eps=0.5, eps_step=0.01, max_iter=50, \n",
    "                               targeted=True, num_random_init=3, batch_size=128)\n",
    "y_test_target = np.zeros((y_test.shape[0], 10))\n",
    "target_label = 7\n",
    "y_test_target[:, target_label] = 1\n",
    "x_test_adv = pgd.generate(x_test, y=y_test_target)\n",
    "score = model.score(x_test_adv, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Targeted Adversarial Test Score: 0.2100\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_test_adv, np.argmax(y_test_target, axis=1))\n",
    "print(\"Targeted Adversarial Test Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAV6ElEQVR4nO3df3RU5ZkH8O+TyS8SQgg/EgIEUIIgZ1ejG8GqdXXZra52q56WKrYeeo5t8JzaVexu67p7qu6edd1WUXfb6uKPI66Ch1210kpVim1RUQSEFRSQXwFCYhCCEAJJmMmzf2Sw0eY+b8idmTvh/X7O4SSZb+bOk5vJw5153/teUVUQkb9yoi6AiKLFJkDkOTYBIs+xCRB5jk2AyHNsAkSei6QJiMjlIrJFRLaJyO1R1GARkXoR2SAi60VkTRbU84SI7BORjT1uGyYiy0Rka/JjWZbVd5eI7E3uw/UickWE9VWJyG9FZJOIvC8ityRvz4p9aNSXkX0omZ4nICIxAB8C+CsADQBWA5ilqh9ktBCDiNQDqFXV/VHXAgAicjGAIwCeUtU/Sd72YwAtqnpvspGWqeoPs6i+uwAcUdX7oqipJxGpBFCpqu+KSAmAtQCuBvAtZME+NOr7OjKwD6M4EpgGYJuq7lDVTgDPArgqgjoGDFVdAaDlczdfBWBB8vMF6H7SRCKgvqyhqk2q+m7y81YAmwCMQZbsQ6O+jIiiCYwBsKfH1w3I4A/cRwrgVRFZKyJ1URcToEJVm4DuJxGA8ojr6c3NIvJe8uVCZC9XehKRCQDOAbAKWbgPP1cfkIF9GEUTkF5uy7a5yxeq6rkA/hrAd5OHu3RyHgYwEUANgCYA90dbDiAigwE8B+BWVT0cdT2f10t9GdmHUTSBBgBVPb4eC6AxgjoCqWpj8uM+AC+g+yVMtmlOvpY88ZpyX8T1fIaqNqtqQlW7ADyKiPehiOSh+w/sGVV9Pnlz1uzD3urL1D6MogmsBjBJRE4TkXwA1wFYEkEdvRKR4uSbMxCRYgBfArDRvlcklgCYnfx8NoAXI6zlj5z440q6BhHuQxERAI8D2KSq83pEWbEPg+rL1D7M+OgAACSHOh4EEAPwhKr+a8aLCCAip6P7f38AyAWwMOr6RGQRgEsAjADQDOBOAL8AsBjAOAC7AcxU1UjenAuo7xJ0H8YqgHoAc068/o6gvosAvA5gA4Cu5M13oPt1d+T70KhvFjKwDyNpAkSUPThjkMhzbAJEnmMTIPIcmwCR59gEiDwXaRPI4im5AFhfWNlcXzbXBmS2vqiPBLL6FwHWF1Y215fNtQEZrC/qJkBEEQs1WUhELgfwELpn/j2mqvda358vBVqI4k+/Po4O5KGg34+fbidbn5YWmXm8oLdzp/7AToG8/cc+83WntiNfCntswLGFHLvn6/HjjgpsnWOKP/N1oq0NseI/3Ja/t83eQPEgM+4os+vPcZSvPe6eONqGWNFn681vsuvrKis285yDjp/vJPT23BPH77djVPDz7/gnLUgcbet1A7n9qO9EQTEAP0OPxUFEZIm1OEghijFdZvT3IbNe+5/b53d8MtHe3dJlxqh8bL19//w8Ox882MzjDXvtAhx2/u0XzPy0298ycz37bDPf8TW7SRQ12k3iuP3jY9zdK828bcZ0My9+bpWZh5VTWGjmu759bmBW/9i8wCzMywEuDkJ0CgjTBAbC4iBE5NDvlwPo4+IgyaGOOgAohP2amYgyL8yRQJ8WB1HV+apaq6q12fwmIJGvwjSBrF4chIj6JuwQ4UktDjKookqrr78tMK9c2Wo+XtMFJWY+dvEOM9djx8x82+1Tzdz17rbLjh/b756f/oNw2w+r/W/s0Y3m82JmntdqD2GNee2QmR+ZYL99n+53310OX3++mR+cYv/8XfbgDaqf/NjME1u22RswrNLlOKwtqR0iBABVXQpgaZhtEFG0OGOQyHNsAkSeYxMg8hybAJHn2ASIPMcmQOS5jF53YIgM0zBnEe76Z3ucvepVex5AzsoN9gN0Jcy448rzzLzgpdX29l0cp4rGzpho5mHGkQlAjj0PwvX8cMk9fYKZx3fUm7leWGPmefXBV1Fb2fwsDnU29/oE45EAkefYBIg8xyZA5Dk2ASLPsQkQeY5NgMhzbAJEngt1KvHJipcXY9+1FwTm5T+1V3sd/6P0nm+/f449D2HEf4V7/KPX2KvVFr1gny+/5yvl9vZHjzDzif9jz6PYPtNezbd67ttmPtDt+Uf79zNmhWMeyu/XmfmHN1Wa+ek/qDdzedNebTpuZKrB67HzSIDIc2wCRJ5jEyDyHJsAkefYBIg8xyZA5Dk2ASLPZXSeQE4nMHhvuHOyQzn/LDN2zQPIqbGvS7BjZqmZD3Gc7t/4gL2ufazDXvsh7Dh+tT1N45RX9S/2DogNH2bmWxy/v+q59vPr0Dft+5c+nZ55GjwSIPIcmwCR59gEiDzHJkDkOTYBIs+xCRB5jk2AyHMZnScQO9KOkhVbA3PnDALHOP/OrxSbeWKQY5zdMQzbtf4DM59gn+7tZI9CU7ptc4zzp1vHUPv/5GNXTbPvXxp83YTEr4Kf3KGagIjUA2hF999vXFVrw2yPiDIvFUcCl6rq/hRsh4giwPcEiDwXtgkogFdFZK2I1KWiICLKrLAvBy5U1UYRKQewTEQ2q+qKnt+QbA51AFCYMzjkwxFRqoU6ElDVxuTHfQBeAPBHb1+q6nxVrVXV2vycwjAPR0Rp0O8mICLFIlJy4nMAXwKwMVWFEVFmhHk5UAHgBRE5sZ2FqvqydQeNJ5A40BKY55x9pvmAnUV5Zj5+qb0ufO67H5p5l5kOfB/dGnzNBwBIXHzIzI/tKjHzEevEzDtL7Lz8Z+ld0CDd8wBGv27PQ3FxXXfDxbpqREzbArN+NwFV3QHg7P7en4iyA4cIiTzHJkDkOTYBIs+xCRB5jk2AyHNsAkSey+h6Ai45La1mnhhrTzvOfW2tmTd9zx4nbz3dnikw+RH7ZMktN40wcxfXdQOGv1lm5u+snGI/gNg/X9Eb9nUTSo/Z4+B51zeb+YENFWZebqbhTX7soJlv+ba9f8Ne1yHdrPUGun4XfM0DHgkQeY5NgMhzbAJEnmMTIPIcmwCR59gEiDzHJkDkuayaJxDf02DmB786zsyPOM4Xn/S043z5iiFm3jp1uJnnHrHPlx/9xnEzf6XRvnBB9cKbzLx4r/34ox4MHitOhdxfjjHzN9552N7AN+z4ymlXmnm8Ya+Za17wuvxA9s8DcBn04juBWY4eDc7SUQwRDRxsAkSeYxMg8hybAJHn2ASIPMcmQOQ5NgEiz2XVPIGm2+zz/SvnhVuX3bUqfPW+sWa++TY7r55rj8Oftz7hqMC27fpHzPzK864w882OeRQFLfb/CR3D7PUIJi0MXtseAM56Z5a9/Q77uhITGt4zc/c8C/vnn2jf/ZTFIwEiz7EJEHmOTYDIc2wCRJ5jEyDyHJsAkefYBIg8l1XzBNpHhru+u9P5Z5nx5muLzNx1vvlXN+0z87rSRjN3ueLSr5l5Yu82M5/8iP3zua6bUNRo/5+hqzeYedtOe5x+5GT7ug6ueQAurnkWl/1dTajtH7s6eN1/ABj0i+Dz/fui/cv29gt/1b/tO48EROQJEdknIht73DZMRJaJyNbkR/uqDUSUtfrycuBJAJd/7rbbASxX1UkAlie/JqIByNkEVHUFgJbP3XwVgAXJzxcAuDrFdRFRhvT3jcEKVW0CgOTHdF9GjojSJO1vDIpIHYA6ACiE/cYUEWVef48EmkWkEgCSHwPfFlfV+apaq6q1eSjo58MRUbr0twksATA7+flsAC+mphwiyjTnywERWQTgEgAjRKQBwJ0A7gWwWERuBLAbwMxUFFO5MuT59o7z5dMt7DyADrWvS+Aax6+ea88TcN3fZfRPwq3nMOU/PzLz+I56ewPhdq9TxxXnmXnB0tVmXvJ/zWYeP+mKPqu/8wBcnE1AVYNWgpiR4lqIKAKcNkzkOTYBIs+xCRB5jk2AyHNsAkSeYxMg8lxWrSfw0XT7+vFFVfZ1Cca/1GHmea/Z56PHRg438y3OeQjhzne/5L3rQt3fNU/CtR5CujnnAaTZHc32ehIFv14TavvxnbtC3T92xkQzT3y4PdT2g/BIgMhzbAJEnmMTIPIcmwCR59gEiDzHJkDkOTYBIs9ldJ6A5MYQKwseiy9qEvP+5T+3z2d3jZOPi51j5rsvt3fH8PV2fbjWjl0Ov15h5pOWtZq5a93/bBf2ugIuv1x0kZmPVvv5dXD2F8y8bMFbJ11TT+maB+DCIwEiz7EJEHmOTYDIc2wCRJ5jEyDyHJsAkefYBIg8l9F5AhpPINHySWAeL7bvf+BGe5wWUDN1zQOYfI89TnvoUvt877DG3mOPU9s/XfT2/JO93kPJbtdPkN55Akcru8x85z3286t0q739WPVpZp7YttPegMP2++15MOOXBl+3QlcFz2HgkQCR59gEiDzHJkDkOTYBIs+xCRB5jk2AyHNsAkSey6rrDox9NXgOAQC0nFVq5oUf2z2tfaQ9Ttw4a5KZJ/LNOLS9P7TH2cf8uz2PIN1yiorM/HdzfmLm5THHRJCQLrjtJvsbpttxYpA9j6Fs8zEzP3ROuZkPDjlP4LQl9nU1cn6/LjATPRp8P9cDi8gTIrJPRDb2uO0uEdkrIuuT/65wbYeIslNfXg48CeDyXm5/QFVrkv+WprYsIsoUZxNQ1RUAWjJQCxFFIMwbgzeLyHvJlwtlKauIiDKqv03gYQATAdQAaAJwf9A3ikidiKwRkTXHYb+xQUSZ168moKrNqppQ1S4AjwKYZnzvfFWtVdXaPBT0t04iSpN+NQERqezx5TUANgZ9LxFlN1G1x0ZFZBGASwCMANAM4M7k1zXoPsW9HsAcVW1yPVjRyCqdcs3cwHz4Y+HWbXeJDRli5u3T7HkCu67MM/Pc0cFjsQCw5YtPmXn1Inuce+xrCTMveGm1mR/4jn2+fPmbB8x86W8Wm7lLhwaf7w4ABWLvX5fLRteEun9YsRHB19QAgMR+e/+GdXhW8HoDG19+EG0H9vR64QznZCFVndXLzY/3vTQiymacNkzkOTYBIs+xCRB5jk2AyHNsAkSeYxMg8lxG1xPIbe9C2Zb2wPzQN+x11Yd+0Grmuu59M99y91Qzr1pmj8Of8eQhM99+7VAzv3uK/fgj15oxGi6N2d9wqb3/tl3/sH3/kFzj9K80pve6AlFr+vpkMy//ub0ehGseS+LwYTMfsujt4G1rW2DGIwEiz7EJEHmOTYDIc2wCRJ5jEyDyHJsAkefYBIg851xPIJWGyDCdLjMCc9c8gVinXWvpss1mfmy6vV5A/itrzNyl4R/s6wa0l9vXPdA8++ebdPOqk66pp67lVWaef2Ovp5t/Kl6/O9Tjh50nEPV6Aem2f4693sOoVxvNPL5zV2C2SpfjsLb0+gvmkQCR59gEiDzHJkDkOTYBIs+xCRB5jk2AyHNsAkSey+h6Ai6lzwSfD90X+79lj7MeONseh69+JdTDY+y/2eeLRy1nxh4zj2eojiDVC+3rLkxEuOdHtjte7JinYcwDCINHAkSeYxMg8hybAJHn2ASIPMcmQOQ5NgEiz7EJEHkus/MERCB5+YGxHu807x6rKLdzx3oDU+6zz4ePepx8oMutGuv4Dns9gS9eZF83wj6bfuDLiegJ6DwSEJEqEfmtiGwSkfdF5Jbk7cNEZJmIbE1+LEt/uUSUan15ORAH8H1VPRPA+QC+KyJTAdwOYLmqTgKwPPk1EQ0wziagqk2q+m7y81YAmwCMAXAVgAXJb1sA4Op0FUlE6XNSbwyKyAQA5wBYBaBCVZuA7kYBwH7BTkRZqc9vDIrIYADPAbhVVQ+L2Cc79LhfHYA6AChEUX9qJKI06tORgIjkobsBPKOqzydvbhaRymReCWBfb/dV1fmqWquqtXlSmIqaiSiF+jI6IAAeB7BJVef1iJYAmJ38fDaAF1NfHhGlW19eDlwI4AYAG0TkxEDvHQDuBbBYRG4EsBvATOeWVM25AJ/cYK8HMPS/3zLzIQt7PRj51OYH7OsaVM891Uei02vQwvZQ9397zwQzH4cNobaf7Sr+w16PIuzfRxBnE1DVNwAEvQEQfCURIhoQOG2YyHNsAkSeYxMg8hybAJHn2ASIPMcmQOS5jK4nIIUFiFVPDsxd45w5NVPNvGv9B2ZesnNg97xtznkO6V2XP/4Xf2bmfzn85VDbHzfz1J4HEDtzkpknNm0189Idx8z8o1suCMyOLwx+bgzsvwoiCo1NgMhzbAJEnmMTIPIcmwCR59gEiDzHJkDkuYzOE+jKi6Fj1ODAPNdedt45D8DFdb521DovqzXzdM8DcMl9ba2Z/+bAmWZ+09C9qSxnwDlYM9zMh7UeNfOW8fbKXKMeCn5+79K2wIxHAkSeYxMg8hybAJHn2ASIPMcmQOQ5NgEiz7EJEHlOVDVjDzZEhul04SrlQWJl9tXdEwcPpvXxXesVTPhT+7oMO/aMNPOyVflmXvnrBjOP79pj5qc613UH4sZV/rY89wCO7tvT66UDeCRA5Dk2ASLPsQkQeY5NgMhzbAJEnmMTIPIcmwCR55zrCYhIFYCnAIwC0AVgvqo+JCJ3AfgOgI+T33qHqi61ttUxthjbbw0e66xefMSsRVfb69Lvuzl43XUAKP9ptOsJ5I4ZbebxqhH2Bt4ON0/ANQ8g1h50Bfpuy6cusR/AviwEcJkjfqTGsYH0OvRNe/+UPm2v59D+5Wlmnn/ouJnntnaYeWep/fupeOtQYLa9LRH8uOZWu8UBfF9V3xWREgBrRWRZMntAVe/rwzaIKEs5m4CqNgFoSn7eKiKbAIxJd2FElBkn9Z6AiEwAcA6AVcmbbhaR90TkCRGx57wSUVbqcxMQkcEAngNwq6oeBvAwgIkAatB9pHB/wP3qRGSNiKxJtAWvc0ZE0ehTExCRPHQ3gGdU9XkAUNVmVU2oaheARwH0+q6Iqs5X1VpVrY0VF6eqbiJKEWcTEBEB8DiATao6r8ftlT2+7RoAG1NfHhGlW19GBy4EcAOADSKyPnnbHQBmiUgNAAVQD2BOWiokorTqy+jAGwB6G6A05wT0pqChDRP//q3APDaqwrx/3LH9qOcBuMbhXca9EjyWCwD22fjhjVxnry1x8dt1Zn50RMzMhz8e/LvPBkfL7QPjnOvs32/Js+GuC/HxjfZ6AYkC+/661rhwh7YHRpwxSOQ5NgEiz7EJEHmOTYDIc2wCRJ5jEyDyHJsAkef6MlkoZbrKitE2Y3pgfug0e5x59H3NZr77R/Z6AqXbu8x86P+uM/OW684185Kd9vneQ7fZ55PDcQ2IzstqzbzlTHsmwZR5u8382GR7nkbsuL3/Br34jpmn27Gr7fP5czrt/Vs5L9w8k+bv2c+/kgZ7HkhU8yh4JEDkOTYBIs+xCRB5jk2AyHNsAkSeYxMg8hybAJHnRB1j0yl9MJGPAezqcdMIAPszVsDJY33hZHN92VwbkPr6xqvqyN6CjDaBP3pwkTWqas+AiRDrCyeb68vm2oDM1seXA0SeYxMg8lzUTWB+xI/vwvrCyeb6srk2IIP1RfqeABFFL+ojASKKGJsAkefYBIg8xyZA5Dk2ASLP/T+6oyk91j0sBwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test_adv[16, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Label: 7\n",
      "Predicted Label: 7\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_test_adv[16:17, :])[0]\n",
    "print(\"Target Label: %i\" % target_label)\n",
    "print(\"Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}