one_shot.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VAT JDP - One shot learning via GPT prompting\n",
    "by Alex Periti, Michele Faedi, Francesco Mistri\n",
    "\n",
    "A study on one shot in-context learning through prompting of GPT. In particular, GPT 3.5 turbo was chosen for its costs/performance ratio. This notebook mirrors `one_shot.ipynb`, refer to it for an overview of the code.\n",
    "\n",
    "The main difference being that two samples (one per class) are extracted from the dataset and used as an extremely small train set. The prompt are then enriched with these two samples along their true labels, in order to enrich the context and help the model.\n",
    "\n",
    "Results are unfortunately very weak, even weaker than the zero shot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "# Populate with your own key to access OpenAI API\n",
    "os.environ['OPENAI_API_KEY'] = 'XXX'\n",
    "\n",
    "import xml.etree.ElementTree as ET\n",
    "from functools import partial\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.dummy import DummyClassifier\n",
    "\n",
    "from vjp.openai_prompt import Prompt\n",
    "from vjp import data, text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_list_et(documents: list[ET.Element]):\n",
    "    \"\"\"Remove intro tags content from the given documents.\"\"\"\n",
    "    for doc in documents:\n",
    "        doc: ET.Element = doc\n",
    "        assert len(list(doc.iter('intro'))) <= 1, doc.attrib\n",
    "\n",
    "    for doc in documents:\n",
    "        doc: ET.Element = doc\n",
    "        parent_map = {c: p for p in doc.iter() for c in p}\n",
    "        intros = list(doc.iter('intro'))\n",
    "        for i in intros:\n",
    "            parent_map[i].remove(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>preliminaries</th>\n",
       "      <th>decisions</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Con sentenza in data DD/MM/2017, la Commission...</td>\n",
       "      <td>L'appello è fondato e va accolto.Preliminarmen...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Con l’avviso di accertamento n°877010100140 no...</td>\n",
       "      <td>Si premette che già nel PVC la Guardia di Fina...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L'Agenzia delle Entrate notificava alla societ...</td>\n",
       "      <td>Osserva questa Commissione, con riguardo al pr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L'Agenzia delle Entrate Ufficio di Piacenza, c...</td>\n",
       "      <td>Osserva questo collegio che dalla lettura degl...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Con rituale ricorso il signor LL, quale Ammini...</td>\n",
       "      <td>Questa Commissione rileva che non è controvers...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       preliminaries  \\\n",
       "0  Con sentenza in data DD/MM/2017, la Commission...   \n",
       "1  Con l’avviso di accertamento n°877010100140 no...   \n",
       "2  L'Agenzia delle Entrate notificava alla societ...   \n",
       "3  L'Agenzia delle Entrate Ufficio di Piacenza, c...   \n",
       "4  Con rituale ricorso il signor LL, quale Ammini...   \n",
       "\n",
       "                                           decisions  label  \n",
       "0  L'appello è fondato e va accolto.Preliminarmen...      1  \n",
       "1  Si premette che già nel PVC la Guardia di Fina...      1  \n",
       "2  Osserva questa Commissione, con riguardo al pr...      1  \n",
       "3  Osserva questo collegio che dalla lettura degl...      1  \n",
       "4  Questa Commissione rileva che non è controvers...      1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upheld, rejected = data.load_second_instance_labeled()\n",
    "clean_list_et(upheld)\n",
    "clean_list_et(rejected)\n",
    "df_all = data.shot_based_dataframe(upheld, rejected)\n",
    "df_all[['preliminaries', 'decisions']] = \\\n",
    "    df_all[['preliminaries', 'decisions']].applymap(\n",
    "        text.shot_normalize_whites_pipeline)\n",
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_shot = 1\n",
    "df_all_proc_len = df_all.sort_values(by=\"preliminaries\", key=lambda x: x.str.len())\n",
    "\n",
    "uphold_example = df_all_proc_len.loc[df_all['label'] == 1].head(n_shot)\n",
    "rejected_example = df_all_proc_len.loc[df_all['label'] == 0].head(n_shot)\n",
    "\n",
    "df_train = pd.concat([uphold_example, rejected_example])\n",
    "df = df_all.drop(list(df_train.index[:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((217, 3), (219, 3), (2, 3))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_all.shape, df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>preliminaries</th>\n",
       "      <th>decisions</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>ZG ha proposto ricorso avverso l’atto di conte...</td>\n",
       "      <td>MOTIVI DELLA DECISIONEL’appellante censura l’o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>L'agenzia delle Entrate di XXX propone appello...</td>\n",
       "      <td>MOTIVI DELLA DECISIONE\\nLa Commissione, letti ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         preliminaries  \\\n",
       "34   ZG ha proposto ricorso avverso l’atto di conte...   \n",
       "191  L'agenzia delle Entrate di XXX propone appello...   \n",
       "\n",
       "                                             decisions  label  \n",
       "34   MOTIVI DELLA DECISIONEL’appellante censura l’o...      1  \n",
       "191  MOTIVI DELLA DECISIONE\\nLa Commissione, letti ...      0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "majority_baseline = DummyClassifier()\n",
    "random_baseline = DummyClassifier(strategy='uniform', random_state=42)\n",
    "baseline_X = np.expand_dims(df.preliminaries.to_numpy(), 1)\n",
    "baseline_y = df.label.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      1.00      0.74       127\n",
      "           1       0.00      0.00      0.00        90\n",
      "\n",
      "    accuracy                           0.59       217\n",
      "   macro avg       0.29      0.50      0.37       217\n",
      "weighted avg       0.34      0.59      0.43       217\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\micfae\\AppData\\Local\\miniconda3\\envs\\vjp\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "c:\\Users\\micfae\\AppData\\Local\\miniconda3\\envs\\vjp\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "c:\\Users\\micfae\\AppData\\Local\\miniconda3\\envs\\vjp\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# Fit and predict on the same data, it doesn't really matter here\n",
    "majority_baseline.fit(baseline_X,\n",
    "                      baseline_y)\n",
    "majority_preds = majority_baseline.predict(baseline_X)\n",
    "print(classification_report(baseline_y, majority_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.54      0.57       127\n",
      "           1       0.44      0.52      0.48        90\n",
      "\n",
      "    accuracy                           0.53       217\n",
      "   macro avg       0.53      0.53      0.53       217\n",
      "weighted avg       0.54      0.53      0.53       217\n",
      "\n"
     ]
    }
   ],
   "source": [
    "random_baseline.fit(baseline_X,\n",
    "                      baseline_y)\n",
    "random_preds = random_baseline.predict(baseline_X)\n",
    "print(classification_report(baseline_y, random_preds))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "verbalizer_ita = 'accolto', 'respinto'\n",
    "template_ita = \"Sei un sistema automatico per l'inferenza di decisioni giudiziarie in Italia. Hai in input un estratto del verbale d'appello. Rispondi con \\\"{}\\\" se è ritieni che l'appello debba essere accolto, con \\\"{}\\\" altrimenti. Rispondi con una sola parola. Il verbale:\"\n",
    "\n",
    "verbalizer_eng = 'upheld', 'rejected'\n",
    "template_eng = \"You are an automated system for inferring judicial decisions in Italy. You are given an excerpt from the appellate protocol as input. Respond with \\\"{}\\\" if you believe the appeal should be granted, or with \\\"{}\\\" otherwise. Respond with a single word. The excerpt:\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_ita = Prompt(template_ita, verbalizer_ita, few_shot_data=df_train.to_dict('records'))\n",
    "prompt_eng = Prompt(template_eng, verbalizer_eng, few_shot_data=df_train.to_dict('records'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_new_column(prompt, row, with_mot):\n",
    "    \"\"\"Apply to a DataFrame column to obtain a series of predictions.\"\"\"\n",
    "    message = prompt.create_prompt(row, with_mot)\n",
    "    response = prompt.send_prompt(message)\n",
    "    try:\n",
    "        msg = prompt.interpret_response(response)\n",
    "        return msg\n",
    "    except:\n",
    "        print(prompt.get_nice_prompt(message))\n",
    "        print(response)\n",
    "        return None"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Italian prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Context size for message too long(6144)\n",
      "Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Thu, 29 Jun 2023 13:03:45 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7dee577f1e8f0e59-MXP', 'alt-svc': 'h3=\":443\"; ma=86400'}\n",
      "WARNING: Context size for message too long(14521)\n",
      "WARNING: Context size for message too long(4938)\n",
      "WARNING: Context size for message too long(4261)\n"
     ]
    }
   ],
   "source": [
    "compute_new_column_no_mot = partial(compute_new_column, prompt_ita,\n",
    "                                    with_mot=False)\n",
    "# compute_new_column_mot = partial(compute_new_column, prompt_ita,\n",
    "#                                  with_mot=True)\n",
    "\n",
    "df['pred_label_no_mot'] = df.apply(compute_new_column_no_mot, axis=1)\n",
    "# df['pred_label_mot'] = df.apply(compute_new_column_mot, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('backup_cache_no_mot_correct_1_italian.pickle', 'wb') as f:\n",
    "    pickle.dump(prompt_ita.cache, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(\"backup_dataframe_no_mot_correct_1shot.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<Axes: title={'center': 'pred_label_no_mot'}>]], dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGxCAYAAABIjE2TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzJElEQVR4nO3de3RU5b3G8WdIJhOCCRhiMonEGGgUFaRcBIW2BCXBKKFIFWp6FDxecIkXihwV0TKpAh48xQsoWquAQgS1gLZSISpEKFglYMvFKmi4lUQkYC4Eh0nY5w9WBseEy4SZyTvh+1lr1up+97vf+e1fp52HvWcyNsuyLAEAABikVXMXAAAA8GMEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUoIXbvn27bDab5syZE9Rj6q1cuVI2m01vvfWW38eebM2VK1cGbM1w9/zzzzfpvx8gXBBQACAMEVDQ0hFQAIMdOnSouUsAgGZBQAGCzOVyyWazacOGDRo2bJji4uLUtm1b/dd//Ze+/fZb77zzzz9fgwcP1qJFi9S9e3dFR0crPz9fklRWVqbRo0erQ4cOioqKUnp6uvLz81VbW+vzXHv27NHw4cMVGxurtm3basSIESorKwvIeWzbtk233HKLMjIyFBMTo3PPPVe5ubnauHFjo/O///57jRs3Tk6nU61bt1b//v21YcOGBvPWrVunIUOGKD4+XtHR0erevbveeOON0663/rbQ66+/rokTJyolJUVxcXEaOHCgvvjiiwbzX3nlFXXr1k3R0dGKj4/Xddddp88//7xJz1lQUKAHH3xQycnJOuuss5Sbm6tvvvlGVVVVuuOOO5SQkKCEhATdcsstqq6u9lnj+++/14QJE5Senq6oqCide+65GjNmjL777jvvnPPPP1+bN29WUVGRbDabbDabzj///Ka0CTBWZHMXAJwprrvuOg0fPlx33nmnNm/erEcffVRbtmzRP/7xD9ntdknS+vXr9fnnn+uRRx5Renq62rRpo7KyMvXu3VutWrXS7373O3Xq1Elr167V448/ru3bt2v27NmSjl5tGThwoPbs2aOpU6fqggsu0LvvvqsRI0YEpP49e/aoffv2euKJJ3TOOedo//79mjt3rvr06aMNGzbowgsv9Jn/8MMPq0ePHvrTn/6kiooKuVwuZWZmasOGDerYsaMkacWKFbr66qvVp08fvfDCC2rbtq0WLFigESNGqKamRqNGjTrtuh9++GH169dPf/rTn1RZWakHH3xQubm5+vzzzxURESFJmjp1qh5++GHdeOONmjp1qsrLy+VyuXTFFVfo008/VUZGht/POWDAAM2ZM0fbt2/X+PHjdeONNyoyMlLdunXT66+/rg0bNujhhx9WbGysnn32WUmSZVkaOnSoPvjgA02YMEE///nP9a9//UuTJk3S2rVrtXbtWjkcDi1evFjXX3+92rZtq+eff16S5HA4TrtXgFEsAEE1adIkS5L129/+1md8/vz5liRr3rx5lmVZVlpamhUREWF98cUXPvNGjx5tnXXWWdaOHTt8xv/v//7PkmRt3rzZsizLmjVrliXJevvtt33m3X777ZYka/bs2adcc0lJyUmPqa2ttQ4fPmxlZGT4nNuKFSssSVaPHj2sI0eOeMe3b99u2e1267bbbvOOde7c2erevbvl8Xh81h48eLCVnJxs1dXV+ay5YsWKUz6H+mOuueYan/E33njDkmStXbvWsizLOnDggNW6desG83bu3Gk5HA4rLy/P7+fMzc31GR87dqwlybr33nt9xocOHWrFx8d7t9977z1LkjVt2jSfeQsXLrQkWX/84x+9Y5dcconVv3//U64NCDfc4gFC5De/+Y3P9vDhwxUZGakVK1Z4xy699FJdcMEFPvP++te/asCAAUpJSVFtba33kZOTI0kqKiqSdPRqRGxsrIYMGeJzfF5eXkDqr62t1ZQpU3TxxRcrKipKkZGRioqK0tatWxu9FZKXlyebzebdTktLU9++fb3nu23bNv373//29uWH53bNNdeotLS00Vsx/vpxPy699FJJ0o4dOyRJa9eu1aFDhxpcrUlNTdWVV16pDz74wO/nHDx4sM/2RRddJEm69tprG4zv37/fe5vnww8/lKQGtdxwww1q06ZNk2oBwhW3eIAQcTqdPtuRkZFq3769ysvLvWPJyckNjvvmm2/0l7/8xXsb6Mf27dsnSSovL1dSUtJJn7epxo0bp+eee04PPvig+vfvr7PPPlutWrXSbbfd1uiHeRt7XqfTqX/+85+Sjp6XJI0fP17jx49v9Dnrz+10tG/f3me7/lZIfc31/W+s9ykpKSosLPT7OePj4322o6KiTjj+/fff66yzzlJ5ebkiIyN1zjnn+Myz2WxyOp0+rxWgpSOgACFSVlamc88917tdW1ur8vJynzfQH15xqJeQkKBLL71UkydPbnTdlJQUSUffiD/55JNGnzcQ5s2bp5tvvllTpkzxGd+3b5/atWt3Ss9bVlbmPd+EhARJ0oQJEzRs2LBGn/PHn2sJhvp6SktLG+zbs2ePt85QaN++vWpra/Xtt9/6hBTLslRWVqbLLrssZLUAzY1bPECIzJ8/32f7jTfeUG1trTIzM0943ODBg7Vp0yZ16tRJvXr1avCoDygDBgxQVVWV3nnnHZ/jCwoKAlK/zWZr8EHMd999V//5z38anf/666/Lsizv9o4dO7RmzRrv+V544YXKyMjQP//5z0bPq1evXoqNjQ1I7SdyxRVXqHXr1po3b57P+O7du/Xhhx/qqquuCnoN9eqf68e1/PnPf9bBgwd9anE4HHwNHS0aV1CAEFm0aJEiIyOVlZXl/RZPt27dNHz48BMe9/vf/16FhYXq27ev7r33Xl144YX6/vvvtX37di1dulQvvPCCOnTooJtvvllPPfWUbr75Zk2ePFkZGRlaunSpli1bFpD6Bw8erDlz5qhz58669NJLVVxcrCeffFIdOnRodP7evXt13XXX6fbbb1dFRYUmTZqk6OhoTZgwwTvnxRdfVE5OjgYNGqRRo0bp3HPP1f79+/X5559r/fr1evPNNwNS+4m0a9dOjz76qB5++GHdfPPNuvHGG1VeXq78/HxFR0dr0qRJQa+hXlZWlgYNGqQHH3xQlZWV6tevn/dbPN27d9dNN93kndu1a1ctWLBACxcuVMeOHRUdHa2uXbuGrFYg2AgoQIgsWrRILpdLs2bNks1mU25urp5++mnv5xCOJzk5WevWrdNjjz2mJ598Urt371ZsbKzS09N19dVX6+yzz5YkxcTE6MMPP9R9992nhx56SDabTdnZ2VqwYIH69u172vU/88wzstvtmjp1qqqrq9WjRw8tWrRIjzzySKPzp0yZok8//VS33HKLKisr1bt3by1YsECdOnXyzhkwYIA++eQTTZ48WWPHjtWBAwfUvn17XXzxxScNboE0YcIEJSYm6tlnn9XChQvVunVrZWZmasqUKX5/xfh02Gw2LVmyRC6XS7Nnz9bkyZOVkJCgm266SVOmTPG5gpWfn6/S0lLdfvvtqqqqUlpamrZv3x6yWoFgs1k/vAYLIOBcLpfy8/P17bffhvTzDAAQzvgMCgAAMA63eIAziGVZqqurO+GciIiIRr9NZIrmOIeW0Dcg3HAFBQgyl8sly7KMuL1TVFQku91+wsfcuXObu8wTao5zaAl9A8INn0EBziBVVVUn/eus6enpDf64mUma4xxaQt+AcENAAQAAxuEWDwAAME5Yfkj2yJEj2rNnj2JjY/lQGgAAYcKyLFVVVSklJUWtWp34GklYBpQ9e/YoNTW1ucsAAABNsGvXruP+Fep6YRlQ6n+fY9euXYqLiwvo2h6PR8uXL1d2dvZxfz0Wp48+hwZ9Dg36HDr0OjSC1efKykqlpqae0u9shWVAqb+tExcXF5SAEhMTo7i4OF78QUSfQ4M+hwZ9Dh16HRrB7vOpfDyDD8kCAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGCeyuQsAAKClO/+hd5u7BL84IixN6928NXAFBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADG8SugTJ06VZdddpliY2OVmJiooUOH6osvvvCZY1mWXC6XUlJS1Lp1a2VmZmrz5s0+c9xut+655x4lJCSoTZs2GjJkiHbv3n36ZwMAAFoEvwJKUVGRxowZo48//liFhYWqra1Vdna2Dh486J0zbdo0TZ8+XTNnztSnn34qp9OprKwsVVVVeeeMHTtWixcv1oIFC7R69WpVV1dr8ODBqqurC9yZAQCAsOXXrxm/9957PtuzZ89WYmKiiouL9Ytf/EKWZenpp5/WxIkTNWzYMEnS3LlzlZSUpIKCAo0ePVoVFRV6+eWX9dprr2ngwIGSpHnz5ik1NVXvv/++Bg0a1OB53W633G63d7uyslKS5PF45PF4/Dvjk6hfL9Drwhd9Dg36HBr0OXTCtdeOCKu5S/CLo9XReoP1HnsqbJZlNblr27ZtU0ZGhjZu3KguXbro66+/VqdOnbR+/Xp1797dO++Xv/yl2rVrp7lz5+rDDz/UVVddpf379+vss8/2zunWrZuGDh2q/Pz8Bs/jcrkaHS8oKFBMTExTywcAACFUU1OjvLw8VVRUKC4u7oRz/bqC8kOWZWncuHH62c9+pi5dukiSysrKJElJSUk+c5OSkrRjxw7vnKioKJ9wUj+n/vgfmzBhgsaNG+fdrqysVGpqqrKzs096gv7yeDwqLCxUVlaW7HZ7QNfGMfQ5NOhzaNDn0AnXXndxLWvuEvziaGXpsV5HAt7n+jsgp6LJAeXuu+/Wv/71L61evbrBPpvN5rNtWVaDsR870RyHwyGHw9Fg3G63B+0FGsy1cQx9Dg36HBr0OXTCrdfuuhO/B5oq0H32Z60mfc34nnvu0TvvvKMVK1aoQ4cO3nGn0ylJDa6E7N2713tVxel06vDhwzpw4MBx5wAAgDObXwHFsizdfffdWrRokT788EOlp6f77E9PT5fT6VRhYaF37PDhwyoqKlLfvn0lST179pTdbveZU1paqk2bNnnnAACAM5tft3jGjBmjgoICvf3224qNjfVeKWnbtq1at24tm82msWPHasqUKcrIyFBGRoamTJmimJgY5eXleefeeuutuv/++9W+fXvFx8dr/Pjx6tq1q/dbPQAA4MzmV0CZNWuWJCkzM9NnfPbs2Ro1apQk6YEHHtChQ4d011136cCBA+rTp4+WL1+u2NhY7/ynnnpKkZGRGj58uA4dOqSrrrpKc+bMUURExOmdDQAAaBH8Ciin8o1km80ml8sll8t13DnR0dGaMWOGZsyY4c/TAwCAMwS/xQMAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMI7fAeWjjz5Sbm6uUlJSZLPZtGTJEp/9Nput0ceTTz7pnZOZmdlg/69//evTPhkAANAy+B1QDh48qG7dumnmzJmN7i8tLfV5vPLKK7LZbPrVr37lM+/222/3mffiiy827QwAAECLE+nvATk5OcrJyTnufqfT6bP99ttva8CAAerYsaPPeExMTIO5AAAAUhMCij+++eYbvfvuu5o7d26DffPnz9e8efOUlJSknJwcTZo0SbGxsY2u43a75Xa7vduVlZWSJI/HI4/HE9Ca69cL9LrwRZ9Dgz6HBn0OnXDttSPCau4S/OJodbTeYL3HngqbZVlN7prNZtPixYs1dOjQRvdPmzZNTzzxhPbs2aPo6Gjv+EsvvaT09HQ5nU5t2rRJEyZM0E9+8hMVFhY2uo7L5VJ+fn6D8YKCAsXExDS1fAAAEEI1NTXKy8tTRUWF4uLiTjg3qAGlc+fOysrK0owZM064TnFxsXr16qXi4mL16NGjwf7GrqCkpqZq3759Jz1Bf3k8HhUWFiorK0t2uz2ga+MY+hwa9Dk06HPohGuvu7iWNXcJfnG0svRYryMB73NlZaUSEhJOKaAE7RbPqlWr9MUXX2jhwoUnndujRw/Z7XZt3bq10YDicDjkcDgajNvt9qC9QIO5No6hz6FBn0ODPodOuPXaXWdr7hKaJNB99metoP0dlJdfflk9e/ZUt27dTjp38+bN8ng8Sk5ODlY5AAAgjPh9BaW6ulrbtm3zbpeUlOizzz5TfHy8zjvvPElHL+G8+eab+sMf/tDg+K+++krz58/XNddco4SEBG3ZskX333+/unfvrn79+p3GqQAAgJbC74Cybt06DRgwwLs9btw4SdLIkSM1Z84cSdKCBQtkWZZuvPHGBsdHRUXpgw8+0DPPPKPq6mqlpqbq2muv1aRJkxQREdHE0wAAAC2J3wElMzNTJ/tc7R133KE77rij0X2pqakqKiry92kBAMAZhN/iAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYx++A8tFHHyk3N1cpKSmy2WxasmSJz/5Ro0bJZrP5PC6//HKfOW63W/fcc48SEhLUpk0bDRkyRLt37z6tEwEAAC2H3wHl4MGD6tatm2bOnHncOVdffbVKS0u9j6VLl/rsHzt2rBYvXqwFCxZo9erVqq6u1uDBg1VXV+f/GQAAgBYn0t8DcnJylJOTc8I5DodDTqez0X0VFRV6+eWX9dprr2ngwIGSpHnz5ik1NVXvv/++Bg0a5G9JAACghfE7oJyKlStXKjExUe3atVP//v01efJkJSYmSpKKi4vl8XiUnZ3tnZ+SkqIuXbpozZo1jQYUt9stt9vt3a6srJQkeTweeTyegNZev16g14Uv+hwa9Dk06HPohGuvHRFWc5fgF0ero/UG6z32VAQ8oOTk5OiGG25QWlqaSkpK9Oijj+rKK69UcXGxHA6HysrKFBUVpbPPPtvnuKSkJJWVlTW65tSpU5Wfn99gfPny5YqJiQn0KUiSCgsLg7IufNHn0KDPoUGfQyfcej2td3NX0DSB7nNNTc0pzw14QBkxYoT3P3fp0kW9evVSWlqa3n33XQ0bNuy4x1mWJZvN1ui+CRMmaNy4cd7tyspKpaamKjs7W3FxcYErXkfTXWFhobKysmS32wO6No6hz6FBn0ODPodOuPa6i2tZc5fgF0crS4/1OhLwPtffATkVQbnF80PJyclKS0vT1q1bJUlOp1OHDx/WgQMHfK6i7N27V3379m10DYfDIYfD0WDcbrcH7QUazLVxDH0ODfocGvQ5dMKt1+66xv8BbrpA99mftYL+d1DKy8u1a9cuJScnS5J69uwpu93uc9motLRUmzZtOm5AAQAAZxa/r6BUV1dr27Zt3u2SkhJ99tlnio+PV3x8vFwul371q18pOTlZ27dv18MPP6yEhARdd911kqS2bdvq1ltv1f3336/27dsrPj5e48ePV9euXb3f6gEAAGc2vwPKunXrNGDAAO92/WdDRo4cqVmzZmnjxo169dVX9d133yk5OVkDBgzQwoULFRsb6z3mqaeeUmRkpIYPH65Dhw7pqquu0pw5cxQRERGAUwIAAOHO74CSmZkpyzr+16WWLTv5B4Gio6M1Y8YMzZgxw9+nBwAAZwB+iwcAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBy/A8pHH32k3NxcpaSkyGazacmSJd59Ho9HDz74oLp27ao2bdooJSVFN998s/bs2eOzRmZmpmw2m8/j17/+9WmfDAAAaBn8DigHDx5Ut27dNHPmzAb7ampqtH79ej366KNav369Fi1apC+//FJDhgxpMPf2229XaWmp9/Hiiy827QwAAECLE+nvATk5OcrJyWl0X9u2bVVYWOgzNmPGDPXu3Vs7d+7Ueeed5x2PiYmR0+n09+kBAMAZwO+A4q+KigrZbDa1a9fOZ3z+/PmaN2+ekpKSlJOTo0mTJik2NrbRNdxut9xut3e7srJS0tFbSh6PJ6D11q8X6HXhiz6HBn0ODfocOuHaa0eE1dwl+MXR6mi9wXqPPRU2y7Ka3DWbzabFixdr6NChje7//vvv9bOf/UydO3fWvHnzvOMvvfSS0tPT5XQ6tWnTJk2YMEE/+clPGlx9qedyuZSfn99gvKCgQDExMU0tHwAAhFBNTY3y8vJUUVGhuLi4E84NWkDxeDy64YYbtHPnTq1cufKEhRQXF6tXr14qLi5Wjx49Guxv7ApKamqq9u3bd9IT9JfH41FhYaGysrJkt9sDujaOoc+hQZ9Dgz6HTrj2uotrWXOX4BdHK0uP9ToS8D5XVlYqISHhlAJKUG7xeDweDR8+XCUlJfrwww9PWkSPHj1kt9u1devWRgOKw+GQw+FoMG6324P2Ag3m2jiGPocGfQ4N+hw64dZrd52tuUtokkD32Z+1Ah5Q6sPJ1q1btWLFCrVv3/6kx2zevFkej0fJycmBLgcAAIQhvwNKdXW1tm3b5t0uKSnRZ599pvj4eKWkpOj666/X+vXr9de//lV1dXUqKyuTJMXHxysqKkpfffWV5s+fr2uuuUYJCQnasmWL7r//fnXv3l39+vUL3JkBAICw5XdAWbdunQYMGODdHjdunCRp5MiRcrlceueddyRJP/3pT32OW7FihTIzMxUVFaUPPvhAzzzzjKqrq5Wamqprr71WkyZNUkRExGmcCgAAaCn8DiiZmZk60edqT/aZ29TUVBUVFfn7tAAA4AzCb/EAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIzjd0D56KOPlJubq5SUFNlsNi1ZssRnv2VZcrlcSklJUevWrZWZmanNmzf7zHG73brnnnuUkJCgNm3aaMiQIdq9e/dpnQgAAGg5/A4oBw8eVLdu3TRz5sxG90+bNk3Tp0/XzJkz9emnn8rpdCorK0tVVVXeOWPHjtXixYu1YMECrV69WtXV1Ro8eLDq6uqafiYAAKDFiPT3gJycHOXk5DS6z7IsPf3005o4caKGDRsmSZo7d66SkpJUUFCg0aNHq6KiQi+//LJee+01DRw4UJI0b948paam6v3339egQYNO43QAAEBL4HdAOZGSkhKVlZUpOzvbO+ZwONS/f3+tWbNGo0ePVnFxsTwej8+clJQUdenSRWvWrGk0oLjdbrndbu92ZWWlJMnj8cjj8QTyFLzrBXpd+KLPoUGfQ4M+h0649toRYTV3CX5xtDpab7DeY09FQANKWVmZJCkpKclnPCkpSTt27PDOiYqK0tlnn91gTv3xPzZ16lTl5+c3GF++fLliYmICUXoDhYWFQVkXvuhzaNDn0KDPoRNuvZ7Wu7kraJpA97mmpuaU5wY0oNSz2Ww+25ZlNRj7sRPNmTBhgsaNG+fdrqysVGpqqrKzsxUXF3f6Bf+Ax+NRYWGhsrKyZLfbA7o2jqHPoUGfQ4M+h0649rqLa1lzl+AXRytLj/U6EvA+198BORUBDShOp1PS0askycnJ3vG9e/d6r6o4nU4dPnxYBw4c8LmKsnfvXvXt27fRdR0OhxwOR4Nxu90etBdoMNfGMfQ5NOhzaNDn0Am3XrvrTvyPdFMFus/+rBXQv4OSnp4up9Ppc0no8OHDKioq8oaPnj17ym63+8wpLS3Vpk2bjhtQAADAmcXvKyjV1dXatm2bd7ukpESfffaZ4uPjdd5552ns2LGaMmWKMjIylJGRoSlTpigmJkZ5eXmSpLZt2+rWW2/V/fffr/bt2ys+Pl7jx49X165dvd/qAQAAZza/A8q6des0YMAA73b9Z0NGjhypOXPm6IEHHtChQ4d011136cCBA+rTp4+WL1+u2NhY7zFPPfWUIiMjNXz4cB06dEhXXXWV5syZo4iIiACcEgAACHd+B5TMzExZ1vG/LmWz2eRyueRyuY47Jzo6WjNmzNCMGTP8fXoAAHAG4Ld4AACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGCXhAOf/882Wz2Ro8xowZI0kaNWpUg32XX355oMsAAABhLDLQC3766aeqq6vzbm/atElZWVm64YYbvGNXX321Zs+e7d2OiooKdBkAACCMBTygnHPOOT7bTzzxhDp16qT+/ft7xxwOh5xOZ6CfGgAAtBABDyg/dPjwYc2bN0/jxo2TzWbzjq9cuVKJiYlq166d+vfvr8mTJysxMfG467jdbrndbu92ZWWlJMnj8cjj8QS05vr1Ar0ufNHn0KDPoUGfQydce+2IsJq7BL84Wh2tN1jvsafCZllW0Lr2xhtvKC8vTzt37lRKSookaeHChTrrrLOUlpamkpISPfroo6qtrVVxcbEcDkej67hcLuXn5zcYLygoUExMTLDKBwAAAVRTU6O8vDxVVFQoLi7uhHODGlAGDRqkqKgo/eUvfznunNLSUqWlpWnBggUaNmxYo3Mau4KSmpqqffv2nfQE/eXxeFRYWKisrCzZ7faAro1j6HNo0OfQoM+hE6697uJa1twl+MXRytJjvY4EvM+VlZVKSEg4pYAStFs8O3bs0Pvvv69FixadcF5ycrLS0tK0devW485xOByNXl2x2+1Be4EGc20cQ59Dgz6HBn0OnXDrtbvOdvJJBgp0n/1ZK2h/B2X27NlKTEzUtddee8J55eXl2rVrl5KTk4NVCgAACDNBCShHjhzR7NmzNXLkSEVGHrtIU11drfHjx2vt2rXavn27Vq5cqdzcXCUkJOi6664LRikAACAMBeUWz/vvv6+dO3fqv//7v33GIyIitHHjRr366qv67rvvlJycrAEDBmjhwoWKjY0NRikAACAMBSWgZGdnq7HP3rZu3VrLloXXB4UAAEDo8Vs8AADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjBDyguFwu2Ww2n4fT6fTutyxLLpdLKSkpat26tTIzM7V58+ZAlwEAAMJYUK6gXHLJJSotLfU+Nm7c6N03bdo0TZ8+XTNnztSnn34qp9OprKwsVVVVBaMUAAAQhoISUCIjI+V0Or2Pc845R9LRqydPP/20Jk6cqGHDhqlLly6aO3euampqVFBQEIxSAABAGIoMxqJbt25VSkqKHA6H+vTpoylTpqhjx44qKSlRWVmZsrOzvXMdDof69++vNWvWaPTo0Y2u53a75Xa7vduVlZWSJI/HI4/HE9Da69cL9LrwRZ9Dgz6HBn0OnXDttSPCau4S/OJodbTeYL3HngqbZVkB7drf/vY31dTU6IILLtA333yjxx9/XP/+97+1efNmffHFF+rXr5/+85//KCUlxXvMHXfcoR07dmjZsmWNrulyuZSfn99gvKCgQDExMYEsHwAABElNTY3y8vJUUVGhuLi4E84NeED5sYMHD6pTp0564IEHdPnll6tfv37as2ePkpOTvXNuv/127dq1S++9916jazR2BSU1NVX79u076Qn6y+PxqLCwUFlZWbLb7QFdG8fQ59Cgz6FBn0MnXHvdxdX4P8BN5Whl6bFeRwLe58rKSiUkJJxSQAnKLZ4fatOmjbp27aqtW7dq6NChkqSysjKfgLJ3714lJSUddw2HwyGHw9Fg3G63B+0FGsy1cQx9Dg36HBr0OXTCrdfuOltzl9Akge6zP2sF/e+guN1uff7550pOTlZ6erqcTqcKCwu9+w8fPqyioiL17ds32KUAAIAwEfArKOPHj1dubq7OO+887d27V48//rgqKys1cuRI2Ww2jR07VlOmTFFGRoYyMjI0ZcoUxcTEKC8vL9ClAACAMBXwgLJ7927deOON2rdvn8455xxdfvnl+vjjj5WWliZJeuCBB3To0CHdddddOnDggPr06aPly5crNjY20KUAAIAwFfCAsmDBghPut9lscrlccrlcgX5qAADQQvBbPAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4wQ8oEydOlWXXXaZYmNjlZiYqKFDh+qLL77wmTNq1CjZbDafx+WXXx7oUgAAQJgKeEApKirSmDFj9PHHH6uwsFC1tbXKzs7WwYMHfeZdffXVKi0t9T6WLl0a6FIAAECYigz0gu+9957P9uzZs5WYmKji4mL94he/8I47HA45nc5APz0AAGgBAh5QfqyiokKSFB8f7zO+cuVKJSYmql27durfv78mT56sxMTERtdwu91yu93e7crKSkmSx+ORx+MJaL316wV6Xfiiz6FBn0ODPodOuPbaEWE1dwl+cbQ6Wm+w3mNPhc2yrKB1zbIs/fKXv9SBAwe0atUq7/jChQt11llnKS0tTSUlJXr00UdVW1ur4uJiORyOBuu4XC7l5+c3GC8oKFBMTEywygcAAAFUU1OjvLw8VVRUKC4u7oRzgxpQxowZo3fffVerV69Whw4djjuvtLRUaWlpWrBggYYNG9Zgf2NXUFJTU7Vv376TnqC/PB6PCgsLlZWVJbvdHtC1cQx9Dg36HBr0OXTCtdddXMuauwS/OFpZeqzXkYD3ubKyUgkJCacUUIJ2i+eee+7RO++8o48++uiE4USSkpOTlZaWpq1btza63+FwNHplxW63B+0FGsy1cQx9Dg36HBr0OXTCrdfuOltzl9Akge6zP2sFPKBYlqV77rlHixcv1sqVK5Wenn7SY8rLy7Vr1y4lJycHuhwAABCGAv414zFjxmjevHkqKChQbGysysrKVFZWpkOHDkmSqqurNX78eK1du1bbt2/XypUrlZubq4SEBF133XWBLgcAAIShgF9BmTVrliQpMzPTZ3z27NkaNWqUIiIitHHjRr366qv67rvvlJycrAEDBmjhwoWKjY0NdDkAACAMBeUWz4m0bt1ay5aF14eFAABAaPFbPAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMaJbO4CTNXFtUzuOltzl3HKtj9xbXOXAABAwHAFBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwTrMGlOeff17p6emKjo5Wz549tWrVquYsBwAAGKLZAsrChQs1duxYTZw4URs2bNDPf/5z5eTkaOfOnc1VEgAAMESzBZTp06fr1ltv1W233aaLLrpITz/9tFJTUzVr1qzmKgkAABiiWX6L5/DhwyouLtZDDz3kM56dna01a9Y0mO92u+V2u73bFRUVkqT9+/fL4/EEtDaPx6OamhpFelqp7kj4/BZPeXl5c5fgl/o+l5eXy263N3c5LRZ9Dg36HDrh2uvI2oPNXYJfIo9Yqqk5EvA+V1VVSZIsyzp5DQF7Vj/s27dPdXV1SkpK8hlPSkpSWVlZg/lTp05Vfn5+g/H09PSg1RhuEv7Q3BUAAFqSvCCuXVVVpbZt255wTrP+mrHN5nuFwrKsBmOSNGHCBI0bN867feTIEe3fv1/t27dvdP7pqKysVGpqqnbt2qW4uLiAro1j6HNo0OfQoM+hQ69DI1h9tixLVVVVSklJOencZgkoCQkJioiIaHC1ZO/evQ2uqkiSw+GQw+HwGWvXrl0wS1RcXBwv/hCgz6FBn0ODPocOvQ6NYPT5ZFdO6jXLh2SjoqLUs2dPFRYW+owXFhaqb9++zVESAAAwSLPd4hk3bpxuuukm9erVS1dccYX++Mc/aufOnbrzzjubqyQAAGCIZgsoI0aMUHl5uX7/+9+rtLRUXbp00dKlS5WWltZcJUk6ejtp0qRJDW4pIbDoc2jQ59Cgz6FDr0PDhD7brFP5rg8AAEAI8Vs8AADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMc0YGlOeff17p6emKjo5Wz549tWrVqhPOLyoqUs+ePRUdHa2OHTvqhRdeCFGl4c2fPi9atEhZWVk655xzFBcXpyuuuELLli0LYbXhy9/Xc72///3vioyM1E9/+tPgFthC+Ntnt9utiRMnKi0tTQ6HQ506ddIrr7wSomrDl799nj9/vrp166aYmBglJyfrlltuCbsfTw21jz76SLm5uUpJSZHNZtOSJUtOekyzvA9aZ5gFCxZYdrvdeumll6wtW7ZY9913n9WmTRtrx44djc7/+uuvrZiYGOu+++6ztmzZYr300kuW3W633nrrrRBXHl787fN9991n/e///q/1ySefWF9++aU1YcIEy263W+vXrw9x5eHF3z7X++6776yOHTta2dnZVrdu3UJTbBhrSp+HDBli9enTxyosLLRKSkqsf/zjH9bf//73EFYdfvzt86pVq6xWrVpZzzzzjPX1119bq1atsi655BJr6NChIa48vCxdutSaOHGi9ec//9mSZC1evPiE85vrffCMCyi9e/e27rzzTp+xzp07Ww899FCj8x944AGrc+fOPmOjR4+2Lr/88qDV2BL42+fGXHzxxVZ+fn6gS2tRmtrnESNGWI888og1adIkAsop8LfPf/vb36y2bdta5eXloSivxfC3z08++aTVsWNHn7Fnn33W6tChQ9BqbGlOJaA01/vgGXWL5/DhwyouLlZ2drbPeHZ2ttasWdPoMWvXrm0wf9CgQVq3bp08Hk/Qag1nTenzjx05ckRVVVWKj48PRoktQlP7PHv2bH311VeaNGlSsEtsEZrS53feeUe9evXStGnTdO655+qCCy7Q+PHjdejQoVCUHJaa0ue+fftq9+7dWrp0qSzL0jfffKO33npL1157bShKPmM01/tgs/2p++awb98+1dXVNfjF5KSkpAa/rFyvrKys0fm1tbXat2+fkpOTg1ZvuGpKn3/sD3/4gw4ePKjhw4cHo8QWoSl93rp1qx566CGtWrVKkZFn1P/8m6wpff7666+1evVqRUdHa/Hixdq3b5/uuusu7d+/n8+hHEdT+ty3b1/Nnz9fI0aM0Pfff6/a2loNGTJEM2bMCEXJZ4zmeh88o66g1LPZbD7blmU1GDvZ/MbG4cvfPtd7/fXX5XK5tHDhQiUmJgarvBbjVPtcV1envLw85efn64ILLghVeS2GP6/nI0eOyGazaf78+erdu7euueYaTZ8+XXPmzOEqykn40+ctW7bo3nvv1e9+9zsVFxfrvffeU0lJCT86GwTN8T54Rv0TKiEhQREREQ3S+N69exukw3pOp7PR+ZGRkWrfvn3Qag1nTelzvYULF+rWW2/Vm2++qYEDBwazzLDnb5+rqqq0bt06bdiwQXfffbeko2+klmUpMjJSy5cv15VXXhmS2sNJU17PycnJOvfcc9W2bVvv2EUXXSTLsrR7925lZGQEteZw1JQ+T506Vf369dP//M//SJIuvfRStWnTRj//+c/1+OOPc4U7QJrrffCMuoISFRWlnj17qrCw0Ge8sLBQffv2bfSYK664osH85cuXq1evXrLb7UGrNZw1pc/S0Ssno0aNUkFBAfeQT4G/fY6Li9PGjRv12WefeR933nmnLrzwQn322Wfq06dPqEoPK015Pffr10979uxRdXW1d+zLL79Uq1at1KFDh6DWG66a0ueamhq1auX7NhYRESHp2L/wcfqa7X0wqB/BNVD919hefvlla8uWLdbYsWOtNm3aWNu3b7csy7Ieeugh66abbvLOr/961W9/+1try5Yt1ssvv8zXjE+Bv30uKCiwIiMjreeee84qLS31Pr777rvmOoWw4G+ff4xv8Zwaf/tcVVVldejQwbr++uutzZs3W0VFRVZGRoZ12223NdcphAV/+zx79mwrMjLSev75562vvvrKWr16tdWrVy+rd+/ezXUKYaGqqsrasGGDtWHDBkuSNX36dGvDhg3er3Ob8j54xgUUy7Ks5557zkpLS7OioqKsHj16WEVFRd59I0eOtPr37+8zf+XKlVb37t2tqKgo6/zzz7dmzZoV4orDkz997t+/vyWpwWPkyJGhLzzM+Pt6/iECyqnzt8+ff/65NXDgQKt169ZWhw4drHHjxlk1NTUhrjr8+NvnZ5991rr44out1q1bW8nJydZvfvMba/fu3SGuOrysWLHihP9/a8r7oM2yuA4GAADMckZ9BgUAAIQHAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGOf/AaSQdOteGLwqAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# df[['pred_label_no_mot', 'pred_label_mot']].hist()\n",
    "df[['pred_label_no_mot']].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\micfae\\AppData\\Local\\Temp\\ipykernel_4504\\851119368.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df.pred_label_no_mot[no_mot_nan_labels] = 1 - df.label[\n"
     ]
    }
   ],
   "source": [
    "# Substitute NaN values always with the wrong label (treat them like\n",
    "# missclassification).\n",
    "no_mot_nan_labels = df.pred_label_no_mot.isna()\n",
    "# mot_nan_labels = df.pred_label_mot.isna()\n",
    "\n",
    "df.pred_label_no_mot[no_mot_nan_labels] = 1 - df.label[\n",
    "    no_mot_nan_labels]\n",
    "# df.pred_label_mot[mot_nan_labels] = 1 - df.label[mot_nan_labels]\n",
    "\n",
    "# df.pred_label_mot = df.pred_label_mot.astype(np.int64)\n",
    "df.pred_label_no_mot = df.pred_label_no_mot.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.06      0.10       127\n",
      "           1       0.43      0.99      0.60        90\n",
      "\n",
      "    accuracy                           0.44       217\n",
      "   macro avg       0.65      0.52      0.35       217\n",
      "weighted avg       0.69      0.44      0.31       217\n",
      "\n",
      "[[  7 120]\n",
      " [  1  89]]\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(df['label'], df['pred_label_no_mot']))\n",
    "print(confusion_matrix(df['label'], df['pred_label_no_mot']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(classification_report(df['label'], df['pred_label_mot']))\n",
    "# print(confusion_matrix(df['label'], df['pred_label_mot']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.iterrows():\n",
    "    message = prompt_ita.create_prompt(row)\n",
    "    print(prompt_ita.get_nice_prompt(message))\n",
    "    \n",
    "    true = verbalizer_ita[1 - row['label']]\n",
    "    pred = verbalizer_ita[1 - row['pred_label_no_mot']]\n",
    "\n",
    "    print(f\"True: {true} - Predicted: {pred}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### English prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The server is overloaded or not ready yet.\n",
      "WARNING: Context size for message too long(6117)\n",
      "The server is overloaded or not ready yet.\n",
      "WARNING: Context size for message too long(14494)\n",
      "WARNING: Context size for message too long(4911)\n",
      "WARNING: Context size for message too long(4234)\n",
      "The server is overloaded or not ready yet.\n",
      "Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Thu, 29 Jun 2023 13:28:12 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7dee7b525aa00e59-MXP', 'alt-svc': 'h3=\":443\"; ma=86400'}\n"
     ]
    }
   ],
   "source": [
    "compute_new_column_no_mot = partial(compute_new_column, prompt_eng,\n",
    "                                    with_mot=False)\n",
    "# compute_new_column_mot = partial(compute_new_column, prompt_eng,\n",
    "#                                  with_mot=True)\n",
    "\n",
    "df['pred_label_no_mot'] = df.apply(compute_new_column_no_mot, axis=1)\n",
    "# df['pred_label_mot'] = df.apply(compute_new_column_mot, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('backup_cache_no_mot_correct_1_english.pickle', 'wb') as f:\n",
    "    pickle.dump(prompt_eng.cache, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle('backup_dataframe_no_mot_correct_1shot_eng.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<Axes: title={'center': 'pred_label_no_mot'}>]], dtype=object)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGxCAYAAABIjE2TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAv6ElEQVR4nO3de3hU9Z3H8c+QTCYEEzAEyARiDGy8gpRLAWG7JEqCUcICVdC4Cq4iProqRVYNSJlUga5uvYGgtQpYiKAWqK1UiMrNQqvcXG5V0HAriUi4JCEwTMLZP3gyOiZAJsxMfhPer+eZ5/H8zu/85nu+HTsfz5nJ2CzLsgQAAGCQZo1dAAAAwI8RUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQgCZu9+7dstlsmjNnTlCPqbFy5UrZbDa99957fh97vjVXrlwZsDXD3cyZMxv0vw8QLggoABCGCCho6ggogMFOnDjR2CUAQKMgoABB5nK5ZLPZtGnTJg0bNkxxcXFq2bKl/uM//kPfffedd97ll1+uQYMGadGiRerWrZuio6OVn58vSSopKdGYMWPUoUMHRUVFKTU1Vfn5+aqqqvJ5rgMHDmj48OGKjY1Vy5YtNWLECJWUlATkPHbt2qV77rlHaWlpiomJUfv27ZWTk6MtW7bUOf/kyZMaN26cEhMT1bx5c/Xv31+bNm2qNW/9+vUaPHiw4uPjFR0drW7duumdd9654Hprbgu9/fbbmjhxopKSkhQXF6cBAwboyy+/rDX/zTffVNeuXRUdHa34+HgNHTpUO3bsaNBzFhQU6IknnpDT6dQll1yinJwcffvttyovL9f999+vhIQEJSQk6J577lFFRYXPGidPnlReXp5SU1MVFRWl9u3b66GHHtLRo0e9cy6//HJt27ZNq1atks1mk81m0+WXX96QNgHGimzsAoCLxdChQzV8+HA98MAD2rZtmyZNmqTt27fr73//u+x2uyRp48aN2rFjh5566imlpqaqRYsWKikpUa9evdSsWTP98pe/VKdOnbRu3To988wz2r17t2bPni3pzNWWAQMG6MCBA5o2bZquuOIKffDBBxoxYkRA6j9w4IBat26tX//612rTpo0OHz6suXPnqnfv3tq0aZOuvPJKn/kTJkxQ9+7d9bvf/U7Hjh2Ty+VSenq6Nm3apI4dO0qSVqxYoZtuukm9e/fWq6++qpYtW2rBggUaMWKEKisrNWrUqAuue8KECerXr59+97vfqaysTE888YRycnK0Y8cORURESJKmTZumCRMm6I477tC0adNUWloql8ul66+/Xp9//rnS0tL8fs6MjAzNmTNHu3fv1vjx43XHHXcoMjJSXbt21dtvv61NmzZpwoQJio2N1csvvyxJsixLQ4YM0ccff6y8vDz97Gc/0//93/9p8uTJWrdundatWyeHw6HFixfr1ltvVcuWLTVz5kxJksPhuOBeAUaxAATV5MmTLUnWL37xC5/x+fPnW5KsefPmWZZlWSkpKVZERIT15Zdf+swbM2aMdckll1h79uzxGf/f//1fS5K1bds2y7Isa9asWZYk649//KPPvNGjR1uSrNmzZ9e75qKiovMeU1VVZZ06dcpKS0vzObcVK1ZYkqzu3btbp0+f9o7v3r3bstvt1n333ecdu+qqq6xu3bpZHo/HZ+1BgwZZTqfTqq6u9llzxYoV9T6HmmNuvvlmn/F33nnHkmStW7fOsizLOnLkiNW8efNa8/bu3Ws5HA4rNzfX7+fMycnxGR87dqwlyXrkkUd8xocMGWLFx8d7tz/88ENLkvXss8/6zFu4cKElyfrtb3/rHbv22mut/v3717s2INxwiwcIkTvvvNNne/jw4YqMjNSKFSu8Y9ddd52uuOIKn3l//vOflZGRoaSkJFVVVXkf2dnZkqRVq1ZJOnM1IjY2VoMHD/Y5Pjc3NyD1V1VVaerUqbrmmmsUFRWlyMhIRUVFaefOnXXeCsnNzZXNZvNup6SkqG/fvt7z3bVrl/7xj394+/LDc7v55ptVXFxc560Yf/24H9ddd50kac+ePZKkdevW6cSJE7Wu1iQnJ+uGG27Qxx9/7PdzDho0yGf76quvliTdcssttcYPHz7svc3zySefSFKtWm677Ta1aNGiQbUA4YpbPECIJCYm+mxHRkaqdevWKi0t9Y45nc5ax3377bf605/+5L0N9GOHDh2SJJWWlqpdu3bnfd6GGjdunF555RU98cQT6t+/vy699FI1a9ZM9913X50f5q3reRMTE/XFF19IOnNekjR+/HiNHz++zuesObcL0bp1a5/tmlshNTXX9L+u3iclJamwsNDv54yPj/fZjoqKOuf4yZMndckll6i0tFSRkZFq06aNzzybzabExESf1wrQ1BFQgBApKSlR+/btvdtVVVUqLS31eQP94RWHGgkJCbruuus0ZcqUOtdNSkqSdOaN+LPPPqvzeQNh3rx5uvvuuzV16lSf8UOHDqlVq1b1et6SkhLv+SYkJEiS8vLyNGzYsDqf88efawmGmnqKi4tr7Ttw4IC3zlBo3bq1qqqq9N133/mEFMuyVFJSop/+9KchqwVobNziAUJk/vz5PtvvvPOOqqqqlJ6efs7jBg0apK1bt6pTp07q2bNnrUdNQMnIyFB5ebnef/99n+MLCgoCUr/NZqv1QcwPPvhA//znP+uc//bbb8uyLO/2nj17tHbtWu/5XnnllUpLS9MXX3xR53n17NlTsbGxAan9XK6//no1b95c8+bN8xnfv3+/PvnkE914441Br6FGzXP9uJY//OEPOn78uE8tDoeDr6GjSeMKChAiixYtUmRkpDIzM73f4unatauGDx9+zuN+9atfqbCwUH379tUjjzyiK6+8UidPntTu3bu1dOlSvfrqq+rQoYPuvvtuvfDCC7r77rs1ZcoUpaWlaenSpVq2bFlA6h80aJDmzJmjq666Stddd502bNig5557Th06dKhz/sGDBzV06FCNHj1ax44d0+TJkxUdHa28vDzvnNdee03Z2dkaOHCgRo0apfbt2+vw4cPasWOHNm7cqHfffTcgtZ9Lq1atNGnSJE2YMEF333237rjjDpWWlio/P1/R0dGaPHly0GuokZmZqYEDB+qJJ55QWVmZ+vXr5/0WT7du3XTXXXd553bp0kULFizQwoUL1bFjR0VHR6tLly4hqxUINgIKECKLFi2Sy+XSrFmzZLPZlJOToxdffNH7OYSzcTqdWr9+vZ5++mk999xz2r9/v2JjY5WamqqbbrpJl156qSQpJiZGn3zyiR599FE9+eSTstlsysrK0oIFC9S3b98Lrv+ll16S3W7XtGnTVFFRoe7du2vRokV66qmn6pw/depUff7557rnnntUVlamXr16acGCBerUqZN3TkZGhj777DNNmTJFY8eO1ZEjR9S6dWtdc8015w1ugZSXl6e2bdvq5Zdf1sKFC9W8eXOlp6dr6tSpfn/F+ELYbDYtWbJELpdLs2fP1pQpU5SQkKC77rpLU6dO9bmClZ+fr+LiYo0ePVrl5eVKSUnR7t27Q1YrEGw264fXYAEEnMvlUn5+vr777ruQfp4BAMIZn0EBAADG4RYPcBGxLEvV1dXnnBMREVHnt4lM0Rjn0BT6BoQbrqAAQeZyuWRZlhG3d1atWiW73X7Ox9y5cxu7zHNqjHNoCn0Dwg2fQQEuIuXl5ef966ypqam1/riZSRrjHJpC34BwQ0ABAADG4RYPAAAwTlh+SPb06dM6cOCAYmNj+VAaAABhwrIslZeXKykpSc2anfsaSVgGlAMHDig5ObmxywAAAA2wb9++s/4V6hphGVBqfp9j3759iouLC+jaHo9Hy5cvV1ZW1ll/PRYXjj6HBn0ODfocOvQ6NILV57KyMiUnJ9frd7bCMqDU3NaJi4sLSkCJiYlRXFwcL/4gos+hQZ9Dgz6HDr0OjWD3uT4fz+BDsgAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGiWzsAgAAaOouf/KDxi7BL44IS8/2atwauIICAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHL8DyurVq5WTk6OkpCTZbDYtWbLEZ7/NZqvz8dxzz3nnpKen19p/++23X/DJAACApsHvgHL8+HF17dpVM2bMqHN/cXGxz+PNN9+UzWbTz3/+c595o0eP9pn32muvNewMAABAkxPp7wHZ2dnKzs4+6/7ExESf7T/+8Y/KyMhQx44dfcZjYmJqzT0bt9stt9vt3S4rK5MkeTweeTye+pZeLzXrBXpd+KLPoUGfQ4M+h0649toRYTV2CX5xNDtTb7DeY+vDZllWg7tms9m0ePFiDRkypM793377rTp06KC5c+cqNzfXO56enq5t27bJsiy1a9dO2dnZmjx5smJjY+tcx+VyKT8/v9Z4QUGBYmJiGlo+AAAIocrKSuXm5urYsWOKi4s751y/r6D4Y+7cuYqNjdWwYcN8xu+8806lpqYqMTFRW7duVV5enr744gsVFhbWuU5eXp7GjRvn3S4rK1NycrKysrLOe4L+8ng8KiwsVGZmpux2e0DXxvfoc2jQ59Cgz6ETrr3u7FrW2CX4xdHM0tM9Twe8zzV3QOojqAHlzTff1J133qno6Gif8dGjR3v/uXPnzkpLS1PPnj21ceNGde/evdY6DodDDoej1rjdbg/aCzSYa+N79Dk06HNo0OfQCbdeu6ttjV1CgwS6z/6sFbSvGa9Zs0Zffvml7rvvvvPO7d69u+x2u3bu3BmscgAAQBgJWkB544031KNHD3Xt2vW8c7dt2yaPxyOn0xmscgAAQBjx+xZPRUWFdu3a5d0uKirS5s2bFR8fr8suu0zSmXtM7777rn7zm9/UOv7rr7/W/PnzdfPNNyshIUHbt2/XY489pm7duqlfv34XcCoAAKCp8DugrF+/XhkZGd7tmg+vjhw5UnPmzJEkLViwQJZl6Y477qh1fFRUlD7++GO99NJLqqioUHJysm655RZNnjxZERERDTwNAADQlPgdUNLT03W+bybff//9uv/+++vcl5ycrFWrVvn7tAAA4CLCb/EAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDh+B5TVq1crJydHSUlJstlsWrJkic/+UaNGyWaz+Tz69OnjM8ftduvhhx9WQkKCWrRoocGDB2v//v0XdCIAAKDp8DugHD9+XF27dtWMGTPOOuemm25ScXGx97F06VKf/WPHjtXixYu1YMECffrpp6qoqNCgQYNUXV3t/xkAAIAmJ9LfA7Kzs5WdnX3OOQ6HQ4mJiXXuO3bsmN544w39/ve/14ABAyRJ8+bNU3Jysj766CMNHDjQ35IAAEAT43dAqY+VK1eqbdu2atWqlfr3768pU6aobdu2kqQNGzbI4/EoKyvLOz8pKUmdO3fW2rVr6wwobrdbbrfbu11WViZJ8ng88ng8Aa29Zr1Arwtf9Dk06HNo0OfQCddeOyKsxi7BL45mZ+oN1ntsfQQ8oGRnZ+u2225TSkqKioqKNGnSJN1www3asGGDHA6HSkpKFBUVpUsvvdTnuHbt2qmkpKTONadNm6b8/Pxa48uXL1dMTEygT0GSVFhYGJR14Ys+hwZ9Dg36HDrh1utnezV2BQ0T6D5XVlbWe27AA8qIESO8/9y5c2f17NlTKSkp+uCDDzRs2LCzHmdZlmw2W5378vLyNG7cOO92WVmZkpOTlZWVpbi4uMAVrzPprrCwUJmZmbLb7QFdG9+jz6FBn0ODPodOuPa6s2tZY5fgF0czS0/3PB3wPtfcAamPoNzi+SGn06mUlBTt3LlTkpSYmKhTp07pyJEjPldRDh48qL59+9a5hsPhkMPhqDVut9uD9gIN5tr4Hn0ODfocGvQ5dMKt1+7quv8D3HSB7rM/awX976CUlpZq3759cjqdkqQePXrIbrf7XDYqLi7W1q1bzxpQAADAxcXvKygVFRXatWuXd7uoqEibN29WfHy84uPj5XK59POf/1xOp1O7d+/WhAkTlJCQoKFDh0qSWrZsqXvvvVePPfaYWrdurfj4eI0fP15dunTxfqsHAABc3PwOKOvXr1dGRoZ3u+azISNHjtSsWbO0ZcsWvfXWWzp69KicTqcyMjK0cOFCxcbGeo954YUXFBkZqeHDh+vEiRO68cYbNWfOHEVERATglAAAQLjzO6Ckp6fLss7+dally87/QaDo6GhNnz5d06dP9/fpAQDARYDf4gEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOH4HlNWrVysnJ0dJSUmy2WxasmSJd5/H49ETTzyhLl26qEWLFkpKStLdd9+tAwcO+KyRnp4um83m87j99tsv+GQAAEDT4HdAOX78uLp27aoZM2bU2ldZWamNGzdq0qRJ2rhxoxYtWqSvvvpKgwcPrjV39OjRKi4u9j5ee+21hp0BAABociL9PSA7O1vZ2dl17mvZsqUKCwt9xqZPn65evXpp7969uuyyy7zjMTExSkxMrNdzut1uud1u73ZZWZmkM1dsPB6Pv6dwTjXrBXpd+KLPoUGfQ4M+h0649toRYTV2CX5xNDtTb7DeY+vDZllWg7tms9m0ePFiDRky5KxzPvroI2VlZeno0aOKi4uTdOYWz7Zt22RZltq1a6fs7GxNnjxZsbGxda7hcrmUn59fa7ygoEAxMTENLR8AAIRQZWWlcnNzdezYMW8mOJugBpSTJ0/qX//1X3XVVVdp3rx53vHXX39dqampSkxM1NatW5WXl6d/+Zd/qXX1pUZdV1CSk5N16NCh856gvzwejwoLC5WZmSm73R7QtfE9+hwa9Dk06HPohGuvO7uWNXYJfnE0s/R0z9MB73NZWZkSEhLqFVD8vsVTXx6PR7fffrtOnz6tmTNn+uwbPXq09587d+6stLQ09ezZUxs3blT37t1rreVwOORwOGqN2+32oL1Ag7k2vkefQ4M+hwZ9Dp1w67W72tbYJTRIoPvsz1pB+Zqxx+PR8OHDVVRUpMLCwvOmpO7du8tut2vnzp3BKAcAAISZgF9BqQknO3fu1IoVK9S6devzHrNt2zZ5PB45nc5AlwMAAMKQ3wGloqJCu3bt8m4XFRVp8+bNio+PV1JSkm699VZt3LhRf/7zn1VdXa2SkhJJUnx8vKKiovT1119r/vz5uvnmm5WQkKDt27frscceU7du3dSvX7/AnRkAAAhbfgeU9evXKyMjw7s9btw4SdLIkSPlcrn0/vvvS5J+8pOf+By3YsUKpaenKyoqSh9//LFeeuklVVRUKDk5WbfccosmT56siIiICzgVAADQVPgdUNLT03WuL/6c70tBycnJWrVqlb9PCwAALiL8Fg8AADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjON3QFm9erVycnKUlJQkm82mJUuW+Oy3LEsul0tJSUlq3ry50tPTtW3bNp85brdbDz/8sBISEtSiRQsNHjxY+/fvv6ATAQAATYffAeX48ePq2rWrZsyYUef+Z599Vs8//7xmzJihzz//XImJicrMzFR5ebl3ztixY7V48WItWLBAn376qSoqKjRo0CBVV1c3/EwAAECTEenvAdnZ2crOzq5zn2VZevHFFzVx4kQNGzZMkjR37ly1a9dOBQUFGjNmjI4dO6Y33nhDv//97zVgwABJ0rx585ScnKyPPvpIAwcOvIDTAQAATYHfAeVcioqKVFJSoqysLO+Yw+FQ//79tXbtWo0ZM0YbNmyQx+PxmZOUlKTOnTtr7dq1dQYUt9stt9vt3S4rK5MkeTweeTyeQJ6Cd71Arwtf9Dk06HNo0OfQCddeOyKsxi7BL45mZ+oN1ntsfQQ0oJSUlEiS2rVr5zPerl077dmzxzsnKipKl156aa05Ncf/2LRp05Sfn19rfPny5YqJiQlE6bUUFhYGZV34os+hQZ9Dgz6HTrj1+tlejV1BwwS6z5WVlfWeG9CAUsNms/lsW5ZVa+zHzjUnLy9P48aN826XlZUpOTlZWVlZiouLu/CCf8Dj8aiwsFCZmZmy2+0BXRvfo8+hQZ9Dgz6HTrj2urNrWWOX4BdHM0tP9zwd8D7X3AGpj4AGlMTERElnrpI4nU7v+MGDB71XVRITE3Xq1CkdOXLE5yrKwYMH1bdv3zrXdTgccjgctcbtdnvQXqDBXBvfo8+hQZ9Dgz6HTrj12l197v9IN1Wg++zPWgH9OyipqalKTEz0uSR06tQprVq1yhs+evToIbvd7jOnuLhYW7duPWtAAQAAFxe/r6BUVFRo165d3u2ioiJt3rxZ8fHxuuyyyzR27FhNnTpVaWlpSktL09SpUxUTE6Pc3FxJUsuWLXXvvffqscceU+vWrRUfH6/x48erS5cu3m/1AACAi5vfAWX9+vXKyMjwbtd8NmTkyJGaM2eOHn/8cZ04cUIPPvigjhw5ot69e2v58uWKjY31HvPCCy8oMjJSw4cP14kTJ3TjjTdqzpw5ioiICMApAQCAcOd3QElPT5dlnf3rUjabTS6XSy6X66xzoqOjNX36dE2fPt3fpwcAABcBfosHAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMEPKBcfvnlstlstR4PPfSQJGnUqFG19vXp0yfQZQAAgDAWGegFP//8c1VXV3u3t27dqszMTN12223esZtuukmzZ8/2bkdFRQW6DAAAEMYCHlDatGnjs/3rX/9anTp1Uv/+/b1jDodDiYmJgX5qAADQRAQ8oPzQqVOnNG/ePI0bN042m807vnLlSrVt21atWrVS//79NWXKFLVt2/as67jdbrndbu92WVmZJMnj8cjj8QS05pr1Ar0ufNHn0KDPoUGfQydce+2IsBq7BL84mp2pN1jvsfVhsywraF175513lJubq7179yopKUmStHDhQl1yySVKSUlRUVGRJk2apKqqKm3YsEEOh6POdVwul/Lz82uNFxQUKCYmJljlAwCAAKqsrFRubq6OHTumuLi4c84NakAZOHCgoqKi9Kc//emsc4qLi5WSkqIFCxZo2LBhdc6p6wpKcnKyDh06dN4T9JfH41FhYaEyMzNlt9sDuja+R59Dgz6HBn0OnXDtdWfXssYuwS+OZpae7nk64H0uKytTQkJCvQJK0G7x7NmzRx999JEWLVp0znlOp1MpKSnauXPnWec4HI46r67Y7fagvUCDuTa+R59Dgz6HBn0OnXDrtbvadv5JBgp0n/1ZK2h/B2X27Nlq27atbrnllnPOKy0t1b59++R0OoNVCgAACDNBCSinT5/W7NmzNXLkSEVGfn+RpqKiQuPHj9e6deu0e/durVy5Ujk5OUpISNDQoUODUQoAAAhDQbnF89FHH2nv3r36z//8T5/xiIgIbdmyRW+99ZaOHj0qp9OpjIwMLVy4ULGxscEoBQAAhKGgBJSsrCzV9dnb5s2ba9my8PqgEAAACD1+iwcAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxgl4QHG5XLLZbD6PxMRE737LsuRyuZSUlKTmzZsrPT1d27ZtC3QZAAAgjAXlCsq1116r4uJi72PLli3efc8++6yef/55zZgxQ59//rkSExOVmZmp8vLyYJQCAADCUFACSmRkpBITE72PNm3aSDpz9eTFF1/UxIkTNWzYMHXu3Flz585VZWWlCgoKglEKAAAIQ5HBWHTnzp1KSkqSw+FQ7969NXXqVHXs2FFFRUUqKSlRVlaWd67D4VD//v21du1ajRkzps713G633G63d7usrEyS5PF45PF4Alp7zXqBXhe+6HNo0OfQoM+hE669dkRYjV2CXxzNztQbrPfY+rBZlhXQrv3lL39RZWWlrrjiCn377bd65pln9I9//EPbtm3Tl19+qX79+umf//ynkpKSvMfcf//92rNnj5YtW1bnmi6XS/n5+bXGCwoKFBMTE8jyAQBAkFRWVio3N1fHjh1TXFzcOecGPKD82PHjx9WpUyc9/vjj6tOnj/r166cDBw7I6XR654wePVr79u3Thx9+WOcadV1BSU5O1qFDh857gv7yeDwqLCxUZmam7HZ7QNfG9+hzaNDn0KDPoROuve7sqvs/wE3laGbp6Z6nA97nsrIyJSQk1CugBOUWzw+1aNFCXbp00c6dOzVkyBBJUklJiU9AOXjwoNq1a3fWNRwOhxwOR61xu90etBdoMNfG9+hzaNDn0KDPoRNuvXZX2xq7hAYJdJ/9WSvofwfF7XZrx44dcjqdSk1NVWJiogoLC737T506pVWrVqlv377BLgUAAISJgF9BGT9+vHJycnTZZZfp4MGDeuaZZ1RWVqaRI0fKZrNp7Nixmjp1qtLS0pSWlqapU6cqJiZGubm5gS4FAACEqYAHlP379+uOO+7QoUOH1KZNG/Xp00d/+9vflJKSIkl6/PHHdeLECT344IM6cuSIevfureXLlys2NjbQpQAAgDAV8ICyYMGCc+632WxyuVxyuVyBfmoAANBE8Fs8AADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGCcyMYuwFSdXcvkrrY1dhn1tvvXtzR2CQAABAxXUAAAgHECHlCmTZumn/70p4qNjVXbtm01ZMgQffnllz5zRo0aJZvN5vPo06dPoEsBAABhKuABZdWqVXrooYf0t7/9TYWFhaqqqlJWVpaOHz/uM++mm25ScXGx97F06dJAlwIAAMJUwD+D8uGHH/psz549W23bttWGDRv0b//2b95xh8OhxMTEQD89AABoAoL+Idljx45JkuLj433GV65cqbZt26pVq1bq37+/pkyZorZt29a5htvtltvt9m6XlZVJkjwejzweT0DrrVnP0cwK6LrBFug+BFtNveFWd7ihz6FBn0MnXHvtiAiv95Sa98BgvcfWh82yrKB1zbIs/fu//7uOHDmiNWvWeMcXLlyoSy65RCkpKSoqKtKkSZNUVVWlDRs2yOFw1FrH5XIpPz+/1nhBQYFiYmKCVT4AAAigyspK5ebm6tixY4qLizvn3KAGlIceekgffPCBPv30U3Xo0OGs84qLi5WSkqIFCxZo2LBhtfbXdQUlOTlZhw4dOu8J+svj8aiwsFCT1jeT+3T4fM14q2tgY5fgl5o+Z2Zmym63N3Y5TRZ9Dg36HDrh2uvOrmWNXYJfHM0sPd3zdMD7XFZWpoSEhHoFlKDd4nn44Yf1/vvva/Xq1ecMJ5LkdDqVkpKinTt31rnf4XDUeWXFbrcH7QXqPm0Lq7+DEk7/ov5QMP83xPfoc2jQ59AJt16H0/vJDwW6z/6sFfCAYlmWHn74YS1evFgrV65UamrqeY8pLS3Vvn375HQ6A10OAAAIQwH/mvFDDz2kefPmqaCgQLGxsSopKVFJSYlOnDghSaqoqND48eO1bt067d69WytXrlROTo4SEhI0dOjQQJcDAADCUMCvoMyaNUuSlJ6e7jM+e/ZsjRo1ShEREdqyZYveeustHT16VE6nUxkZGVq4cKFiY2MDXQ4AAAhDQbnFcy7NmzfXsmXh9WEhAAAQWvwWDwAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACM06gBZebMmUpNTVV0dLR69OihNWvWNGY5AADAEI0WUBYuXKixY8dq4sSJ2rRpk372s58pOztbe/fubaySAACAIRotoDz//PO69957dd999+nqq6/Wiy++qOTkZM2aNauxSgIAAIaIbIwnPXXqlDZs2KAnn3zSZzwrK0tr166tNd/tdsvtdnu3jx07Jkk6fPiwPB5PQGvzeDyqrKxUpKeZqk/bArp2MJWWljZ2CX6p6XNpaansdntjl9Nk0efQoM+hE669jqw63tgl+CXytKXKytMB73N5ebkkybKs89cQsGf1w6FDh1RdXa127dr5jLdr104lJSW15k+bNk35+fm1xlNTU4NWY7hJ+E1jVwAAaEpyg7h2eXm5WrZsec45jRJQathsvlcoLMuqNSZJeXl5GjdunHf79OnTOnz4sFq3bl3n/AtRVlam5ORk7du3T3FxcQFdG9+jz6FBn0ODPocOvQ6NYPXZsiyVl5crKSnpvHMbJaAkJCQoIiKi1tWSgwcP1rqqIkkOh0MOh8NnrFWrVsEsUXFxcbz4Q4A+hwZ9Dg36HDr0OjSC0efzXTmp0Sgfko2KilKPHj1UWFjoM15YWKi+ffs2RkkAAMAgjXaLZ9y4cbrrrrvUs2dPXX/99frtb3+rvXv36oEHHmiskgAAgCEaLaCMGDFCpaWl+tWvfqXi4mJ17txZS5cuVUpKSmOVJOnM7aTJkyfXuqWEwKLPoUGfQ4M+hw69Dg0T+myz6vNdHwAAgBDit3gAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABjnogwoM2fOVGpqqqKjo9WjRw+tWbPmnPNXrVqlHj16KDo6Wh07dtSrr74aokrDmz99XrRokTIzM9WmTRvFxcXp+uuv17Jly0JYbfjy9/Vc469//asiIyP1k5/8JLgFNhH+9tntdmvixIlKSUmRw+FQp06d9Oabb4ao2vDlb5/nz5+vrl27KiYmRk6nU/fcc0/Y/XhqqK1evVo5OTlKSkqSzWbTkiVLzntMo7wPWheZBQsWWHa73Xr99det7du3W48++qjVokULa8+ePXXO/+abb6yYmBjr0UcftbZv3269/vrrlt1ut957770QVx5e/O3zo48+av3P//yP9dlnn1lfffWVlZeXZ9ntdmvjxo0hrjy8+NvnGkePHrU6duxoZWVlWV27dg1NsWGsIX0ePHiw1bt3b6uwsNAqKiqy/v73v1t//etfQ1h1+PG3z2vWrLGaNWtmvfTSS9Y333xjrVmzxrr22mutIUOGhLjy8LJ06VJr4sSJ1h/+8AdLkrV48eJzzm+s98GLLqD06tXLeuCBB3zGrrrqKuvJJ5+sc/7jjz9uXXXVVT5jY8aMsfr06RO0GpsCf/tcl2uuucbKz88PdGlNSkP7PGLECOupp56yJk+eTECpB3/7/Je//MVq2bKlVVpaGorymgx/+/zcc89ZHTt29Bl7+eWXrQ4dOgStxqamPgGlsd4HL6pbPKdOndKGDRuUlZXlM56VlaW1a9fWecy6detqzR84cKDWr18vj8cTtFrDWUP6/GOnT59WeXm54uPjg1Fik9DQPs+ePVtff/21Jk+eHOwSm4SG9Pn9999Xz5499eyzz6p9+/a64oorNH78eJ04cSIUJYelhvS5b9++2r9/v5YuXSrLsvTtt9/qvffe0y233BKKki8ajfU+2Gh/6r4xHDp0SNXV1bV+Mbldu3a1flm5RklJSZ3zq6qqdOjQITmdzqDVG64a0ucf+81vfqPjx49r+PDhwSixSWhIn3fu3Kknn3xSa9asUWTkRfWvf4M1pM/ffPONPv30U0VHR2vx4sU6dOiQHnzwQR0+fJjPoZxFQ/rct29fzZ8/XyNGjNDJkydVVVWlwYMHa/r06aEo+aLRWO+DF9UVlBo2m81n27KsWmPnm1/XOHz52+cab7/9tlwulxYuXKi2bdsGq7wmo759rq6uVm5urvLz83XFFVeEqrwmw5/X8+nTp2Wz2TR//nz16tVLN998s55//nnNmTOHqyjn4U+ft2/frkceeUS//OUvtWHDBn344YcqKiriR2eDoDHeBy+q/4RKSEhQRERErTR+8ODBWumwRmJiYp3zIyMj1bp166DVGs4a0ucaCxcu1L333qt3331XAwYMCGaZYc/fPpeXl2v9+vXatGmT/uu//kvSmTdSy7IUGRmp5cuX64YbbghJ7eGkIa9np9Op9u3bq2XLlt6xq6++WpZlaf/+/UpLSwtqzeGoIX2eNm2a+vXrp//+7/+WJF133XVq0aKFfvazn+mZZ57hCneANNb74EV1BSUqKko9evRQYWGhz3hhYaH69u1b5zHXX399rfnLly9Xz549Zbfbg1ZrOGtIn6UzV05GjRqlgoIC7iHXg799jouL05YtW7R582bv44EHHtCVV16pzZs3q3fv3qEqPaw05PXcr18/HThwQBUVFd6xr776Ss2aNVOHDh2CWm+4akifKysr1ayZ79tYRESEpO//Cx8XrtHeB4P6EVwD1XyN7Y033rC2b99ujR071mrRooW1e/duy7Is68knn7Tuuusu7/yar1f94he/sLZv32698cYbfM24Hvztc0FBgRUZGWm98sorVnFxsfdx9OjRxjqFsOBvn3+Mb/HUj799Li8vtzp06GDdeuut1rZt26xVq1ZZaWlp1n333ddYpxAW/O3z7NmzrcjISGvmzJnW119/bX366adWz549rV69ejXWKYSF8vJya9OmTdamTZssSdbzzz9vbdq0yft1blPeBy+6gGJZlvXKK69YKSkpVlRUlNW9e3dr1apV3n0jR460+vfv7zN/5cqVVrdu3ayoqCjr8ssvt2bNmhXiisOTP33u37+/JanWY+TIkaEvPMz4+3r+IQJK/fnb5x07dlgDBgywmjdvbnXo0MEaN26cVVlZGeKqw4+/fX755Zeta665xmrevLnldDqtO++809q/f3+Iqw4vK1asOOf/35ryPmizLK6DAQAAs1xUn0EBAADhgYACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMb5f4TDAejxvL1lAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df[['pred_label_no_mot']].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[['pred_label_no_mot', 'pred_label_mot']].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\micfae\\AppData\\Local\\Temp\\ipykernel_4504\\851119368.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df.pred_label_no_mot[no_mot_nan_labels] = 1 - df.label[\n"
     ]
    }
   ],
   "source": [
    "# Substitute NaN values always with the wrong label (treat them like\n",
    "# missclassification).\n",
    "no_mot_nan_labels = df.pred_label_no_mot.isna()\n",
    "# mot_nan_labels = df.pred_label_mot.isna()\n",
    "\n",
    "df.pred_label_no_mot[no_mot_nan_labels] = 1 - df.label[\n",
    "    no_mot_nan_labels]\n",
    "# df.pred_label_mot[mot_nan_labels] = 1 - df.label[mot_nan_labels]\n",
    "\n",
    "# df.pred_label_mot = df.pred_label_mot.astype(np.int64)\n",
    "df.pred_label_no_mot = df.pred_label_no_mot.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.15      0.25       127\n",
      "           1       0.43      0.90      0.58        90\n",
      "\n",
      "    accuracy                           0.46       217\n",
      "   macro avg       0.55      0.52      0.41       217\n",
      "weighted avg       0.57      0.46      0.38       217\n",
      "\n",
      "[[ 19 108]\n",
      " [  9  81]]\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(df['label'], df['pred_label_no_mot']))\n",
    "print(confusion_matrix(df['label'], df['pred_label_no_mot']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(classification_report(df['label'], df['pred_label_mot']))\n",
    "# print(confusion_matrix(df['label'], df['pred_label_mot']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.iterrows():\n",
    "    message = prompt_eng.create_prompt(row)\n",
    "    print(prompt_eng.get_nice_prompt(message))\n",
    "    \n",
    "    true = verbalizer_eng[1 - row['label']]\n",
    "    pred = verbalizer_eng[1 - row['pred_label_no_mot']]\n",
    "\n",
    "    print(f\"True: {true} - Predicted: {pred}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vjp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}