13_GermEval17_Experiment_Single.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import time\n",
    "import logging\n",
    "from data.data_loader import Dataset\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import get_default_params, OutputLayerType, LearningSchedulerType, OptimizerType, hyperOpt_goodParams, default_params\n",
    "from misc import utils\n",
    "\n",
    "from optimizer import get_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from trainer.train import Trainer\n",
    "import pprint\n",
    "from data.germeval2017 import germeval2017_dataset as dsl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Functions\n",
    "\n",
    "These functions will load the dataset and the model. The run configuration will determine the architecture and hyper parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_model(dataset, rc, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=rc)\n",
    "    model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names)\n",
    "    optimizer = get_optimizer(model, rc)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        rc,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer\n",
    "\n",
    "def load_dataset(rc, logger, task):\n",
    "    dataset = Dataset(\n",
    "        task,\n",
    "        logger,\n",
    "        rc,\n",
    "        source_index=PREFERENCES.source_index,\n",
    "        target_vocab_index=PREFERENCES.target_vocab_index,\n",
    "        data_path=PREFERENCES.data_root,\n",
    "        train_file=PREFERENCES.data_train,\n",
    "        valid_file=PREFERENCES.data_validation,\n",
    "        test_file=PREFERENCES.data_test,\n",
    "        file_format=PREFERENCES.file_format,\n",
    "        init_token=None,\n",
    "        eos_token=None\n",
    "    )\n",
    "    dataset.load_data(dsl, verbose=False)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preamble - Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/data/germeval2017',\n",
    "    data_train='train_v1.4.tsv',    \n",
    "    data_validation='dev_v1.4.tsv',\n",
    "    data_test='test_TIMESTAMP1.tsv',\n",
    "    source_index=0,\n",
    "    target_vocab_index=2,\n",
    "    file_format='csv'\n",
    ")\n",
    "main_experiment_name = 'GermEval7_Experiments'\n",
    "use_cuda = True\n",
    "STATUS_FAIL = 'fail'\n",
    "STATUS_OK = 'ok'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Definition of experiments\n",
    " \n",
    " - SpellChecker On\n",
    " - Fasttext\n",
    " - Single Sentence\n",
    " - Combined Sentence\n",
    " - Stop Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{ 'att_d_k': 300,\n",
      "  'att_d_v': 300,\n",
      "  'batch_size': 12,\n",
      "  'clip_comments_to': 113,\n",
      "  'dropout_rate': 0.302424,\n",
      "  'early_stopping': 5,\n",
      "  'embedding_dim': 300,\n",
      "  'embedding_name': '6B',\n",
      "  'embedding_type': 'fasttext',\n",
      "  'harmonize_bahn': True,\n",
      "  'language': 'de',\n",
      "  'learning_rate_scheduler': { 'noam_learning_rate_factor': 1.418,\n",
      "                               'noam_learning_rate_warmup': 8000},\n",
      "  'learning_rate_scheduler_type': <LearningSchedulerType.Noam: 1>,\n",
      "  'log_every_xth_iteration': -1,\n",
      "  'model_size': 300,\n",
      "  'num_encoder_blocks': 2,\n",
      "  'num_epochs': 25,\n",
      "  'num_heads': 1,\n",
      "  'optimizer': { 'adam_beta1': 0.81,\n",
      "                 'adam_beta2': 0.7173,\n",
      "                 'adam_eps': 0.000814,\n",
      "                 'learning_rate': 7.2e-05},\n",
      "  'optimizer_type': <OptimizerType.Adam: 1>,\n",
      "  'organic_text_cleaning': False,\n",
      "  'output_dropout_rate': 0.79602089766246,\n",
      "  'output_layer': { 'output_conv_kernel_size': 5,\n",
      "                    'output_conv_num_filters': 300,\n",
      "                    'output_conv_padding': 0,\n",
      "                    'output_conv_stride': 1},\n",
      "  'output_layer_type': <OutputLayerType.LinearSum: 1>,\n",
      "  'pointwise_layer_size': 405,\n",
      "  'task': 'absa',\n",
      "  'use_stop_words': True}\n"
     ]
    }
   ],
   "source": [
    "baseline = {**default_params, **hyperOpt_goodParams}\n",
    "print(pprint.pformat(baseline, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'name': 'Baseline',\n",
    "        'description': 'Baseline. Uses hyperOpt_goodParams without any changes',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {}\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Current commit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'e6a4eff'\n"
     ]
    }
   ],
   "source": [
    "utils.get_current_git_commit()\n",
    "print('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def objective(rc, experiment):\n",
    "    run_time = time.time()\n",
    "    \n",
    "    # reset loggers\n",
    "    utils.reset_loggers()\n",
    "    experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    dataset_logger = logging.getLogger('data_loader')\n",
    "    \n",
    "    logger.info('Experiment: ' + experiment['name'])\n",
    "    logger.info('Description: ' + experiment['description'])\n",
    "    \n",
    "    logger.info('Parameters')\n",
    "    logger.info(rc)\n",
    "    print('\\n\\n#########################################################################')\n",
    "    print('Name: ' + experiment['name'])\n",
    "    print('Description: ' + experiment['description'])\n",
    "    print('#########################################################################\\n\\n')\n",
    "    print(rc)\n",
    "\n",
    "    logger.debug('Load dataset')\n",
    "    try:\n",
    "        dataset = load_dataset(rc, dataset_logger, rc.task)\n",
    "    except Exception as err:\n",
    "        print('Could load dataset: ' + str(err))\n",
    "        logger.exception(\"Could not load dataset\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "\n",
    "    try:\n",
    "        trainer = load_model(dataset, rc, experiment_name)\n",
    "    except Exception as err:\n",
    "        print('Could not load model: ' + str(err))\n",
    "        logger.exception(\"Could not load model\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "\n",
    "    logger.debug('model loaded')\n",
    "\n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=rc.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        print('Exception while training: ' + str(err))\n",
    "        logger.exception(\"Could not complete iteration\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    if math.isnan(trainer.get_best_loss()):\n",
    "        print('Loss is nan')\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=True, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation.\")\n",
    "        print('Could not complete iteration evaluation: ' + str(err))\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "    print(f'VAL f1\\t{trainer.get_best_f1()} - ({result[1][1]})')\n",
    "    print(f'VAL loss\\t{trainer.get_best_loss()}')\n",
    "    return {\n",
    "            'loss': result[1][0],\n",
    "            'status': STATUS_OK,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1(),\n",
    "            'sample_iterations': trainer.get_num_samples_seen(),\n",
    "            'iterations': trainer.get_num_iterations(),\n",
    "            'rc': rc,\n",
    "            'results': {\n",
    "                'train': {\n",
    "                    'loss': result[0][0],\n",
    "                    'f1': result[0][1]\n",
    "                },\n",
    "                'validation': {\n",
    "                    'loss': result[1][0],\n",
    "                    'f1': result[1][1]\n",
    "                },\n",
    "                'test': {\n",
    "                    'loss': result[2][0],\n",
    "                    'f1': result[2][1]\n",
    "                }\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval7_Experiments\\20190401\\0\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Baseline\n",
      "Description: Baseline. Uses hyperOpt_goodParams without any changes\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 12, 'learning_rate_schedu[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                        absa                       |\n",
      "|          batch_size          |                         12                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                      7.2e-05                      |\n",
      "|  noam_learning_rate_warmup   |                        8000                       |\n",
      "|  noam_learning_rate_factor   |                       1.418                       |\n",
      "|          adam_beta1          |                        0.81                       |\n",
      "|          adam_beta2          |                       0.7173                      |\n",
      "|           adam_eps           |                      0.000814                     |\n",
      "|      adam_weight_decay       |                         0                         |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                      0.302424                     |\n",
      "|     pointwise_layer_size     |                        405                        |\n",
      "|      last_layer_dropout      |                  0.79602089766246                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         25                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        113                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                        True                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9720404e314347fd8185c54f74b5f005",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t21k\t0.77\t\t0.64\t\t0.314\t\t0.895\t\t4.41m - 4.4m / 0.0m\n",
      "2\t42k\t0.60\t\t0.65\t\t0.381\t\t0.909\t\t4.37m - 8.9m / 110.4m\n",
      "3\t63k\t0.59\t\t0.59\t\t0.432\t\t0.933\t\t4.27m - 13.2m / 109.4m\n",
      "4\t84k\t0.61\t\t0.69\t\t0.527\t\t0.959\t\t4.51m - 17.8m / 107.2m\n",
      "5\t105k\t0.61\t\t0.72\t\t0.435\t\t0.933\t\t4.27m - 22.2m / 112.6m\n",
      "6\t126k\t0.57\t\t0.64\t\t0.425\t\t0.935\t\t4.28m - 26.5m / 107.7m\n",
      "7\t147k\t0.54\t\t0.67\t\t0.362\t\t0.901\t\t4.31m - 30.9m / 108.0m\n",
      "8\t168k\t0.53\t\t0.83\t\t0.274\t\t0.839\t\t4.32m - 35.3m / 108.5m\n",
      "9\t189k\t0.52\t\t0.65\t\t0.508\t\t0.946\t\t4.32m - 39.7m / 108.7m\n",
      "VAL f1\t0.5270945810837833 - (0.5270945810837833)\n",
      "VAL loss\t0.5930153871484009\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Baseline\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.5930153871484009\n",
      "        | \\     )|_\tf1: 0.5270945810837833\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for e in experiments:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    \n",
    "    # generate rc\n",
    "    rc = get_default_params(use_cuda=True, overwrite=e['rc'], from_default=baseline)\n",
    "    result = objective(rc, e)\n",
    "    \n",
    "    if result['status'] == STATUS_OK:\n",
    "        print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\tExperiment: {e['name']}\\n\\\n",
    "          /`\\\\_/`\\\\\\tStatus: {result['status']}\\n\\\n",
    "         //  _  \\\\\\\\\\tLoss: {result['best_loss']}\\n\\\n",
    "        | \\\\     )|_\\tf1: {result['best_f1']}\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    else:\n",
    "        print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\tExperiment: {e['name']} (FAIL)\\n\\\n",
    "          /`\\\\_/`\\\\\\n\\\n",
    "         //  _  \\\\\\\\\\\\n\\\n",
    "        | \\\\     )|_\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}