12_Organic19_Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import time\n",
    "import logging\n",
    "from data.data_loader import Dataset\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import get_default_params, OutputLayerType, LearningSchedulerType, OptimizerType, good_organic_hp_params\n",
    "from misc import utils\n",
    "\n",
    "from optimizer import get_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from trainer.train import Trainer\n",
    "import pprint\n",
    "from data.organic2019 import organic_dataset as dsl\n",
    "from data.organic2019 import ORGANIC_TASK_ALL, ORGANIC_TASK_ENTITIES, ORGANIC_TASK_ATTRIBUTES, ORGANIC_TASK_ENTITIES_COMBINE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Functions\n",
    "\n",
    "These functions will load the dataset and the model. The run configuration will determine the architecture and hyper parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_model(dataset, rc, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=rc)\n",
    "    model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names)\n",
    "    optimizer = get_optimizer(model, rc)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        rc,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer\n",
    "\n",
    "def load_dataset(rc, logger, task):\n",
    "    dataset = Dataset(\n",
    "        task,\n",
    "        logger,\n",
    "        rc,\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        data_path=PREFERENCES.data_root,\n",
    "        train_file=PREFERENCES.data_train,\n",
    "        valid_file=PREFERENCES.data_validation,\n",
    "        test_file=PREFERENCES.data_test,\n",
    "        file_format='.tsv',\n",
    "        init_token=None,\n",
    "        eos_token=None\n",
    "    )\n",
    "    dataset.load_data(dsl, verbose=False)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preamble - Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/data/organic2019',\n",
    "    data_train='train.csv',    \n",
    "    data_validation='validation.csv',\n",
    "    data_test='test.csv',\n",
    "    early_stopping='highest_5_F1'\n",
    ")\n",
    "main_experiment_name = 'Organic19_Experiments'\n",
    "use_cuda = True\n",
    "STATUS_FAIL = 'fail'\n",
    "STATUS_OK = 'ok'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Definition of experiments\n",
    " \n",
    " - SpellChecker On\n",
    " - Fasttext\n",
    " - Single Sentence\n",
    " - Combined Sentence\n",
    " - Stop Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{ 'att_d_k': 100,\n",
      "  'att_d_v': 100,\n",
      "  'batch_size': 20,\n",
      "  'clip_comments_to': 195,\n",
      "  'dropout_rate': 0.392996573831,\n",
      "  'early_stopping': 5,\n",
      "  'embedding_dim': 300,\n",
      "  'embedding_name': '6B',\n",
      "  'embedding_type': 'glove',\n",
      "  'language': 'en',\n",
      "  'learning_rate_scheduler': { 'noam_learning_rate_factor': 3.3368149482,\n",
      "                               'noam_learning_rate_warmup': 4631},\n",
      "  'learning_rate_scheduler_type': <LearningSchedulerType.Noam: 1>,\n",
      "  'log_every_xth_iteration': -1,\n",
      "  'model_size': 300,\n",
      "  'num_encoder_blocks': 2,\n",
      "  'num_epochs': 35,\n",
      "  'num_heads': 3,\n",
      "  'optimizer': { 'adam_beta1': 0.89178641984,\n",
      "                 'adam_beta2': 0.83491754824,\n",
      "                 'adam_eps': 8.734158747166484e-09,\n",
      "                 'adam_weight_decay': 1e-08,\n",
      "                 'learning_rate': 0.001},\n",
      "  'optimizer_type': <OptimizerType.Adam: 1>,\n",
      "  'output_dropout_rate': 0.7608194889605,\n",
      "  'output_layer_type': <OutputLayerType.LinearSum: 1>,\n",
      "  'pointwise_layer_size': 195,\n",
      "  'task': 'entities',\n",
      "  'use_spell_checkers': False}\n"
     ]
    }
   ],
   "source": [
    "baseline = good_organic_hp_params\n",
    "print(pprint.pformat(baseline, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'name': 'Baseline',\n",
    "        'description': 'Baseline. Uses good_organic_hp_params without any changes',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {}\n",
    "    },\n",
    "    {\n",
    "        'name': 'SpellChecker On',\n",
    "        'description': 'Uses the baseline parameters but with the spellchecker enabled',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'use_spell_checkers': True\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Fasttext',\n",
    "        'description': 'Uses english Fasttext embeddings',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'embedding_type': 'fasttext'\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Rolling Sentences',\n",
    "        'description': 'Uses a combination of the last and the current sentence instead of just the sentence alone',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'task': 'entities_combine'\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Stop Words',\n",
    "        'description': 'Uses stop words removal',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'use_stop_words': True\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Embedding finetuning Off',\n",
    "        'description': 'This experiment prevents finetuning of the embedding layer',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'finetune_embedding': False\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Organic Text Cleaning',\n",
    "        'description': 'Remove certain words and replace them with their correct counterparts',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'organic_text_cleaning': True\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Full Text Cleaning',\n",
    "        'description': 'Features all text cleaning techniques including organic text cleaning, spell checkers, contraction removal, url token replacement, stop words',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'organic_text_cleaning': True,\n",
    "            'use_stop_words': True,\n",
    "            'use_spell_checkers': True,\n",
    "            'contraction_removal': True,\n",
    "            'replace_url_tokens': True\n",
    "        }\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Current commit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'50e2eba'\n"
     ]
    }
   ],
   "source": [
    "utils.get_current_git_commit()\n",
    "print('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def objective(rc, experiment):\n",
    "    run_time = time.time()\n",
    "    \n",
    "    # reset loggers\n",
    "    utils.reset_loggers()\n",
    "    experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    dataset_logger = logging.getLogger('data_loader')\n",
    "    \n",
    "    logger.info('Experiment: ' + experiment['name'])\n",
    "    logger.info('Description: ' + experiment['description'])\n",
    "    \n",
    "    logger.info('Parameters')\n",
    "    logger.info(rc)\n",
    "    print('\\n\\n#########################################################################')\n",
    "    print('Name: ' + experiment['name'])\n",
    "    print('Description: ' + experiment['description'])\n",
    "    print('#########################################################################\\n\\n')\n",
    "    print(rc)\n",
    "\n",
    "    logger.debug('Load dataset')\n",
    "    try:\n",
    "        dataset = load_dataset(rc, dataset_logger, rc.task)\n",
    "    except Exception as err:\n",
    "        print('Could load dataset: ' + str(err))\n",
    "        logger.exception(\"Could not load dataset\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "\n",
    "    try:\n",
    "        trainer = load_model(dataset, rc, experiment_name)\n",
    "    except Exception as err:\n",
    "        print('Could not load model: ' + str(err))\n",
    "        logger.exception(\"Could not load model\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "\n",
    "    logger.debug('model loaded')\n",
    "\n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=rc.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        print('Exception while training: ' + str(err))\n",
    "        logger.exception(\"Could not complete iteration\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    if math.isnan(trainer.get_best_loss()):\n",
    "        print('Loss is nan')\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=True, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation.\")\n",
    "        print('Could not complete iteration evaluation: ' + str(err))\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "    print(f'VAL f1\\t{trainer.get_best_f1()} - ({result[1][1]})')\n",
    "    print(f'VAL loss\\t{trainer.get_best_loss()}')\n",
    "    return {\n",
    "            'loss': result[1][0],\n",
    "            'status': STATUS_OK,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1(),\n",
    "            'sample_iterations': trainer.get_num_samples_seen(),\n",
    "            'iterations': trainer.get_num_iterations(),\n",
    "            'rc': rc,\n",
    "            'results': {\n",
    "                'train': {\n",
    "                    'loss': result[0][0],\n",
    "                    'f1': result[0][1]\n",
    "                },\n",
    "                'validation': {\n",
    "                    'loss': result[1][0],\n",
    "                    'f1': result[1][1]\n",
    "                },\n",
    "                'test': {\n",
    "                    'loss': result[2][0],\n",
    "                    'f1': result[2][1]\n",
    "                }\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\14\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Baseline\n",
      "Description: Baseline. Uses good_organic_hp_params without any changes\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]es'} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b368edeea9d3438c952f9ca6395b9ec3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.49\t\t0.38\t\t0.053\t\t0.872\t\t0.51m - 0.5m / 0.0m\n",
      "2\t9k\t0.42\t\t0.34\t\t0.181\t\t0.816\t\t0.48m - 1.0m / 17.9m\n",
      "3\t13k\t0.39\t\t0.36\t\t0.230\t\t0.798\t\t0.44m - 1.5m / 17.0m\n",
      "4\t17k\t0.38\t\t0.37\t\t0.214\t\t0.800\t\t0.44m - 2.0m / 15.7m\n",
      "5\t22k\t0.36\t\t0.34\t\t0.266\t\t0.814\t\t0.44m - 2.5m / 15.7m\n",
      "6\t26k\t0.35\t\t0.38\t\t0.206\t\t0.763\t\t0.43m - 3.0m / 15.7m\n",
      "7\t30k\t0.33\t\t0.38\t\t0.247\t\t0.788\t\t0.44m - 3.4m / 15.5m\n",
      "8\t35k\t0.32\t\t0.35\t\t0.246\t\t0.799\t\t0.43m - 3.9m / 15.7m\n",
      "9\t39k\t0.31\t\t0.43\t\t0.224\t\t0.782\t\t0.43m - 4.4m / 15.6m\n",
      "10\t43k\t0.30\t\t0.38\t\t0.243\t\t0.793\t\t0.43m - 4.8m / 15.6m\n",
      "VAL f1\t0.2657952069716776 - (0.2657952069716776)\n",
      "VAL loss\t0.33774394708521227\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Baseline\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.33774394708521227\n",
      "        | \\     )|_\tf1: 0.2657952069716776\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: SpellChecker On\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\15\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: SpellChecker On\n",
      "Description: Uses the baseline parameters but with the spellchecker enabled\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]es'} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                        True                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "463c3f660e3d40c8a444c8c5c9830afe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.48\t\t0.40\t\t0.180\t\t0.842\t\t0.45m - 0.5m / 0.0m\n",
      "2\t9k\t0.43\t\t0.37\t\t0.176\t\t0.786\t\t0.44m - 0.9m / 15.8m\n",
      "3\t13k\t0.39\t\t0.37\t\t0.225\t\t0.798\t\t0.44m - 1.4m / 15.6m\n",
      "4\t17k\t0.38\t\t0.38\t\t0.244\t\t0.805\t\t0.44m - 1.9m / 15.7m\n",
      "5\t22k\t0.36\t\t0.34\t\t0.284\t\t0.820\t\t0.45m - 2.4m / 15.7m\n",
      "6\t26k\t0.35\t\t0.36\t\t0.213\t\t0.757\t\t0.46m - 2.9m / 15.9m\n",
      "7\t30k\t0.33\t\t0.38\t\t0.263\t\t0.798\t\t0.42m - 3.4m / 16.2m\n",
      "8\t35k\t0.32\t\t0.33\t\t0.270\t\t0.817\t\t0.43m - 3.8m / 15.1m\n",
      "9\t39k\t0.30\t\t0.36\t\t0.218\t\t0.805\t\t0.42m - 4.3m / 15.4m\n",
      "10\t43k\t0.29\t\t0.36\t\t0.275\t\t0.830\t\t0.42m - 4.7m / 15.3m\n",
      "VAL f1\t0.2838283828382838 - (0.2838283828382838)\n",
      "VAL loss\t0.3339613486738766\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: SpellChecker On\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.3339613486738766\n",
      "        | \\     )|_\tf1: 0.2838283828382838\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Fasttext\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\16\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Fasttext\n",
      "Description: Uses english Fasttext embeddings\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'fasttext', 'learning[...]es'} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24a520abc9ed484b820771c6e782a822",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.50\t\t0.39\t\t0.006\t\t0.885\t\t0.42m - 0.4m / 0.0m\n",
      "2\t9k\t0.42\t\t0.35\t\t0.204\t\t0.815\t\t0.43m - 0.9m / 14.7m\n",
      "3\t13k\t0.37\t\t0.35\t\t0.240\t\t0.799\t\t0.42m - 1.3m / 15.1m\n",
      "4\t17k\t0.36\t\t0.37\t\t0.225\t\t0.795\t\t0.43m - 1.8m / 14.8m\n",
      "5\t22k\t0.34\t\t0.33\t\t0.292\t\t0.830\t\t0.43m - 2.3m / 15.1m\n",
      "6\t26k\t0.32\t\t0.37\t\t0.269\t\t0.780\t\t0.42m - 2.7m / 15.2m\n",
      "7\t30k\t0.30\t\t0.40\t\t0.249\t\t0.771\t\t0.42m - 3.2m / 15.0m\n",
      "8\t35k\t0.28\t\t0.39\t\t0.253\t\t0.773\t\t0.42m - 3.7m / 15.1m\n",
      "9\t39k\t0.26\t\t0.45\t\t0.264\t\t0.800\t\t0.43m - 4.1m / 15.0m\n",
      "10\t43k\t0.24\t\t0.43\t\t0.268\t\t0.809\t\t0.42m - 4.6m / 15.3m\n",
      "VAL f1\t0.29161882893226176 - (0.29161882893226176)\n",
      "VAL loss\t0.3286756620687597\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Fasttext\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.3286756620687597\n",
      "        | \\     )|_\tf1: 0.29161882893226176\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Rolling Sentences\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\17\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Rolling Sentences\n",
      "Description: Uses a combination of the last and the current sentence instead of just the sentence alone\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]ne'} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                  entities_combine                 |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac3914cdab304d0f97c6da0c72e44da3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.51\t\t0.38\t\t0.065\t\t0.875\t\t0.42m - 0.4m / 0.0m\n",
      "2\t9k\t0.42\t\t0.36\t\t0.146\t\t0.813\t\t0.42m - 0.9m / 14.9m\n",
      "3\t13k\t0.39\t\t0.37\t\t0.235\t\t0.790\t\t0.42m - 1.4m / 14.9m\n",
      "4\t17k\t0.38\t\t0.40\t\t0.197\t\t0.770\t\t0.42m - 1.8m / 14.9m\n",
      "5\t22k\t0.37\t\t0.35\t\t0.238\t\t0.809\t\t0.43m - 2.3m / 15.0m\n",
      "6\t26k\t0.35\t\t0.43\t\t0.182\t\t0.716\t\t0.42m - 2.7m / 15.1m\n",
      "7\t30k\t0.34\t\t0.38\t\t0.241\t\t0.790\t\t0.42m - 3.2m / 15.0m\n",
      "8\t35k\t0.33\t\t0.41\t\t0.217\t\t0.780\t\t0.43m - 3.7m / 15.1m\n",
      "9\t39k\t0.31\t\t0.43\t\t0.213\t\t0.784\t\t0.42m - 4.1m / 15.2m\n",
      "10\t43k\t0.30\t\t0.44\t\t0.220\t\t0.809\t\t0.43m - 4.6m / 15.2m\n",
      "11\t48k\t0.28\t\t0.46\t\t0.247\t\t0.793\t\t0.43m - 5.1m / 15.3m\n",
      "12\t52k\t0.27\t\t0.47\t\t0.236\t\t0.778\t\t0.42m - 5.5m / 15.3m\n",
      "13\t56k\t0.26\t\t0.48\t\t0.165\t\t0.761\t\t0.42m - 6.0m / 15.2m\n",
      "14\t60k\t0.25\t\t0.49\t\t0.251\t\t0.799\t\t0.42m - 6.5m / 15.3m\n",
      "15\t65k\t0.25\t\t0.47\t\t0.198\t\t0.781\t\t0.43m - 6.9m / 15.4m\n",
      "16\t69k\t0.23\t\t0.58\t\t0.206\t\t0.789\t\t0.42m - 7.4m / 15.5m\n",
      "17\t73k\t0.24\t\t0.61\t\t0.143\t\t0.701\t\t0.43m - 7.8m / 15.4m\n",
      "18\t78k\t0.23\t\t0.55\t\t0.226\t\t0.798\t\t0.42m - 8.3m / 15.6m\n",
      "19\t82k\t0.22\t\t0.53\t\t0.249\t\t0.793\t\t0.42m - 8.8m / 15.6m\n",
      "VAL f1\t0.2510729613733906 - (0.2510729613733906)\n",
      "VAL loss\t0.3534780880984138\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Rolling Sentences\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.3534780880984138\n",
      "        | \\     )|_\tf1: 0.2510729613733906\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Stop Words\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\18\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Stop Words\n",
      "Description: Uses stop words removal\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3eb93ddcd1014868a1b47aac575f9240",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.50\t\t0.37\t\t0.210\t\t0.870\t\t0.41m - 0.4m / 0.0m\n",
      "2\t9k\t0.40\t\t0.35\t\t0.201\t\t0.793\t\t0.42m - 0.9m / 14.4m\n",
      "3\t13k\t0.37\t\t0.36\t\t0.262\t\t0.791\t\t0.42m - 1.3m / 14.9m\n",
      "4\t17k\t0.37\t\t0.37\t\t0.254\t\t0.813\t\t0.42m - 1.8m / 14.9m\n",
      "5\t22k\t0.35\t\t0.36\t\t0.265\t\t0.806\t\t0.42m - 2.3m / 14.9m\n",
      "6\t26k\t0.33\t\t0.41\t\t0.221\t\t0.757\t\t0.43m - 2.7m / 15.0m\n",
      "7\t30k\t0.32\t\t0.39\t\t0.268\t\t0.798\t\t0.43m - 3.2m / 15.1m\n",
      "8\t35k\t0.31\t\t0.37\t\t0.281\t\t0.806\t\t0.42m - 3.7m / 15.2m\n",
      "9\t39k\t0.29\t\t0.39\t\t0.273\t\t0.824\t\t0.43m - 4.1m / 15.1m\n",
      "10\t43k\t0.27\t\t0.44\t\t0.247\t\t0.808\t\t0.43m - 4.6m / 15.3m\n",
      "11\t48k\t0.26\t\t0.42\t\t0.265\t\t0.807\t\t0.42m - 5.1m / 15.3m\n",
      "12\t52k\t0.24\t\t0.48\t\t0.244\t\t0.803\t\t0.42m - 5.5m / 15.1m\n",
      "13\t56k\t0.23\t\t0.47\t\t0.255\t\t0.801\t\t0.42m - 6.0m / 15.1m\n",
      "VAL f1\t0.281282316442606 - (0.281282316442606)\n",
      "VAL loss\t0.34873716550714823\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Stop Words\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.34873716550714823\n",
      "        | \\     )|_\tf1: 0.281282316442606\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Embedding finetuning Off\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\19\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Embedding finetuning Off\n",
      "Description: This experiment prevents finetuning of the embedding layer\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                       False                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0ccb53303704827a438ac8504effc50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.49\t\t0.39\t\t0.048\t\t0.871\t\t0.40m - 0.4m / 0.0m\n",
      "2\t9k\t0.42\t\t0.35\t\t0.180\t\t0.819\t\t0.41m - 0.9m / 14.1m\n",
      "3\t13k\t0.39\t\t0.37\t\t0.212\t\t0.795\t\t0.41m - 1.3m / 14.4m\n",
      "4\t17k\t0.39\t\t0.37\t\t0.224\t\t0.801\t\t0.41m - 1.7m / 14.4m\n",
      "5\t22k\t0.38\t\t0.33\t\t0.250\t\t0.819\t\t0.41m - 2.2m / 14.5m\n",
      "6\t26k\t0.38\t\t0.39\t\t0.186\t\t0.745\t\t0.40m - 2.6m / 14.4m\n",
      "7\t30k\t0.38\t\t0.35\t\t0.212\t\t0.802\t\t0.40m - 3.1m / 14.3m\n",
      "8\t35k\t0.38\t\t0.36\t\t0.224\t\t0.805\t\t0.40m - 3.5m / 14.3m\n",
      "9\t39k\t0.37\t\t0.36\t\t0.214\t\t0.805\t\t0.40m - 4.0m / 14.4m\n",
      "10\t43k\t0.38\t\t0.37\t\t0.219\t\t0.803\t\t0.40m - 4.4m / 14.4m\n",
      "VAL f1\t0.24971363115693013 - (0.24971363115693013)\n",
      "VAL loss\t0.3311501334695255\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Embedding finetuning Off\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.3311501334695255\n",
      "        | \\     )|_\tf1: 0.24971363115693013\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Organic Text Cleaning\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\20\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Organic Text Cleaning\n",
      "Description: Remove certain words and replace them with their correct counterparts\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                        True                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a06aa387359d4a15bf732e5de19301b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t4k\t0.50\t\t0.38\t\t0.110\t\t0.871\t\t0.41m - 0.4m / 0.0m\n",
      "2\t9k\t0.42\t\t0.34\t\t0.183\t\t0.819\t\t0.43m - 0.9m / 14.5m\n",
      "3\t13k\t0.39\t\t0.35\t\t0.230\t\t0.811\t\t0.43m - 1.3m / 15.0m\n",
      "4\t17k\t0.38\t\t0.38\t\t0.231\t\t0.805\t\t0.42m - 1.8m / 15.0m\n",
      "5\t22k\t0.36\t\t0.35\t\t0.262\t\t0.803\t\t0.41m - 2.3m / 14.8m\n",
      "6\t26k\t0.35\t\t0.37\t\t0.240\t\t0.768\t\t0.42m - 2.7m / 14.7m\n",
      "7\t30k\t0.34\t\t0.35\t\t0.269\t\t0.821\t\t0.42m - 3.2m / 15.0m\n",
      "8\t35k\t0.32\t\t0.34\t\t0.235\t\t0.804\t\t0.41m - 3.6m / 14.9m\n",
      "9\t39k\t0.30\t\t0.36\t\t0.248\t\t0.799\t\t0.42m - 4.1m / 14.9m\n",
      "10\t43k\t0.30\t\t0.35\t\t0.273\t\t0.835\t\t0.42m - 4.6m / 14.9m\n",
      "11\t48k\t0.28\t\t0.38\t\t0.265\t\t0.833\t\t0.41m - 5.0m / 15.1m\n",
      "12\t52k\t0.28\t\t0.39\t\t0.238\t\t0.806\t\t0.42m - 5.5m / 15.0m\n",
      "13\t56k\t0.26\t\t0.42\t\t0.254\t\t0.801\t\t0.41m - 5.9m / 15.1m\n",
      "14\t60k\t0.26\t\t0.45\t\t0.242\t\t0.799\t\t0.42m - 6.4m / 15.0m\n",
      "15\t65k\t0.24\t\t0.40\t\t0.254\t\t0.827\t\t0.41m - 6.8m / 15.1m\n",
      "VAL f1\t0.2730627306273063 - (0.2730627306273063)\n",
      "VAL loss\t0.34360285716898303\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Organic Text Cleaning\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.34360285716898303\n",
      "        | \\     )|_\tf1: 0.2730627306273063\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Full Text Cleaning\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190331\\21\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Full Text Cleaning\n",
      "Description: Features all text cleaning techniques including organic text cleaning, spell checkers, contraction removal, url token replacement, stop words\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      entities                     |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.001                       |\n",
      "|  noam_learning_rate_warmup   |                        4631                       |\n",
      "|  noam_learning_rate_factor   |                    3.3368149482                   |\n",
      "|          adam_beta1          |                   0.89178641984                   |\n",
      "|          adam_beta2          |                   0.83491754824                   |\n",
      "|           adam_eps           |               8.734158747166484e-09               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                   0.392996573831                  |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                  0.7608194889605                  |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        195                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                        True                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                        True                       |\n",
      "|    organic_text_cleaning     |                        True                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "for e in experiments:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    \n",
    "    # generate rc\n",
    "    rc = get_default_params(use_cuda=True, overwrite=e['rc'], from_default=baseline)\n",
    "    result = objective(rc, e)\n",
    "    \n",
    "    if result['status'] == STATUS_OK:\n",
    "        print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\tExperiment: {e['name']}\\n\\\n",
    "          /`\\\\_/`\\\\\\tStatus: {result['status']}\\n\\\n",
    "         //  _  \\\\\\\\\\tLoss: {result['best_loss']}\\n\\\n",
    "        | \\\\     )|_\\tf1: {result['best_f1']}\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    else:\n",
    "        print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\tExperiment: {e['name']} (FAIL)\\n\\\n",
    "          /`\\\\_/`\\\\\\n\\\n",
    "         //  _  \\\\\\\\\\\\n\\\n",
    "        | \\\\     )|_\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}