19_HyperoptExecutor_NotbookForm - Organic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import os\n",
    "import time\n",
    "import logging\n",
    "from hyperopt.plotting import *\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials, base\n",
    "from data.data_loader import Dataset\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import from_hyperopt, OutputLayerType, LearningSchedulerType, OptimizerType, default_params\n",
    "from misc import utils\n",
    "from misc.hyperopt_space import *\n",
    "\n",
    "from optimizer import get_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from trainer.train import Trainer\n",
    "import pprint\n",
    "import argparse\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "POSSIBLE_DATASET_VALUES = ['germeval', 'organic', 'amazon']\n",
    "\n",
    "\n",
    "def load_model(dataset, rc, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=rc)\n",
    "    model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names)\n",
    "    optimizer = get_optimizer(model, rc)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        rc,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer\n",
    "\n",
    "def load_dataset(rc, logger, task):\n",
    "    dataset = Dataset(\n",
    "            task,\n",
    "            logger,\n",
    "            rc,\n",
    "            source_index=PREFERENCES.source_index,\n",
    "            target_vocab_index=PREFERENCES.target_vocab_index,\n",
    "            data_path=PREFERENCES.data_root,\n",
    "            train_file=PREFERENCES.data_train,\n",
    "            valid_file=PREFERENCES.data_validation,\n",
    "            test_file=PREFERENCES.data_test,\n",
    "            file_format=PREFERENCES.file_format,\n",
    "            init_token=None,\n",
    "            eos_token=None\n",
    "        )\n",
    "    dataset.load_data(dsl, verbose=False)\n",
    "    return dataset\n",
    "\n",
    "def objective(parameters):\n",
    "    run_time = time.time()\n",
    "\n",
    "    utils.reset_loggers()\n",
    "    experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    dataset_logger = logging.getLogger('data_loader')\n",
    "\n",
    "    # generate hp's from parameters\n",
    "    try:\n",
    "        rc = from_hyperopt(parameters, use_cuda, model_size=300, early_stopping=5, num_epochs=35, log_every_xth_iteration=-1, language=PREFERENCES.language)\n",
    "    except Exception as err:\n",
    "        print('Could not convert params: ' + str(err))\n",
    "        logger.exception(\"Could not load parameters from hyperopt configuration: \" + parameters)\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.info('New Params:')\n",
    "    logger.info(rc)\n",
    "    print('\\n\\n#########################################################################')\n",
    "    print(rc)\n",
    "\n",
    "    logger.debug('Load dataset')\n",
    "    try:\n",
    "        dataset = load_dataset(rc, dataset_logger, rc.task)\n",
    "    except Exception as err:\n",
    "        print('Could not load dataset: ' + str(err))\n",
    "        logger.exception(\"Could not load dataset\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "\n",
    "    try:\n",
    "        trainer = load_model(dataset, rc, experiment_name)\n",
    "    except Exception as err:\n",
    "        print('Could not load model: ' + str(err))\n",
    "        logger.exception(\"Could not load model\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "\n",
    "    logger.debug('model loaded')\n",
    "\n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=rc.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        print('Exception while training: ' + str(err))\n",
    "        logger.exception(\"Could not complete iteration\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    if math.isnan(trainer.get_best_loss()):\n",
    "        print('Loss is nan')\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=True, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation.\")\n",
    "        print('Could not complete iteration evaluation: ' + str(err))\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "    print(f'VAL f1\\t{trainer.get_best_f1()} - ({result[1][1]})')\n",
    "    print(f'VAL loss\\t{trainer.get_best_loss()}')\n",
    "\n",
    "    print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\n\\\n",
    "          /`\\\\_/`\\\\\\n\\\n",
    "         //  _  \\\\\\\\\\tLoss: {trainer.get_best_loss()}\\n\\\n",
    "        | \\\\     )|_\\tf1: {trainer.get_best_f1()}\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "\n",
    "    return {\n",
    "            'loss': result[1][0],\n",
    "            'status': STATUS_OK,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1(),\n",
    "            'sample_iterations': trainer.get_num_samples_seen(),\n",
    "            'iterations': trainer.get_num_iterations(),\n",
    "            'rc': rc,\n",
    "            'results': {\n",
    "                'train': {\n",
    "                    'loss': result[0][0],\n",
    "                    'f1': result[0][1]\n",
    "                },\n",
    "                'validation': {\n",
    "                    'loss': result[1][0],\n",
    "                    'f1': result[1][1]\n",
    "                },\n",
    "                'test': {\n",
    "                    'loss': result[2][0],\n",
    "                    'f1': result[2][1]\n",
    "                }\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset_choice = 'organic'\n",
    "runs = 100\n",
    "main_experiment_name = 'OrganicCoarseHyperopt'\n",
    "use_cuda = True\n",
    "description = 'HyperOpt Run on the organic coarse dataset'\n",
    "\n",
    "if dataset_choice not in POSSIBLE_DATASET_VALUES:\n",
    "    raise Error()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if dataset_choice == POSSIBLE_DATASET_VALUES[0]:\n",
    "    PREFERENCES.defaults(\n",
    "        data_root='./data/data/germeval2017',\n",
    "        data_train='train_v1.4.tsv',    \n",
    "        data_validation='dev_v1.4.tsv',\n",
    "        data_test='test_TIMESTAMP1.tsv',\n",
    "        source_index=0,\n",
    "        target_vocab_index=2,\n",
    "        file_format='csv',\n",
    "        language='de'\n",
    "    )\n",
    "    from data.germeval2017 import germeval2017_dataset as dsl\n",
    "\n",
    "    search_space = {\n",
    "        'batch_size': hp.quniform('batch_size', 10, 100, 1),\n",
    "        'num_encoder_blocks': hp.quniform('num_encoder_blocks', 1, 8, 1),\n",
    "        'pointwise_layer_size': hp.quniform('pointwise_layer_size', 32, 256, 1),\n",
    "        'clip_comments_to': hp.quniform('clip_comments_to', 10, 250, 1),\n",
    "        'dropout_rate': hp.uniform('dropout_rate', 0.0, 0.8),\n",
    "        'output_dropout_rate': hp.uniform('last_layer_dropout', 0.0, 0.8),\n",
    "        'num_heads': hp.choice('num_heads', [1, 2, 3, 4, 5]),\n",
    "        'transformer_use_bias': hp_bool('transformer_use_bias'),\n",
    "        'output_layer': hp.choice('output_layer', [\n",
    "            {\n",
    "                'type': OutputLayerType.Convolutions,\n",
    "                'output_conv_num_filters': hp.quniform('output_conv_num_filters', 1, 400, 1),\n",
    "                'output_conv_kernel_size': hp.quniform('output_conv_kernel_size', 1, 10, 1),\n",
    "                'output_conv_stride': hp.quniform('output_conv_stride', 1, 10, 1),\n",
    "                'output_conv_padding': hp.quniform('output_conv_padding', 0, 5, 1),\n",
    "            },\n",
    "            {\n",
    "                'type': OutputLayerType.LinearSum\n",
    "            }\n",
    "        ]),\n",
    "        'learning_rate_scheduler': hp.choice('learning_rate_scheduler', [\n",
    "            {\n",
    "                'type': LearningSchedulerType.Noam,\n",
    "                'noam_learning_rate_warmup': hp.quniform('noam_learning_rate_warmup', 1000, 9000, 1),\n",
    "                'noam_learning_rate_factor': hp.uniform('noam_learning_rate_factor', 0.01, 4)\n",
    "            }\n",
    "        ]),\n",
    "        'optimizer': hp.choice('optimizer', [\n",
    "            {\n",
    "                'type': OptimizerType.Adam,\n",
    "                'adam_beta1': hp.uniform('adam_beta1', 0.7, 0.999),\n",
    "                'adam_beta2': hp.uniform('adam_beta2', 0.7, 0.999),\n",
    "                'adam_eps': hp.loguniform('adam_eps', np.log(1e-10), np.log(1)),\n",
    "                'learning_rate': hp.lognormal('adam_learning_rate', np.log(0.01), np.log(10)),\n",
    "                'adam_weight_decay': 1*10**hp.quniform('adam_weight_decay', -8, -3, 1)\n",
    "            },\n",
    "            #{\n",
    "            #    'type': OptimizerType.SGD,\n",
    "            #    'sgd_momentum': hp.uniform('sgd_momentum', 0.4, 1),\n",
    "            #    'sgd_weight_decay': hp.loguniform('sgd_weight_decay', np.log(1e-4), np.log(1)),\n",
    "            #    'sgd_nesterov': hp_bool('sgd_nesterov'),\n",
    "            #    'learning_rate': hp.lognormal('sgd_learning_rate', np.log(0.01), np.log(10))\n",
    "        ]),\n",
    "        'replace_url_tokens': hp_bool('replace_url_tokens'),\n",
    "        'harmonize_bahn': hp_bool('harmonize_bahn'),\n",
    "        'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove']),\n",
    "        'embedding_name': hp.choice('embedding_name', ['6B']),\n",
    "        'embedding_dim': hp.choice('embedding_dim', [300]),\n",
    "        'use_stop_words': hp_bool('use_stop_words'),\n",
    "        'use_spell_checker': hp_bool('use_spell_checker'),\n",
    "        'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove']),\n",
    "        'task': 'germeval'\n",
    "    }\n",
    "\n",
    "elif dataset_choice == POSSIBLE_DATASET_VALUES[1]:\n",
    "     from data.organic2019 import organic_dataset as dsl\n",
    "     from data.organic2019 import ORGANIC_TASK_ALL, ORGANIC_TASK_ENTITIES, ORGANIC_TASK_ATTRIBUTES, ORGANIC_TASK_ENTITIES_COMBINE, ORGANIC_TASK_COARSE\n",
    "     PREFERENCES.defaults(\n",
    "        data_root='./data/data/organic2019',\n",
    "        data_train='train.csv',    \n",
    "        data_validation='validation.csv',\n",
    "        data_test='test.csv',\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        file_format='csv',\n",
    "        language='en'\n",
    "     )\n",
    "\n",
    "     search_space = {\n",
    "        'batch_size': hp.quniform('batch_size', 10, 64, 1),\n",
    "        'num_encoder_blocks': hp.quniform('num_encoder_blocks', 1, 4, 1),\n",
    "        'pointwise_layer_size': hp.quniform('pointwise_layer_size', 32, 350, 1),\n",
    "        'clip_comments_to': hp.quniform('clip_comments_to', 45, 180, 1),\n",
    "        'dropout_rate': hp.uniform('dropout_rate', 0.0, 0.8),\n",
    "        'output_dropout_rate': hp.uniform('last_layer_dropout', 0.0, 0.8),\n",
    "        'num_heads': hp.choice('num_heads', [1, 2, 3, 4, 5]),\n",
    "        'transformer_use_bias': hp_bool('transformer_use_bias'),\n",
    "        'output_layer': hp.choice('output_layer', [\n",
    "            {\n",
    "                'type': OutputLayerType.Convolutions,\n",
    "                'output_conv_num_filters': hp.quniform('output_conv_num_filters', 10, 400, 1),\n",
    "                'output_conv_kernel_size': hp.quniform('output_conv_kernel_size', 1, 10, 1),\n",
    "                'output_conv_stride': hp.quniform('output_conv_stride', 1, 10, 1),\n",
    "                'output_conv_padding': hp.quniform('output_conv_padding', 0, 5, 1),\n",
    "            },\n",
    "            {\n",
    "                'type': OutputLayerType.LinearSum\n",
    "            }\n",
    "        ]),\n",
    "        'learning_rate_scheduler': hp.choice('learning_rate_scheduler', [\n",
    "            {\n",
    "                'type': LearningSchedulerType.Noam,\n",
    "                'noam_learning_rate_warmup': hp.quniform('noam_learning_rate_warmup', 1000, 9000, 1),\n",
    "                'noam_learning_rate_factor': hp.uniform('noam_learning_rate_factor', 0.01, 4)\n",
    "            }\n",
    "        ]),\n",
    "        'optimizer': hp.choice('optimizer', [\n",
    "            {\n",
    "                'type': OptimizerType.Adam,\n",
    "                'adam_beta1': hp.uniform('adam_beta1', 0.7, 0.999),\n",
    "                'adam_beta2': hp.uniform('adam_beta2', 0.7, 0.999),\n",
    "                'adam_eps': hp.loguniform('adam_eps', np.log(1e-10), np.log(1)),\n",
    "                'learning_rate': hp.lognormal('adam_learning_rate', np.log(0.01), np.log(10)),\n",
    "                'adam_weight_decay': 1*10**hp.quniform('adam_weight_decay', -8, -3, 1)\n",
    "            },\n",
    "            #{\n",
    "            #    'type': OptimizerType.SGD,\n",
    "            #    'sgd_momentum': hp.uniform('sgd_momentum', 0.4, 1),\n",
    "            #    'sgd_weight_decay': hp.loguniform('sgd_weight_decay', np.log(1e-4), np.log(1)),\n",
    "            #    'sgd_nesterov': hp_bool('sgd_nesterov'),\n",
    "            #    'learning_rate': hp.lognormal('sgd_learning_rate', np.log(0.01), np.log(10))\n",
    "        ]),\n",
    "        'task': ORGANIC_TASK_COARSE,\n",
    "        'use_stop_words': hp_bool('use_stop_words'),\n",
    "        'use_spell_checker': hp_bool('use_spell_checker'),\n",
    "        'embedding_type': hp.choice('embedding_type', ['fasttext', 'glove'])\n",
    "    }\n",
    "else:\n",
    "    PREFERENCES.defaults(\n",
    "        data_root='./data/data/amazon/splits',\n",
    "        data_train='train.pkl',    \n",
    "        data_validation='val.pkl',\n",
    "        data_test='test.pkl',\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        file_format='pkl',\n",
    "        language='en'\n",
    "    )\n",
    "    from data.amazon import amazon_dataset as dsl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\0\n"
     ]
    }
   ],
   "source": [
    "experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "logger = logging.getLogger(__name__)\n",
    "dataset_logger = logging.getLogger('data_loader')\n",
    "logger.info('Run hyper parameter random grid search for experiment with name ' + main_experiment_name)\n",
    "logger.info('num_optim_iterations: ' + str(runs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'57a7aff'\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\1  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 13.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         13                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.003624452706648444               |\n",
      "|  noam_learning_rate_warmup   |                        4099                       |\n",
      "|  noam_learning_rate_factor   |                 1.3791717554363747                |\n",
      "|          adam_beta1          |                 0.9666867424753487                |\n",
      "|          adam_beta2          |                 0.769948897530184                 |\n",
      "|           adam_eps           |                0.013650212023316677               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.09594804585175237                |\n",
      "|     pointwise_layer_size     |                         87                        |\n",
      "|      last_layer_dropout      |                0.09232513876943012                |\n",
      "|   output_conv_num_filters    |                        326                        |\n",
      "|   output_conv_kernel_size    |                         2                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        161                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  0%|                                                                            | 0/100 [00:00<?, ?it/s, best loss: ?]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bdbc658a6794e89a9a1a38e1e735410",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.07\t\t0.88\t\t0.061\t\t0.939\t\t3.94m - 3.9m / 0.0m                                                                     \n",
      "2\t9k\t0.87\t\t0.78\t\t0.215\t\t0.882\t\t3.87m - 7.9m / 137.9m                                                                   \n",
      "3\t13k\t0.85\t\t0.73\t\t0.294\t\t0.919\t\t3.82m - 11.8m / 135.6m                                                                 \n",
      "4\t17k\t0.81\t\t0.73\t\t0.249\t\t0.913\t\t3.83m - 15.6m / 134.1m                                                                 \n",
      "5\t22k\t0.78\t\t0.78\t\t0.262\t\t0.901\t\t3.83m - 19.5m / 134.4m                                                                 \n",
      "6\t26k\t0.77\t\t0.73\t\t0.293\t\t0.897\t\t3.82m - 23.4m / 134.5m                                                                 \n",
      "7\t30k\t0.75\t\t0.71\t\t0.265\t\t0.898\t\t3.80m - 27.3m / 134.3m                                                                 \n",
      "8\t35k\t0.74\t\t0.77\t\t0.237\t\t0.911\t\t3.80m - 31.1m / 133.8m                                                                 \n",
      "Could not complete iteration evaluation: Dimension out of range (expected to be in range of [-1, 0], but got 1)        \n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\2  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 59.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         59                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.010694169090689556               |\n",
      "|  noam_learning_rate_warmup   |                        7747                       |\n",
      "|  noam_learning_rate_factor   |                 3.1337506723115482                |\n",
      "|          adam_beta1          |                 0.9808633405906884                |\n",
      "|          adam_beta2          |                 0.9001432425019027                |\n",
      "|           adam_eps           |               4.350098715378944e-09               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.7253856314738492                |\n",
      "|     pointwise_layer_size     |                        299                        |\n",
      "|      last_layer_dropout      |                 0.1871541616852569                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        113                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  1%|▋                                                              | 1/100 [31:52<52:35:05, 1912.18s/it, best loss: ?]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "447f36cf9fc34b01800be383bb449122",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.39\t\t0.29\t\t0.033\t\t0.927\t\t0.28m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.30\t\t0.29\t\t0.026\t\t0.937\t\t0.28m - 0.6m / 9.8m                                                                     \n",
      "3\t13k\t0.29\t\t0.28\t\t0.207\t\t0.913\t\t0.28m - 1.0m / 10.0m                                                                   \n",
      "4\t17k\t0.30\t\t0.27\t\t0.005\t\t0.940\t\t0.28m - 1.3m / 10.0m                                                                   \n",
      "5\t22k\t0.29\t\t0.27\t\t0.010\t\t0.934\t\t0.28m - 1.6m / 9.9m                                                                    \n",
      "6\t26k\t0.29\t\t0.26\t\t0.164\t\t0.933\t\t0.28m - 1.9m / 10.1m                                                                   \n",
      "7\t31k\t0.29\t\t0.26\t\t0.130\t\t0.919\t\t0.27m - 2.3m / 10.1m                                                                   \n",
      "8\t35k\t0.28\t\t0.27\t\t0.155\t\t0.810\t\t0.27m - 2.6m / 9.9m                                                                    \n",
      "VAL f1\t0.20719424460431654 - (0.20719424460431654)                                                                     \n",
      "VAL loss\t0.25547231792730124                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.25547231792730124\n",
      "        | \\     )|_\tf1: 0.20719424460431654\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\3  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 42.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         42                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 19.24853043698165                 |\n",
      "|  noam_learning_rate_warmup   |                        2893                       |\n",
      "|  noam_learning_rate_factor   |                 3.787549565487592                 |\n",
      "|          adam_beta1          |                 0.9010498865628063                |\n",
      "|          adam_beta2          |                 0.8086112620950472                |\n",
      "|           adam_eps           |               1.6612148679985779e-10              |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.24655637916593892                |\n",
      "|     pointwise_layer_size     |                        190                        |\n",
      "|      last_layer_dropout      |                 0.6307896152480591                |\n",
      "|   output_conv_num_filters    |                        204                        |\n",
      "|   output_conv_kernel_size    |                         4                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         5                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        124                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  2%|▉                                             | 2/100 [34:49<37:53:19, 1391.84s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8654d475c56d45d3b4ebb726f2b95244",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.43\t\t0.38\t\t0.227\t\t0.872\t\t0.43m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.36\t\t0.37\t\t0.164\t\t0.819\t\t0.43m - 0.9m / 15.3m                                                                    \n",
      "3\t13k\t0.37\t\t0.32\t\t0.233\t\t0.855\t\t0.44m - 1.4m / 15.1m                                                                   \n",
      "4\t17k\t0.35\t\t0.32\t\t0.217\t\t0.882\t\t0.44m - 1.9m / 15.5m                                                                   \n",
      "5\t22k\t0.34\t\t0.32\t\t0.195\t\t0.866\t\t0.43m - 2.4m / 15.6m                                                                   \n",
      "6\t26k\t0.32\t\t0.30\t\t0.259\t\t0.886\t\t0.44m - 2.9m / 15.4m                                                                   \n",
      "7\t30k\t0.31\t\t0.32\t\t0.276\t\t0.880\t\t0.43m - 3.4m / 15.7m                                                                   \n",
      "8\t35k\t0.30\t\t0.31\t\t0.272\t\t0.881\t\t0.42m - 3.9m / 15.3m                                                                   \n",
      "9\t39k\t0.33\t\t0.33\t\t0.160\t\t0.806\t\t0.43m - 4.3m / 15.4m                                                                   \n",
      "10\t43k\t0.30\t\t0.32\t\t0.201\t\t0.813\t\t0.44m - 4.8m / 15.5m                                                                  \n",
      "11\t48k\t0.32\t\t0.32\t\t0.199\t\t0.840\t\t0.42m - 5.3m / 15.8m                                                                  \n",
      "12\t52k\t0.33\t\t0.32\t\t0.219\t\t0.839\t\t0.43m - 5.8m / 15.5m                                                                  \n",
      "VAL f1\t0.2763507528786537 - (0.2763507528786537)                                                                       \n",
      "VAL loss\t0.29754717860903057                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.29754717860903057\n",
      "        | \\     )|_\tf1: 0.2763507528786537\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\4  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 30.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         30                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.001480894509624945               |\n",
      "|  noam_learning_rate_warmup   |                        8216                       |\n",
      "|  noam_learning_rate_factor   |                 1.7659608488327836                |\n",
      "|          adam_beta1          |                 0.9722293928877346                |\n",
      "|          adam_beta2          |                 0.7887634161390427                |\n",
      "|           adam_eps           |                1.80717023609247e-05               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.36185852522276934                |\n",
      "|     pointwise_layer_size     |                        239                        |\n",
      "|      last_layer_dropout      |                 0.3704424835646637                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        164                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  3%|█▍                                            | 3/100 [41:09<29:19:16, 1088.21s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cee3fb776c14727919b32cd47aafef4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.71\t\t0.44\t\t0.000\t\t0.937\t\t0.50m - 0.5m / 0.0m                                                                     \n",
      "2\t9k\t0.52\t\t0.42\t\t0.021\t\t0.937\t\t0.50m - 1.1m / 17.6m                                                                    \n",
      "3\t13k\t0.50\t\t0.41\t\t0.122\t\t0.931\t\t0.48m - 1.6m / 17.6m                                                                   \n",
      "4\t17k\t0.48\t\t0.42\t\t0.181\t\t0.876\t\t0.49m - 2.1m / 17.0m                                                                   \n",
      "5\t22k\t0.44\t\t0.40\t\t0.190\t\t0.875\t\t0.50m - 2.7m / 17.3m                                                                   \n",
      "6\t26k\t0.42\t\t0.43\t\t0.206\t\t0.831\t\t0.48m - 3.2m / 17.7m                                                                   \n",
      "7\t30k\t0.38\t\t0.44\t\t0.243\t\t0.861\t\t0.48m - 3.7m / 17.0m                                                                   \n",
      "8\t35k\t0.35\t\t0.45\t\t0.207\t\t0.840\t\t0.50m - 4.3m / 17.2m                                                                   \n",
      "9\t39k\t0.38\t\t0.47\t\t0.176\t\t0.821\t\t0.50m - 4.9m / 17.8m                                                                   \n",
      "10\t43k\t0.32\t\t0.49\t\t0.224\t\t0.849\t\t0.48m - 5.4m / 18.0m                                                                  \n",
      "11\t48k\t0.26\t\t0.49\t\t0.266\t\t0.865\t\t0.48m - 5.9m / 17.4m                                                                  \n",
      "12\t52k\t0.26\t\t0.52\t\t0.258\t\t0.872\t\t0.50m - 6.5m / 17.5m                                                                  \n",
      "13\t56k\t0.25\t\t0.56\t\t0.174\t\t0.825\t\t0.52m - 7.1m / 18.1m                                                                  \n",
      "14\t60k\t0.25\t\t0.58\t\t0.256\t\t0.863\t\t0.49m - 7.6m / 18.4m                                                                  \n",
      "15\t65k\t0.22\t\t0.58\t\t0.244\t\t0.867\t\t0.50m - 8.1m / 17.9m                                                                  \n",
      "16\t69k\t0.24\t\t0.62\t\t0.211\t\t0.833\t\t0.50m - 8.7m / 18.1m                                                                  \n",
      "VAL f1\t0.2662440570522979 - (0.2662440570522979)                                                                       \n",
      "VAL loss\t0.4018554806709289                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4018554806709289\n",
      "        | \\     )|_\tf1: 0.2662440570522979\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\5  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 53.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         53                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.021371999957633767               |\n",
      "|  noam_learning_rate_warmup   |                        8136                       |\n",
      "|  noam_learning_rate_factor   |                 2.187407914965786                 |\n",
      "|          adam_beta1          |                 0.9830078087458741                |\n",
      "|          adam_beta2          |                 0.7285894186423103                |\n",
      "|           adam_eps           |               9.030620863862147e-05               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.7043925761655404                |\n",
      "|     pointwise_layer_size     |                        176                        |\n",
      "|      last_layer_dropout      |                 0.6444015457352118                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         66                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  4%|█▉                                             | 4/100 [50:21<24:43:48, 927.38s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4108a9f0c2c946beaac4966f1ec4e66f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.47\t\t0.39\t\t0.098\t\t0.674\t\t0.31m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.36\t\t0.31\t\t0.004\t\t0.911\t\t0.31m - 0.7m / 10.8m                                                                    \n",
      "3\t13k\t0.33\t\t0.29\t\t0.207\t\t0.913\t\t0.31m - 1.0m / 11.0m                                                                   \n",
      "4\t17k\t0.33\t\t0.29\t\t0.000\t\t0.940\t\t0.31m - 1.4m / 10.9m                                                                   \n",
      "5\t22k\t0.33\t\t0.28\t\t0.000\t\t0.940\t\t0.29m - 1.7m / 11.0m                                                                   \n",
      "6\t26k\t0.33\t\t0.29\t\t0.000\t\t0.940\t\t0.29m - 2.1m / 10.6m                                                                   \n",
      "7\t30k\t0.33\t\t0.28\t\t0.015\t\t0.936\t\t0.29m - 2.4m / 10.7m                                                                   \n",
      "8\t35k\t0.32\t\t0.27\t\t0.047\t\t0.879\t\t0.31m - 2.8m / 10.7m                                                                   \n",
      "VAL f1\t0.20719424460431654 - (0.20719424460431654)                                                                     \n",
      "VAL loss\t0.2744849711415581                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2744849711415581\n",
      "        | \\     )|_\tf1: 0.20719424460431654\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\6  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 30.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         30                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.1365772892353055                |\n",
      "|  noam_learning_rate_warmup   |                        8714                       |\n",
      "|  noam_learning_rate_factor   |                 2.3812986692350075                |\n",
      "|          adam_beta1          |                 0.7436637072010549                |\n",
      "|          adam_beta2          |                 0.9252625927137365                |\n",
      "|           adam_eps           |               1.0582863737002922e-06              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.21199762493291657                |\n",
      "|     pointwise_layer_size     |                        170                        |\n",
      "|      last_layer_dropout      |                0.09041753402047244                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         77                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  5%|██▎                                            | 5/100 [53:40<18:42:09, 708.73s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b30e42b51e04b548d4502afc4ff26b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.81\t\t0.42\t\t0.055\t\t0.928\t\t0.43m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.50\t\t0.40\t\t0.141\t\t0.899\t\t0.42m - 0.9m / 15.1m                                                                    \n",
      "3\t13k\t0.46\t\t0.37\t\t0.227\t\t0.890\t\t0.41m - 1.4m / 14.9m                                                                   \n",
      "4\t17k\t0.41\t\t0.37\t\t0.255\t\t0.885\t\t0.43m - 1.9m / 14.4m                                                                   \n",
      "5\t22k\t0.38\t\t0.36\t\t0.246\t\t0.878\t\t0.41m - 2.3m / 15.2m                                                                   \n",
      "6\t26k\t0.36\t\t0.38\t\t0.221\t\t0.864\t\t0.41m - 2.8m / 14.8m                                                                   \n",
      "7\t30k\t0.34\t\t0.38\t\t0.248\t\t0.872\t\t0.40m - 3.2m / 14.6m                                                                   \n",
      "8\t35k\t0.31\t\t0.38\t\t0.264\t\t0.886\t\t0.42m - 3.7m / 14.6m                                                                   \n",
      "9\t39k\t0.30\t\t0.41\t\t0.239\t\t0.859\t\t0.42m - 4.2m / 15.1m                                                                   \n",
      "10\t43k\t0.28\t\t0.41\t\t0.225\t\t0.859\t\t0.40m - 4.6m / 15.0m                                                                  \n",
      "11\t48k\t0.27\t\t0.42\t\t0.235\t\t0.864\t\t0.40m - 5.1m / 14.8m                                                                  \n",
      "12\t52k\t0.26\t\t0.42\t\t0.270\t\t0.880\t\t0.42m - 5.6m / 14.8m                                                                  \n",
      "13\t56k\t0.24\t\t0.43\t\t0.220\t\t0.855\t\t0.43m - 6.1m / 15.2m                                                                  \n",
      "14\t60k\t0.23\t\t0.46\t\t0.255\t\t0.886\t\t0.41m - 6.5m / 15.5m                                                                  \n",
      "15\t65k\t0.22\t\t0.46\t\t0.238\t\t0.867\t\t0.40m - 7.0m / 15.2m                                                                  \n",
      "16\t69k\t0.21\t\t0.49\t\t0.239\t\t0.869\t\t0.41m - 7.4m / 15.0m                                                                  \n",
      "17\t73k\t0.20\t\t0.49\t\t0.237\t\t0.873\t\t0.43m - 7.9m / 15.2m                                                                  \n",
      "VAL f1\t0.27007943512797883 - (0.27007943512797883)                                                                     \n",
      "VAL loss\t0.3584907399283515                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3584907399283515\n",
      "        | \\     )|_\tf1: 0.27007943512797883\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\7  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 45.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         45                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.006777405471435764               |\n",
      "|  noam_learning_rate_warmup   |                        8675                       |\n",
      "|  noam_learning_rate_factor   |                 2.416064122469982                 |\n",
      "|          adam_beta1          |                 0.8894439201125701                |\n",
      "|          adam_beta2          |                 0.7923608753762482                |\n",
      "|           adam_eps           |               5.303766820286891e-08               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.047927471328082664               |\n",
      "|     pointwise_layer_size     |                        134                        |\n",
      "|      last_layer_dropout      |                 0.6801952496416432                |\n",
      "|   output_conv_num_filters    |                        386                        |\n",
      "|   output_conv_kernel_size    |                         10                        |\n",
      "|      output_conv_stride      |                         6                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         90                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  6%|██▋                                          | 6/100 [1:02:04<16:54:01, 647.25s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d052d8c100594182853966dd1477d532",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.40\t\t0.35\t\t0.223\t\t0.908\t\t0.49m - 0.5m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.32\t\t0.224\t\t0.880\t\t0.50m - 1.1m / 17.4m                                                                    \n",
      "3\t13k\t0.28\t\t0.30\t\t0.252\t\t0.897\t\t0.50m - 1.6m / 17.6m                                                                   \n",
      "4\t17k\t0.26\t\t0.30\t\t0.218\t\t0.868\t\t0.50m - 2.2m / 17.6m                                                                   \n",
      "5\t22k\t0.22\t\t0.29\t\t0.196\t\t0.869\t\t0.50m - 2.7m / 17.7m                                                                   \n",
      "6\t26k\t0.17\t\t0.28\t\t0.222\t\t0.878\t\t0.50m - 3.3m / 17.7m                                                                   \n",
      "7\t30k\t0.15\t\t0.29\t\t0.201\t\t0.869\t\t0.50m - 3.8m / 17.7m                                                                   \n",
      "8\t35k\t0.14\t\t0.29\t\t0.200\t\t0.879\t\t0.50m - 4.4m / 17.8m                                                                   \n",
      "VAL f1\t0.25240641711229944 - (0.25240641711229944)                                                                     \n",
      "VAL loss\t0.2844473335478041                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2844473335478041\n",
      "        | \\     )|_\tf1: 0.25240641711229944\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\8  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 15.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         15                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00036152526839919997              |\n",
      "|  noam_learning_rate_warmup   |                        4082                       |\n",
      "|  noam_learning_rate_factor   |                 1.7313879499582208                |\n",
      "|          adam_beta1          |                 0.881440230686323                 |\n",
      "|          adam_beta2          |                 0.7135520481042981                |\n",
      "|           adam_eps           |               8.542780313207992e-09               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.01934779140515408                |\n",
      "|     pointwise_layer_size     |                         34                        |\n",
      "|      last_layer_dropout      |                 0.4806455393299134                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        103                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  7%|███▏                                         | 7/100 [1:06:54<13:57:12, 540.14s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e254eb80c1e34f2e902be0eb658b92a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.12\t\t0.72\t\t0.145\t\t0.933\t\t0.57m - 0.6m / 0.0m                                                                     \n",
      "2\t9k\t0.81\t\t0.63\t\t0.244\t\t0.915\t\t0.57m - 1.2m / 20.0m                                                                    \n",
      "3\t13k\t0.71\t\t0.61\t\t0.307\t\t0.919\t\t0.55m - 1.8m / 20.1m                                                                   \n",
      "4\t17k\t0.65\t\t0.63\t\t0.244\t\t0.894\t\t0.55m - 2.4m / 19.4m                                                                   \n",
      "5\t22k\t0.60\t\t0.65\t\t0.240\t\t0.881\t\t0.56m - 3.0m / 19.6m                                                                   \n",
      "6\t26k\t0.56\t\t0.68\t\t0.209\t\t0.872\t\t0.55m - 3.6m / 19.9m                                                                   \n",
      "7\t30k\t0.55\t\t0.73\t\t0.183\t\t0.861\t\t0.55m - 4.2m / 19.6m                                                                   \n",
      "8\t35k\t0.57\t\t0.68\t\t0.246\t\t0.900\t\t0.57m - 4.9m / 19.6m                                                                   \n",
      "VAL f1\t0.30673316708229426 - (0.30673316708229426)                                                                     \n",
      "VAL loss\t0.6114724504774895                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.6114724504774895\n",
      "        | \\     )|_\tf1: 0.30673316708229426\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\9  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 24.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         24                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.00174170725929656                |\n",
      "|  noam_learning_rate_warmup   |                        8163                       |\n",
      "|  noam_learning_rate_factor   |                 3.9494684572251155                |\n",
      "|          adam_beta1          |                 0.9435479160374373                |\n",
      "|          adam_beta2          |                 0.9742476354351262                |\n",
      "|           adam_eps           |                0.15447222027359325                |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.15420598932592533                |\n",
      "|     pointwise_layer_size     |                         42                        |\n",
      "|      last_layer_dropout      |                0.14397304944001377                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        129                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  8%|███▌                                         | 8/100 [1:12:19<12:09:31, 475.78s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9bbe2917d8b0412395325313a3cdd49d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.87\t\t0.52\t\t0.048\t\t0.935\t\t0.55m - 0.6m / 0.0m                                                                     \n",
      "2\t9k\t0.62\t\t0.51\t\t0.067\t\t0.910\t\t0.56m - 1.2m / 19.4m                                                                    \n",
      "3\t13k\t0.60\t\t0.50\t\t0.102\t\t0.935\t\t0.56m - 1.8m / 19.8m                                                                   \n",
      "4\t17k\t0.58\t\t0.47\t\t0.178\t\t0.894\t\t0.53m - 2.4m / 19.8m                                                                   \n",
      "5\t22k\t0.52\t\t0.44\t\t0.248\t\t0.906\t\t0.54m - 3.0m / 18.9m                                                                   \n",
      "6\t26k\t0.48\t\t0.45\t\t0.207\t\t0.891\t\t0.55m - 3.6m / 19.2m                                                                   \n",
      "7\t30k\t0.46\t\t0.45\t\t0.263\t\t0.889\t\t0.52m - 4.1m / 19.5m                                                                   \n",
      "8\t35k\t0.44\t\t0.44\t\t0.278\t\t0.900\t\t0.53m - 4.7m / 18.9m                                                                   \n",
      "9\t39k\t0.41\t\t0.45\t\t0.247\t\t0.885\t\t0.56m - 5.3m / 19.1m                                                                   \n",
      "10\t43k\t0.40\t\t0.45\t\t0.239\t\t0.878\t\t0.54m - 5.9m / 19.9m                                                                  \n",
      "11\t48k\t0.38\t\t0.46\t\t0.226\t\t0.878\t\t0.52m - 6.5m / 19.4m                                                                  \n",
      "12\t52k\t0.36\t\t0.49\t\t0.229\t\t0.876\t\t0.57m - 7.1m / 19.1m                                                                  \n",
      "13\t56k\t0.34\t\t0.47\t\t0.238\t\t0.867\t\t0.55m - 7.7m / 20.2m                                                                  \n",
      "VAL f1\t0.2775423728813559 - (0.2775423728813559)                                                                       \n",
      "VAL loss\t0.4386093048822313                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4386093048822313\n",
      "        | \\     )|_\tf1: 0.2775423728813559\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\10 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 15.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         15                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.06183089952080932                |\n",
      "|  noam_learning_rate_warmup   |                        8042                       |\n",
      "|  noam_learning_rate_factor   |                 0.6866620392021984                |\n",
      "|          adam_beta1          |                 0.9884360715252452                |\n",
      "|          adam_beta2          |                 0.8251599518469881                |\n",
      "|           adam_eps           |               2.783145744465062e-09               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.06007340618677422                |\n",
      "|     pointwise_layer_size     |                        148                        |\n",
      "|      last_layer_dropout      |                 0.4356708440819974                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        149                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "  9%|████                                         | 9/100 [1:20:39<12:12:17, 482.83s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1863088768a147eb97076d75f40251d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.44\t\t0.77\t\t0.096\t\t0.930\t\t0.73m - 0.7m / 0.0m                                                                     \n",
      "2\t9k\t1.02\t\t0.73\t\t0.029\t\t0.934\t\t0.74m - 1.5m / 25.6m                                                                    \n",
      "3\t13k\t0.87\t\t0.93\t\t0.210\t\t0.919\t\t0.71m - 2.3m / 26.0m                                                                   \n",
      "4\t17k\t0.68\t\t0.76\t\t0.272\t\t0.910\t\t0.70m - 3.1m / 25.1m                                                                   \n",
      "5\t22k\t0.57\t\t0.76\t\t0.271\t\t0.923\t\t0.72m - 3.8m / 24.7m                                                                   \n",
      "6\t26k\t0.49\t\t0.85\t\t0.307\t\t0.920\t\t0.72m - 4.6m / 25.5m                                                                   \n",
      "7\t30k\t0.44\t\t0.88\t\t0.282\t\t0.916\t\t0.70m - 5.4m / 25.7m                                                                   \n",
      "8\t35k\t0.55\t\t0.97\t\t0.210\t\t0.910\t\t0.69m - 6.1m / 24.9m                                                                   \n",
      "9\t39k\t0.48\t\t1.06\t\t0.221\t\t0.909\t\t0.69m - 6.9m / 24.9m                                                                   \n",
      "10\t43k\t0.75\t\t1.00\t\t0.242\t\t0.887\t\t0.69m - 7.6m / 24.8m                                                                  \n",
      "11\t48k\t0.75\t\t1.16\t\t0.187\t\t0.911\t\t0.70m - 8.4m / 25.0m                                                                  \n",
      "VAL f1\t0.30710659898477155 - (0.30710659898477155)                                                                     \n",
      "VAL loss\t0.7265787677488464                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.7265787677488464\n",
      "        | \\     )|_\tf1: 0.30710659898477155\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\11 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 32.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         32                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.1139863898508334                |\n",
      "|  noam_learning_rate_warmup   |                        6926                       |\n",
      "|  noam_learning_rate_factor   |                 3.0484568025077667                |\n",
      "|          adam_beta1          |                 0.7308681348184356                |\n",
      "|          adam_beta2          |                 0.7998331311931998                |\n",
      "|           adam_eps           |               3.5429440866616314e-09              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.5256175315652177                |\n",
      "|     pointwise_layer_size     |                        264                        |\n",
      "|      last_layer_dropout      |                 0.6460708478561524                |\n",
      "|   output_conv_num_filters    |                        373                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         5                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         64                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 10%|████▍                                       | 10/100 [1:29:44<12:32:14, 501.50s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5743b8cb7f594e119b735036c45be51d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.54\t\t0.56\t\t0.097\t\t0.922\t\t0.56m - 0.6m / 0.0m                                                                     \n",
      "2\t9k\t0.47\t\t0.55\t\t0.153\t\t0.743\t\t0.56m - 1.2m / 19.6m                                                                    \n",
      "3\t13k\t0.44\t\t0.51\t\t0.149\t\t0.745\t\t0.57m - 1.8m / 19.8m                                                                   \n",
      "4\t17k\t0.44\t\t0.45\t\t0.174\t\t0.793\t\t0.56m - 2.4m / 20.0m                                                                   \n",
      "5\t22k\t0.44\t\t0.43\t\t0.188\t\t0.814\t\t0.56m - 3.0m / 20.0m                                                                   \n",
      "6\t26k\t0.43\t\t0.43\t\t0.199\t\t0.817\t\t0.56m - 3.7m / 20.0m                                                                   \n",
      "7\t30k\t0.43\t\t0.45\t\t0.204\t\t0.830\t\t0.57m - 4.3m / 20.0m                                                                   \n",
      "8\t35k\t0.42\t\t0.49\t\t0.173\t\t0.809\t\t0.57m - 4.9m / 20.4m                                                                   \n",
      "9\t39k\t0.43\t\t0.49\t\t0.250\t\t0.852\t\t0.56m - 5.5m / 20.2m                                                                   \n",
      "10\t43k\t0.43\t\t0.51\t\t0.177\t\t0.790\t\t0.57m - 6.2m / 20.3m                                                                  \n",
      "11\t48k\t0.43\t\t0.54\t\t0.183\t\t0.787\t\t0.56m - 6.8m / 20.4m                                                                  \n",
      "12\t52k\t0.43\t\t0.53\t\t0.201\t\t0.802\t\t0.56m - 7.4m / 20.3m                                                                  \n",
      "13\t56k\t0.42\t\t0.49\t\t0.218\t\t0.820\t\t0.56m - 8.0m / 20.4m                                                                  \n",
      "14\t60k\t0.44\t\t0.59\t\t0.134\t\t0.720\t\t0.56m - 8.6m / 20.4m                                                                  \n",
      "VAL f1\t0.24981188863807374 - (0.24981188863807374)                                                                     \n",
      "VAL loss\t0.4290249401872808                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4290249401872808\n",
      "        | \\     )|_\tf1: 0.24981188863807374\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\12 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 28.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         28                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.5837220835755537                |\n",
      "|  noam_learning_rate_warmup   |                        6109                       |\n",
      "|  noam_learning_rate_factor   |                 2.9330050909458643                |\n",
      "|          adam_beta1          |                 0.9853407645612071                |\n",
      "|          adam_beta2          |                 0.990163815818136                 |\n",
      "|           adam_eps           |               0.0005561832504798763               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.5771518369825807                |\n",
      "|     pointwise_layer_size     |                        238                        |\n",
      "|      last_layer_dropout      |                0.16280349697213525                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         49                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 11%|████▊                                       | 11/100 [1:38:56<12:46:35, 516.80s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e744fa5568943b499e041dd180c4eb7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.69\t\t0.50\t\t0.000\t\t0.940\t\t0.36m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.55\t\t0.47\t\t0.005\t\t0.940\t\t0.35m - 0.8m / 12.5m                                                                    \n",
      "3\t13k\t0.54\t\t0.46\t\t0.188\t\t0.935\t\t0.36m - 1.2m / 12.4m                                                                   \n",
      "4\t17k\t0.52\t\t0.44\t\t0.236\t\t0.879\t\t0.35m - 1.6m / 12.8m                                                                   \n",
      "5\t22k\t0.46\t\t0.45\t\t0.195\t\t0.844\t\t0.36m - 2.0m / 12.5m                                                                   \n",
      "6\t26k\t0.43\t\t0.51\t\t0.180\t\t0.801\t\t0.36m - 2.4m / 12.7m                                                                   \n",
      "7\t30k\t0.40\t\t0.51\t\t0.196\t\t0.807\t\t0.40m - 2.9m / 12.9m                                                                   \n",
      "8\t35k\t0.39\t\t0.51\t\t0.221\t\t0.829\t\t0.39m - 3.3m / 14.1m                                                                   \n",
      "9\t39k\t0.39\t\t0.51\t\t0.221\t\t0.825\t\t0.37m - 3.7m / 13.8m                                                                   \n",
      "VAL f1\t0.2360717658168083 - (0.2360717658168083)                                                                       \n",
      "VAL loss\t0.4356100261211395                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4356100261211395\n",
      "        | \\     )|_\tf1: 0.2360717658168083\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\13 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 20.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         20                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.24050309176662543                |\n",
      "|  noam_learning_rate_warmup   |                        4068                       |\n",
      "|  noam_learning_rate_factor   |                 0.7293559133043482                |\n",
      "|          adam_beta1          |                 0.8627041554732364                |\n",
      "|          adam_beta2          |                 0.7294891170734115                |\n",
      "|           adam_eps           |               3.4985520788660714e-07              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.28960802027019356                |\n",
      "|     pointwise_layer_size     |                        164                        |\n",
      "|      last_layer_dropout      |                0.41458372422864953                |\n",
      "|   output_conv_num_filters    |                        256                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         9                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        133                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 12%|█████▎                                      | 12/100 [1:43:06<10:40:35, 436.77s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed0fe0aa34824547ab003aeb5f4b85cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.79\t\t0.67\t\t0.234\t\t0.907\t\t0.82m - 0.8m / 0.0m                                                                     \n",
      "2\t9k\t0.61\t\t0.63\t\t0.177\t\t0.833\t\t0.92m - 1.8m / 28.9m                                                                    \n",
      "3\t13k\t0.57\t\t0.55\t\t0.262\t\t0.872\t\t0.94m - 2.8m / 32.2m                                                                   \n",
      "4\t17k\t0.57\t\t0.53\t\t0.228\t\t0.877\t\t0.88m - 3.8m / 32.9m                                                                   \n",
      "5\t22k\t0.57\t\t0.56\t\t0.232\t\t0.850\t\t0.88m - 4.7m / 31.1m                                                                   \n",
      "6\t26k\t0.60\t\t0.53\t\t0.208\t\t0.853\t\t0.90m - 5.7m / 31.3m                                                                   \n",
      "7\t30k\t0.59\t\t0.55\t\t0.218\t\t0.857\t\t0.90m - 6.7m / 31.8m                                                                   \n",
      "8\t35k\t0.56\t\t0.53\t\t0.245\t\t0.855\t\t0.95m - 7.7m / 31.8m                                                                   \n",
      "VAL f1\t0.2619647355163728 - (0.2619647355163728)                                                                       \n",
      "VAL loss\t0.5283116396735696                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.5283116396735696\n",
      "        | \\     )|_\tf1: 0.2619647355163728\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\14 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 52.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         52                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0008370993110519924               |\n",
      "|  noam_learning_rate_warmup   |                        2174                       |\n",
      "|  noam_learning_rate_factor   |                 2.096914024881706                 |\n",
      "|          adam_beta1          |                 0.9009467826261766                |\n",
      "|          adam_beta2          |                 0.9721552213276276                |\n",
      "|           adam_eps           |               0.0012316823256927963               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.3380260839987898                |\n",
      "|     pointwise_layer_size     |                        110                        |\n",
      "|      last_layer_dropout      |                 0.1805580351596955                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        171                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 13%|█████▋                                      | 13/100 [1:51:38<11:06:02, 459.34s/it, best loss: 0.2751564790973556]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ea11550c8a14b4b9c9c2c3f681e49b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.41\t\t0.27\t\t0.137\t\t0.929\t\t0.31m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.33\t\t0.29\t\t0.162\t\t0.872\t\t0.29m - 0.7m / 10.9m                                                                    \n",
      "3\t13k\t0.30\t\t0.24\t\t0.182\t\t0.902\t\t0.31m - 1.0m / 10.2m                                                                   \n",
      "4\t17k\t0.27\t\t0.24\t\t0.240\t\t0.886\t\t0.31m - 1.4m / 10.9m                                                                   \n",
      "5\t22k\t0.25\t\t0.26\t\t0.196\t\t0.878\t\t0.30m - 1.8m / 11.1m                                                                   \n",
      "6\t26k\t0.23\t\t0.27\t\t0.213\t\t0.826\t\t0.30m - 2.2m / 11.0m                                                                   \n",
      "7\t31k\t0.23\t\t0.25\t\t0.237\t\t0.849\t\t0.32m - 2.5m / 10.9m                                                                   \n",
      "8\t35k\t0.21\t\t0.27\t\t0.232\t\t0.855\t\t0.30m - 2.9m / 11.6m                                                                   \n",
      "9\t39k\t0.20\t\t0.28\t\t0.209\t\t0.828\t\t0.31m - 3.3m / 11.1m                                                                   \n",
      "VAL f1\t0.24024024024024024 - (0.24024024024024024)                                                                     \n",
      "VAL loss\t0.24273466932904592                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24273466932904592\n",
      "        | \\     )|_\tf1: 0.24024024024024024\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\15 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 42.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         42                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.03916887979812515                |\n",
      "|  noam_learning_rate_warmup   |                        4698                       |\n",
      "|  noam_learning_rate_factor   |                 3.440268957522829                 |\n",
      "|          adam_beta1          |                 0.954144772863434                 |\n",
      "|          adam_beta2          |                 0.9157301369380269                |\n",
      "|           adam_eps           |                3.07329809221697e-08               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.1828306063378282                |\n",
      "|     pointwise_layer_size     |                        339                        |\n",
      "|      last_layer_dropout      |                 0.2642167409231512                |\n",
      "|   output_conv_num_filters    |                         27                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        150                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 14%|██████▏                                     | 14/100 [1:55:22<9:17:00, 388.61s/it, best loss: 0.24273466932904592]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f0c7f25a68049aabd65f9503380bbf9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.45\t\t0.36\t\t0.095\t\t0.933\t\t0.32m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.35\t\t0.31\t\t0.251\t\t0.877\t\t0.31m - 0.7m / 11.4m                                                                    \n",
      "3\t13k\t0.28\t\t0.28\t\t0.252\t\t0.883\t\t0.29m - 1.0m / 11.0m                                                                   \n",
      "4\t17k\t0.23\t\t0.28\t\t0.265\t\t0.897\t\t0.29m - 1.4m / 10.4m                                                                   \n",
      "5\t22k\t0.18\t\t0.30\t\t0.268\t\t0.891\t\t0.29m - 1.7m / 10.4m                                                                   \n",
      "6\t26k\t0.15\t\t0.30\t\t0.265\t\t0.887\t\t0.29m - 2.1m / 10.4m                                                                   \n",
      "7\t30k\t0.13\t\t0.34\t\t0.274\t\t0.894\t\t0.29m - 2.4m / 10.4m                                                                   \n",
      "8\t35k\t0.11\t\t0.36\t\t0.297\t\t0.909\t\t0.31m - 2.8m / 10.5m                                                                   \n",
      "9\t39k\t0.09\t\t0.41\t\t0.275\t\t0.904\t\t0.30m - 3.2m / 11.2m                                                                   \n",
      "10\t43k\t0.08\t\t0.45\t\t0.266\t\t0.905\t\t0.29m - 3.5m / 11.0m                                                                  \n",
      "11\t48k\t0.08\t\t0.42\t\t0.259\t\t0.908\t\t0.30m - 3.9m / 10.9m                                                                  \n",
      "12\t52k\t0.08\t\t0.48\t\t0.248\t\t0.898\t\t0.30m - 4.2m / 11.2m                                                                  \n",
      "13\t56k\t0.07\t\t0.47\t\t0.268\t\t0.910\t\t0.30m - 4.6m / 11.3m                                                                  \n",
      "VAL f1\t0.29705215419501135 - (0.29705215419501135)                                                                     \n",
      "VAL loss\t0.2811305295853388                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2811305295853388\n",
      "        | \\     )|_\tf1: 0.29705215419501135\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\16 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 60.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         60                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.0033314453174766                |\n",
      "|  noam_learning_rate_warmup   |                        4460                       |\n",
      "|  noam_learning_rate_factor   |                 1.722253589104732                 |\n",
      "|          adam_beta1          |                 0.8144290890458296                |\n",
      "|          adam_beta2          |                 0.9933615810981813                |\n",
      "|           adam_eps           |               1.9143803893200448e-09              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.0847620034316531                |\n",
      "|     pointwise_layer_size     |                        103                        |\n",
      "|      last_layer_dropout      |                0.11329632267881014                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        109                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 15%|██████▌                                     | 15/100 [2:00:34<8:37:54, 365.58s/it, best loss: 0.24273466932904592]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cbd6fce2b874424b09878948ea0e29f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.44\t\t0.26\t\t0.054\t\t0.882\t\t0.32m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.25\t\t0.047\t\t0.920\t\t0.33m - 0.7m / 11.3m                                                                    \n",
      "3\t13k\t0.28\t\t0.24\t\t0.056\t\t0.913\t\t0.26m - 1.1m / 11.7m                                                                   \n",
      "4\t17k\t0.26\t\t0.22\t\t0.200\t\t0.896\t\t0.31m - 1.4m / 9.5m                                                                    \n",
      "5\t22k\t0.23\t\t0.21\t\t0.218\t\t0.890\t\t0.32m - 1.8m / 11.2m                                                                   \n",
      "6\t26k\t0.21\t\t0.23\t\t0.207\t\t0.850\t\t0.39m - 2.4m / 11.7m                                                                   \n",
      "7\t30k\t0.19\t\t0.22\t\t0.201\t\t0.857\t\t0.40m - 2.9m / 13.8m                                                                   \n",
      "8\t35k\t0.18\t\t0.22\t\t0.212\t\t0.859\t\t0.36m - 3.3m / 14.2m                                                                   \n",
      "9\t39k\t0.16\t\t0.22\t\t0.260\t\t0.891\t\t0.35m - 3.7m / 13.0m                                                                   \n",
      "10\t43k\t0.14\t\t0.23\t\t0.223\t\t0.871\t\t0.33m - 4.1m / 12.9m                                                                  \n",
      "11\t48k\t0.13\t\t0.24\t\t0.274\t\t0.895\t\t0.37m - 4.6m / 12.5m                                                                  \n",
      "12\t52k\t0.12\t\t0.25\t\t0.220\t\t0.871\t\t0.32m - 5.0m / 13.5m                                                                  \n",
      "13\t56k\t0.11\t\t0.26\t\t0.227\t\t0.870\t\t0.30m - 5.3m / 12.5m                                                                  \n",
      "14\t60k\t0.10\t\t0.27\t\t0.236\t\t0.877\t\t0.27m - 5.7m / 12.0m                                                                  \n",
      "15\t65k\t0.09\t\t0.29\t\t0.243\t\t0.890\t\t0.27m - 6.0m / 11.4m                                                                  \n",
      "16\t69k\t0.08\t\t0.28\t\t0.262\t\t0.887\t\t0.27m - 6.3m / 11.5m                                                                  \n",
      "VAL f1\t0.27364185110663986 - (0.27364185110663986)                                                                     \n",
      "VAL loss\t0.21310370365778605                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21310370365778605\n",
      "        | \\     )|_\tf1: 0.27364185110663986\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\17 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 56.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         56                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0014279987833730617               |\n",
      "|  noam_learning_rate_warmup   |                        8125                       |\n",
      "|  noam_learning_rate_factor   |                 3.379653308042748                 |\n",
      "|          adam_beta1          |                 0.8170830219057682                |\n",
      "|          adam_beta2          |                 0.903223580173617                 |\n",
      "|           adam_eps           |                0.00618374631257556                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.2563647066896527                |\n",
      "|     pointwise_layer_size     |                        235                        |\n",
      "|      last_layer_dropout      |                 0.335876312748293                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        157                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 16%|███████▏                                     | 16/100 [2:07:18<8:47:53, 377.07s/it, best loss: 0.2369892239570618]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "140946330ab44ae789caaf05e06a4b9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.45\t\t0.28\t\t0.036\t\t0.889\t\t0.40m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.31\t\t0.27\t\t0.086\t\t0.902\t\t0.41m - 0.9m / 14.0m                                                                    \n",
      "3\t13k\t0.30\t\t0.26\t\t0.089\t\t0.933\t\t0.40m - 1.3m / 14.4m                                                                   \n",
      "4\t17k\t0.29\t\t0.27\t\t0.170\t\t0.834\t\t0.41m - 1.8m / 14.2m                                                                   \n",
      "5\t22k\t0.27\t\t0.25\t\t0.184\t\t0.857\t\t0.40m - 2.3m / 14.7m                                                                   \n",
      "6\t26k\t0.25\t\t0.25\t\t0.196\t\t0.823\t\t0.39m - 2.7m / 14.2m                                                                   \n",
      "7\t31k\t0.24\t\t0.24\t\t0.200\t\t0.838\t\t0.39m - 3.2m / 14.1m                                                                   \n",
      "8\t35k\t0.23\t\t0.24\t\t0.214\t\t0.863\t\t0.39m - 3.6m / 14.0m                                                                   \n",
      "9\t39k\t0.22\t\t0.26\t\t0.190\t\t0.830\t\t0.40m - 4.1m / 14.2m                                                                   \n",
      "10\t44k\t0.21\t\t0.26\t\t0.195\t\t0.834\t\t0.40m - 4.5m / 14.6m                                                                  \n",
      "11\t48k\t0.20\t\t0.25\t\t0.229\t\t0.863\t\t0.40m - 5.0m / 14.5m                                                                  \n",
      "12\t52k\t0.19\t\t0.26\t\t0.222\t\t0.857\t\t0.39m - 5.4m / 14.6m                                                                  \n",
      "13\t57k\t0.18\t\t0.29\t\t0.187\t\t0.835\t\t0.39m - 5.9m / 14.5m                                                                  \n",
      "14\t61k\t0.18\t\t0.27\t\t0.209\t\t0.850\t\t0.39m - 6.3m / 14.5m                                                                  \n",
      "15\t66k\t0.17\t\t0.29\t\t0.207\t\t0.847\t\t0.40m - 6.8m / 14.6m                                                                  \n",
      "16\t70k\t0.17\t\t0.30\t\t0.186\t\t0.855\t\t0.48m - 7.3m / 14.9m                                                                  \n",
      "VAL f1\t0.2292358803986711 - (0.2292358803986711)                                                                       \n",
      "VAL loss\t0.24277191218875704                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24277191218875704\n",
      "        | \\     )|_\tf1: 0.2292358803986711\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\18 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 13.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         13                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.28510939079013875                |\n",
      "|  noam_learning_rate_warmup   |                        6146                       |\n",
      "|  noam_learning_rate_factor   |                 2.1335985432536715                |\n",
      "|          adam_beta1          |                 0.7240559614797965                |\n",
      "|          adam_beta2          |                 0.793957049053619                 |\n",
      "|           adam_eps           |                0.009169829043029607               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.6115443843094416                |\n",
      "|     pointwise_layer_size     |                        330                        |\n",
      "|      last_layer_dropout      |                 0.5125664908151606                |\n",
      "|   output_conv_num_filters    |                        366                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         50                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 17%|███████▋                                     | 17/100 [2:15:07<9:20:01, 404.83s/it, best loss: 0.2369892239570618]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f46905b430414947aabce45cdad7ce13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.08\t\t1.01\t\t0.096\t\t0.882\t\t1.11m - 1.1m / 0.0m                                                                     \n",
      "2\t9k\t0.97\t\t0.85\t\t0.173\t\t0.837\t\t1.12m - 2.3m / 38.8m                                                                    \n",
      "3\t13k\t0.95\t\t0.79\t\t0.147\t\t0.841\t\t1.12m - 3.5m / 39.3m                                                                   \n",
      "4\t17k\t0.96\t\t0.79\t\t0.238\t\t0.851\t\t1.07m - 4.6m / 39.4m                                                                   \n",
      "5\t22k\t0.98\t\t0.77\t\t0.260\t\t0.862\t\t1.14m - 5.8m / 37.9m                                                                   \n",
      "6\t26k\t0.98\t\t1.00\t\t0.196\t\t0.813\t\t1.16m - 7.0m / 40.1m                                                                   \n",
      "7\t30k\t0.97\t\t0.93\t\t0.215\t\t0.829\t\t1.10m - 8.2m / 40.7m                                                                   \n",
      "8\t35k\t0.96\t\t1.01\t\t0.214\t\t0.836\t\t1.10m - 9.3m / 39.0m                                                                   \n",
      "9\t39k\t0.95\t\t0.98\t\t0.192\t\t0.800\t\t1.11m - 10.5m / 39.0m                                                                  \n",
      "10\t43k\t0.95\t\t0.93\t\t0.230\t\t0.822\t\t1.15m - 11.7m / 39.5m                                                                 \n",
      "Could not complete iteration evaluation: Dimension out of range (expected to be in range of [-1, 0], but got 1)        \n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190423\\19 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 22.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         22                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.007772569060423516               |\n",
      "|  noam_learning_rate_warmup   |                        1676                       |\n",
      "|  noam_learning_rate_factor   |                 0.3094946770236759                |\n",
      "|          adam_beta1          |                 0.8407289702333527                |\n",
      "|          adam_beta2          |                 0.8484729317324506                |\n",
      "|           adam_eps           |               2.7173032523623712e-08              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.450320133870874                 |\n",
      "|     pointwise_layer_size     |                         61                        |\n",
      "|      last_layer_dropout      |                0.06674080003605339                |\n",
      "|   output_conv_num_filters    |                        240                        |\n",
      "|   output_conv_kernel_size    |                         4                         |\n",
      "|      output_conv_stride      |                         8                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        105                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 18%|███████▉                                    | 18/100 [2:27:40<11:35:52, 509.18s/it, best loss: 0.2369892239570618]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "940a38f3115e4d488eff423fe5ce7e87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.72\t\t0.61\t\t0.134\t\t0.912\t\t0.74m - 0.7m / 0.0m                                                                     \n",
      "2\t9k\t0.58\t\t0.56\t\t0.194\t\t0.812\t\t0.75m - 1.6m / 26.1m                                                                    \n",
      "3\t13k\t0.53\t\t0.58\t\t0.187\t\t0.790\t\t0.74m - 2.4m / 26.4m                                                                   \n",
      "4\t17k\t0.52\t\t0.56\t\t0.166\t\t0.810\t\t0.73m - 3.1m / 26.1m                                                                   \n",
      "5\t22k\t0.53\t\t0.56\t\t0.199\t\t0.828\t\t0.74m - 3.9m / 25.7m                                                                   \n",
      "6\t26k\t0.54\t\t0.54\t\t0.208\t\t0.844\t\t0.74m - 4.7m / 26.1m                                                                   \n",
      "7\t30k\t0.55\t\t0.53\t\t0.235\t\t0.857\t\t0.73m - 5.5m / 26.3m                                                                   \n",
      "8\t35k\t0.53\t\t0.53\t\t0.220\t\t0.826\t\t0.74m - 6.3m / 26.0m                                                                   \n",
      "9\t39k\t0.51\t\t0.52\t\t0.245\t\t0.842\t\t0.74m - 7.1m / 26.4m                                                                   \n",
      "10\t43k\t0.48\t\t0.50\t\t0.239\t\t0.831\t\t0.73m - 7.9m / 26.3m                                                                  \n",
      "11\t48k\t0.45\t\t0.54\t\t0.217\t\t0.836\t\t0.73m - 8.7m / 26.1m                                                                  \n",
      "12\t52k\t0.42\t\t0.55\t\t0.237\t\t0.814\t\t0.73m - 9.5m / 26.2m                                                                  \n",
      "13\t56k\t0.40\t\t0.51\t\t0.239\t\t0.836\t\t0.73m - 10.2m / 26.3m                                                                 \n",
      "14\t61k\t0.37\t\t0.55\t\t0.245\t\t0.845\t\t0.88m - 11.2m / 26.3m                                                                 \n",
      "VAL f1\t0.2446351931330472 - (0.2446351931330472)                                                                       \n",
      "VAL loss\t0.5007812827825546                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.5007812827825546\n",
      "        | \\     )|_\tf1: 0.2446351931330472\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\0  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 39.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         39                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.05809791629217533                |\n",
      "|  noam_learning_rate_warmup   |                        5180                       |\n",
      "|  noam_learning_rate_factor   |                 2.5252152231060685                |\n",
      "|          adam_beta1          |                 0.769500048476924                 |\n",
      "|          adam_beta2          |                  0.96295738416793                 |\n",
      "|           adam_eps           |                5.70676031805328e-06               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.03247853579654292                |\n",
      "|     pointwise_layer_size     |                        330                        |\n",
      "|      last_layer_dropout      |                0.042124975091598675               |\n",
      "|   output_conv_num_filters    |                        336                        |\n",
      "|   output_conv_kernel_size    |                         3                         |\n",
      "|      output_conv_stride      |                         6                         |\n",
      "|     output_conv_padding      |                         1                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        140                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 19%|████████▎                                   | 19/100 [2:39:27<12:47:28, 568.50s/it, best loss: 0.2369892239570618]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27ee2bee183e4adc922b4534d3bbd8ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.46\t\t0.35\t\t0.117\t\t0.933\t\t0.42m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.36\t\t0.32\t\t0.230\t\t0.859\t\t0.43m - 0.9m / 14.6m                                                                    \n",
      "3\t13k\t0.29\t\t0.29\t\t0.261\t\t0.870\t\t0.42m - 1.4m / 15.2m                                                                   \n",
      "4\t17k\t0.23\t\t0.31\t\t0.273\t\t0.889\t\t0.42m - 1.9m / 14.9m                                                                   \n",
      "5\t22k\t0.17\t\t0.31\t\t0.269\t\t0.894\t\t0.42m - 2.3m / 15.0m                                                                   \n",
      "6\t26k\t0.13\t\t0.34\t\t0.263\t\t0.897\t\t0.42m - 2.8m / 14.9m                                                                   \n",
      "7\t30k\t0.09\t\t0.37\t\t0.275\t\t0.909\t\t0.42m - 3.3m / 14.9m                                                                   \n",
      "8\t35k\t0.07\t\t0.42\t\t0.269\t\t0.918\t\t0.42m - 3.8m / 15.0m                                                                   \n",
      "9\t39k\t0.06\t\t0.38\t\t0.277\t\t0.912\t\t0.42m - 4.2m / 15.3m                                                                   \n",
      "10\t43k\t0.04\t\t0.45\t\t0.243\t\t0.902\t\t0.42m - 4.7m / 15.3m                                                                  \n",
      "11\t48k\t0.04\t\t0.53\t\t0.256\t\t0.912\t\t0.42m - 5.2m / 15.3m                                                                  \n",
      "12\t52k\t0.04\t\t0.57\t\t0.231\t\t0.915\t\t0.42m - 5.6m / 15.3m                                                                  \n",
      "13\t56k\t0.03\t\t0.57\t\t0.252\t\t0.917\t\t0.41m - 6.1m / 15.3m                                                                  \n",
      "14\t61k\t0.03\t\t0.65\t\t0.261\t\t0.929\t\t0.42m - 6.6m / 15.3m                                                                  \n",
      "VAL f1\t0.27699530516431925 - (0.27699530516431925)                                                                     \n",
      "VAL loss\t0.29391550134729455                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.29391550134729455\n",
      "        | \\     )|_\tf1: 0.27699530516431925\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\1  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 50.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         50                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0004236187853383737               |\n",
      "|  noam_learning_rate_warmup   |                        1450                       |\n",
      "|  noam_learning_rate_factor   |                 1.3012515341482187                |\n",
      "|          adam_beta1          |                 0.802573282931043                 |\n",
      "|          adam_beta2          |                 0.9485505321106119                |\n",
      "|           adam_eps           |                 0.864621228000028                 |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.36033011227777023                |\n",
      "|     pointwise_layer_size     |                        112                        |\n",
      "|      last_layer_dropout      |                 0.2754115017292377                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        174                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 20%|████████▊                                   | 20/100 [2:46:29<11:39:26, 524.58s/it, best loss: 0.2369892239570618]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88219ffe4f0f41c288d71ad0f43e4da6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.52\t\t0.38\t\t0.055\t\t0.726\t\t0.22m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.37\t\t0.29\t\t0.000\t\t0.929\t\t0.22m - 0.5m / 7.6m                                                                     \n",
      "3\t13k\t0.34\t\t0.28\t\t0.000\t\t0.937\t\t0.22m - 0.7m / 7.7m                                                                    \n",
      "4\t17k\t0.34\t\t0.28\t\t0.000\t\t0.931\t\t0.22m - 1.0m / 7.7m                                                                    \n",
      "5\t22k\t0.34\t\t0.28\t\t0.000\t\t0.937\t\t0.22m - 1.3m / 7.8m                                                                    \n",
      "6\t26k\t0.33\t\t0.28\t\t0.000\t\t0.935\t\t0.22m - 1.6m / 7.8m                                                                    \n",
      "VAL f1\t0.054945054945054944 - (0.054945054945054944)                                                                   \n",
      "VAL loss\t0.2807151767185756                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2807151767185756\n",
      "        | \\     )|_\tf1: 0.054945054945054944\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\2  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 64.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00021139179749148622              |\n",
      "|  noam_learning_rate_warmup   |                        2562                       |\n",
      "|  noam_learning_rate_factor   |                 1.3552126355378677                |\n",
      "|          adam_beta1          |                 0.9231713755194506                |\n",
      "|          adam_beta2          |                 0.8726230515689922                |\n",
      "|           adam_eps           |               0.0007591109054994114               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.44549440629435716                |\n",
      "|     pointwise_layer_size     |                         91                        |\n",
      "|      last_layer_dropout      |                0.24476239033721275                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        176                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 21%|█████████▍                                   | 21/100 [2:48:23<8:48:21, 401.29s/it, best loss: 0.2369892239570618]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a8dd7c16ca54d478ecfe9d88e8bf8e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.36\t\t0.24\t\t0.080\t\t0.891\t\t0.27m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.27\t\t0.23\t\t0.173\t\t0.898\t\t0.27m - 0.6m / 9.6m                                                                     \n",
      "3\t13k\t0.27\t\t0.23\t\t0.150\t\t0.890\t\t0.28m - 0.9m / 9.6m                                                                    \n",
      "4\t17k\t0.27\t\t0.21\t\t0.240\t\t0.916\t\t0.28m - 1.3m / 9.8m                                                                    \n",
      "5\t22k\t0.24\t\t0.21\t\t0.221\t\t0.836\t\t0.27m - 1.6m / 9.9m                                                                    \n",
      "6\t26k\t0.22\t\t0.21\t\t0.210\t\t0.825\t\t0.28m - 1.9m / 9.8m                                                                    \n",
      "7\t30k\t0.21\t\t0.21\t\t0.246\t\t0.832\t\t0.28m - 2.2m / 10.1m                                                                   \n",
      "8\t35k\t0.19\t\t0.23\t\t0.188\t\t0.807\t\t0.28m - 2.6m / 10.1m                                                                   \n",
      "9\t39k\t0.19\t\t0.24\t\t0.170\t\t0.774\t\t0.28m - 2.9m / 10.1m                                                                   \n",
      "10\t44k\t0.18\t\t0.27\t\t0.184\t\t0.778\t\t0.27m - 3.2m / 10.1m                                                                  \n",
      "11\t48k\t0.17\t\t0.25\t\t0.231\t\t0.827\t\t0.27m - 3.6m / 10.1m                                                                  \n",
      "12\t52k\t0.16\t\t0.28\t\t0.166\t\t0.771\t\t0.27m - 3.9m / 10.2m                                                                  \n",
      "VAL f1\t0.24575695858791582 - (0.24575695858791582)                                                                     \n",
      "VAL loss\t0.20778764660159746                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.20778764660159746\n",
      "        | \\     )|_\tf1: 0.24575695858791582\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\3  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 61.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         61                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               3.858478694795554e-06               |\n",
      "|  noam_learning_rate_warmup   |                        2974                       |\n",
      "|  noam_learning_rate_factor   |                 1.1727779826454476                |\n",
      "|          adam_beta1          |                 0.9274072150890583                |\n",
      "|          adam_beta2          |                 0.8769360099881707                |\n",
      "|           adam_eps           |               5.438404631201256e-10               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.4559634908857557                |\n",
      "|     pointwise_layer_size     |                         65                        |\n",
      "|      last_layer_dropout      |                 0.2623394791135295                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        116                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 22%|█████████▉                                   | 22/100 [2:52:39<7:45:01, 357.71s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4515c39b682d47428700e1db894ebfb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.40\t\t0.26\t\t0.021\t\t0.864\t\t0.22m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.24\t\t0.008\t\t0.923\t\t0.22m - 0.5m / 7.8m                                                                     \n",
      "3\t13k\t0.28\t\t0.24\t\t0.010\t\t0.934\t\t0.22m - 0.8m / 7.8m                                                                    \n",
      "4\t17k\t0.28\t\t0.24\t\t0.048\t\t0.916\t\t0.22m - 1.0m / 7.9m                                                                    \n",
      "5\t22k\t0.26\t\t0.22\t\t0.173\t\t0.913\t\t0.22m - 1.3m / 7.9m                                                                    \n",
      "6\t26k\t0.24\t\t0.22\t\t0.234\t\t0.891\t\t0.22m - 1.6m / 8.0m                                                                    \n",
      "7\t30k\t0.22\t\t0.23\t\t0.179\t\t0.834\t\t0.22m - 1.9m / 8.1m                                                                    \n",
      "8\t35k\t0.21\t\t0.22\t\t0.185\t\t0.826\t\t0.22m - 2.1m / 8.1m                                                                    \n",
      "9\t39k\t0.20\t\t0.23\t\t0.230\t\t0.849\t\t0.22m - 2.4m / 8.2m                                                                    \n",
      "10\t43k\t0.19\t\t0.24\t\t0.220\t\t0.824\t\t0.22m - 2.7m / 8.3m                                                                   \n",
      "11\t48k\t0.18\t\t0.23\t\t0.240\t\t0.846\t\t0.22m - 2.9m / 8.3m                                                                   \n",
      "12\t52k\t0.17\t\t0.24\t\t0.241\t\t0.840\t\t0.22m - 3.2m / 8.3m                                                                   \n",
      "13\t56k\t0.16\t\t0.26\t\t0.194\t\t0.777\t\t0.22m - 3.5m / 8.4m                                                                   \n",
      "14\t61k\t0.15\t\t0.27\t\t0.232\t\t0.828\t\t0.22m - 3.8m / 8.5m                                                                   \n",
      "15\t65k\t0.15\t\t0.26\t\t0.233\t\t0.851\t\t0.22m - 4.0m / 8.5m                                                                   \n",
      "16\t69k\t0.14\t\t0.27\t\t0.221\t\t0.837\t\t0.22m - 4.3m / 8.5m                                                                   \n",
      "17\t74k\t0.14\t\t0.29\t\t0.193\t\t0.786\t\t0.22m - 4.6m / 8.6m                                                                   \n",
      "VAL f1\t0.24089635854341737 - (0.24089635854341737)                                                                     \n",
      "VAL loss\t0.21792900106294555                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21792900106294555\n",
      "        | \\     )|_\tf1: 0.24089635854341737\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\4  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 61.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         61                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               5.171757660773677e-05               |\n",
      "|  noam_learning_rate_warmup   |                        3054                       |\n",
      "|  noam_learning_rate_factor   |                0.09177997263287185                |\n",
      "|          adam_beta1          |                 0.7889069824373885                |\n",
      "|          adam_beta2          |                 0.8620793117320669                |\n",
      "|           adam_eps           |               9.527708259433331e-05               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.43874179241074324                |\n",
      "|     pointwise_layer_size     |                         95                        |\n",
      "|      last_layer_dropout      |                0.008935698158383354               |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         99                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 23%|██████████▎                                  | 23/100 [2:57:35<7:15:18, 339.20s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc268a753fdd49e3b01a5fb9f1cf9a75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.47\t\t0.52\t\t0.067\t\t0.239\t\t0.17m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.44\t\t0.44\t\t0.074\t\t0.372\t\t0.18m - 0.4m / 6.0m                                                                     \n",
      "3\t13k\t0.40\t\t0.35\t\t0.094\t\t0.623\t\t0.17m - 0.6m / 6.2m                                                                    \n",
      "4\t17k\t0.35\t\t0.29\t\t0.080\t\t0.754\t\t0.17m - 0.8m / 6.0m                                                                    \n",
      "5\t22k\t0.32\t\t0.26\t\t0.070\t\t0.839\t\t0.17m - 1.1m / 6.1m                                                                    \n",
      "6\t26k\t0.30\t\t0.25\t\t0.072\t\t0.920\t\t0.18m - 1.3m / 6.1m                                                                    \n",
      "7\t30k\t0.29\t\t0.25\t\t0.060\t\t0.933\t\t0.17m - 1.6m / 6.5m                                                                    \n",
      "8\t35k\t0.28\t\t0.25\t\t0.048\t\t0.934\t\t0.16m - 1.8m / 6.4m                                                                    \n",
      "VAL f1\t0.09378574151734655 - (0.09378574151734655)                                                                     \n",
      "VAL loss\t0.24694807021344298                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24694807021344298\n",
      "        | \\     )|_\tf1: 0.09378574151734655\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\5  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 63.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         63                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               9.957857670718118e-05               |\n",
      "|  noam_learning_rate_warmup   |                        2246                       |\n",
      "|  noam_learning_rate_factor   |                 0.9524004750182544                |\n",
      "|          adam_beta1          |                 0.8344745902874439                |\n",
      "|          adam_beta2          |                 0.9979033876682725                |\n",
      "|           adam_eps           |                0.13748740278263477                |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.5282578315198135                |\n",
      "|     pointwise_layer_size     |                        132                        |\n",
      "|      last_layer_dropout      |                 0.7602153947337376                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         86                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 24%|██████████▊                                  | 24/100 [2:59:41<5:48:37, 275.23s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b894795c45604a9398dea5563a9f3e3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.41\t\t0.34\t\t0.063\t\t0.519\t\t0.28m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.25\t\t0.062\t\t0.852\t\t0.29m - 0.6m / 9.9m                                                                     \n",
      "3\t13k\t0.29\t\t0.24\t\t0.170\t\t0.885\t\t0.29m - 1.0m / 10.1m                                                                   \n",
      "4\t17k\t0.28\t\t0.24\t\t0.188\t\t0.900\t\t0.28m - 1.3m / 10.1m                                                                   \n",
      "5\t22k\t0.28\t\t0.24\t\t0.162\t\t0.883\t\t0.29m - 1.6m / 10.2m                                                                   \n",
      "6\t26k\t0.28\t\t0.25\t\t0.189\t\t0.893\t\t0.29m - 2.0m / 10.3m                                                                   \n",
      "7\t30k\t0.28\t\t0.24\t\t0.150\t\t0.901\t\t0.29m - 2.3m / 10.4m                                                                   \n",
      "8\t35k\t0.28\t\t0.24\t\t0.165\t\t0.881\t\t0.29m - 2.6m / 10.4m                                                                   \n",
      "9\t39k\t0.28\t\t0.24\t\t0.177\t\t0.888\t\t0.29m - 3.0m / 10.5m                                                                   \n",
      "10\t43k\t0.28\t\t0.24\t\t0.210\t\t0.892\t\t0.29m - 3.3m / 10.5m                                                                  \n",
      "11\t48k\t0.27\t\t0.25\t\t0.161\t\t0.849\t\t0.29m - 3.7m / 10.5m                                                                  \n",
      "12\t52k\t0.26\t\t0.23\t\t0.190\t\t0.856\t\t0.29m - 4.0m / 10.6m                                                                  \n",
      "13\t57k\t0.25\t\t0.24\t\t0.185\t\t0.793\t\t0.29m - 4.3m / 10.6m                                                                  \n",
      "14\t61k\t0.24\t\t0.24\t\t0.173\t\t0.793\t\t0.29m - 4.7m / 10.7m                                                                  \n",
      "15\t65k\t0.23\t\t0.24\t\t0.151\t\t0.773\t\t0.29m - 5.0m / 10.7m                                                                  \n",
      "VAL f1\t0.209919261822376 - (0.209919261822376)                                                                         \n",
      "VAL loss\t0.23247834866639802                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.23247834866639802\n",
      "        | \\     )|_\tf1: 0.209919261822376\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\6  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 58.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         58                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               9.049567474754483e-05               |\n",
      "|  noam_learning_rate_warmup   |                        5196                       |\n",
      "|  noam_learning_rate_factor   |                 1.7444734495256162                |\n",
      "|          adam_beta1          |                 0.7026488813554167                |\n",
      "|          adam_beta2          |                 0.9416085580751713                |\n",
      "|           adam_eps           |               3.571317307222086e-07               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.7986579144601773                |\n",
      "|     pointwise_layer_size     |                         72                        |\n",
      "|      last_layer_dropout      |                0.33423320286739866                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        119                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 25%|███████████▎                                 | 25/100 [3:05:04<6:02:02, 289.63s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87c7987f04a849b2a7588cd9dea2ae75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.45\t\t0.41\t\t0.078\t\t0.554\t\t0.23m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.36\t\t0.28\t\t0.187\t\t0.895\t\t0.23m - 0.5m / 8.1m                                                                     \n",
      "3\t13k\t0.31\t\t0.29\t\t0.049\t\t0.936\t\t0.23m - 0.8m / 8.2m                                                                    \n",
      "4\t17k\t0.30\t\t0.30\t\t0.008\t\t0.919\t\t0.23m - 1.1m / 8.3m                                                                    \n",
      "5\t22k\t0.30\t\t0.32\t\t0.016\t\t0.896\t\t0.23m - 1.4m / 8.3m                                                                    \n",
      "6\t26k\t0.30\t\t0.31\t\t0.037\t\t0.898\t\t0.23m - 1.6m / 8.3m                                                                    \n",
      "7\t30k\t0.30\t\t0.30\t\t0.015\t\t0.916\t\t0.23m - 1.9m / 8.4m                                                                    \n",
      "VAL f1\t0.18689320388349515 - (0.18689320388349515)                                                                     \n",
      "VAL loss\t0.27630521785253764                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.27630521785253764\n",
      "        | \\     )|_\tf1: 0.18689320388349515\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\7  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 64.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               2.682781325956491e-05               |\n",
      "|  noam_learning_rate_warmup   |                        1016                       |\n",
      "|  noam_learning_rate_factor   |                 1.5273587474522115                |\n",
      "|          adam_beta1          |                 0.9265504504654037                |\n",
      "|          adam_beta2          |                 0.7538459935910318                |\n",
      "|           adam_eps           |               7.399675192012443e-06               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.12663472413218407                |\n",
      "|     pointwise_layer_size     |                        207                        |\n",
      "|      last_layer_dropout      |                0.004239087770033581               |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         85                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 26%|███████████▋                                 | 26/100 [3:07:20<5:00:07, 243.34s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ea3fbdb8e544f569fef0bdb5c98b14a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.33\t\t0.23\t\t0.044\t\t0.915\t\t0.20m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.28\t\t0.24\t\t0.139\t\t0.868\t\t0.20m - 0.4m / 7.0m                                                                     \n",
      "3\t13k\t0.24\t\t0.24\t\t0.112\t\t0.860\t\t0.20m - 0.7m / 7.1m                                                                    \n",
      "4\t17k\t0.22\t\t0.22\t\t0.210\t\t0.871\t\t0.20m - 1.0m / 7.1m                                                                    \n",
      "5\t22k\t0.19\t\t0.24\t\t0.185\t\t0.845\t\t0.20m - 1.2m / 7.3m                                                                    \n",
      "6\t26k\t0.17\t\t0.30\t\t0.151\t\t0.837\t\t0.20m - 1.5m / 7.2m                                                                    \n",
      "7\t30k\t0.14\t\t0.29\t\t0.241\t\t0.884\t\t0.20m - 1.7m / 7.3m                                                                    \n",
      "8\t35k\t0.14\t\t0.30\t\t0.273\t\t0.878\t\t0.20m - 2.0m / 7.4m                                                                    \n",
      "9\t39k\t0.15\t\t0.30\t\t0.254\t\t0.903\t\t0.20m - 2.2m / 7.4m                                                                    \n",
      "10\t44k\t0.13\t\t0.34\t\t0.207\t\t0.854\t\t0.20m - 2.5m / 7.4m                                                                   \n",
      "11\t48k\t0.12\t\t0.34\t\t0.232\t\t0.868\t\t0.20m - 2.7m / 7.5m                                                                   \n",
      "12\t52k\t0.15\t\t0.36\t\t0.207\t\t0.878\t\t0.20m - 3.0m / 7.6m                                                                   \n",
      "13\t57k\t0.11\t\t0.29\t\t0.215\t\t0.885\t\t0.20m - 3.2m / 7.6m                                                                   \n",
      "VAL f1\t0.2732049036777583 - (0.2732049036777583)                                                                       \n",
      "VAL loss\t0.22373954703410467                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22373954703410467\n",
      "        | \\     )|_\tf1: 0.2732049036777583\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\8  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 49.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         49                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00035735731027185637              |\n",
      "|  noam_learning_rate_warmup   |                        3487                       |\n",
      "|  noam_learning_rate_factor   |                 0.352538530069161                 |\n",
      "|          adam_beta1          |                 0.8610717605088936                |\n",
      "|          adam_beta2          |                 0.8373991985807954                |\n",
      "|           adam_eps           |               0.0002707152148350563               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.29772085964402706                |\n",
      "|     pointwise_layer_size     |                         36                        |\n",
      "|      last_layer_dropout      |                0.21514251443618093                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        143                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 27%|████████████▏                                | 27/100 [3:10:52<4:44:55, 234.18s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9608717129d443cdb6086b7a5ff24d54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.56\t\t0.51\t\t0.050\t\t0.500\t\t0.20m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.43\t\t0.32\t\t0.059\t\t0.850\t\t0.19m - 0.5m / 6.9m                                                                     \n",
      "3\t13k\t0.35\t\t0.29\t\t0.047\t\t0.940\t\t0.20m - 0.7m / 6.9m                                                                    \n",
      "4\t17k\t0.34\t\t0.29\t\t0.059\t\t0.937\t\t0.20m - 1.0m / 7.1m                                                                    \n",
      "5\t22k\t0.33\t\t0.29\t\t0.150\t\t0.926\t\t0.20m - 1.2m / 7.2m                                                                    \n",
      "6\t26k\t0.33\t\t0.29\t\t0.159\t\t0.901\t\t0.21m - 1.5m / 7.3m                                                                    \n",
      "7\t31k\t0.32\t\t0.28\t\t0.194\t\t0.901\t\t0.20m - 1.7m / 7.5m                                                                    \n",
      "8\t35k\t0.30\t\t0.28\t\t0.199\t\t0.895\t\t0.20m - 2.0m / 7.3m                                                                    \n",
      "9\t39k\t0.28\t\t0.27\t\t0.239\t\t0.903\t\t0.20m - 2.2m / 7.3m                                                                    \n",
      "10\t44k\t0.27\t\t0.27\t\t0.229\t\t0.881\t\t0.20m - 2.5m / 7.5m                                                                   \n",
      "11\t48k\t0.26\t\t0.26\t\t0.275\t\t0.898\t\t0.19m - 2.7m / 7.5m                                                                   \n",
      "12\t52k\t0.25\t\t0.26\t\t0.249\t\t0.888\t\t0.20m - 3.0m / 7.4m                                                                   \n",
      "13\t57k\t0.24\t\t0.26\t\t0.282\t\t0.897\t\t0.20m - 3.2m / 7.6m                                                                   \n",
      "14\t61k\t0.23\t\t0.26\t\t0.279\t\t0.886\t\t0.20m - 3.5m / 7.6m                                                                   \n",
      "15\t65k\t0.22\t\t0.27\t\t0.251\t\t0.880\t\t0.20m - 3.7m / 7.7m                                                                   \n",
      "16\t70k\t0.21\t\t0.27\t\t0.260\t\t0.874\t\t0.20m - 4.0m / 7.7m                                                                   \n",
      "17\t74k\t0.20\t\t0.27\t\t0.276\t\t0.897\t\t0.20m - 4.2m / 7.8m                                                                   \n",
      "18\t78k\t0.20\t\t0.27\t\t0.264\t\t0.885\t\t0.20m - 4.5m / 7.8m                                                                   \n",
      "VAL f1\t0.282183316168898 - (0.282183316168898)                                                                         \n",
      "VAL loss\t0.2560727061057578                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2560727061057578\n",
      "        | \\     )|_\tf1: 0.282183316168898\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\9  \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 55.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         55                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.003041583437150142               |\n",
      "|  noam_learning_rate_warmup   |                        5841                       |\n",
      "|  noam_learning_rate_factor   |                 2.6735579668330414                |\n",
      "|          adam_beta1          |                 0.763392530871501                 |\n",
      "|          adam_beta2          |                 0.8804772364293801                |\n",
      "|           adam_eps           |                0.002669725781553501               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.6769952532933439                |\n",
      "|     pointwise_layer_size     |                         92                        |\n",
      "|      last_layer_dropout      |                0.12251793743756852                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         95                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 28%|████████████▌                                | 28/100 [3:15:43<5:01:21, 251.13s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a647ca6e480746b5abd01e18ef1dd64d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.41\t\t0.29\t\t0.046\t\t0.854\t\t0.31m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.28\t\t0.000\t\t0.933\t\t0.31m - 0.7m / 10.9m                                                                    \n",
      "3\t13k\t0.31\t\t0.28\t\t0.000\t\t0.940\t\t0.32m - 1.0m / 10.9m                                                                   \n",
      "4\t17k\t0.32\t\t0.28\t\t0.000\t\t0.940\t\t0.31m - 1.4m / 11.2m                                                                   \n",
      "5\t22k\t0.31\t\t0.29\t\t0.051\t\t0.891\t\t0.31m - 1.8m / 11.1m                                                                   \n",
      "6\t26k\t0.30\t\t0.27\t\t0.173\t\t0.853\t\t0.31m - 2.1m / 11.1m                                                                   \n",
      "7\t30k\t0.29\t\t0.28\t\t0.165\t\t0.844\t\t0.31m - 2.5m / 11.2m                                                                   \n",
      "8\t35k\t0.27\t\t0.37\t\t0.134\t\t0.717\t\t0.31m - 2.9m / 11.3m                                                                   \n",
      "9\t39k\t0.27\t\t0.37\t\t0.140\t\t0.717\t\t0.32m - 3.2m / 11.3m                                                                   \n",
      "10\t43k\t0.26\t\t0.33\t\t0.152\t\t0.734\t\t0.31m - 3.6m / 11.5m                                                                  \n",
      "11\t48k\t0.25\t\t0.35\t\t0.144\t\t0.735\t\t0.31m - 3.9m / 11.4m                                                                  \n",
      "VAL f1\t0.17288135593220338 - (0.17288135593220338)                                                                     \n",
      "VAL loss\t0.26967865832440263                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.26967865832440263\n",
      "        | \\     )|_\tf1: 0.17288135593220338\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\10 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 47.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         47                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.004085582299999                 |\n",
      "|  noam_learning_rate_warmup   |                        4629                       |\n",
      "|  noam_learning_rate_factor   |                 1.4576514849577438                |\n",
      "|          adam_beta1          |                 0.9204265258944597                |\n",
      "|          adam_beta2          |                 0.8819237961903031                |\n",
      "|           adam_eps           |                0.03154510346917735                |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.40822640460193543                |\n",
      "|     pointwise_layer_size     |                        121                        |\n",
      "|      last_layer_dropout      |                0.23805509542564157                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         70                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 29%|█████████████                                | 29/100 [3:20:01<4:59:48, 253.36s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72c2181919e04c259b7cc3681a0f4748",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.56\t\t0.36\t\t0.031\t\t0.777\t\t0.24m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.37\t\t0.28\t\t0.110\t\t0.923\t\t0.24m - 0.5m / 8.4m                                                                     \n",
      "3\t13k\t0.35\t\t0.29\t\t0.080\t\t0.911\t\t0.24m - 0.8m / 8.5m                                                                    \n",
      "4\t17k\t0.34\t\t0.28\t\t0.117\t\t0.903\t\t0.24m - 1.1m / 8.4m                                                                    \n",
      "5\t22k\t0.33\t\t0.27\t\t0.206\t\t0.886\t\t0.24m - 1.4m / 8.7m                                                                    \n",
      "6\t26k\t0.31\t\t0.26\t\t0.236\t\t0.875\t\t0.24m - 1.7m / 8.6m                                                                    \n",
      "7\t30k\t0.29\t\t0.26\t\t0.237\t\t0.878\t\t0.24m - 2.0m / 8.7m                                                                    \n",
      "8\t35k\t0.28\t\t0.27\t\t0.229\t\t0.864\t\t0.25m - 2.3m / 8.8m                                                                    \n",
      "9\t39k\t0.27\t\t0.26\t\t0.249\t\t0.868\t\t0.25m - 2.6m / 9.1m                                                                    \n",
      "10\t43k\t0.25\t\t0.28\t\t0.237\t\t0.843\t\t0.25m - 2.9m / 9.2m                                                                   \n",
      "11\t48k\t0.24\t\t0.28\t\t0.230\t\t0.841\t\t0.25m - 3.2m / 9.2m                                                                   \n",
      "12\t52k\t0.23\t\t0.30\t\t0.195\t\t0.832\t\t0.25m - 3.5m / 9.3m                                                                   \n",
      "13\t56k\t0.23\t\t0.30\t\t0.217\t\t0.822\t\t0.24m - 3.8m / 9.3m                                                                   \n",
      "14\t61k\t0.22\t\t0.31\t\t0.215\t\t0.829\t\t0.25m - 4.1m / 9.2m                                                                   \n",
      "VAL f1\t0.24896265560165975 - (0.24896265560165975)                                                                     \n",
      "VAL loss\t0.25734429790618574                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.25734429790618574\n",
      "        | \\     )|_\tf1: 0.24896265560165975\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\11 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 36.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         36                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               1.316587255591905e-05               |\n",
      "|  noam_learning_rate_warmup   |                        3415                       |\n",
      "|  noam_learning_rate_factor   |                 1.0464460237347049                |\n",
      "|          adam_beta1          |                 0.818910718031624                 |\n",
      "|          adam_beta2          |                 0.7616398335292971                |\n",
      "|           adam_eps           |               1.5898550591822665e-10              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.522789437105804                 |\n",
      "|     pointwise_layer_size     |                        149                        |\n",
      "|      last_layer_dropout      |                 0.3176222288906266                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        110                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 30%|█████████████▌                               | 30/100 [3:24:28<5:00:28, 257.54s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "453460aa6d354a00b46e607a76f8b7df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.61\t\t0.37\t\t0.000\t\t0.936\t\t0.38m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.45\t\t0.37\t\t0.047\t\t0.940\t\t0.38m - 0.8m / 13.4m                                                                    \n",
      "3\t13k\t0.43\t\t0.35\t\t0.212\t\t0.900\t\t0.38m - 1.2m / 13.3m                                                                   \n",
      "4\t17k\t0.40\t\t0.35\t\t0.225\t\t0.849\t\t0.39m - 1.7m / 13.4m                                                                   \n",
      "5\t22k\t0.37\t\t0.37\t\t0.195\t\t0.827\t\t0.38m - 2.1m / 13.7m                                                                   \n",
      "6\t26k\t0.35\t\t0.40\t\t0.165\t\t0.754\t\t0.38m - 2.6m / 13.7m                                                                   \n",
      "7\t30k\t0.34\t\t0.39\t\t0.184\t\t0.790\t\t0.38m - 3.0m / 13.7m                                                                   \n",
      "8\t35k\t0.33\t\t0.42\t\t0.199\t\t0.784\t\t0.38m - 3.4m / 13.7m                                                                   \n",
      "9\t39k\t0.33\t\t0.40\t\t0.188\t\t0.806\t\t0.38m - 3.9m / 13.9m                                                                   \n",
      "VAL f1\t0.22477064220183487 - (0.22477064220183487)                                                                     \n",
      "VAL loss\t0.3488379054599339                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3488379054599339\n",
      "        | \\     )|_\tf1: 0.22477064220183487\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\12 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 64.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.015512784979028261               |\n",
      "|  noam_learning_rate_warmup   |                        2359                       |\n",
      "|  noam_learning_rate_factor   |                 0.7838677230325278                |\n",
      "|          adam_beta1          |                 0.8600074163310488                |\n",
      "|          adam_beta2          |                 0.9327796061291654                |\n",
      "|           adam_eps           |               2.4327792488314483e-05              |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                 0.7678309241772228                |\n",
      "|     pointwise_layer_size     |                         76                        |\n",
      "|      last_layer_dropout      |                 0.5474983531002635                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        135                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 31%|█████████████▉                               | 31/100 [3:28:44<4:55:27, 256.92s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95e77b24612e43a18b56f8332f9ad6c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.39\t\t0.25\t\t0.049\t\t0.861\t\t0.27m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.26\t\t0.000\t\t0.940\t\t0.27m - 0.6m / 9.4m                                                                     \n",
      "3\t13k\t0.27\t\t0.27\t\t0.000\t\t0.940\t\t0.27m - 0.9m / 9.7m                                                                    \n",
      "4\t17k\t0.28\t\t0.26\t\t0.005\t\t0.937\t\t0.27m - 1.2m / 9.5m                                                                    \n",
      "5\t22k\t0.27\t\t0.26\t\t0.029\t\t0.934\t\t0.27m - 1.6m / 9.7m                                                                    \n",
      "6\t26k\t0.27\t\t0.25\t\t0.096\t\t0.930\t\t0.27m - 1.9m / 9.7m                                                                    \n",
      "7\t30k\t0.26\t\t0.25\t\t0.122\t\t0.857\t\t0.27m - 2.2m / 9.7m                                                                    \n",
      "8\t35k\t0.24\t\t0.28\t\t0.158\t\t0.808\t\t0.26m - 2.5m / 9.8m                                                                    \n",
      "9\t39k\t0.24\t\t0.28\t\t0.177\t\t0.801\t\t0.26m - 2.8m / 9.7m                                                                    \n",
      "10\t44k\t0.23\t\t0.30\t\t0.174\t\t0.776\t\t0.27m - 3.2m / 9.8m                                                                   \n",
      "11\t48k\t0.22\t\t0.28\t\t0.193\t\t0.794\t\t0.27m - 3.5m / 9.9m                                                                   \n",
      "12\t52k\t0.22\t\t0.34\t\t0.145\t\t0.723\t\t0.26m - 3.8m / 9.9m                                                                   \n",
      "13\t57k\t0.22\t\t0.29\t\t0.165\t\t0.762\t\t0.27m - 4.1m / 9.9m                                                                   \n",
      "14\t61k\t0.21\t\t0.29\t\t0.186\t\t0.776\t\t0.27m - 4.4m / 10.0m                                                                  \n",
      "15\t65k\t0.21\t\t0.33\t\t0.156\t\t0.750\t\t0.27m - 4.8m / 10.1m                                                                  \n",
      "16\t70k\t0.21\t\t0.29\t\t0.196\t\t0.785\t\t0.27m - 5.1m / 10.2m                                                                  \n",
      "17\t74k\t0.20\t\t0.31\t\t0.189\t\t0.780\t\t0.27m - 5.4m / 10.2m                                                                  \n",
      "18\t78k\t0.20\t\t0.29\t\t0.175\t\t0.786\t\t0.27m - 5.7m / 10.2m                                                                  \n",
      "19\t83k\t0.20\t\t0.30\t\t0.174\t\t0.771\t\t0.27m - 6.0m / 10.3m                                                                  \n",
      "20\t87k\t0.20\t\t0.31\t\t0.178\t\t0.774\t\t0.27m - 6.4m / 10.4m                                                                  \n",
      "21\t91k\t0.20\t\t0.30\t\t0.194\t\t0.810\t\t0.27m - 6.7m / 10.4m                                                                  \n",
      "VAL f1\t0.19631901840490798 - (0.19631901840490798)                                                                     \n",
      "VAL loss\t0.24698136001825333                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24698136001825333\n",
      "        | \\     )|_\tf1: 0.19631901840490798\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\13 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 59.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         59                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.000215891948844624               |\n",
      "|  noam_learning_rate_warmup   |                        6719                       |\n",
      "|  noam_learning_rate_factor   |                 1.9256926757489452                |\n",
      "|          adam_beta1          |                 0.8797185944250261                |\n",
      "|          adam_beta2          |                 0.8213414946324453                |\n",
      "|           adam_eps           |                 0.8807091497004007                |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.6226106546459327                |\n",
      "|     pointwise_layer_size     |                        205                        |\n",
      "|      last_layer_dropout      |                0.11284552439445762                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        123                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 32%|██████████████▍                              | 32/100 [3:35:46<5:47:23, 306.52s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c2db352ce8f499abab943d5a9151fbd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.46\t\t0.58\t\t0.059\t\t0.265\t\t0.30m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.43\t\t0.46\t\t0.063\t\t0.380\t\t0.30m - 0.6m / 10.5m                                                                    \n",
      "3\t13k\t0.38\t\t0.35\t\t0.103\t\t0.680\t\t0.30m - 1.0m / 10.6m                                                                   \n",
      "4\t17k\t0.34\t\t0.30\t\t0.121\t\t0.843\t\t0.30m - 1.4m / 10.7m                                                                   \n",
      "5\t22k\t0.32\t\t0.29\t\t0.145\t\t0.896\t\t0.30m - 1.7m / 10.7m                                                                   \n",
      "6\t26k\t0.30\t\t0.28\t\t0.123\t\t0.901\t\t0.30m - 2.1m / 10.9m                                                                   \n",
      "7\t31k\t0.30\t\t0.28\t\t0.114\t\t0.899\t\t0.30m - 2.4m / 10.9m                                                                   \n",
      "8\t35k\t0.30\t\t0.28\t\t0.126\t\t0.906\t\t0.30m - 2.8m / 10.9m                                                                   \n",
      "9\t39k\t0.30\t\t0.28\t\t0.109\t\t0.895\t\t0.30m - 3.1m / 11.0m                                                                   \n",
      "10\t44k\t0.29\t\t0.27\t\t0.112\t\t0.901\t\t0.30m - 3.5m / 11.0m                                                                  \n",
      "VAL f1\t0.14507772020725387 - (0.14507772020725387)                                                                     \n",
      "VAL loss\t0.27467229810811705                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.27467229810811705\n",
      "        | \\     )|_\tf1: 0.14507772020725387\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\14 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 44.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         44                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0008266402664130388               |\n",
      "|  noam_learning_rate_warmup   |                        3584                       |\n",
      "|  noam_learning_rate_factor   |                0.42151381048810155                |\n",
      "|          adam_beta1          |                 0.7841969612043149                |\n",
      "|          adam_beta2          |                 0.9935243362850416                |\n",
      "|           adam_eps           |               1.6191190232519173e-06              |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.22791248338639158                |\n",
      "|     pointwise_layer_size     |                         51                        |\n",
      "|      last_layer_dropout      |                0.19825677574463346                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        163                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 33%|██████████████▊                              | 33/100 [3:39:37<5:16:59, 283.88s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8eb0e35789774ac18d342df214a42053",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.60\t\t0.36\t\t0.074\t\t0.851\t\t0.45m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.39\t\t0.31\t\t0.000\t\t0.938\t\t0.45m - 1.0m / 15.7m                                                                    \n",
      "3\t13k\t0.37\t\t0.31\t\t0.059\t\t0.938\t\t0.45m - 1.5m / 15.8m                                                                   \n",
      "4\t17k\t0.36\t\t0.29\t\t0.165\t\t0.918\t\t0.45m - 2.0m / 15.9m                                                                   \n",
      "5\t22k\t0.33\t\t0.28\t\t0.216\t\t0.908\t\t0.45m - 2.5m / 15.9m                                                                   \n",
      "6\t26k\t0.31\t\t0.26\t\t0.248\t\t0.895\t\t0.45m - 3.0m / 16.0m                                                                   \n",
      "7\t30k\t0.29\t\t0.26\t\t0.243\t\t0.874\t\t0.45m - 3.5m / 16.0m                                                                   \n",
      "8\t35k\t0.27\t\t0.27\t\t0.233\t\t0.867\t\t0.45m - 4.0m / 16.2m                                                                   \n",
      "9\t39k\t0.26\t\t0.27\t\t0.217\t\t0.857\t\t0.45m - 4.5m / 16.1m                                                                   \n",
      "10\t44k\t0.24\t\t0.27\t\t0.307\t\t0.894\t\t0.45m - 5.0m / 16.2m                                                                  \n",
      "11\t48k\t0.23\t\t0.28\t\t0.273\t\t0.876\t\t0.45m - 5.5m / 16.2m                                                                  \n",
      "12\t52k\t0.21\t\t0.28\t\t0.225\t\t0.856\t\t0.45m - 6.0m / 16.3m                                                                  \n",
      "13\t57k\t0.20\t\t0.28\t\t0.256\t\t0.878\t\t0.45m - 6.5m / 16.4m                                                                  \n",
      "14\t61k\t0.20\t\t0.30\t\t0.250\t\t0.877\t\t0.45m - 7.0m / 16.4m                                                                  \n",
      "15\t65k\t0.19\t\t0.29\t\t0.273\t\t0.880\t\t0.45m - 7.5m / 16.5m                                                                  \n",
      "VAL f1\t0.30666666666666664 - (0.30666666666666664)                                                                     \n",
      "VAL loss\t0.2618545185435902                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2618545185435902\n",
      "        | \\     )|_\tf1: 0.30666666666666664\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\15 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 37.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         37                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0046324775219240225               |\n",
      "|  noam_learning_rate_warmup   |                        2586                       |\n",
      "|  noam_learning_rate_factor   |                 2.7954229917522335                |\n",
      "|          adam_beta1          |                 0.9124003172456776                |\n",
      "|          adam_beta2          |                 0.9021737713101919                |\n",
      "|           adam_eps           |               7.041533155900994e-10               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.4831245384027619                |\n",
      "|     pointwise_layer_size     |                         96                        |\n",
      "|      last_layer_dropout      |                 0.3694205762178871                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        110                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 34%|███████████████▎                             | 34/100 [3:47:34<6:15:58, 341.80s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "293664ee2e5d422280d40ba6fc472c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.50\t\t0.37\t\t0.084\t\t0.915\t\t0.39m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.43\t\t0.32\t\t0.264\t\t0.876\t\t0.39m - 0.8m / 13.7m                                                                    \n",
      "3\t13k\t0.38\t\t0.36\t\t0.188\t\t0.829\t\t0.39m - 1.3m / 13.6m                                                                   \n",
      "4\t17k\t0.36\t\t0.40\t\t0.191\t\t0.796\t\t0.38m - 1.7m / 13.9m                                                                   \n",
      "5\t22k\t0.35\t\t0.34\t\t0.254\t\t0.867\t\t0.37m - 2.1m / 13.6m                                                                   \n",
      "6\t26k\t0.35\t\t0.38\t\t0.211\t\t0.872\t\t0.38m - 2.6m / 13.4m                                                                   \n",
      "7\t30k\t0.34\t\t0.41\t\t0.208\t\t0.863\t\t0.37m - 3.0m / 13.7m                                                                   \n",
      "VAL f1\t0.2641843971631206 - (0.2641843971631206)                                                                       \n",
      "VAL loss\t0.3237404874853186                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3237404874853186\n",
      "        | \\     )|_\tf1: 0.2641843971631206\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\16 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 54.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         54                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.015137463096661063               |\n",
      "|  noam_learning_rate_warmup   |                        1623                       |\n",
      "|  noam_learning_rate_factor   |                 1.529468439529946                 |\n",
      "|          adam_beta1          |                 0.9562585262191479                |\n",
      "|          adam_beta2          |                 0.9547700355649685                |\n",
      "|           adam_eps           |                0.033785311604393006               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.09346526705625494                |\n",
      "|     pointwise_layer_size     |                        204                        |\n",
      "|      last_layer_dropout      |                0.06989565853956162                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         78                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 35%|███████████████▋                             | 35/100 [3:50:58<5:25:16, 300.25s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a88c8a004f5472aa1411c60507a5415",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.42\t\t0.27\t\t0.119\t\t0.857\t\t0.23m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.31\t\t0.25\t\t0.149\t\t0.890\t\t0.23m - 0.5m / 8.0m                                                                     \n",
      "3\t13k\t0.28\t\t0.23\t\t0.227\t\t0.883\t\t0.24m - 0.8m / 8.3m                                                                    \n",
      "4\t17k\t0.24\t\t0.22\t\t0.163\t\t0.870\t\t0.24m - 1.1m / 8.4m                                                                    \n",
      "5\t22k\t0.22\t\t0.24\t\t0.200\t\t0.879\t\t0.24m - 1.4m / 8.5m                                                                    \n",
      "6\t26k\t0.20\t\t0.22\t\t0.219\t\t0.881\t\t0.23m - 1.7m / 8.5m                                                                    \n",
      "7\t30k\t0.19\t\t0.25\t\t0.227\t\t0.847\t\t0.23m - 1.9m / 8.3m                                                                    \n",
      "8\t35k\t0.17\t\t0.26\t\t0.179\t\t0.867\t\t0.23m - 2.2m / 8.3m                                                                    \n",
      "VAL f1\t0.22682445759368836 - (0.22682445759368836)                                                                     \n",
      "VAL loss\t0.2209954980819944                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2209954980819944\n",
      "        | \\     )|_\tf1: 0.22682445759368836\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\17 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 54.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         54                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.015152970465789556               |\n",
      "|  noam_learning_rate_warmup   |                        1696                       |\n",
      "|  noam_learning_rate_factor   |                 1.3727178331876861                |\n",
      "|          adam_beta1          |                 0.9552014183096693                |\n",
      "|          adam_beta2          |                 0.8635653979316168                |\n",
      "|           adam_eps           |                0.09732344402813746                |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.4031520484427492                |\n",
      "|     pointwise_layer_size     |                        266                        |\n",
      "|      last_layer_dropout      |                0.04506226207990453                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         76                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 36%|████████████████▏                            | 36/100 [3:53:31<4:33:19, 256.25s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdca9937e2734bb6a72c15247db86637",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.48\t\t0.28\t\t0.159\t\t0.873\t\t0.18m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.26\t\t0.177\t\t0.924\t\t0.18m - 0.4m / 6.3m                                                                     \n",
      "3\t13k\t0.31\t\t0.25\t\t0.209\t\t0.924\t\t0.18m - 0.6m / 6.3m                                                                    \n",
      "4\t17k\t0.30\t\t0.25\t\t0.201\t\t0.907\t\t0.18m - 0.9m / 6.5m                                                                    \n",
      "5\t22k\t0.29\t\t0.24\t\t0.236\t\t0.909\t\t0.18m - 1.1m / 6.5m                                                                    \n",
      "6\t26k\t0.27\t\t0.24\t\t0.253\t\t0.882\t\t0.18m - 1.3m / 6.4m                                                                    \n",
      "7\t30k\t0.25\t\t0.24\t\t0.217\t\t0.861\t\t0.18m - 1.6m / 6.5m                                                                    \n",
      "8\t35k\t0.24\t\t0.24\t\t0.232\t\t0.873\t\t0.18m - 1.8m / 6.6m                                                                    \n",
      "9\t39k\t0.23\t\t0.25\t\t0.231\t\t0.852\t\t0.18m - 2.0m / 6.7m                                                                    \n",
      "10\t43k\t0.22\t\t0.25\t\t0.262\t\t0.872\t\t0.18m - 2.3m / 6.8m                                                                   \n",
      "11\t48k\t0.21\t\t0.26\t\t0.243\t\t0.863\t\t0.18m - 2.5m / 6.8m                                                                   \n",
      "12\t52k\t0.20\t\t0.26\t\t0.234\t\t0.860\t\t0.18m - 2.7m / 6.8m                                                                   \n",
      "13\t56k\t0.19\t\t0.29\t\t0.217\t\t0.829\t\t0.18m - 2.9m / 7.0m                                                                   \n",
      "14\t60k\t0.19\t\t0.29\t\t0.200\t\t0.807\t\t0.18m - 3.2m / 6.9m                                                                   \n",
      "15\t65k\t0.18\t\t0.30\t\t0.199\t\t0.820\t\t0.18m - 3.4m / 7.0m                                                                   \n",
      "VAL f1\t0.26174496644295303 - (0.26174496644295303)                                                                     \n",
      "VAL loss\t0.2354254646906777                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2354254646906777\n",
      "        | \\     )|_\tf1: 0.26174496644295303\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\18 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 51.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         51                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.023360344108494226               |\n",
      "|  noam_learning_rate_warmup   |                        1205                       |\n",
      "|  noam_learning_rate_factor   |                0.05738955063171858                |\n",
      "|          adam_beta1          |                 0.9696022242845097                |\n",
      "|          adam_beta2          |                 0.9543257327608305                |\n",
      "|           adam_eps           |                0.02818602065426046                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.325491392551484                 |\n",
      "|     pointwise_layer_size     |                        296                        |\n",
      "|      last_layer_dropout      |                 0.5607731355522034                |\n",
      "|   output_conv_num_filters    |                         16                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         3                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         54                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 37%|████████████████▋                            | 37/100 [3:57:15<4:18:46, 246.45s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1601702841df4f2ba7e09810c071ac8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.54\t\t0.40\t\t0.033\t\t0.769\t\t0.27m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.37\t\t0.34\t\t0.082\t\t0.873\t\t0.27m - 0.6m / 9.4m                                                                     \n",
      "3\t13k\t0.34\t\t0.34\t\t0.089\t\t0.913\t\t0.26m - 0.9m / 9.6m                                                                    \n",
      "4\t17k\t0.34\t\t0.34\t\t0.139\t\t0.902\t\t0.27m - 1.2m / 9.4m                                                                    \n",
      "5\t22k\t0.33\t\t0.34\t\t0.124\t\t0.891\t\t0.26m - 1.6m / 9.6m                                                                    \n",
      "6\t26k\t0.33\t\t0.33\t\t0.156\t\t0.902\t\t0.26m - 1.9m / 9.5m                                                                    \n",
      "7\t30k\t0.33\t\t0.33\t\t0.163\t\t0.878\t\t0.26m - 2.2m / 9.6m                                                                    \n",
      "8\t35k\t0.31\t\t0.32\t\t0.218\t\t0.871\t\t0.27m - 2.5m / 9.6m                                                                    \n",
      "9\t39k\t0.30\t\t0.31\t\t0.204\t\t0.870\t\t0.27m - 2.8m / 9.8m                                                                    \n",
      "10\t43k\t0.29\t\t0.29\t\t0.209\t\t0.859\t\t0.29m - 3.2m / 9.9m                                                                   \n",
      "11\t48k\t0.28\t\t0.29\t\t0.222\t\t0.844\t\t0.27m - 3.5m / 10.4m                                                                  \n",
      "12\t52k\t0.27\t\t0.28\t\t0.198\t\t0.839\t\t0.27m - 4.0m / 10.3m                                                                  \n",
      "13\t56k\t0.26\t\t0.27\t\t0.217\t\t0.848\t\t0.26m - 4.3m / 10.3m                                                                  \n",
      "14\t61k\t0.25\t\t0.27\t\t0.199\t\t0.842\t\t0.27m - 4.6m / 10.1m                                                                  \n",
      "15\t65k\t0.24\t\t0.27\t\t0.220\t\t0.842\t\t0.26m - 4.9m / 10.3m                                                                  \n",
      "16\t69k\t0.23\t\t0.27\t\t0.196\t\t0.825\t\t0.26m - 5.2m / 10.3m                                                                  \n",
      "VAL f1\t0.2220555138784696 - (0.2220555138784696)                                                                       \n",
      "VAL loss\t0.2680471938531272                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2680471938531272\n",
      "        | \\     )|_\tf1: 0.2220555138784696\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\19 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 57.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         57                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.011755382916119973               |\n",
      "|  noam_learning_rate_warmup   |                        1918                       |\n",
      "|  noam_learning_rate_factor   |                 2.358234069439205                 |\n",
      "|          adam_beta1          |                 0.9492210873826735                |\n",
      "|          adam_beta2          |                 0.9165092856513793                |\n",
      "|           adam_eps           |               8.014542612552399e-05               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.11593744762519548                |\n",
      "|     pointwise_layer_size     |                        194                        |\n",
      "|      last_layer_dropout      |                0.44681258242410204                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         57                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 38%|█████████████████                            | 38/100 [4:02:50<4:42:17, 273.19s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27fb8ac2008843a8b7ce564b7c734ec9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.39\t\t0.26\t\t0.151\t\t0.922\t\t0.17m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.25\t\t0.194\t\t0.901\t\t0.16m - 0.4m / 5.9m                                                                     \n",
      "3\t13k\t0.25\t\t0.23\t\t0.199\t\t0.893\t\t0.16m - 0.6m / 5.8m                                                                    \n",
      "4\t17k\t0.21\t\t0.24\t\t0.269\t\t0.887\t\t0.16m - 0.8m / 5.9m                                                                    \n",
      "5\t22k\t0.19\t\t0.26\t\t0.245\t\t0.881\t\t0.16m - 1.0m / 5.9m                                                                    \n",
      "6\t26k\t0.17\t\t0.27\t\t0.240\t\t0.880\t\t0.17m - 1.3m / 6.0m                                                                    \n",
      "7\t30k\t0.15\t\t0.26\t\t0.245\t\t0.869\t\t0.16m - 1.5m / 6.2m                                                                    \n",
      "8\t35k\t0.13\t\t0.28\t\t0.256\t\t0.900\t\t0.17m - 1.7m / 6.1m                                                                    \n",
      "9\t39k\t0.12\t\t0.28\t\t0.255\t\t0.888\t\t0.16m - 1.9m / 6.2m                                                                    \n",
      "VAL f1\t0.2688588007736944 - (0.2688588007736944)                                                                       \n",
      "VAL loss\t0.2313747963710138                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2313747963710138\n",
      "        | \\     )|_\tf1: 0.2688588007736944\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\20 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 47.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         47                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.019934722458405996               |\n",
      "|  noam_learning_rate_warmup   |                        2637                       |\n",
      "|  noam_learning_rate_factor   |                 1.8879027825212498                |\n",
      "|          adam_beta1          |                  0.93884401051719                 |\n",
      "|          adam_beta2          |                 0.8912916580387842                |\n",
      "|           adam_eps           |                 0.4048573100654902                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |               0.0007236441174616887               |\n",
      "|     pointwise_layer_size     |                        258                        |\n",
      "|      last_layer_dropout      |                0.07704781040549244                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         80                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 39%|█████████████████▌                           | 39/100 [4:05:02<3:54:44, 230.89s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d959c0fdaeb45c5a953ca400b560968",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.48\t\t0.31\t\t0.091\t\t0.910\t\t0.25m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.36\t\t0.30\t\t0.121\t\t0.913\t\t0.25m - 0.6m / 8.9m                                                                     \n",
      "3\t13k\t0.34\t\t0.29\t\t0.088\t\t0.913\t\t0.25m - 0.9m / 9.0m                                                                    \n",
      "4\t17k\t0.33\t\t0.29\t\t0.080\t\t0.883\t\t0.25m - 1.2m / 8.9m                                                                    \n",
      "5\t22k\t0.32\t\t0.27\t\t0.166\t\t0.906\t\t0.26m - 1.5m / 9.0m                                                                    \n",
      "6\t26k\t0.30\t\t0.26\t\t0.220\t\t0.905\t\t0.25m - 1.8m / 9.3m                                                                    \n",
      "7\t30k\t0.28\t\t0.25\t\t0.210\t\t0.884\t\t0.25m - 2.1m / 9.2m                                                                    \n",
      "8\t35k\t0.26\t\t0.26\t\t0.176\t\t0.866\t\t0.26m - 2.4m / 9.2m                                                                    \n",
      "9\t39k\t0.25\t\t0.24\t\t0.224\t\t0.889\t\t0.25m - 2.7m / 9.4m                                                                    \n",
      "10\t43k\t0.23\t\t0.25\t\t0.244\t\t0.885\t\t0.25m - 3.0m / 9.4m                                                                   \n",
      "11\t48k\t0.22\t\t0.25\t\t0.236\t\t0.889\t\t0.25m - 3.3m / 9.3m                                                                   \n",
      "12\t52k\t0.21\t\t0.25\t\t0.207\t\t0.889\t\t0.26m - 3.6m / 9.4m                                                                   \n",
      "13\t56k\t0.20\t\t0.27\t\t0.236\t\t0.878\t\t0.25m - 3.9m / 9.6m                                                                   \n",
      "14\t61k\t0.19\t\t0.26\t\t0.288\t\t0.906\t\t0.25m - 4.2m / 9.6m                                                                   \n",
      "15\t65k\t0.18\t\t0.26\t\t0.256\t\t0.900\t\t0.25m - 4.5m / 9.6m                                                                   \n",
      "16\t69k\t0.17\t\t0.28\t\t0.224\t\t0.872\t\t0.25m - 4.8m / 9.7m                                                                   \n",
      "17\t74k\t0.16\t\t0.27\t\t0.261\t\t0.891\t\t0.25m - 5.1m / 9.7m                                                                   \n",
      "18\t78k\t0.15\t\t0.28\t\t0.233\t\t0.874\t\t0.25m - 5.5m / 9.8m                                                                   \n",
      "19\t82k\t0.14\t\t0.28\t\t0.246\t\t0.896\t\t0.26m - 5.8m / 9.8m                                                                   \n",
      "VAL f1\t0.287596048298573 - (0.287596048298573)                                                                         \n",
      "VAL loss\t0.24452989786229234                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24452989786229234\n",
      "        | \\     )|_\tf1: 0.287596048298573\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\21 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 33.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         33                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.008679685943678324               |\n",
      "|  noam_learning_rate_warmup   |                        1222                       |\n",
      "|  noam_learning_rate_factor   |                 0.6046176406059689                |\n",
      "|          adam_beta1          |                 0.9983625768200569                |\n",
      "|          adam_beta2          |                 0.8190970302318883                |\n",
      "|           adam_eps           |                0.002338253622388337               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.6558834604330337                |\n",
      "|     pointwise_layer_size     |                        219                        |\n",
      "|      last_layer_dropout      |                0.29538733278277657                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        178                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 40%|██████████████████                           | 40/100 [4:11:10<4:31:59, 271.99s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65f38c7c244847779d31211e5d798448",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.61\t\t0.47\t\t0.105\t\t0.906\t\t0.29m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.50\t\t0.52\t\t0.051\t\t0.651\t\t0.30m - 0.6m / 10.3m                                                                    \n",
      "3\t13k\t0.50\t\t0.49\t\t0.000\t\t0.940\t\t0.30m - 1.0m / 10.6m                                                                   \n",
      "4\t17k\t0.50\t\t0.43\t\t0.000\t\t0.940\t\t0.30m - 1.3m / 10.6m                                                                   \n",
      "5\t22k\t0.50\t\t0.41\t\t0.000\t\t0.940\t\t0.30m - 1.7m / 10.6m                                                                   \n",
      "6\t26k\t0.50\t\t0.44\t\t0.000\t\t0.940\t\t0.30m - 2.0m / 10.8m                                                                   \n",
      "VAL f1\t0.10479041916167664 - (0.10479041916167664)                                                                     \n",
      "VAL loss\t0.41341030761886893                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.41341030761886893\n",
      "        | \\     )|_\tf1: 0.10479041916167664\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\22 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 62.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         62                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.02899251880210331                |\n",
      "|  noam_learning_rate_warmup   |                        3836                       |\n",
      "|  noam_learning_rate_factor   |                 1.5733172576455363                |\n",
      "|          adam_beta1          |                 0.8947907481175669                |\n",
      "|          adam_beta2          |                 0.7737347560683068                |\n",
      "|           adam_eps           |                0.040046922931353646               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.16555331951810104                |\n",
      "|     pointwise_layer_size     |                        178                        |\n",
      "|      last_layer_dropout      |                0.15041815027056968                |\n",
      "|   output_conv_num_filters    |                        104                        |\n",
      "|   output_conv_kernel_size    |                         7                         |\n",
      "|      output_conv_stride      |                         1                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         66                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 41%|██████████████████▍                          | 41/100 [4:13:45<3:52:56, 236.89s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f34aa7d7d0ba4189b466e9014d4fafb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.32\t\t0.23\t\t0.070\t\t0.935\t\t0.29m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.27\t\t0.23\t\t0.149\t\t0.913\t\t0.30m - 0.6m / 10.3m                                                                    \n",
      "3\t13k\t0.24\t\t0.21\t\t0.206\t\t0.881\t\t0.29m - 1.0m / 10.5m                                                                   \n",
      "4\t17k\t0.21\t\t0.20\t\t0.246\t\t0.871\t\t0.30m - 1.3m / 10.4m                                                                   \n",
      "5\t22k\t0.19\t\t0.20\t\t0.224\t\t0.845\t\t0.29m - 1.7m / 10.6m                                                                   \n",
      "6\t26k\t0.17\t\t0.19\t\t0.255\t\t0.868\t\t0.29m - 2.0m / 10.6m                                                                   \n",
      "7\t30k\t0.16\t\t0.20\t\t0.234\t\t0.854\t\t0.30m - 2.4m / 10.6m                                                                   \n",
      "8\t35k\t0.14\t\t0.20\t\t0.252\t\t0.868\t\t0.30m - 2.7m / 10.8m                                                                   \n",
      "9\t39k\t0.13\t\t0.20\t\t0.256\t\t0.888\t\t0.30m - 3.1m / 10.9m                                                                   \n",
      "10\t43k\t0.11\t\t0.21\t\t0.260\t\t0.876\t\t0.30m - 3.4m / 10.9m                                                                  \n",
      "11\t48k\t0.10\t\t0.20\t\t0.298\t\t0.894\t\t0.30m - 3.8m / 10.9m                                                                  \n",
      "12\t52k\t0.09\t\t0.21\t\t0.274\t\t0.880\t\t0.30m - 4.2m / 11.0m                                                                  \n",
      "13\t56k\t0.08\t\t0.21\t\t0.295\t\t0.890\t\t0.30m - 4.5m / 11.0m                                                                  \n",
      "14\t61k\t0.07\t\t0.21\t\t0.288\t\t0.892\t\t0.29m - 4.8m / 11.1m                                                                  \n",
      "15\t65k\t0.06\t\t0.22\t\t0.292\t\t0.897\t\t0.30m - 5.2m / 11.1m                                                                  \n",
      "16\t69k\t0.05\t\t0.23\t\t0.329\t\t0.913\t\t0.29m - 5.5m / 11.1m                                                                  \n",
      "17\t74k\t0.05\t\t0.22\t\t0.334\t\t0.918\t\t0.30m - 5.9m / 11.2m                                                                  \n",
      "18\t78k\t0.04\t\t0.23\t\t0.297\t\t0.915\t\t0.30m - 6.2m / 11.3m                                                                  \n",
      "19\t82k\t0.04\t\t0.24\t\t0.339\t\t0.921\t\t0.30m - 6.6m / 11.3m                                                                  \n",
      "20\t87k\t0.03\t\t0.23\t\t0.317\t\t0.915\t\t0.30m - 6.9m / 11.4m                                                                  \n",
      "21\t91k\t0.03\t\t0.24\t\t0.314\t\t0.921\t\t0.30m - 7.3m / 11.5m                                                                  \n",
      "22\t95k\t0.02\t\t0.25\t\t0.328\t\t0.921\t\t0.30m - 7.6m / 11.5m                                                                  \n",
      "23\t100k\t0.02\t\t0.26\t\t0.329\t\t0.925\t\t0.30m - 8.0m / 11.6m                                                                 \n",
      "24\t104k\t0.02\t\t0.26\t\t0.343\t\t0.925\t\t0.30m - 8.4m / 11.6m                                                                 \n",
      "25\t108k\t0.02\t\t0.26\t\t0.308\t\t0.928\t\t0.30m - 8.7m / 11.7m                                                                 \n",
      "26\t113k\t0.01\t\t0.27\t\t0.332\t\t0.931\t\t0.30m - 9.1m / 11.7m                                                                 \n",
      "27\t117k\t0.01\t\t0.26\t\t0.297\t\t0.926\t\t0.30m - 9.4m / 11.8m                                                                 \n",
      "28\t122k\t0.01\t\t0.27\t\t0.324\t\t0.929\t\t0.30m - 9.8m / 11.8m                                                                 \n",
      "29\t126k\t0.01\t\t0.28\t\t0.328\t\t0.928\t\t0.30m - 10.1m / 11.9m                                                                \n",
      "VAL f1\t0.3425692695214106 - (0.3425692695214106)                                                                       \n",
      "VAL loss\t0.19295881896890618                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19295881896890618\n",
      "        | \\     )|_\tf1: 0.3425692695214106\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\23 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 40.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         40                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.012932589200628218               |\n",
      "|  noam_learning_rate_warmup   |                        3119                       |\n",
      "|  noam_learning_rate_factor   |                 1.142143132814446                 |\n",
      "|          adam_beta1          |                 0.9958683784390912                |\n",
      "|          adam_beta2          |                 0.7007934114225352                |\n",
      "|           adam_eps           |               0.00033582547679550357              |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.7429241840556284                |\n",
      "|     pointwise_layer_size     |                        302                        |\n",
      "|      last_layer_dropout      |                 0.3772534049251174                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         93                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 42%|██████████████████▉                          | 42/100 [4:24:21<5:44:39, 356.55s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22a55aa55e96470eb8689e248a28181b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.55\t\t0.42\t\t0.000\t\t0.937\t\t0.29m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.44\t\t0.45\t\t0.147\t\t0.859\t\t0.29m - 0.6m / 10.2m                                                                    \n",
      "3\t13k\t0.43\t\t0.46\t\t0.025\t\t0.783\t\t0.29m - 1.0m / 10.3m                                                                   \n",
      "4\t17k\t0.49\t\t0.42\t\t0.079\t\t0.849\t\t0.29m - 1.3m / 10.4m                                                                   \n",
      "5\t22k\t0.49\t\t0.43\t\t0.000\t\t0.940\t\t0.29m - 1.7m / 10.4m                                                                   \n",
      "6\t26k\t0.51\t\t0.81\t\t0.000\t\t0.940\t\t0.29m - 2.0m / 10.4m                                                                   \n",
      "7\t30k\t0.51\t\t0.93\t\t0.000\t\t0.940\t\t0.29m - 2.3m / 10.5m                                                                   \n",
      "VAL f1\t0.14714424007744434 - (0.14714424007744434)                                                                     \n",
      "VAL loss\t0.41704958015018034                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.41704958015018034\n",
      "        | \\     )|_\tf1: 0.14714424007744434\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\24 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 54.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         54                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0053052455981467935               |\n",
      "|  noam_learning_rate_warmup   |                        1857                       |\n",
      "|  noam_learning_rate_factor   |                 0.8979442290300521                |\n",
      "|          adam_beta1          |                 0.9698469333725502                |\n",
      "|          adam_beta2          |                 0.9732398462823315                |\n",
      "|           adam_eps           |               4.1499546174071495e-05              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.26814038961550635                |\n",
      "|     pointwise_layer_size     |                        151                        |\n",
      "|      last_layer_dropout      |                0.22561293261178117                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         74                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 43%|███████████████████▎                         | 43/100 [4:27:04<4:43:34, 298.50s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed13d37caa56444396eb8743277235f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.40\t\t0.27\t\t0.110\t\t0.910\t\t0.17m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.26\t\t0.070\t\t0.906\t\t0.18m - 0.4m / 6.0m                                                                     \n",
      "3\t13k\t0.31\t\t0.25\t\t0.112\t\t0.915\t\t0.17m - 0.6m / 6.2m                                                                    \n",
      "4\t17k\t0.28\t\t0.24\t\t0.253\t\t0.901\t\t0.17m - 0.8m / 6.2m                                                                    \n",
      "5\t22k\t0.25\t\t0.22\t\t0.249\t\t0.899\t\t0.18m - 1.1m / 6.3m                                                                    \n",
      "6\t26k\t0.23\t\t0.21\t\t0.261\t\t0.897\t\t0.17m - 1.3m / 6.4m                                                                    \n",
      "7\t30k\t0.20\t\t0.22\t\t0.218\t\t0.886\t\t0.18m - 1.5m / 6.3m                                                                    \n",
      "8\t35k\t0.18\t\t0.22\t\t0.272\t\t0.882\t\t0.18m - 1.8m / 6.6m                                                                    \n",
      "9\t39k\t0.16\t\t0.23\t\t0.274\t\t0.891\t\t0.18m - 2.0m / 6.6m                                                                    \n",
      "10\t43k\t0.14\t\t0.24\t\t0.296\t\t0.904\t\t0.18m - 2.2m / 6.7m                                                                   \n",
      "11\t48k\t0.13\t\t0.25\t\t0.280\t\t0.895\t\t0.17m - 2.5m / 6.7m                                                                   \n",
      "12\t52k\t0.11\t\t0.25\t\t0.270\t\t0.874\t\t0.18m - 2.7m / 6.7m                                                                   \n",
      "13\t56k\t0.10\t\t0.27\t\t0.269\t\t0.881\t\t0.17m - 2.9m / 6.8m                                                                   \n",
      "14\t60k\t0.09\t\t0.29\t\t0.275\t\t0.877\t\t0.17m - 3.1m / 6.8m                                                                   \n",
      "15\t65k\t0.08\t\t0.31\t\t0.259\t\t0.875\t\t0.18m - 3.4m / 6.8m                                                                   \n",
      "VAL f1\t0.29559748427672955 - (0.29559748427672955)                                                                     \n",
      "VAL loss\t0.20946792572263687                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.20946792572263687\n",
      "        | \\     )|_\tf1: 0.29559748427672955\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\25 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 44.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         44                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.009891211883135081               |\n",
      "|  noam_learning_rate_warmup   |                        2640                       |\n",
      "|  noam_learning_rate_factor   |                 2.212930974930818                 |\n",
      "|          adam_beta1          |                 0.8735835663484197                |\n",
      "|          adam_beta2          |                 0.9243381781010245                |\n",
      "|           adam_eps           |               0.0008996414355070907               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                 0.5521639041091726                |\n",
      "|     pointwise_layer_size     |                        279                        |\n",
      "|      last_layer_dropout      |                 0.7982753005481383                |\n",
      "|   output_conv_num_filters    |                        124                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         1                         |\n",
      "|     output_conv_padding      |                         5                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         62                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 44%|███████████████████▊                         | 44/100 [4:30:54<4:19:27, 277.99s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2f7771e96f7448a96afd87f17496b3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [5456 x 11], m2: [124 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\26 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 27.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         27                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.018647176356428045               |\n",
      "|  noam_learning_rate_warmup   |                        4416                       |\n",
      "|  noam_learning_rate_factor   |                 0.5348227172372164                |\n",
      "|          adam_beta1          |                 0.9091379679230784                |\n",
      "|          adam_beta2          |                 0.8402577072742837                |\n",
      "|           adam_eps           |                0.003594509237579648               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.36752492874244547                |\n",
      "|     pointwise_layer_size     |                        222                        |\n",
      "|      last_layer_dropout      |                 0.1732478885872665                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        155                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 45%|████████████████████▎                        | 45/100 [4:30:58<2:59:21, 195.67s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c25eabd02c194312af07884b453a7459",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.93\t\t0.57\t\t0.125\t\t0.853\t\t0.35m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.58\t\t0.45\t\t0.005\t\t0.940\t\t0.36m - 0.8m / 12.4m                                                                    \n",
      "3\t13k\t0.55\t\t0.45\t\t0.005\t\t0.939\t\t0.35m - 1.2m / 12.8m                                                                   \n",
      "4\t17k\t0.54\t\t0.45\t\t0.114\t\t0.931\t\t0.34m - 1.6m / 12.6m                                                                   \n",
      "5\t22k\t0.51\t\t0.42\t\t0.187\t\t0.921\t\t0.34m - 2.0m / 12.2m                                                                   \n",
      "6\t26k\t0.48\t\t0.40\t\t0.252\t\t0.900\t\t0.35m - 2.4m / 12.3m                                                                   \n",
      "7\t30k\t0.45\t\t0.41\t\t0.250\t\t0.895\t\t0.35m - 2.8m / 12.5m                                                                   \n",
      "8\t35k\t0.42\t\t0.41\t\t0.258\t\t0.883\t\t0.36m - 3.2m / 12.7m                                                                   \n",
      "9\t39k\t0.41\t\t0.41\t\t0.248\t\t0.869\t\t0.36m - 3.6m / 12.8m                                                                   \n",
      "10\t43k\t0.39\t\t0.42\t\t0.262\t\t0.873\t\t0.36m - 4.0m / 13.0m                                                                  \n",
      "11\t48k\t0.37\t\t0.44\t\t0.246\t\t0.858\t\t0.34m - 4.4m / 13.0m                                                                  \n",
      "12\t52k\t0.36\t\t0.46\t\t0.229\t\t0.845\t\t0.35m - 4.8m / 12.7m                                                                  \n",
      "13\t56k\t0.34\t\t0.46\t\t0.259\t\t0.860\t\t0.36m - 5.2m / 13.0m                                                                  \n",
      "14\t60k\t0.33\t\t0.47\t\t0.266\t\t0.863\t\t0.35m - 5.6m / 13.2m                                                                  \n",
      "15\t65k\t0.32\t\t0.48\t\t0.241\t\t0.834\t\t0.34m - 6.0m / 13.0m                                                                  \n",
      "16\t69k\t0.31\t\t0.48\t\t0.247\t\t0.839\t\t0.34m - 6.4m / 12.9m                                                                  \n",
      "17\t73k\t0.30\t\t0.49\t\t0.243\t\t0.843\t\t0.34m - 6.8m / 13.0m                                                                  \n",
      "18\t78k\t0.29\t\t0.52\t\t0.242\t\t0.831\t\t0.35m - 7.2m / 13.0m                                                                  \n",
      "19\t82k\t0.29\t\t0.48\t\t0.229\t\t0.848\t\t0.34m - 7.6m / 13.2m                                                                  \n",
      "VAL f1\t0.266147859922179 - (0.266147859922179)                                                                         \n",
      "VAL loss\t0.39859656662682863                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.39859656662682863\n",
      "        | \\     )|_\tf1: 0.266147859922179\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\27 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 47.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         47                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.031406541356049755               |\n",
      "|  noam_learning_rate_warmup   |                        3900                       |\n",
      "|  noam_learning_rate_factor   |                 1.2547245775961176                |\n",
      "|          adam_beta1          |                 0.9361757321751978                |\n",
      "|          adam_beta2          |                 0.865920233039387                 |\n",
      "|           adam_eps           |                 0.3274935091532244                |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.19515532720348586                |\n",
      "|     pointwise_layer_size     |                        187                        |\n",
      "|      last_layer_dropout      |                0.05061401757580655                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         45                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 46%|████████████████████▋                        | 46/100 [4:39:00<4:13:30, 281.68s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1da58361b6342dab81e2adca844396b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.58\t\t0.44\t\t0.062\t\t0.534\t\t0.23m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.42\t\t0.31\t\t0.119\t\t0.891\t\t0.23m - 0.5m / 8.2m                                                                     \n",
      "3\t13k\t0.37\t\t0.30\t\t0.095\t\t0.922\t\t0.23m - 0.8m / 8.1m                                                                    \n",
      "4\t17k\t0.36\t\t0.29\t\t0.019\t\t0.931\t\t0.23m - 1.1m / 8.1m                                                                    \n",
      "5\t22k\t0.36\t\t0.29\t\t0.015\t\t0.933\t\t0.23m - 1.4m / 8.3m                                                                    \n",
      "6\t26k\t0.35\t\t0.29\t\t0.039\t\t0.935\t\t0.23m - 1.6m / 8.3m                                                                    \n",
      "7\t30k\t0.35\t\t0.28\t\t0.129\t\t0.932\t\t0.23m - 1.9m / 8.4m                                                                    \n",
      "8\t35k\t0.35\t\t0.28\t\t0.114\t\t0.936\t\t0.24m - 2.2m / 8.4m                                                                    \n",
      "9\t39k\t0.34\t\t0.27\t\t0.099\t\t0.935\t\t0.24m - 2.5m / 8.8m                                                                    \n",
      "10\t43k\t0.33\t\t0.27\t\t0.199\t\t0.917\t\t0.24m - 2.8m / 8.8m                                                                   \n",
      "11\t48k\t0.31\t\t0.25\t\t0.201\t\t0.920\t\t0.25m - 3.1m / 8.9m                                                                   \n",
      "12\t52k\t0.30\t\t0.25\t\t0.204\t\t0.908\t\t0.24m - 3.4m / 9.3m                                                                   \n",
      "13\t56k\t0.29\t\t0.26\t\t0.258\t\t0.894\t\t0.24m - 3.7m / 9.0m                                                                   \n",
      "14\t61k\t0.29\t\t0.25\t\t0.266\t\t0.896\t\t0.24m - 4.0m / 8.9m                                                                   \n",
      "15\t65k\t0.28\t\t0.24\t\t0.274\t\t0.890\t\t0.24m - 4.3m / 9.0m                                                                   \n",
      "16\t69k\t0.26\t\t0.25\t\t0.284\t\t0.887\t\t0.25m - 4.6m / 9.1m                                                                   \n",
      "17\t74k\t0.26\t\t0.24\t\t0.275\t\t0.890\t\t0.24m - 4.9m / 9.3m                                                                   \n",
      "18\t78k\t0.25\t\t0.24\t\t0.266\t\t0.884\t\t0.23m - 5.2m / 9.4m                                                                   \n",
      "19\t82k\t0.24\t\t0.24\t\t0.284\t\t0.890\t\t0.23m - 5.5m / 9.2m                                                                   \n",
      "20\t86k\t0.24\t\t0.24\t\t0.248\t\t0.878\t\t0.23m - 5.8m / 9.2m                                                                   \n",
      "21\t91k\t0.23\t\t0.24\t\t0.272\t\t0.877\t\t0.23m - 6.0m / 9.3m                                                                   \n",
      "VAL f1\t0.28435114503816794 - (0.28435114503816794)                                                                     \n",
      "VAL loss\t0.235140286861582                                                                                             \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.235140286861582\n",
      "        | \\     )|_\tf1: 0.28435114503816794\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\28 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 58.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         58                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0025468637533307245               |\n",
      "|  noam_learning_rate_warmup   |                        1044                       |\n",
      "|  noam_learning_rate_factor   |                 3.250977447846374                 |\n",
      "|          adam_beta1          |                 0.8903526481380039                |\n",
      "|          adam_beta2          |                 0.8074150392877963                |\n",
      "|           adam_eps           |               0.00018372035887484746              |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.4943116166317052                |\n",
      "|     pointwise_layer_size     |                        160                        |\n",
      "|      last_layer_dropout      |                0.007888052646187221               |\n",
      "|   output_conv_num_filters    |                        100                        |\n",
      "|   output_conv_kernel_size    |                         1                         |\n",
      "|      output_conv_stride      |                         3                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        169                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 47%|█████████████████████▏                       | 47/100 [4:45:31<4:37:50, 314.54s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4fa53cb136142888255d705d52d9e01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.33\t\t0.28\t\t0.141\t\t0.847\t\t0.35m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.28\t\t0.26\t\t0.171\t\t0.751\t\t0.35m - 0.8m / 12.2m                                                                    \n",
      "3\t13k\t0.26\t\t0.25\t\t0.189\t\t0.820\t\t0.35m - 1.1m / 12.3m                                                                   \n",
      "4\t17k\t0.26\t\t0.25\t\t0.150\t\t0.752\t\t0.35m - 1.5m / 12.3m                                                                   \n",
      "5\t22k\t0.26\t\t0.24\t\t0.202\t\t0.820\t\t0.35m - 1.9m / 12.4m                                                                   \n",
      "6\t26k\t0.26\t\t0.24\t\t0.236\t\t0.867\t\t0.35m - 2.3m / 12.4m                                                                   \n",
      "7\t30k\t0.27\t\t0.28\t\t0.145\t\t0.716\t\t0.35m - 2.8m / 12.5m                                                                   \n",
      "8\t35k\t0.26\t\t0.29\t\t0.149\t\t0.758\t\t0.35m - 3.2m / 12.5m                                                                   \n",
      "9\t39k\t0.26\t\t0.24\t\t0.221\t\t0.855\t\t0.35m - 3.6m / 12.7m                                                                   \n",
      "10\t44k\t0.25\t\t0.25\t\t0.194\t\t0.826\t\t0.35m - 4.0m / 12.7m                                                                  \n",
      "11\t48k\t0.25\t\t0.26\t\t0.172\t\t0.786\t\t0.35m - 4.4m / 12.8m                                                                  \n",
      "VAL f1\t0.23619371282922685 - (0.23619371282922685)                                                                     \n",
      "VAL loss\t0.23655833869144838                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.23655833869144838\n",
      "        | \\     )|_\tf1: 0.23619371282922685\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\29 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 10.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         10                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.016051978418494407               |\n",
      "|  noam_learning_rate_warmup   |                        7665                       |\n",
      "|  noam_learning_rate_factor   |                 1.5735092364012782                |\n",
      "|          adam_beta1          |                 0.9769092272878527                |\n",
      "|          adam_beta2          |                 0.9103389122063278                |\n",
      "|           adam_eps           |                0.016896520225581524               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.5938340681899237                |\n",
      "|     pointwise_layer_size     |                        134                        |\n",
      "|      last_layer_dropout      |                0.47793096210282754                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        101                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 48%|█████████████████████▌                       | 48/100 [4:50:17<4:25:10, 305.98s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9cf3599ab79842b593c1575dddddd633",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.73\t\t1.03\t\t0.000\t\t0.940\t\t0.82m - 0.8m / 0.0m                                                                     \n",
      "2\t9k\t1.18\t\t1.04\t\t0.145\t\t0.931\t\t0.82m - 1.7m / 28.9m                                                                    \n",
      "3\t13k\t1.18\t\t1.01\t\t0.011\t\t0.939\t\t0.83m - 2.6m / 29.0m                                                                   \n",
      "4\t17k\t1.14\t\t0.95\t\t0.205\t\t0.923\t\t0.82m - 3.5m / 29.1m                                                                   \n",
      "5\t22k\t1.03\t\t0.93\t\t0.259\t\t0.905\t\t0.81m - 4.3m / 28.8m                                                                   \n",
      "6\t26k\t0.97\t\t1.04\t\t0.216\t\t0.848\t\t0.82m - 5.2m / 28.7m                                                                   \n",
      "7\t30k\t0.92\t\t1.05\t\t0.256\t\t0.857\t\t0.82m - 6.1m / 28.9m                                                                   \n",
      "8\t35k\t0.89\t\t1.11\t\t0.239\t\t0.832\t\t0.83m - 6.9m / 29.1m                                                                   \n",
      "9\t39k\t0.85\t\t1.35\t\t0.182\t\t0.773\t\t0.82m - 7.8m / 29.3m                                                                   \n",
      "10\t43k\t0.82\t\t1.20\t\t0.229\t\t0.811\t\t0.82m - 8.7m / 29.2m                                                                  \n",
      "VAL f1\t0.2590909090909091 - (0.2590909090909091)                                                                       \n",
      "VAL loss\t0.9262410907184375                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.9262410907184375\n",
      "        | \\     )|_\tf1: 0.2590909090909091\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\30 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 52.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         52                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.0062216203838028                |\n",
      "|  noam_learning_rate_warmup   |                        5714                       |\n",
      "|  noam_learning_rate_factor   |                 1.9601691639323977                |\n",
      "|          adam_beta1          |                 0.9546400616803203                |\n",
      "|          adam_beta2          |                 0.9367617717387493                |\n",
      "|           adam_eps           |               1.1297085116455954e-06              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.21485419135567144                |\n",
      "|     pointwise_layer_size     |                        240                        |\n",
      "|      last_layer_dropout      |                0.13477142444973417                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         82                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 49%|██████████████████████                       | 49/100 [4:59:48<5:27:28, 385.26s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7355e7c9d18749ffa06fe5f6c3992e7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.48\t\t0.31\t\t0.084\t\t0.877\t\t0.23m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.33\t\t0.27\t\t0.083\t\t0.916\t\t0.24m - 0.5m / 8.2m                                                                     \n",
      "3\t13k\t0.31\t\t0.27\t\t0.110\t\t0.918\t\t0.24m - 0.8m / 8.5m                                                                    \n",
      "4\t17k\t0.29\t\t0.25\t\t0.198\t\t0.902\t\t0.24m - 1.1m / 8.5m                                                                    \n",
      "5\t22k\t0.26\t\t0.25\t\t0.218\t\t0.887\t\t0.24m - 1.4m / 8.4m                                                                    \n",
      "6\t26k\t0.24\t\t0.25\t\t0.220\t\t0.867\t\t0.24m - 1.7m / 8.6m                                                                    \n",
      "7\t31k\t0.22\t\t0.25\t\t0.242\t\t0.876\t\t0.24m - 2.0m / 8.6m                                                                    \n",
      "8\t35k\t0.20\t\t0.27\t\t0.212\t\t0.852\t\t0.24m - 2.3m / 8.8m                                                                    \n",
      "9\t39k\t0.19\t\t0.27\t\t0.230\t\t0.872\t\t0.26m - 2.6m / 8.9m                                                                    \n",
      "10\t44k\t0.18\t\t0.28\t\t0.232\t\t0.853\t\t0.25m - 2.9m / 9.4m                                                                   \n",
      "11\t48k\t0.16\t\t0.29\t\t0.208\t\t0.837\t\t0.24m - 3.2m / 9.1m                                                                   \n",
      "12\t52k\t0.15\t\t0.29\t\t0.226\t\t0.863\t\t0.24m - 3.5m / 9.0m                                                                   \n",
      "VAL f1\t0.24187725631768953 - (0.24187725631768953)                                                                     \n",
      "VAL loss\t0.251385775241223                                                                                             \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.251385775241223\n",
      "        | \\     )|_\tf1: 0.24187725631768953\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\31 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 17.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         17                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.013467044269955633               |\n",
      "|  noam_learning_rate_warmup   |                        1427                       |\n",
      "|  noam_learning_rate_factor   |                 3.715200885627908                 |\n",
      "|          adam_beta1          |                 0.9902066062153156                |\n",
      "|          adam_beta2          |                 0.9844163919518004                |\n",
      "|           adam_eps           |               3.1267311004522968e-06              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.06748624381700824                |\n",
      "|     pointwise_layer_size     |                        252                        |\n",
      "|      last_layer_dropout      |                0.39541278724401296                |\n",
      "|   output_conv_num_filters    |                        171                        |\n",
      "|   output_conv_kernel_size    |                         9                         |\n",
      "|      output_conv_stride      |                         10                        |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        127                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 50%|██████████████████████▌                      | 50/100 [5:03:41<4:43:09, 339.78s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "274c84a17ce6402ba1456e18d9c57b46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.99\t\t0.72\t\t0.123\t\t0.924\t\t0.74m - 0.7m / 0.0m                                                                     \n",
      "2\t9k\t0.84\t\t0.76\t\t0.110\t\t0.907\t\t0.75m - 1.5m / 25.8m                                                                    \n",
      "3\t13k\t0.77\t\t0.72\t\t0.223\t\t0.912\t\t0.75m - 2.3m / 26.4m                                                                   \n",
      "4\t17k\t0.75\t\t0.69\t\t0.010\t\t0.938\t\t0.74m - 3.1m / 26.5m                                                                   \n",
      "5\t22k\t0.80\t\t0.68\t\t0.195\t\t0.907\t\t0.73m - 3.9m / 26.1m                                                                   \n",
      "6\t26k\t0.75\t\t0.83\t\t0.005\t\t0.939\t\t0.74m - 4.7m / 26.0m                                                                   \n",
      "7\t30k\t0.82\t\t0.84\t\t0.011\t\t0.938\t\t0.74m - 5.5m / 26.2m                                                                   \n",
      "8\t35k\t0.78\t\t0.73\t\t0.005\t\t0.938\t\t0.73m - 6.3m / 26.2m                                                                   \n",
      "VAL f1\t0.22283356258596973 - (0.22283356258596973)                                                                     \n",
      "VAL loss\t0.6816407063428094                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.6816407063428094\n",
      "        | \\     )|_\tf1: 0.22283356258596973\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\32 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 49.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         49                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0010697947578799914               |\n",
      "|  noam_learning_rate_warmup   |                        2160                       |\n",
      "|  noam_learning_rate_factor   |                 0.2325361388020708                |\n",
      "|          adam_beta1          |                 0.8466663296582907                |\n",
      "|          adam_beta2          |                 0.7783217435539207                |\n",
      "|           adam_eps           |               1.4352204975278237e-07              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.30114772137017476                |\n",
      "|     pointwise_layer_size     |                        173                        |\n",
      "|      last_layer_dropout      |                 0.595926551733155                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         59                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 51%|██████████████████████▉                      | 51/100 [5:10:32<4:54:54, 361.12s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "025667e26d87436a895046073b48ee2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.56\t\t0.46\t\t0.039\t\t0.520\t\t0.18m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.40\t\t0.31\t\t0.014\t\t0.928\t\t0.19m - 0.4m / 6.5m                                                                     \n",
      "3\t13k\t0.35\t\t0.29\t\t0.005\t\t0.939\t\t0.19m - 0.7m / 6.7m                                                                    \n",
      "4\t17k\t0.34\t\t0.30\t\t0.023\t\t0.932\t\t0.19m - 0.9m / 6.7m                                                                    \n",
      "5\t22k\t0.33\t\t0.29\t\t0.079\t\t0.926\t\t0.19m - 1.2m / 6.9m                                                                    \n",
      "6\t26k\t0.32\t\t0.28\t\t0.126\t\t0.921\t\t0.18m - 1.4m / 6.8m                                                                    \n",
      "7\t31k\t0.30\t\t0.28\t\t0.159\t\t0.911\t\t0.19m - 1.6m / 6.8m                                                                    \n",
      "8\t35k\t0.28\t\t0.28\t\t0.167\t\t0.898\t\t0.18m - 1.9m / 7.1m                                                                    \n",
      "9\t39k\t0.27\t\t0.27\t\t0.240\t\t0.910\t\t0.19m - 2.1m / 6.9m                                                                    \n",
      "10\t44k\t0.25\t\t0.27\t\t0.192\t\t0.892\t\t0.19m - 2.4m / 7.1m                                                                   \n",
      "11\t48k\t0.23\t\t0.27\t\t0.266\t\t0.904\t\t0.19m - 2.6m / 7.0m                                                                   \n",
      "12\t52k\t0.22\t\t0.28\t\t0.246\t\t0.885\t\t0.19m - 2.8m / 7.1m                                                                   \n",
      "13\t57k\t0.21\t\t0.28\t\t0.255\t\t0.892\t\t0.20m - 3.1m / 7.2m                                                                   \n",
      "14\t61k\t0.20\t\t0.29\t\t0.239\t\t0.888\t\t0.20m - 3.3m / 7.5m                                                                   \n",
      "15\t65k\t0.19\t\t0.29\t\t0.247\t\t0.872\t\t0.20m - 3.6m / 7.6m                                                                   \n",
      "16\t70k\t0.18\t\t0.30\t\t0.234\t\t0.872\t\t0.20m - 3.9m / 7.7m                                                                   \n",
      "VAL f1\t0.26550598476605003 - (0.26550598476605003)                                                                     \n",
      "VAL loss\t0.27057932834235987                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.27057932834235987\n",
      "        | \\     )|_\tf1: 0.26550598476605003\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\33 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 42.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         42                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.02375911657531403                |\n",
      "|  noam_learning_rate_warmup   |                        4877                       |\n",
      "|  noam_learning_rate_factor   |                 2.5576260825191364                |\n",
      "|          adam_beta1          |                 0.9068413137505199                |\n",
      "|          adam_beta2          |                 0.7448874185528804                |\n",
      "|           adam_eps           |               0.0074830717283358465               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.36534072952455754                |\n",
      "|     pointwise_layer_size     |                        305                        |\n",
      "|      last_layer_dropout      |                0.08629377589883158                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        180                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 52%|███████████████████████▍                     | 52/100 [5:14:53<4:24:45, 330.94s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee5094eb1061423abcae6f9955bb9e2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.47\t\t0.35\t\t0.103\t\t0.910\t\t0.41m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.39\t\t0.35\t\t0.184\t\t0.876\t\t0.40m - 0.9m / 14.4m                                                                    \n",
      "3\t13k\t0.38\t\t0.33\t\t0.173\t\t0.867\t\t0.39m - 1.3m / 14.0m                                                                   \n",
      "4\t17k\t0.35\t\t0.34\t\t0.172\t\t0.819\t\t0.40m - 1.7m / 13.9m                                                                   \n",
      "5\t22k\t0.33\t\t0.35\t\t0.163\t\t0.813\t\t0.39m - 2.2m / 14.1m                                                                   \n",
      "6\t26k\t0.30\t\t0.35\t\t0.186\t\t0.797\t\t0.39m - 2.6m / 13.9m                                                                   \n",
      "7\t30k\t0.30\t\t0.32\t\t0.228\t\t0.839\t\t0.39m - 3.1m / 14.0m                                                                   \n",
      "8\t35k\t0.28\t\t0.32\t\t0.229\t\t0.846\t\t0.39m - 3.5m / 14.2m                                                                   \n",
      "9\t39k\t0.28\t\t0.34\t\t0.198\t\t0.829\t\t0.40m - 4.0m / 14.1m                                                                   \n",
      "10\t43k\t0.27\t\t0.35\t\t0.167\t\t0.847\t\t0.39m - 4.4m / 14.3m                                                                  \n",
      "11\t48k\t0.27\t\t0.34\t\t0.217\t\t0.815\t\t0.39m - 4.9m / 14.3m                                                                  \n",
      "12\t52k\t0.27\t\t0.35\t\t0.202\t\t0.824\t\t0.39m - 5.3m / 14.4m                                                                  \n",
      "13\t56k\t0.26\t\t0.35\t\t0.240\t\t0.863\t\t0.39m - 5.8m / 14.4m                                                                  \n",
      "14\t61k\t0.25\t\t0.38\t\t0.190\t\t0.813\t\t0.40m - 6.2m / 14.5m                                                                  \n",
      "15\t65k\t0.25\t\t0.36\t\t0.173\t\t0.820\t\t0.39m - 6.7m / 14.6m                                                                  \n",
      "16\t69k\t0.25\t\t0.36\t\t0.223\t\t0.862\t\t0.39m - 7.1m / 14.6m                                                                  \n",
      "17\t74k\t0.24\t\t0.38\t\t0.174\t\t0.810\t\t0.40m - 7.6m / 14.6m                                                                  \n",
      "18\t78k\t0.23\t\t0.38\t\t0.187\t\t0.831\t\t0.39m - 8.0m / 14.7m                                                                  \n",
      "VAL f1\t0.23973727422003285 - (0.23973727422003285)                                                                     \n",
      "VAL loss\t0.31573494843074257                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.31573494843074257\n",
      "        | \\     )|_\tf1: 0.23973727422003285\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\34 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 35.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         35                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00018994532580661653              |\n",
      "|  noam_learning_rate_warmup   |                        8966                       |\n",
      "|  noam_learning_rate_factor   |                 2.318324317744684                 |\n",
      "|          adam_beta1          |                 0.9610173366079859                |\n",
      "|          adam_beta2          |                 0.9595431729801437                |\n",
      "|           adam_eps           |                 0.0773089487739663                |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.1328153545737963                |\n",
      "|     pointwise_layer_size     |                        126                        |\n",
      "|      last_layer_dropout      |                 0.6914605356801423                |\n",
      "|   output_conv_num_filters    |                         53                        |\n",
      "|   output_conv_kernel_size    |                         8                         |\n",
      "|      output_conv_stride      |                         3                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         71                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 53%|███████████████████████▊                     | 53/100 [5:23:19<5:00:32, 383.66s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42d82bf94c694b2e8785605ac5298260",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.54\t\t0.51\t\t0.031\t\t0.857\t\t0.40m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.43\t\t0.50\t\t0.136\t\t0.869\t\t0.40m - 0.9m / 14.2m                                                                    \n",
      "3\t13k\t0.39\t\t0.46\t\t0.152\t\t0.868\t\t0.40m - 1.3m / 14.2m                                                                   \n",
      "4\t17k\t0.35\t\t0.42\t\t0.188\t\t0.868\t\t0.40m - 1.8m / 14.1m                                                                   \n",
      "5\t22k\t0.32\t\t0.38\t\t0.216\t\t0.884\t\t0.41m - 2.2m / 14.3m                                                                   \n",
      "6\t26k\t0.29\t\t0.37\t\t0.243\t\t0.890\t\t0.43m - 2.7m / 14.5m                                                                   \n",
      "7\t30k\t0.26\t\t0.37\t\t0.226\t\t0.872\t\t0.41m - 3.2m / 15.1m                                                                   \n",
      "8\t35k\t0.24\t\t0.36\t\t0.211\t\t0.878\t\t0.40m - 3.6m / 14.8m                                                                   \n",
      "9\t39k\t0.23\t\t0.36\t\t0.195\t\t0.880\t\t0.39m - 4.1m / 14.4m                                                                   \n",
      "10\t43k\t0.21\t\t0.35\t\t0.222\t\t0.873\t\t0.40m - 4.5m / 14.3m                                                                  \n",
      "11\t48k\t0.19\t\t0.34\t\t0.238\t\t0.880\t\t0.40m - 5.0m / 14.5m                                                                  \n",
      "VAL f1\t0.24346076458752516 - (0.24346076458752516)                                                                     \n",
      "VAL loss\t0.3440527016775949                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3440527016775949\n",
      "        | \\     )|_\tf1: 0.24346076458752516\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\35 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 25.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         25                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.010281253462766869               |\n",
      "|  noam_learning_rate_warmup   |                        4277                       |\n",
      "|  noam_learning_rate_factor   |                 1.6767916287158053                |\n",
      "|          adam_beta1          |                 0.9365467749006705                |\n",
      "|          adam_beta2          |                 0.9263485794087469                |\n",
      "|           adam_eps           |               1.5736345611131243e-05              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |               0.0027049462755303066               |\n",
      "|     pointwise_layer_size     |                        283                        |\n",
      "|      last_layer_dropout      |                 0.2494135183882299                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        151                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 54%|████████████████████████▎                    | 54/100 [5:28:42<4:40:08, 365.40s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dc61e3ffbb94d829b8af62093b7b5b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.81\t\t0.48\t\t0.019\t\t0.932\t\t0.36m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.59\t\t0.49\t\t0.069\t\t0.919\t\t0.35m - 0.8m / 12.7m                                                                    \n",
      "3\t13k\t0.52\t\t0.46\t\t0.194\t\t0.898\t\t0.36m - 1.2m / 12.4m                                                                   \n",
      "4\t17k\t0.46\t\t0.42\t\t0.267\t\t0.886\t\t0.36m - 1.6m / 12.9m                                                                   \n",
      "5\t22k\t0.40\t\t0.42\t\t0.224\t\t0.885\t\t0.36m - 2.0m / 12.9m                                                                   \n",
      "6\t26k\t0.34\t\t0.43\t\t0.260\t\t0.897\t\t0.37m - 2.4m / 12.9m                                                                   \n",
      "7\t30k\t0.28\t\t0.42\t\t0.279\t\t0.889\t\t0.37m - 2.9m / 13.1m                                                                   \n",
      "8\t35k\t0.23\t\t0.45\t\t0.318\t\t0.904\t\t0.36m - 3.3m / 13.1m                                                                   \n",
      "9\t39k\t0.19\t\t0.49\t\t0.297\t\t0.913\t\t0.36m - 3.7m / 13.1m                                                                   \n",
      "10\t43k\t0.16\t\t0.52\t\t0.272\t\t0.905\t\t0.36m - 4.1m / 13.1m                                                                  \n",
      "11\t48k\t0.14\t\t0.53\t\t0.252\t\t0.905\t\t0.37m - 4.5m / 13.2m                                                                  \n",
      "12\t52k\t0.11\t\t0.59\t\t0.269\t\t0.908\t\t0.38m - 5.0m / 13.6m                                                                  \n",
      "13\t56k\t0.10\t\t0.64\t\t0.278\t\t0.918\t\t0.37m - 5.4m / 13.8m                                                                  \n",
      "VAL f1\t0.3176593521421108 - (0.3176593521421108)                                                                       \n",
      "VAL loss\t0.4188421317509243                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4188421317509243\n",
      "        | \\     )|_\tf1: 0.3176593521421108\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\36 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 60.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         60                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.054757425538301305               |\n",
      "|  noam_learning_rate_warmup   |                        7124                       |\n",
      "|  noam_learning_rate_factor   |                 2.918122280714181                 |\n",
      "|          adam_beta1          |                 0.9814545424163847                |\n",
      "|          adam_beta2          |                 0.8557505996893672                |\n",
      "|           adam_eps           |                 0.2788247011061983                |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.6493111884451359                |\n",
      "|     pointwise_layer_size     |                        113                        |\n",
      "|      last_layer_dropout      |                0.19270782423718708                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         91                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 55%|████████████████████████▊                    | 55/100 [5:34:41<4:32:38, 363.53s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d41c0669a22b4bb28a4cbbcbb2174355",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.46\t\t0.54\t\t0.066\t\t0.362\t\t0.25m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.39\t\t0.34\t\t0.103\t\t0.619\t\t0.26m - 0.6m / 8.9m                                                                     \n",
      "3\t13k\t0.32\t\t0.27\t\t0.160\t\t0.818\t\t0.26m - 0.9m / 9.1m                                                                    \n",
      "4\t17k\t0.30\t\t0.26\t\t0.051\t\t0.933\t\t0.26m - 1.2m / 9.2m                                                                    \n",
      "5\t22k\t0.29\t\t0.26\t\t0.057\t\t0.930\t\t0.26m - 1.5m / 9.2m                                                                    \n",
      "6\t26k\t0.29\t\t0.26\t\t0.055\t\t0.933\t\t0.26m - 1.8m / 9.2m                                                                    \n",
      "7\t30k\t0.29\t\t0.26\t\t0.139\t\t0.934\t\t0.26m - 2.1m / 9.4m                                                                    \n",
      "8\t35k\t0.29\t\t0.26\t\t0.059\t\t0.932\t\t0.26m - 2.4m / 9.4m                                                                    \n",
      "VAL f1\t0.16045845272206305 - (0.16045845272206305)                                                                     \n",
      "VAL loss\t0.2619714498519898                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2619714498519898\n",
      "        | \\     )|_\tf1: 0.16045845272206305\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\37 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 63.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         63                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0005697436975151285               |\n",
      "|  noam_learning_rate_warmup   |                        3348                       |\n",
      "|  noam_learning_rate_factor   |                 0.8349319090572298                |\n",
      "|          adam_beta1          |                 0.8320985588977997                |\n",
      "|          adam_beta2          |                 0.9473716920343452                |\n",
      "|           adam_eps           |                0.000757405575246576               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.7070731109467372                |\n",
      "|     pointwise_layer_size     |                         51                        |\n",
      "|      last_layer_dropout      |                 0.3151881059953121                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        143                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 56%|█████████████████████████▏                   | 56/100 [5:37:29<3:43:23, 304.63s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c55698619624aebb60d4fe854dbdd89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.40\t\t0.32\t\t0.058\t\t0.577\t\t0.23m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.31\t\t0.26\t\t0.203\t\t0.910\t\t0.23m - 0.5m / 8.1m                                                                     \n",
      "3\t13k\t0.28\t\t0.27\t\t0.030\t\t0.936\t\t0.23m - 0.8m / 8.2m                                                                    \n",
      "4\t17k\t0.28\t\t0.27\t\t0.188\t\t0.919\t\t0.23m - 1.1m / 8.2m                                                                    \n",
      "5\t22k\t0.28\t\t0.27\t\t0.005\t\t0.934\t\t0.23m - 1.4m / 8.3m                                                                    \n",
      "6\t26k\t0.27\t\t0.27\t\t0.067\t\t0.923\t\t0.23m - 1.6m / 8.3m                                                                    \n",
      "7\t30k\t0.27\t\t0.27\t\t0.079\t\t0.931\t\t0.23m - 1.9m / 8.4m                                                                    \n",
      "VAL f1\t0.20334261838440112 - (0.20334261838440112)                                                                     \n",
      "VAL loss\t0.26282673285751745                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.26282673285751745\n",
      "        | \\     )|_\tf1: 0.20334261838440112\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\38 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 57.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         57                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0021460038692543435               |\n",
      "|  noam_learning_rate_warmup   |                        2870                       |\n",
      "|  noam_learning_rate_factor   |                 1.0105537112459753                |\n",
      "|          adam_beta1          |                 0.9182210581268296                |\n",
      "|          adam_beta2          |                 0.8931433304473059                |\n",
      "|           adam_eps           |                0.001785274607485937               |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                0.43004890907213134                |\n",
      "|     pointwise_layer_size     |                        349                        |\n",
      "|      last_layer_dropout      |                0.35518259442254974                |\n",
      "|   output_conv_num_filters    |                        282                        |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         5                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         97                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 57%|█████████████████████████▋                   | 57/100 [5:39:43<3:01:46, 253.64s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae69384c46d743c99db997f3afebb78b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.33\t\t0.27\t\t0.011\t\t0.940\t\t0.39m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.26\t\t0.228\t\t0.876\t\t0.39m - 0.9m / 13.8m                                                                    \n",
      "3\t13k\t0.25\t\t0.24\t\t0.194\t\t0.819\t\t0.39m - 1.3m / 13.9m                                                                   \n",
      "4\t17k\t0.23\t\t0.24\t\t0.214\t\t0.807\t\t0.39m - 1.7m / 14.0m                                                                   \n",
      "5\t22k\t0.22\t\t0.25\t\t0.183\t\t0.776\t\t0.39m - 2.2m / 13.9m                                                                   \n",
      "6\t26k\t0.21\t\t0.28\t\t0.143\t\t0.688\t\t0.39m - 2.6m / 14.0m                                                                   \n",
      "7\t30k\t0.20\t\t0.28\t\t0.153\t\t0.694\t\t0.39m - 3.1m / 14.1m                                                                   \n",
      "VAL f1\t0.22753346080305928 - (0.22753346080305928)                                                                     \n",
      "VAL loss\t0.23629493601838047                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.23629493601838047\n",
      "        | \\     )|_\tf1: 0.22753346080305928\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\39 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 40.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         40                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               4.166220632553607e-08               |\n",
      "|  noam_learning_rate_warmup   |                        5232                       |\n",
      "|  noam_learning_rate_factor   |                 2.0382524931806167                |\n",
      "|          adam_beta1          |                 0.8705637608209986                |\n",
      "|          adam_beta2          |                 0.8329988810554201                |\n",
      "|           adam_eps           |                0.011879481336139947               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.4819585828747207                |\n",
      "|     pointwise_layer_size     |                         81                        |\n",
      "|      last_layer_dropout      |                0.28627843284283727                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        115                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 58%|██████████████████████████                   | 58/100 [5:43:14<2:48:30, 240.72s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f80ba1f910e84079bfac1e47c976b60c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.62\t\t0.35\t\t0.036\t\t0.922\t\t0.31m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.41\t\t0.35\t\t0.008\t\t0.921\t\t0.31m - 0.7m / 11.1m                                                                    \n",
      "3\t13k\t0.40\t\t0.34\t\t0.110\t\t0.922\t\t0.31m - 1.1m / 11.0m                                                                   \n",
      "4\t17k\t0.40\t\t0.33\t\t0.140\t\t0.915\t\t0.31m - 1.4m / 11.2m                                                                   \n",
      "5\t22k\t0.37\t\t0.31\t\t0.186\t\t0.867\t\t0.30m - 1.8m / 10.9m                                                                   \n",
      "6\t26k\t0.35\t\t0.31\t\t0.226\t\t0.847\t\t0.30m - 2.1m / 10.8m                                                                   \n",
      "7\t30k\t0.32\t\t0.34\t\t0.211\t\t0.819\t\t0.30m - 2.5m / 10.9m                                                                   \n",
      "8\t35k\t0.30\t\t0.33\t\t0.226\t\t0.819\t\t0.30m - 2.8m / 11.0m                                                                   \n",
      "9\t39k\t0.29\t\t0.38\t\t0.180\t\t0.779\t\t0.30m - 3.2m / 11.1m                                                                   \n",
      "10\t43k\t0.28\t\t0.34\t\t0.221\t\t0.819\t\t0.30m - 3.5m / 11.1m                                                                  \n",
      "11\t48k\t0.27\t\t0.38\t\t0.192\t\t0.764\t\t0.30m - 3.9m / 11.2m                                                                  \n",
      "VAL f1\t0.2259287338893101 - (0.2259287338893101)                                                                       \n",
      "VAL loss\t0.30866569942898214                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.30866569942898214\n",
      "        | \\     )|_\tf1: 0.2259287338893101\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\40 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 32.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         32                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               6.922812734457648e-06               |\n",
      "|  noam_learning_rate_warmup   |                        1470                       |\n",
      "|  noam_learning_rate_factor   |                 0.6640677836401946                |\n",
      "|          adam_beta1          |                 0.8949837728149985                |\n",
      "|          adam_beta2          |                 0.9838909454598133                |\n",
      "|           adam_eps           |               4.622767403743653e-05               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.24081670922672005                |\n",
      "|     pointwise_layer_size     |                        222                        |\n",
      "|      last_layer_dropout      |                 0.4244142412336679                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        105                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 59%|██████████████████████████▌                  | 59/100 [5:47:39<2:49:24, 247.92s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40ced33d4a6c4a9eb49a0a5573df2c96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.60\t\t0.40\t\t0.000\t\t0.939\t\t0.27m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.49\t\t0.40\t\t0.035\t\t0.927\t\t0.27m - 0.6m / 9.5m                                                                     \n",
      "3\t13k\t0.47\t\t0.36\t\t0.182\t\t0.923\t\t0.27m - 0.9m / 9.6m                                                                    \n",
      "4\t17k\t0.41\t\t0.36\t\t0.234\t\t0.897\t\t0.27m - 1.3m / 9.6m                                                                    \n",
      "5\t22k\t0.36\t\t0.36\t\t0.281\t\t0.899\t\t0.27m - 1.6m / 9.8m                                                                    \n",
      "6\t26k\t0.33\t\t0.37\t\t0.277\t\t0.890\t\t0.27m - 1.9m / 9.8m                                                                    \n",
      "7\t30k\t0.30\t\t0.39\t\t0.248\t\t0.863\t\t0.27m - 2.2m / 9.8m                                                                    \n",
      "8\t35k\t0.27\t\t0.39\t\t0.245\t\t0.879\t\t0.28m - 2.6m / 9.9m                                                                    \n",
      "9\t39k\t0.25\t\t0.42\t\t0.274\t\t0.882\t\t0.27m - 2.9m / 10.3m                                                                   \n",
      "10\t43k\t0.23\t\t0.45\t\t0.230\t\t0.864\t\t0.27m - 3.2m / 10.0m                                                                  \n",
      "VAL f1\t0.2814814814814815 - (0.2814814814814815)                                                                       \n",
      "VAL loss\t0.3576152663339268                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3576152663339268\n",
      "        | \\     )|_\tf1: 0.2814814814814815\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\41 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 53.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         53                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.671973885743481                 |\n",
      "|  noam_learning_rate_warmup   |                        2131                       |\n",
      "|  noam_learning_rate_factor   |                 1.1110008180892361                |\n",
      "|          adam_beta1          |                 0.9295570324159511                |\n",
      "|          adam_beta2          |                 0.967547262263042                 |\n",
      "|           adam_eps           |               0.00014275419982413053              |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.5491150617296866                |\n",
      "|     pointwise_layer_size     |                        316                        |\n",
      "|      last_layer_dropout      |                0.46265749387952687                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         87                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 60%|███████████████████████████                  | 60/100 [5:51:15<2:38:53, 238.33s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7777d6e8d9d42c0b627aab4a46a85d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.40\t\t0.28\t\t0.128\t\t0.924\t\t0.28m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.27\t\t0.151\t\t0.880\t\t0.28m - 0.6m / 9.8m                                                                     \n",
      "3\t13k\t0.30\t\t0.26\t\t0.207\t\t0.819\t\t0.28m - 1.0m / 9.9m                                                                    \n",
      "4\t17k\t0.27\t\t0.28\t\t0.157\t\t0.774\t\t0.28m - 1.3m / 10.0m                                                                   \n",
      "5\t22k\t0.26\t\t0.28\t\t0.162\t\t0.797\t\t0.28m - 1.6m / 9.9m                                                                    \n",
      "6\t26k\t0.24\t\t0.28\t\t0.204\t\t0.799\t\t0.28m - 2.0m / 10.0m                                                                   \n",
      "7\t30k\t0.24\t\t0.30\t\t0.193\t\t0.801\t\t0.28m - 2.3m / 10.2m                                                                   \n",
      "8\t35k\t0.23\t\t0.29\t\t0.180\t\t0.800\t\t0.28m - 2.6m / 10.2m                                                                   \n",
      "VAL f1\t0.20680628272251309 - (0.20680628272251309)                                                                     \n",
      "VAL loss\t0.2624898355283506                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2624898355283506\n",
      "        | \\     )|_\tf1: 0.20680628272251309\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\42 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 56.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         56                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.12410932263394828                |\n",
      "|  noam_learning_rate_warmup   |                        3856                       |\n",
      "|  noam_learning_rate_factor   |                 1.8001092497007456                |\n",
      "|          adam_beta1          |                 0.9449448821512489                |\n",
      "|          adam_beta2          |                 0.8726419485082249                |\n",
      "|           adam_eps           |               0.0004047177835454019               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.33256500100753705                |\n",
      "|     pointwise_layer_size     |                        140                        |\n",
      "|      last_layer_dropout      |                0.09902213215437725                |\n",
      "|   output_conv_num_filters    |                        169                        |\n",
      "|   output_conv_kernel_size    |                         6                         |\n",
      "|      output_conv_stride      |                         2                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        167                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 61%|███████████████████████████▍                 | 61/100 [5:54:14<2:23:20, 220.52s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21be5d1ade1c4cdcac4f88405c7ecfef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [9464 x 2], m2: [169 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\43 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 46.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         46                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               2.655462720664518e-05               |\n",
      "|  noam_learning_rate_warmup   |                        5539                       |\n",
      "|  noam_learning_rate_factor   |                0.18530423115502703                |\n",
      "|          adam_beta1          |                 0.7095650668518018                |\n",
      "|          adam_beta2          |                 0.7865199972141835                |\n",
      "|           adam_eps           |                0.05502070525782414                |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         1                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.2760989149189484                |\n",
      "|     pointwise_layer_size     |                        202                        |\n",
      "|      last_layer_dropout      |                0.026899494443174896               |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        136                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 62%|███████████████████████████▉                 | 62/100 [5:54:16<1:38:14, 155.13s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9c1da865f7340f895c5b4acdc150d66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.70\t\t0.74\t\t0.054\t\t0.223\t\t0.22m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.63\t\t0.60\t\t0.054\t\t0.281\t\t0.22m - 0.5m / 7.7m                                                                     \n",
      "3\t13k\t0.53\t\t0.45\t\t0.051\t\t0.585\t\t0.22m - 0.8m / 7.7m                                                                    \n",
      "4\t17k\t0.44\t\t0.35\t\t0.080\t\t0.902\t\t0.22m - 1.0m / 7.9m                                                                    \n",
      "5\t22k\t0.39\t\t0.31\t\t0.039\t\t0.935\t\t0.22m - 1.3m / 7.9m                                                                    \n",
      "6\t26k\t0.37\t\t0.30\t\t0.010\t\t0.938\t\t0.22m - 1.6m / 7.9m                                                                    \n",
      "7\t30k\t0.37\t\t0.30\t\t0.016\t\t0.938\t\t0.22m - 1.9m / 8.0m                                                                    \n",
      "8\t35k\t0.36\t\t0.29\t\t0.010\t\t0.937\t\t0.22m - 2.1m / 8.0m                                                                    \n",
      "9\t39k\t0.36\t\t0.29\t\t0.015\t\t0.935\t\t0.22m - 2.4m / 8.1m                                                                    \n",
      "VAL f1\t0.08035714285714286 - (0.08035714285714286)                                                                     \n",
      "VAL loss\t0.2945459225903387                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2945459225903387\n",
      "        | \\     )|_\tf1: 0.08035714285714286\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\44 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 50.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         50                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.046904392469897215               |\n",
      "|  noam_learning_rate_warmup   |                        6199                       |\n",
      "|  noam_learning_rate_factor   |                 1.3945909046622287                |\n",
      "|          adam_beta1          |                 0.8518683496541234                |\n",
      "|          adam_beta2          |                 0.8486679455300932                |\n",
      "|           adam_eps           |                0.00564218130961315                |\n",
      "|      adam_weight_decay       |                       0.001                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.03873588850754687                |\n",
      "|     pointwise_layer_size     |                        162                        |\n",
      "|      last_layer_dropout      |                 0.5195667641275727                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        121                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                       False                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 63%|████████████████████████████▎                | 63/100 [5:57:08<1:38:46, 160.18s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d5ee8a53173403182717bedd6902592",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.55\t\t0.34\t\t0.100\t\t0.787\t\t0.25m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.35\t\t0.29\t\t0.033\t\t0.924\t\t0.26m - 0.6m / 8.8m                                                                     \n",
      "3\t13k\t0.34\t\t0.29\t\t0.014\t\t0.931\t\t0.25m - 0.9m / 9.0m                                                                    \n",
      "4\t17k\t0.33\t\t0.29\t\t0.121\t\t0.890\t\t0.25m - 1.2m / 8.8m                                                                    \n",
      "5\t22k\t0.32\t\t0.27\t\t0.154\t\t0.909\t\t0.25m - 1.5m / 8.9m                                                                    \n",
      "6\t26k\t0.30\t\t0.25\t\t0.180\t\t0.901\t\t0.25m - 1.8m / 9.0m                                                                    \n",
      "7\t30k\t0.27\t\t0.26\t\t0.202\t\t0.889\t\t0.25m - 2.1m / 9.1m                                                                    \n",
      "8\t35k\t0.26\t\t0.24\t\t0.247\t\t0.890\t\t0.25m - 2.4m / 9.1m                                                                    \n",
      "9\t39k\t0.24\t\t0.25\t\t0.209\t\t0.880\t\t0.25m - 2.7m / 9.2m                                                                    \n",
      "10\t44k\t0.23\t\t0.25\t\t0.213\t\t0.872\t\t0.25m - 3.0m / 9.2m                                                                   \n",
      "11\t48k\t0.22\t\t0.26\t\t0.232\t\t0.881\t\t0.25m - 3.3m / 9.3m                                                                   \n",
      "12\t52k\t0.21\t\t0.26\t\t0.215\t\t0.888\t\t0.25m - 3.6m / 9.4m                                                                   \n",
      "13\t57k\t0.20\t\t0.26\t\t0.264\t\t0.898\t\t0.25m - 3.9m / 9.4m                                                                   \n",
      "14\t61k\t0.19\t\t0.27\t\t0.229\t\t0.882\t\t0.25m - 4.2m / 9.5m                                                                   \n",
      "15\t65k\t0.18\t\t0.26\t\t0.226\t\t0.875\t\t0.25m - 4.5m / 9.5m                                                                   \n",
      "16\t70k\t0.17\t\t0.27\t\t0.234\t\t0.894\t\t0.26m - 4.8m / 9.6m                                                                   \n",
      "17\t74k\t0.16\t\t0.28\t\t0.240\t\t0.882\t\t0.25m - 5.1m / 9.7m                                                                   \n",
      "18\t78k\t0.15\t\t0.30\t\t0.264\t\t0.901\t\t0.25m - 5.4m / 9.7m                                                                   \n",
      "VAL f1\t0.2644281217208814 - (0.2644281217208814)                                                                       \n",
      "VAL loss\t0.24494093486240934                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24494093486240934\n",
      "        | \\     )|_\tf1: 0.2644281217208814\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\45 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 61.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         61                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                 0.0072615586841429                |\n",
      "|  noam_learning_rate_warmup   |                        2341                       |\n",
      "|  noam_learning_rate_factor   |                 2.2046335114295217                |\n",
      "|          adam_beta1          |                 0.7519915911384191                |\n",
      "|          adam_beta2          |                 0.8867627111772415                |\n",
      "|           adam_eps           |               1.723230749093638e-08               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.09084595283787206                |\n",
      "|     pointwise_layer_size     |                        182                        |\n",
      "|      last_layer_dropout      |                0.21399973796992244                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                         51                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 64%|████████████████████████████▊                | 64/100 [6:02:56<2:09:53, 216.48s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d7c2df1c809496884defca045529380",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.38\t\t0.24\t\t0.129\t\t0.912\t\t0.23m - 0.2m / 0.0m                                                                     \n",
      "2\t9k\t0.27\t\t0.22\t\t0.188\t\t0.874\t\t0.22m - 0.5m / 8.2m                                                                     \n",
      "3\t13k\t0.24\t\t0.23\t\t0.144\t\t0.810\t\t0.23m - 0.8m / 7.9m                                                                    \n",
      "4\t17k\t0.22\t\t0.24\t\t0.203\t\t0.840\t\t0.23m - 1.1m / 8.1m                                                                    \n",
      "5\t22k\t0.19\t\t0.22\t\t0.245\t\t0.873\t\t0.23m - 1.3m / 8.2m                                                                    \n",
      "6\t26k\t0.18\t\t0.23\t\t0.239\t\t0.875\t\t0.24m - 1.7m / 8.3m                                                                    \n",
      "7\t30k\t0.17\t\t0.26\t\t0.257\t\t0.887\t\t0.24m - 2.0m / 8.7m                                                                    \n",
      "8\t35k\t0.15\t\t0.28\t\t0.234\t\t0.868\t\t0.24m - 2.2m / 8.8m                                                                    \n",
      "9\t39k\t0.14\t\t0.28\t\t0.238\t\t0.875\t\t0.24m - 2.6m / 8.8m                                                                    \n",
      "10\t43k\t0.13\t\t0.27\t\t0.229\t\t0.868\t\t0.23m - 2.8m / 8.8m                                                                   \n",
      "11\t48k\t0.13\t\t0.29\t\t0.244\t\t0.874\t\t0.23m - 3.1m / 8.6m                                                                   \n",
      "12\t52k\t0.12\t\t0.28\t\t0.221\t\t0.872\t\t0.23m - 3.4m / 8.6m                                                                   \n",
      "VAL f1\t0.25680933852140075 - (0.25680933852140075)                                                                     \n",
      "VAL loss\t0.22109904315302278                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22109904315302278\n",
      "        | \\     )|_\tf1: 0.25680933852140075\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\46 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 60.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         60                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.001379005725185856               |\n",
      "|  noam_learning_rate_warmup   |                        4472                       |\n",
      "|  noam_learning_rate_factor   |                 1.2880381707634754                |\n",
      "|          adam_beta1          |                 0.8089208489689219                |\n",
      "|          adam_beta2          |                 0.9806935600963501                |\n",
      "|           adam_eps           |               9.130114382704545e-08               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.08848847320219734                |\n",
      "|     pointwise_layer_size     |                        102                        |\n",
      "|      last_layer_dropout      |                0.13069020211606258                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        108                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 65%|█████████████████████████████▎               | 65/100 [6:06:40<2:07:37, 218.79s/it, best loss: 0.2133179878195127]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "135cf76ea27f4971b024d4f9d36117b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.43\t\t0.29\t\t0.025\t\t0.848\t\t0.26m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.30\t\t0.25\t\t0.044\t\t0.923\t\t0.26m - 0.6m / 9.3m                                                                     \n",
      "3\t13k\t0.28\t\t0.23\t\t0.170\t\t0.921\t\t0.27m - 0.9m / 9.4m                                                                    \n",
      "4\t17k\t0.26\t\t0.21\t\t0.216\t\t0.911\t\t0.26m - 1.2m / 9.5m                                                                    \n",
      "5\t22k\t0.23\t\t0.21\t\t0.229\t\t0.899\t\t0.26m - 1.5m / 9.4m                                                                    \n",
      "6\t26k\t0.21\t\t0.22\t\t0.195\t\t0.863\t\t0.27m - 1.9m / 9.5m                                                                    \n",
      "7\t30k\t0.19\t\t0.22\t\t0.218\t\t0.866\t\t0.27m - 2.2m / 9.6m                                                                    \n",
      "8\t35k\t0.18\t\t0.22\t\t0.211\t\t0.855\t\t0.27m - 2.5m / 9.7m                                                                    \n",
      "9\t39k\t0.17\t\t0.21\t\t0.270\t\t0.890\t\t0.26m - 2.8m / 9.7m                                                                    \n",
      "10\t43k\t0.15\t\t0.23\t\t0.250\t\t0.882\t\t0.27m - 3.1m / 9.8m                                                                   \n",
      "11\t48k\t0.14\t\t0.23\t\t0.267\t\t0.884\t\t0.26m - 3.5m / 9.8m                                                                   \n",
      "12\t52k\t0.13\t\t0.24\t\t0.218\t\t0.871\t\t0.27m - 3.8m / 9.8m                                                                   \n",
      "13\t56k\t0.12\t\t0.26\t\t0.220\t\t0.872\t\t0.27m - 4.1m / 9.9m                                                                   \n",
      "14\t60k\t0.11\t\t0.25\t\t0.261\t\t0.883\t\t0.27m - 4.4m / 10.0m                                                                  \n",
      "VAL f1\t0.26990291262135924 - (0.26990291262135924)                                                                     \n",
      "VAL loss\t0.20803299082650076                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.20803299082650076\n",
      "        | \\     )|_\tf1: 0.26990291262135924\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\47 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 59.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         59                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00027127939792023553              |\n",
      "|  noam_learning_rate_warmup   |                        3220                       |\n",
      "|  noam_learning_rate_factor   |                 1.644534134985935                 |\n",
      "|          adam_beta1          |                 0.8099738063354094                |\n",
      "|          adam_beta2          |                 0.9831091445920633                |\n",
      "|           adam_eps           |               9.922399965905717e-08               |\n",
      "|      adam_weight_decay       |                       1e-05                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.1562088522722633                |\n",
      "|     pointwise_layer_size     |                        102                        |\n",
      "|      last_layer_dropout      |                0.15692071729444793                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        104                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 66%|█████████████████████████████               | 66/100 [6:11:27<2:15:30, 239.14s/it, best loss: 0.21180716885460746]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5969d66700bc4f208a3d1d1d330f4be1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.41\t\t0.26\t\t0.106\t\t0.916\t\t0.27m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.25\t\t0.118\t\t0.900\t\t0.27m - 0.6m / 9.5m                                                                     \n",
      "3\t13k\t0.27\t\t0.23\t\t0.200\t\t0.907\t\t0.27m - 0.9m / 9.7m                                                                    \n",
      "4\t17k\t0.24\t\t0.22\t\t0.239\t\t0.875\t\t0.27m - 1.3m / 9.7m                                                                    \n",
      "5\t22k\t0.22\t\t0.22\t\t0.233\t\t0.883\t\t0.28m - 1.6m / 9.8m                                                                    \n",
      "6\t26k\t0.20\t\t0.24\t\t0.220\t\t0.875\t\t0.27m - 1.9m / 10.0m                                                                   \n",
      "7\t31k\t0.18\t\t0.24\t\t0.208\t\t0.843\t\t0.27m - 2.2m / 9.9m                                                                    \n",
      "8\t35k\t0.17\t\t0.24\t\t0.224\t\t0.855\t\t0.28m - 2.6m / 9.8m                                                                    \n",
      "9\t39k\t0.15\t\t0.24\t\t0.193\t\t0.849\t\t0.28m - 2.9m / 10.0m                                                                   \n",
      "VAL f1\t0.23883318140382861 - (0.23883318140382861)                                                                     \n",
      "VAL loss\t0.221194182412099                                                                                             \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.221194182412099\n",
      "        | \\     )|_\tf1: 0.23883318140382861\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\48 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 63.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         63                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0002115213863987488               |\n",
      "|  noam_learning_rate_warmup   |                        4200                       |\n",
      "|  noam_learning_rate_factor   |                 0.4545648347058415                |\n",
      "|          adam_beta1          |                 0.8051873637370621                |\n",
      "|          adam_beta2          |                 0.9987305942853002                |\n",
      "|           adam_eps           |               3.436843689786432e-07               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.1703828993335869                |\n",
      "|     pointwise_layer_size     |                        106                        |\n",
      "|      last_layer_dropout      |                 0.1669893988471761                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        131                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 67%|█████████████████████████████▍              | 67/100 [6:14:45<2:04:44, 226.81s/it, best loss: 0.21180716885460746]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "049e6ad0eb6c42eda3db78d4338b7615",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.42\t\t0.33\t\t0.065\t\t0.619\t\t0.35m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.31\t\t0.24\t\t0.086\t\t0.850\t\t0.34m - 0.7m / 12.2m                                                                    \n",
      "3\t13k\t0.28\t\t0.23\t\t0.100\t\t0.919\t\t0.35m - 1.1m / 12.1m                                                                   \n",
      "4\t17k\t0.27\t\t0.22\t\t0.068\t\t0.920\t\t0.34m - 1.5m / 12.2m                                                                   \n",
      "5\t22k\t0.26\t\t0.22\t\t0.116\t\t0.890\t\t0.34m - 1.9m / 12.2m                                                                   \n",
      "6\t26k\t0.25\t\t0.21\t\t0.142\t\t0.873\t\t0.34m - 2.3m / 12.2m                                                                   \n",
      "7\t30k\t0.23\t\t0.20\t\t0.170\t\t0.876\t\t0.34m - 2.7m / 12.3m                                                                   \n",
      "8\t35k\t0.21\t\t0.21\t\t0.158\t\t0.840\t\t0.35m - 3.1m / 12.4m                                                                   \n",
      "9\t39k\t0.20\t\t0.20\t\t0.225\t\t0.873\t\t0.34m - 3.5m / 12.7m                                                                   \n",
      "10\t43k\t0.19\t\t0.20\t\t0.204\t\t0.852\t\t0.34m - 3.9m / 12.5m                                                                  \n",
      "11\t48k\t0.18\t\t0.20\t\t0.223\t\t0.850\t\t0.34m - 4.3m / 12.6m                                                                  \n",
      "12\t52k\t0.17\t\t0.20\t\t0.208\t\t0.851\t\t0.35m - 4.7m / 12.6m                                                                  \n",
      "13\t57k\t0.16\t\t0.21\t\t0.223\t\t0.864\t\t0.35m - 5.1m / 12.8m                                                                  \n",
      "14\t61k\t0.15\t\t0.20\t\t0.226\t\t0.851\t\t0.34m - 5.5m / 12.9m                                                                  \n",
      "15\t65k\t0.14\t\t0.22\t\t0.188\t\t0.840\t\t0.34m - 5.9m / 12.8m                                                                  \n",
      "16\t70k\t0.14\t\t0.21\t\t0.250\t\t0.882\t\t0.34m - 6.3m / 12.8m                                                                  \n",
      "17\t74k\t0.13\t\t0.22\t\t0.235\t\t0.867\t\t0.35m - 6.7m / 12.9m                                                                  \n",
      "18\t78k\t0.12\t\t0.22\t\t0.242\t\t0.867\t\t0.35m - 7.1m / 13.0m                                                                  \n",
      "19\t83k\t0.11\t\t0.22\t\t0.240\t\t0.866\t\t0.34m - 7.5m / 13.1m                                                                  \n",
      "20\t87k\t0.11\t\t0.23\t\t0.237\t\t0.866\t\t0.34m - 7.9m / 13.1m                                                                  \n",
      "21\t91k\t0.10\t\t0.24\t\t0.239\t\t0.860\t\t0.34m - 8.3m / 13.1m                                                                  \n",
      "VAL f1\t0.25 - (0.25)                                                                                                   \n",
      "VAL loss\t0.19766364652643756                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19766364652643756\n",
      "        | \\     )|_\tf1: 0.25\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\49 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 63.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         63                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0001454667189625517               |\n",
      "|  noam_learning_rate_warmup   |                        4856                       |\n",
      "|  noam_learning_rate_factor   |                 1.2713285520197564                |\n",
      "|          adam_beta1          |                 0.7906886611503945                |\n",
      "|          adam_beta2          |                 0.7125440237651945                |\n",
      "|           adam_eps           |               5.326542145752844e-09               |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.380862771951625                 |\n",
      "|     pointwise_layer_size     |                         65                        |\n",
      "|      last_layer_dropout      |                0.17603503976489368                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        130                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 68%|█████████████████████████████▉              | 68/100 [6:23:31<2:48:53, 316.67s/it, best loss: 0.21180716885460746]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "102d6c22ae4244599314608fc48ab305",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.42\t\t0.27\t\t0.041\t\t0.799\t\t0.34m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.23\t\t0.166\t\t0.882\t\t0.34m - 0.7m / 11.9m                                                                    \n",
      "3\t13k\t0.27\t\t0.23\t\t0.193\t\t0.895\t\t0.34m - 1.1m / 12.0m                                                                   \n",
      "4\t17k\t0.26\t\t0.21\t\t0.206\t\t0.874\t\t0.34m - 1.5m / 12.1m                                                                   \n",
      "5\t22k\t0.24\t\t0.21\t\t0.211\t\t0.854\t\t0.34m - 1.9m / 12.1m                                                                   \n",
      "6\t26k\t0.22\t\t0.22\t\t0.165\t\t0.799\t\t0.33m - 2.3m / 12.1m                                                                   \n",
      "7\t30k\t0.21\t\t0.22\t\t0.169\t\t0.803\t\t0.34m - 2.7m / 12.0m                                                                   \n",
      "8\t35k\t0.20\t\t0.23\t\t0.177\t\t0.789\t\t0.34m - 3.1m / 12.1m                                                                   \n",
      "9\t39k\t0.19\t\t0.22\t\t0.203\t\t0.824\t\t0.34m - 3.5m / 12.2m                                                                   \n",
      "10\t43k\t0.18\t\t0.22\t\t0.228\t\t0.825\t\t0.34m - 3.8m / 12.3m                                                                  \n",
      "11\t48k\t0.17\t\t0.23\t\t0.209\t\t0.821\t\t0.34m - 4.2m / 12.3m                                                                  \n",
      "12\t52k\t0.17\t\t0.25\t\t0.152\t\t0.767\t\t0.34m - 4.6m / 12.4m                                                                  \n",
      "13\t57k\t0.16\t\t0.23\t\t0.234\t\t0.838\t\t0.34m - 5.0m / 12.4m                                                                  \n",
      "14\t61k\t0.16\t\t0.23\t\t0.211\t\t0.819\t\t0.34m - 5.4m / 12.6m                                                                  \n",
      "15\t65k\t0.15\t\t0.24\t\t0.182\t\t0.799\t\t0.34m - 5.8m / 12.6m                                                                  \n",
      "16\t70k\t0.14\t\t0.24\t\t0.230\t\t0.853\t\t0.34m - 6.2m / 12.7m                                                                  \n",
      "17\t74k\t0.14\t\t0.24\t\t0.211\t\t0.822\t\t0.35m - 6.6m / 12.8m                                                                  \n",
      "18\t78k\t0.14\t\t0.23\t\t0.250\t\t0.862\t\t0.35m - 7.0m / 12.9m                                                                  \n",
      "19\t83k\t0.13\t\t0.25\t\t0.224\t\t0.837\t\t0.34m - 7.4m / 13.0m                                                                  \n",
      "20\t87k\t0.13\t\t0.26\t\t0.233\t\t0.852\t\t0.34m - 7.8m / 13.0m                                                                  \n",
      "21\t91k\t0.12\t\t0.28\t\t0.226\t\t0.835\t\t0.35m - 8.2m / 13.0m                                                                  \n",
      "22\t96k\t0.12\t\t0.28\t\t0.252\t\t0.862\t\t0.35m - 8.6m / 13.2m                                                                  \n",
      "23\t100k\t0.12\t\t0.28\t\t0.249\t\t0.861\t\t0.34m - 9.0m / 13.2m                                                                 \n",
      "24\t104k\t0.11\t\t0.29\t\t0.259\t\t0.859\t\t0.35m - 9.5m / 13.2m                                                                 \n",
      "25\t109k\t0.11\t\t0.29\t\t0.221\t\t0.831\t\t0.34m - 9.9m / 13.3m                                                                 \n",
      "26\t113k\t0.10\t\t0.30\t\t0.235\t\t0.845\t\t0.35m - 10.3m / 13.4m                                                                \n",
      "27\t117k\t0.10\t\t0.31\t\t0.231\t\t0.855\t\t0.35m - 10.7m / 13.5m                                                                \n",
      "28\t122k\t0.10\t\t0.32\t\t0.250\t\t0.862\t\t0.34m - 11.1m / 13.5m                                                                \n",
      "29\t126k\t0.09\t\t0.32\t\t0.231\t\t0.855\t\t0.34m - 11.5m / 13.5m                                                                \n",
      "VAL f1\t0.2585139318885449 - (0.2585139318885449)                                                                       \n",
      "VAL loss\t0.2097190498674988                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2097190498674988\n",
      "        | \\     )|_\tf1: 0.2585139318885449\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\50 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 62.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         62                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00016840973956259016              |\n",
      "|  noam_learning_rate_warmup   |                        3623                       |\n",
      "|  noam_learning_rate_factor   |                 0.4150230798936053                |\n",
      "|          adam_beta1          |                 0.7767090225510399                |\n",
      "|          adam_beta2          |                 0.9777552715687797                |\n",
      "|           adam_eps           |               1.5205703918047645e-09              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.18454418705535447                |\n",
      "|     pointwise_layer_size     |                         83                        |\n",
      "|      last_layer_dropout      |                 0.3063965836318771                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        156                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 69%|██████████████████████████████▎             | 69/100 [6:35:26<3:45:21, 436.19s/it, best loss: 0.21180716885460746]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43f65dd44e574f91ae55b4a937412a7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.45\t\t0.35\t\t0.040\t\t0.589\t\t0.32m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.24\t\t0.044\t\t0.921\t\t0.31m - 0.7m / 11.1m                                                                    \n",
      "3\t13k\t0.28\t\t0.24\t\t0.023\t\t0.929\t\t0.31m - 1.2m / 11.2m                                                                   \n",
      "4\t17k\t0.28\t\t0.24\t\t0.095\t\t0.920\t\t0.31m - 1.5m / 11.3m                                                                   \n",
      "5\t22k\t0.27\t\t0.23\t\t0.129\t\t0.914\t\t0.31m - 1.9m / 11.2m                                                                   \n",
      "6\t26k\t0.26\t\t0.22\t\t0.187\t\t0.896\t\t0.31m - 2.3m / 11.3m                                                                   \n",
      "7\t30k\t0.24\t\t0.21\t\t0.163\t\t0.875\t\t0.31m - 2.6m / 11.4m                                                                   \n",
      "8\t35k\t0.23\t\t0.21\t\t0.230\t\t0.880\t\t0.31m - 3.0m / 11.4m                                                                   \n",
      "9\t39k\t0.21\t\t0.20\t\t0.217\t\t0.884\t\t0.31m - 3.4m / 11.5m                                                                   \n",
      "10\t43k\t0.20\t\t0.20\t\t0.249\t\t0.876\t\t0.31m - 3.7m / 11.6m                                                                  \n",
      "11\t48k\t0.19\t\t0.20\t\t0.266\t\t0.887\t\t0.31m - 4.1m / 11.5m                                                                  \n",
      "12\t52k\t0.18\t\t0.21\t\t0.222\t\t0.863\t\t0.31m - 4.5m / 11.7m                                                                  \n",
      "13\t56k\t0.17\t\t0.21\t\t0.251\t\t0.862\t\t0.31m - 4.8m / 11.7m                                                                  \n",
      "14\t61k\t0.16\t\t0.21\t\t0.245\t\t0.872\t\t0.32m - 5.2m / 11.8m                                                                  \n",
      "15\t65k\t0.15\t\t0.21\t\t0.223\t\t0.852\t\t0.32m - 5.6m / 12.0m                                                                  \n",
      "16\t69k\t0.15\t\t0.22\t\t0.251\t\t0.871\t\t0.32m - 6.0m / 12.1m                                                                  \n",
      "VAL f1\t0.2664129400570885 - (0.2664129400570885)                                                                       \n",
      "VAL loss\t0.19632605711619058                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19632605711619058\n",
      "        | \\     )|_\tf1: 0.2664129400570885\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\51 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 62.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         62                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00013055363147430223              |\n",
      "|  noam_learning_rate_warmup   |                        4274                       |\n",
      "|  noam_learning_rate_factor   |                 0.4036889297283957                |\n",
      "|          adam_beta1          |                 0.7367859924043584                |\n",
      "|          adam_beta2          |                 0.992109859916319                 |\n",
      "|           adam_eps           |               1.1535314390209003e-09              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.18146659944928234                |\n",
      "|     pointwise_layer_size     |                         41                        |\n",
      "|      last_layer_dropout      |                 0.2734411331263888                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        147                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 70%|██████████████████████████████▊             | 70/100 [6:41:51<3:30:25, 420.84s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2219c141f2764f43a241c31724a44b96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.44\t\t0.37\t\t0.038\t\t0.568\t\t0.36m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.26\t\t0.037\t\t0.915\t\t0.37m - 0.8m / 12.8m                                                                    \n",
      "3\t13k\t0.29\t\t0.24\t\t0.053\t\t0.930\t\t0.36m - 1.2m / 13.0m                                                                   \n",
      "4\t17k\t0.28\t\t0.24\t\t0.114\t\t0.924\t\t0.36m - 1.6m / 12.9m                                                                   \n",
      "5\t22k\t0.27\t\t0.23\t\t0.103\t\t0.917\t\t0.37m - 2.1m / 13.0m                                                                   \n",
      "6\t26k\t0.26\t\t0.22\t\t0.178\t\t0.898\t\t0.36m - 2.5m / 13.2m                                                                   \n",
      "7\t30k\t0.25\t\t0.22\t\t0.186\t\t0.880\t\t0.37m - 2.9m / 13.0m                                                                   \n",
      "8\t35k\t0.23\t\t0.21\t\t0.189\t\t0.882\t\t0.37m - 3.3m / 13.4m                                                                   \n",
      "9\t39k\t0.22\t\t0.21\t\t0.226\t\t0.879\t\t0.37m - 3.8m / 13.5m                                                                   \n",
      "10\t43k\t0.21\t\t0.21\t\t0.211\t\t0.882\t\t0.37m - 4.2m / 13.5m                                                                  \n",
      "11\t48k\t0.20\t\t0.20\t\t0.230\t\t0.884\t\t0.37m - 4.6m / 13.6m                                                                  \n",
      "12\t52k\t0.19\t\t0.21\t\t0.236\t\t0.871\t\t0.36m - 5.0m / 13.6m                                                                  \n",
      "13\t56k\t0.18\t\t0.22\t\t0.236\t\t0.865\t\t0.34m - 5.5m / 13.4m                                                                  \n",
      "14\t61k\t0.17\t\t0.21\t\t0.241\t\t0.870\t\t0.36m - 5.9m / 13.0m                                                                  \n",
      "15\t65k\t0.16\t\t0.21\t\t0.225\t\t0.856\t\t0.37m - 6.3m / 13.5m                                                                  \n",
      "16\t69k\t0.15\t\t0.22\t\t0.237\t\t0.878\t\t0.36m - 6.7m / 13.7m                                                                  \n",
      "17\t74k\t0.14\t\t0.21\t\t0.259\t\t0.869\t\t0.36m - 7.1m / 13.7m                                                                  \n",
      "18\t78k\t0.14\t\t0.22\t\t0.235\t\t0.874\t\t0.37m - 7.6m / 13.7m                                                                  \n",
      "19\t82k\t0.13\t\t0.22\t\t0.239\t\t0.859\t\t0.36m - 8.0m / 13.8m                                                                  \n",
      "20\t87k\t0.13\t\t0.24\t\t0.240\t\t0.863\t\t0.37m - 8.4m / 13.9m                                                                  \n",
      "21\t91k\t0.12\t\t0.23\t\t0.235\t\t0.862\t\t0.37m - 8.8m / 14.0m                                                                  \n",
      "22\t95k\t0.11\t\t0.23\t\t0.273\t\t0.880\t\t0.38m - 9.3m / 14.1m                                                                  \n",
      "23\t100k\t0.11\t\t0.24\t\t0.243\t\t0.863\t\t0.37m - 9.7m / 14.2m                                                                 \n",
      "24\t104k\t0.10\t\t0.24\t\t0.262\t\t0.881\t\t0.36m - 10.1m / 14.2m                                                                \n",
      "25\t108k\t0.10\t\t0.23\t\t0.228\t\t0.862\t\t0.36m - 10.5m / 14.2m                                                                \n",
      "26\t113k\t0.09\t\t0.25\t\t0.235\t\t0.861\t\t0.37m - 11.0m / 14.2m                                                                \n",
      "27\t117k\t0.09\t\t0.25\t\t0.226\t\t0.872\t\t0.36m - 11.4m / 14.3m                                                                \n",
      "VAL f1\t0.2732811140121845 - (0.2732811140121845)                                                                       \n",
      "VAL loss\t0.20496040646747873                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.20496040646747873\n",
      "        | \\     )|_\tf1: 0.2732811140121845\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\52 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 55.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         55                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               6.995517674560595e-05               |\n",
      "|  noam_learning_rate_warmup   |                        6364                       |\n",
      "|  noam_learning_rate_factor   |                 0.512588777336199                 |\n",
      "|          adam_beta1          |                 0.7758706984103994                |\n",
      "|          adam_beta2          |                 0.9789930161028432                |\n",
      "|           adam_eps           |               4.092908118052232e-07               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.06765074068073688                |\n",
      "|     pointwise_layer_size     |                         85                        |\n",
      "|      last_layer_dropout      |                 0.3468390382990714                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        126                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 71%|███████████████████████████████▏            | 71/100 [6:53:44<4:05:44, 508.43s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de477382f8834f5589affb7a65562361",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.56\t\t0.50\t\t0.054\t\t0.490\t\t0.35m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.39\t\t0.29\t\t0.067\t\t0.816\t\t0.35m - 0.7m / 12.2m                                                                    \n",
      "3\t13k\t0.32\t\t0.26\t\t0.128\t\t0.906\t\t0.35m - 1.2m / 12.3m                                                                   \n",
      "4\t17k\t0.31\t\t0.25\t\t0.114\t\t0.925\t\t0.35m - 1.6m / 12.4m                                                                   \n",
      "5\t22k\t0.29\t\t0.25\t\t0.149\t\t0.907\t\t0.35m - 2.0m / 12.5m                                                                   \n",
      "6\t26k\t0.28\t\t0.23\t\t0.219\t\t0.909\t\t0.35m - 2.4m / 12.5m                                                                   \n",
      "7\t30k\t0.27\t\t0.23\t\t0.206\t\t0.893\t\t0.35m - 2.8m / 12.6m                                                                   \n",
      "8\t35k\t0.25\t\t0.23\t\t0.181\t\t0.875\t\t0.35m - 3.2m / 12.6m                                                                   \n",
      "9\t39k\t0.24\t\t0.22\t\t0.213\t\t0.875\t\t0.35m - 3.6m / 12.7m                                                                   \n",
      "10\t43k\t0.22\t\t0.22\t\t0.207\t\t0.880\t\t0.35m - 4.0m / 12.8m                                                                  \n",
      "11\t48k\t0.21\t\t0.22\t\t0.226\t\t0.875\t\t0.36m - 4.4m / 12.8m                                                                  \n",
      "12\t52k\t0.20\t\t0.22\t\t0.236\t\t0.870\t\t0.35m - 4.8m / 13.0m                                                                  \n",
      "13\t56k\t0.19\t\t0.23\t\t0.217\t\t0.851\t\t0.36m - 5.2m / 12.9m                                                                  \n",
      "14\t61k\t0.18\t\t0.23\t\t0.232\t\t0.868\t\t0.37m - 5.6m / 13.1m                                                                  \n",
      "15\t65k\t0.17\t\t0.22\t\t0.246\t\t0.882\t\t0.37m - 6.1m / 13.4m                                                                  \n",
      "16\t70k\t0.16\t\t0.23\t\t0.252\t\t0.883\t\t0.37m - 6.5m / 13.4m                                                                  \n",
      "17\t74k\t0.15\t\t0.23\t\t0.261\t\t0.873\t\t0.36m - 6.9m / 13.6m                                                                  \n",
      "18\t78k\t0.14\t\t0.23\t\t0.246\t\t0.881\t\t0.36m - 7.3m / 13.5m                                                                  \n",
      "19\t83k\t0.14\t\t0.23\t\t0.254\t\t0.875\t\t0.36m - 7.7m / 13.5m                                                                  \n",
      "20\t87k\t0.13\t\t0.24\t\t0.227\t\t0.872\t\t0.37m - 8.2m / 13.6m                                                                  \n",
      "21\t91k\t0.12\t\t0.24\t\t0.285\t\t0.892\t\t0.36m - 8.6m / 13.7m                                                                  \n",
      "22\t96k\t0.11\t\t0.24\t\t0.261\t\t0.874\t\t0.35m - 9.0m / 13.6m                                                                  \n",
      "23\t100k\t0.10\t\t0.24\t\t0.251\t\t0.885\t\t0.35m - 9.4m / 13.7m                                                                 \n",
      "24\t104k\t0.10\t\t0.24\t\t0.282\t\t0.889\t\t0.36m - 9.8m / 13.7m                                                                 \n",
      "25\t109k\t0.09\t\t0.25\t\t0.271\t\t0.891\t\t0.36m - 10.2m / 13.8m                                                                \n",
      "26\t113k\t0.08\t\t0.26\t\t0.236\t\t0.880\t\t0.36m - 10.6m / 13.8m                                                                \n",
      "VAL f1\t0.2854349951124145 - (0.2854349951124145)                                                                       \n",
      "VAL loss\t0.21803426123284672                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21803426123284672\n",
      "        | \\     )|_\tf1: 0.2854349951124145\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\53 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 59.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         59                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00015823085070310392              |\n",
      "|  noam_learning_rate_warmup   |                        5417                       |\n",
      "|  noam_learning_rate_factor   |                0.16761224473412284                |\n",
      "|          adam_beta1          |                 0.7530595962908538                |\n",
      "|          adam_beta2          |                 0.9669517513304332                |\n",
      "|           adam_eps           |               1.2363923572442724e-08              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.11705113358133515                |\n",
      "|     pointwise_layer_size     |                         33                        |\n",
      "|      last_layer_dropout      |                0.12785351388697952                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        160                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 72%|███████████████████████████████▋            | 72/100 [7:04:50<4:19:22, 555.82s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61564f6d10574a939a9a49fd2a46168c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.50\t\t0.45\t\t0.057\t\t0.336\t\t0.38m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.40\t\t0.32\t\t0.110\t\t0.723\t\t0.39m - 0.8m / 13.5m                                                                    \n",
      "3\t13k\t0.33\t\t0.27\t\t0.135\t\t0.875\t\t0.39m - 1.3m / 13.6m                                                                   \n",
      "4\t17k\t0.30\t\t0.25\t\t0.057\t\t0.925\t\t0.39m - 1.7m / 13.6m                                                                   \n",
      "5\t22k\t0.29\t\t0.25\t\t0.054\t\t0.926\t\t0.39m - 2.2m / 13.8m                                                                   \n",
      "6\t26k\t0.28\t\t0.25\t\t0.032\t\t0.922\t\t0.39m - 2.6m / 14.0m                                                                   \n",
      "7\t31k\t0.28\t\t0.25\t\t0.089\t\t0.927\t\t0.39m - 3.0m / 13.9m                                                                   \n",
      "8\t35k\t0.27\t\t0.25\t\t0.098\t\t0.920\t\t0.39m - 3.5m / 14.0m                                                                   \n",
      "VAL f1\t0.1346578366445916 - (0.1346578366445916)                                                                       \n",
      "VAL loss\t0.24578220022600247                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24578220022600247\n",
      "        | \\     )|_\tf1: 0.1346578366445916\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\54 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 19.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         19                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00026693178690373026              |\n",
      "|  noam_learning_rate_warmup   |                        3643                       |\n",
      "|  noam_learning_rate_factor   |                 0.7295773843035682                |\n",
      "|          adam_beta1          |                 0.7984040478332852                |\n",
      "|          adam_beta2          |                 0.9434715912441379                |\n",
      "|           adam_eps           |               3.9746149877437554e-10              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.025976326760238505               |\n",
      "|     pointwise_layer_size     |                         55                        |\n",
      "|      last_layer_dropout      |                 0.310395970729297                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        132                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 73%|████████████████████████████████            | 73/100 [7:08:47<3:27:04, 460.18s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f10f78b346b742c285b3dc7db066ce1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.99\t\t0.62\t\t0.076\t\t0.913\t\t0.77m - 0.8m / 0.0m                                                                     \n",
      "2\t9k\t0.68\t\t0.51\t\t0.290\t\t0.915\t\t0.78m - 1.6m / 27.1m                                                                    \n",
      "3\t13k\t0.59\t\t0.48\t\t0.301\t\t0.909\t\t0.77m - 2.4m / 27.3m                                                                   \n",
      "4\t17k\t0.55\t\t0.51\t\t0.249\t\t0.911\t\t0.77m - 3.3m / 27.2m                                                                   \n",
      "5\t22k\t0.51\t\t0.48\t\t0.267\t\t0.910\t\t0.78m - 4.1m / 27.1m                                                                   \n",
      "6\t26k\t0.47\t\t0.55\t\t0.207\t\t0.863\t\t0.77m - 4.9m / 27.5m                                                                   \n",
      "7\t30k\t0.42\t\t0.55\t\t0.248\t\t0.883\t\t0.78m - 5.7m / 27.4m                                                                   \n",
      "8\t35k\t0.39\t\t0.62\t\t0.231\t\t0.866\t\t0.77m - 6.6m / 27.6m                                                                   \n",
      "VAL f1\t0.30056497175141245 - (0.30056497175141245)                                                                     \n",
      "VAL loss\t0.4766918148910791                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4766918148910791\n",
      "        | \\     )|_\tf1: 0.30056497175141245\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\55 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 57.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         57                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00010278345209666197              |\n",
      "|  noam_learning_rate_warmup   |                        4617                       |\n",
      "|  noam_learning_rate_factor   |                 0.8405179738876511                |\n",
      "|          adam_beta1          |                 0.7262083210003027                |\n",
      "|          adam_beta2          |                 0.9539957992477722                |\n",
      "|           adam_eps           |                5.09020375539717e-08               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.1314810126251545                |\n",
      "|     pointwise_layer_size     |                         71                        |\n",
      "|      last_layer_dropout      |                0.20089208694953561                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        118                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 74%|████████████████████████████████▌           | 74/100 [7:15:58<3:15:31, 451.21s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36ab48b5840745cf97cb338dbc6be563",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.54\t\t0.34\t\t0.056\t\t0.685\t\t0.33m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.27\t\t0.055\t\t0.926\t\t0.34m - 0.7m / 11.6m                                                                    \n",
      "3\t13k\t0.30\t\t0.26\t\t0.084\t\t0.922\t\t0.33m - 1.1m / 11.9m                                                                   \n",
      "4\t17k\t0.28\t\t0.25\t\t0.145\t\t0.905\t\t0.33m - 1.5m / 11.8m                                                                   \n",
      "5\t22k\t0.26\t\t0.23\t\t0.204\t\t0.892\t\t0.34m - 1.9m / 11.8m                                                                   \n",
      "6\t26k\t0.24\t\t0.23\t\t0.199\t\t0.870\t\t0.33m - 2.3m / 12.0m                                                                   \n",
      "7\t30k\t0.23\t\t0.22\t\t0.232\t\t0.864\t\t0.33m - 2.7m / 12.0m                                                                   \n",
      "8\t35k\t0.21\t\t0.22\t\t0.241\t\t0.883\t\t0.34m - 3.1m / 12.1m                                                                   \n",
      "9\t39k\t0.20\t\t0.23\t\t0.227\t\t0.864\t\t0.33m - 3.4m / 12.2m                                                                   \n",
      "10\t43k\t0.18\t\t0.22\t\t0.219\t\t0.875\t\t0.33m - 3.8m / 12.1m                                                                  \n",
      "11\t48k\t0.17\t\t0.23\t\t0.230\t\t0.849\t\t0.34m - 4.2m / 12.2m                                                                  \n",
      "12\t52k\t0.16\t\t0.23\t\t0.257\t\t0.874\t\t0.33m - 4.6m / 12.3m                                                                  \n",
      "13\t56k\t0.15\t\t0.24\t\t0.216\t\t0.853\t\t0.34m - 5.0m / 12.2m                                                                  \n",
      "14\t61k\t0.14\t\t0.25\t\t0.206\t\t0.839\t\t0.34m - 5.4m / 12.4m                                                                  \n",
      "15\t65k\t0.13\t\t0.25\t\t0.235\t\t0.873\t\t0.34m - 5.8m / 12.5m                                                                  \n",
      "16\t69k\t0.13\t\t0.25\t\t0.233\t\t0.864\t\t0.34m - 6.2m / 12.6m                                                                  \n",
      "17\t74k\t0.12\t\t0.26\t\t0.253\t\t0.878\t\t0.34m - 6.6m / 12.6m                                                                  \n",
      "VAL f1\t0.2566295979469632 - (0.2566295979469632)                                                                       \n",
      "VAL loss\t0.21725485478228299                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21725485478228299\n",
      "        | \\     )|_\tf1: 0.2566295979469632\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\56 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 52.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         52                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.001488187039190986               |\n",
      "|  noam_learning_rate_warmup   |                        4174                       |\n",
      "|  noam_learning_rate_factor   |                0.03090775783160482                |\n",
      "|          adam_beta1          |                 0.827284878208874                 |\n",
      "|          adam_beta2          |                 0.998486018323427                 |\n",
      "|           adam_eps           |               1.9301610210082968e-07              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.054662208525527445               |\n",
      "|     pointwise_layer_size     |                        119                        |\n",
      "|      last_layer_dropout      |                0.40980323785100636                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        140                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 75%|█████████████████████████████████           | 75/100 [7:22:57<3:04:00, 441.62s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e56c0b840864309aae84e0ed05a6eaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.63\t\t0.60\t\t0.038\t\t0.178\t\t0.39m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.57\t\t0.50\t\t0.042\t\t0.300\t\t0.39m - 0.8m / 13.7m                                                                    \n",
      "3\t13k\t0.48\t\t0.41\t\t0.064\t\t0.535\t\t0.39m - 1.3m / 13.7m                                                                   \n",
      "4\t17k\t0.40\t\t0.34\t\t0.123\t\t0.763\t\t0.40m - 1.7m / 13.9m                                                                   \n",
      "5\t22k\t0.36\t\t0.31\t\t0.135\t\t0.878\t\t0.41m - 2.2m / 14.2m                                                                   \n",
      "6\t26k\t0.34\t\t0.29\t\t0.113\t\t0.912\t\t0.40m - 2.7m / 14.4m                                                                   \n",
      "7\t31k\t0.33\t\t0.28\t\t0.085\t\t0.920\t\t0.39m - 3.1m / 14.3m                                                                   \n",
      "8\t35k\t0.32\t\t0.28\t\t0.065\t\t0.926\t\t0.39m - 3.5m / 14.2m                                                                   \n",
      "9\t39k\t0.32\t\t0.28\t\t0.044\t\t0.928\t\t0.39m - 4.0m / 14.2m                                                                   \n",
      "10\t44k\t0.32\t\t0.27\t\t0.089\t\t0.927\t\t0.39m - 4.5m / 14.2m                                                                  \n",
      "VAL f1\t0.13498312710911137 - (0.13498312710911137)                                                                     \n",
      "VAL loss\t0.27325048551454645                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.27325048551454645\n",
      "        | \\     )|_\tf1: 0.13498312710911137\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\57 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 64.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0004966323382999671               |\n",
      "|  noam_learning_rate_warmup   |                        5856                       |\n",
      "|  noam_learning_rate_factor   |                0.30132868493296155                |\n",
      "|          adam_beta1          |                 0.7645418885674832                |\n",
      "|          adam_beta2          |                 0.9107953452986163                |\n",
      "|           adam_eps           |               2.2540832542433917e-10              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.25138743230321264                |\n",
      "|     pointwise_layer_size     |                        101                        |\n",
      "|      last_layer_dropout      |                 0.2508972720766134                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        155                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 76%|█████████████████████████████████▍          | 76/100 [7:27:51<2:38:59, 397.49s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55d8ed61fdfc49378bc49da7d3dcddec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.54\t\t0.53\t\t0.036\t\t0.198\t\t0.31m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.43\t\t0.34\t\t0.037\t\t0.549\t\t0.31m - 0.7m / 10.8m                                                                    \n",
      "3\t13k\t0.32\t\t0.25\t\t0.039\t\t0.921\t\t0.31m - 1.0m / 10.8m                                                                   \n",
      "4\t17k\t0.28\t\t0.23\t\t0.035\t\t0.937\t\t0.31m - 1.4m / 10.9m                                                                   \n",
      "5\t22k\t0.27\t\t0.22\t\t0.055\t\t0.938\t\t0.31m - 1.7m / 11.0m                                                                   \n",
      "6\t26k\t0.27\t\t0.22\t\t0.071\t\t0.933\t\t0.31m - 2.1m / 11.0m                                                                   \n",
      "7\t30k\t0.27\t\t0.22\t\t0.075\t\t0.932\t\t0.31m - 2.5m / 11.1m                                                                   \n",
      "8\t35k\t0.26\t\t0.22\t\t0.107\t\t0.929\t\t0.31m - 2.8m / 11.1m                                                                   \n",
      "9\t39k\t0.26\t\t0.22\t\t0.147\t\t0.910\t\t0.31m - 3.2m / 11.2m                                                                   \n",
      "10\t44k\t0.25\t\t0.21\t\t0.200\t\t0.903\t\t0.31m - 3.6m / 11.3m                                                                  \n",
      "11\t48k\t0.24\t\t0.21\t\t0.192\t\t0.890\t\t0.31m - 3.9m / 11.4m                                                                  \n",
      "12\t52k\t0.23\t\t0.21\t\t0.192\t\t0.883\t\t0.31m - 4.3m / 11.5m                                                                  \n",
      "13\t57k\t0.22\t\t0.21\t\t0.205\t\t0.869\t\t0.31m - 4.7m / 11.5m                                                                  \n",
      "14\t61k\t0.21\t\t0.20\t\t0.213\t\t0.868\t\t0.31m - 5.0m / 11.5m                                                                  \n",
      "15\t65k\t0.20\t\t0.20\t\t0.226\t\t0.874\t\t0.31m - 5.4m / 11.6m                                                                  \n",
      "16\t70k\t0.20\t\t0.20\t\t0.222\t\t0.872\t\t0.31m - 5.8m / 11.7m                                                                  \n",
      "17\t74k\t0.19\t\t0.20\t\t0.232\t\t0.869\t\t0.31m - 6.1m / 11.7m                                                                  \n",
      "18\t78k\t0.19\t\t0.20\t\t0.221\t\t0.856\t\t0.31m - 6.5m / 11.8m                                                                  \n",
      "19\t83k\t0.18\t\t0.21\t\t0.210\t\t0.842\t\t0.31m - 6.9m / 11.8m                                                                  \n",
      "20\t87k\t0.17\t\t0.21\t\t0.195\t\t0.836\t\t0.31m - 7.2m / 11.9m                                                                  \n",
      "21\t91k\t0.17\t\t0.21\t\t0.225\t\t0.846\t\t0.31m - 7.6m / 11.9m                                                                  \n",
      "22\t96k\t0.17\t\t0.21\t\t0.220\t\t0.846\t\t0.31m - 8.0m / 12.0m                                                                  \n",
      "VAL f1\t0.23150565709312446 - (0.23150565709312446)                                                                     \n",
      "VAL loss\t0.19904887676239014                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19904887676239014\n",
      "        | \\     )|_\tf1: 0.23150565709312446\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\58 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 61.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         61                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0005437683275309334               |\n",
      "|  noam_learning_rate_warmup   |                        6793                       |\n",
      "|  noam_learning_rate_factor   |                 0.5729003093512732                |\n",
      "|          adam_beta1          |                 0.7147035716759778                |\n",
      "|          adam_beta2          |                 0.9122061752204602                |\n",
      "|           adam_eps           |               3.6114129454293317e-10              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.2084622668067005                |\n",
      "|     pointwise_layer_size     |                        142                        |\n",
      "|      last_layer_dropout      |                 0.2576210001651822                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        173                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 77%|█████████████████████████████████▉          | 77/100 [7:36:15<2:44:35, 429.36s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1ed6d49605046e8a7fc4d9d1e3ced12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.48\t\t0.42\t\t0.071\t\t0.537\t\t0.34m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.35\t\t0.28\t\t0.135\t\t0.874\t\t0.33m - 0.7m / 11.8m                                                                    \n",
      "3\t13k\t0.29\t\t0.24\t\t0.047\t\t0.923\t\t0.34m - 1.1m / 11.8m                                                                   \n",
      "4\t17k\t0.29\t\t0.24\t\t0.022\t\t0.928\t\t0.34m - 1.5m / 12.1m                                                                   \n",
      "5\t22k\t0.28\t\t0.24\t\t0.021\t\t0.937\t\t0.34m - 1.9m / 12.2m                                                                   \n",
      "6\t26k\t0.28\t\t0.24\t\t0.097\t\t0.927\t\t0.34m - 2.3m / 12.3m                                                                   \n",
      "7\t30k\t0.27\t\t0.23\t\t0.102\t\t0.897\t\t0.34m - 2.7m / 12.2m                                                                   \n",
      "VAL f1\t0.1345093715545755 - (0.1345093715545755)                                                                       \n",
      "VAL loss\t0.2342609082414804                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2342609082414804\n",
      "        | \\     )|_\tf1: 0.1345093715545755\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\59 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 50.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         50                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0007821329914894593               |\n",
      "|  noam_learning_rate_warmup   |                        6516                       |\n",
      "|  noam_learning_rate_factor   |                0.28030252234706715                |\n",
      "|          adam_beta1          |                 0.7671183075463001                |\n",
      "|          adam_beta2          |                 0.922689511119231                 |\n",
      "|           adam_eps           |               2.0744032065405092e-09              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.2406919583903814                |\n",
      "|     pointwise_layer_size     |                        100                        |\n",
      "|      last_layer_dropout      |                0.38558646239141536                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        159                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 78%|██████████████████████████████████▎         | 78/100 [7:39:24<2:11:00, 357.29s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7240acc79e942d69ec48575d2d8bdfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.66\t\t0.64\t\t0.039\t\t0.347\t\t0.35m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.51\t\t0.42\t\t0.045\t\t0.641\t\t0.35m - 0.8m / 12.2m                                                                    \n",
      "3\t13k\t0.38\t\t0.31\t\t0.048\t\t0.899\t\t0.35m - 1.2m / 12.4m                                                                   \n",
      "4\t17k\t0.34\t\t0.29\t\t0.012\t\t0.920\t\t0.35m - 1.6m / 12.4m                                                                   \n",
      "5\t22k\t0.33\t\t0.28\t\t0.023\t\t0.931\t\t0.35m - 2.0m / 12.5m                                                                   \n",
      "6\t26k\t0.33\t\t0.28\t\t0.035\t\t0.929\t\t0.35m - 2.4m / 12.5m                                                                   \n",
      "7\t30k\t0.32\t\t0.27\t\t0.096\t\t0.925\t\t0.35m - 2.8m / 12.5m                                                                   \n",
      "8\t35k\t0.31\t\t0.27\t\t0.136\t\t0.924\t\t0.35m - 3.2m / 12.7m                                                                   \n",
      "9\t39k\t0.30\t\t0.26\t\t0.149\t\t0.918\t\t0.35m - 3.6m / 12.6m                                                                   \n",
      "10\t44k\t0.29\t\t0.26\t\t0.134\t\t0.883\t\t0.35m - 4.0m / 12.8m                                                                  \n",
      "11\t48k\t0.28\t\t0.25\t\t0.181\t\t0.887\t\t0.35m - 4.4m / 12.8m                                                                  \n",
      "12\t52k\t0.27\t\t0.26\t\t0.163\t\t0.878\t\t0.36m - 4.8m / 12.9m                                                                  \n",
      "13\t57k\t0.26\t\t0.25\t\t0.196\t\t0.877\t\t0.35m - 5.2m / 13.1m                                                                  \n",
      "14\t61k\t0.25\t\t0.25\t\t0.191\t\t0.882\t\t0.36m - 5.6m / 13.1m                                                                  \n",
      "15\t65k\t0.24\t\t0.25\t\t0.216\t\t0.878\t\t0.35m - 6.0m / 13.1m                                                                  \n",
      "16\t70k\t0.23\t\t0.26\t\t0.218\t\t0.870\t\t0.35m - 6.4m / 13.2m                                                                  \n",
      "17\t74k\t0.23\t\t0.25\t\t0.216\t\t0.865\t\t0.36m - 6.9m / 13.2m                                                                  \n",
      "18\t78k\t0.22\t\t0.26\t\t0.205\t\t0.857\t\t0.37m - 7.3m / 13.4m                                                                  \n",
      "19\t83k\t0.21\t\t0.26\t\t0.215\t\t0.861\t\t0.37m - 7.7m / 13.6m                                                                  \n",
      "20\t87k\t0.21\t\t0.27\t\t0.205\t\t0.851\t\t0.36m - 8.1m / 13.6m                                                                  \n",
      "21\t91k\t0.20\t\t0.27\t\t0.218\t\t0.860\t\t0.36m - 8.5m / 13.6m                                                                  \n",
      "22\t96k\t0.19\t\t0.27\t\t0.223\t\t0.865\t\t0.36m - 9.0m / 13.7m                                                                  \n",
      "23\t100k\t0.19\t\t0.28\t\t0.207\t\t0.856\t\t0.35m - 9.4m / 13.7m                                                                 \n",
      "24\t104k\t0.18\t\t0.27\t\t0.204\t\t0.859\t\t0.36m - 9.8m / 13.7m                                                                 \n",
      "25\t109k\t0.18\t\t0.28\t\t0.220\t\t0.856\t\t0.35m - 10.2m / 13.8m                                                                \n",
      "26\t113k\t0.17\t\t0.27\t\t0.237\t\t0.873\t\t0.36m - 10.6m / 13.8m                                                                \n",
      "27\t117k\t0.17\t\t0.28\t\t0.195\t\t0.847\t\t0.36m - 11.0m / 13.9m                                                                \n",
      "28\t122k\t0.16\t\t0.28\t\t0.215\t\t0.853\t\t0.36m - 11.4m / 13.9m                                                                \n",
      "29\t126k\t0.16\t\t0.30\t\t0.219\t\t0.855\t\t0.36m - 11.9m / 14.0m                                                                \n",
      "30\t130k\t0.15\t\t0.29\t\t0.204\t\t0.849\t\t0.36m - 12.3m / 14.1m                                                                \n",
      "31\t135k\t0.15\t\t0.29\t\t0.217\t\t0.858\t\t0.36m - 12.7m / 14.1m                                                                \n",
      "VAL f1\t0.23743500866551126 - (0.23743500866551126)                                                                     \n",
      "VAL loss\t0.24710835320608956                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.24710835320608956\n",
      "        | \\     )|_\tf1: 0.23743500866551126\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\60 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 48.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         48                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0004161578583856497               |\n",
      "|  noam_learning_rate_warmup   |                        7410                       |\n",
      "|  noam_learning_rate_factor   |                 0.9723992290647475                |\n",
      "|          adam_beta1          |                 0.7756860327865825                |\n",
      "|          adam_beta2          |                 0.9048527155891969                |\n",
      "|           adam_eps           |               2.1725123232243628e-10              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.30957540325424965                |\n",
      "|     pointwise_layer_size     |                        154                        |\n",
      "|      last_layer_dropout      |                0.22728894002393574                |\n",
      "|   output_conv_num_filters    |                        301                        |\n",
      "|   output_conv_kernel_size    |                         1                         |\n",
      "|      output_conv_stride      |                         10                        |\n",
      "|     output_conv_padding      |                         5                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        154                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n",
      " 79%|██████████████████████████████████▊         | 79/100 [7:52:32<2:50:17, 486.55s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "978d275e41714a63b0b24ccc7873b04a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.47\t\t0.40\t\t0.000\t\t0.940\t\t0.42m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.36\t\t0.33\t\t0.000\t\t0.940\t\t0.42m - 0.9m / 14.7m                                                                    \n",
      "3\t13k\t0.34\t\t0.31\t\t0.011\t\t0.939\t\t0.42m - 1.4m / 14.8m                                                                   \n",
      "4\t17k\t0.33\t\t0.31\t\t0.238\t\t0.922\t\t0.42m - 1.9m / 14.9m                                                                   \n",
      "5\t22k\t0.30\t\t0.29\t\t0.229\t\t0.873\t\t0.42m - 2.3m / 15.1m                                                                   \n",
      "6\t26k\t0.28\t\t0.27\t\t0.239\t\t0.877\t\t0.42m - 2.8m / 15.0m                                                                   \n",
      "7\t30k\t0.27\t\t0.28\t\t0.220\t\t0.845\t\t0.42m - 3.3m / 15.0m                                                                   \n",
      "8\t35k\t0.26\t\t0.28\t\t0.210\t\t0.835\t\t0.42m - 3.8m / 15.1m                                                                   \n",
      "9\t39k\t0.25\t\t0.27\t\t0.229\t\t0.845\t\t0.43m - 4.3m / 15.3m                                                                   \n",
      "10\t43k\t0.24\t\t0.27\t\t0.203\t\t0.821\t\t0.44m - 4.7m / 15.6m                                                                  \n",
      "11\t48k\t0.23\t\t0.27\t\t0.227\t\t0.842\t\t0.44m - 5.2m / 15.8m                                                                  \n",
      "VAL f1\t0.23926940639269406 - (0.23926940639269406)                                                                     \n",
      "VAL loss\t0.26675399428322205                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.26675399428322205\n",
      "        | \\     )|_\tf1: 0.23926940639269406\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\61 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 56.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         56                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0006804409733657097               |\n",
      "|  noam_learning_rate_warmup   |                        5938                       |\n",
      "|  noam_learning_rate_factor   |                0.01638348510114701                |\n",
      "|          adam_beta1          |                 0.7505507772062753                |\n",
      "|          adam_beta2          |                 0.9365044790673744                |\n",
      "|           adam_eps           |               1.1165892506424887e-10              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.10298000192174028                |\n",
      "|     pointwise_layer_size     |                         44                        |\n",
      "|      last_layer_dropout      |                 0.2866890601596484                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        165                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 80%|███████████████████████████████████▏        | 80/100 [7:58:15<2:27:45, 443.29s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "347f1072031b484b8b13032bab88a039",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.62\t\t0.63\t\t0.057\t\t0.079\t\t0.33m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.60\t\t0.60\t\t0.058\t\t0.102\t\t0.34m - 0.7m / 11.7m                                                                    \n",
      "3\t13k\t0.58\t\t0.57\t\t0.059\t\t0.139\t\t0.34m - 1.1m / 12.0m                                                                   \n",
      "4\t17k\t0.54\t\t0.52\t\t0.063\t\t0.219\t\t0.34m - 1.5m / 12.1m                                                                   \n",
      "5\t22k\t0.50\t\t0.47\t\t0.064\t\t0.324\t\t0.34m - 1.9m / 12.2m                                                                   \n",
      "6\t26k\t0.46\t\t0.41\t\t0.059\t\t0.448\t\t0.34m - 2.3m / 12.1m                                                                   \n",
      "7\t31k\t0.42\t\t0.37\t\t0.061\t\t0.578\t\t0.33m - 2.7m / 12.3m                                                                   \n",
      "8\t35k\t0.38\t\t0.33\t\t0.068\t\t0.715\t\t0.34m - 3.1m / 12.2m                                                                   \n",
      "9\t39k\t0.36\t\t0.31\t\t0.062\t\t0.851\t\t0.34m - 3.5m / 12.3m                                                                   \n",
      "10\t44k\t0.34\t\t0.29\t\t0.067\t\t0.919\t\t0.34m - 3.9m / 12.4m                                                                  \n",
      "11\t48k\t0.33\t\t0.28\t\t0.032\t\t0.931\t\t0.34m - 4.3m / 12.3m                                                                  \n",
      "12\t52k\t0.32\t\t0.27\t\t0.015\t\t0.934\t\t0.34m - 4.7m / 12.4m                                                                  \n",
      "13\t57k\t0.32\t\t0.27\t\t0.010\t\t0.935\t\t0.34m - 5.1m / 12.5m                                                                  \n",
      "VAL f1\t0.06788511749347259 - (0.06788511749347259)                                                                     \n",
      "VAL loss\t0.2687655772481646                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2687655772481646\n",
      "        | \\     )|_\tf1: 0.06788511749347259\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\62 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 64.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0009967502667345904               |\n",
      "|  noam_learning_rate_warmup   |                        5018                       |\n",
      "|  noam_learning_rate_factor   |                 1.1865904172338655                |\n",
      "|          adam_beta1          |                 0.740112968157727                 |\n",
      "|          adam_beta2          |                 0.9300330377099486                |\n",
      "|           adam_eps           |               4.1373629548949504e-09              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.14510605239642932                |\n",
      "|     pointwise_layer_size     |                        127                        |\n",
      "|      last_layer_dropout      |                 0.4950939712723986                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        137                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 81%|███████████████████████████████████▋        | 81/100 [8:03:54<2:10:29, 412.06s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f029cc6fc884daf8d065305b37538e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.37\t\t0.25\t\t0.047\t\t0.828\t\t0.29m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.28\t\t0.22\t\t0.107\t\t0.935\t\t0.29m - 0.6m / 10.2m                                                                    \n",
      "3\t13k\t0.27\t\t0.22\t\t0.114\t\t0.933\t\t0.29m - 1.0m / 10.3m                                                                   \n",
      "4\t17k\t0.26\t\t0.21\t\t0.208\t\t0.917\t\t0.30m - 1.3m / 10.4m                                                                   \n",
      "5\t22k\t0.24\t\t0.21\t\t0.183\t\t0.886\t\t0.29m - 1.7m / 10.5m                                                                   \n",
      "6\t26k\t0.23\t\t0.20\t\t0.205\t\t0.888\t\t0.29m - 2.0m / 10.5m                                                                   \n",
      "7\t30k\t0.21\t\t0.20\t\t0.232\t\t0.877\t\t0.29m - 2.4m / 10.6m                                                                   \n",
      "8\t35k\t0.19\t\t0.20\t\t0.246\t\t0.870\t\t0.29m - 2.7m / 10.7m                                                                   \n",
      "9\t39k\t0.18\t\t0.20\t\t0.216\t\t0.868\t\t0.29m - 3.1m / 10.7m                                                                   \n",
      "10\t44k\t0.17\t\t0.20\t\t0.224\t\t0.875\t\t0.29m - 3.4m / 10.7m                                                                  \n",
      "11\t48k\t0.16\t\t0.21\t\t0.245\t\t0.882\t\t0.30m - 3.8m / 10.8m                                                                  \n",
      "12\t52k\t0.15\t\t0.23\t\t0.195\t\t0.829\t\t0.29m - 4.1m / 11.0m                                                                  \n",
      "13\t57k\t0.14\t\t0.22\t\t0.232\t\t0.849\t\t0.29m - 4.5m / 10.9m                                                                  \n",
      "VAL f1\t0.24552429667519182 - (0.24552429667519182)                                                                     \n",
      "VAL loss\t0.19635533541440964                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.19635533541440964\n",
      "        | \\     )|_\tf1: 0.24552429667519182\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\63 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 64.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                5.94938502611485e-05               |\n",
      "|  noam_learning_rate_warmup   |                        7116                       |\n",
      "|  noam_learning_rate_factor   |                 0.1368735568346256                |\n",
      "|          adam_beta1          |                 0.7362791870941333                |\n",
      "|          adam_beta2          |                 0.9303605545860207                |\n",
      "|           adam_eps           |               5.459508023491661e-09               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.2656161367746275                |\n",
      "|     pointwise_layer_size     |                        118                        |\n",
      "|      last_layer_dropout      |                 0.5079450934546857                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        148                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 82%|████████████████████████████████████        | 82/100 [8:08:48<1:52:59, 376.65s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "295a60a0e75644c8826d981660003c88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.44\t\t0.44\t\t0.027\t\t0.432\t\t0.31m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.41\t\t0.38\t\t0.027\t\t0.456\t\t0.30m - 0.7m / 10.7m                                                                    \n",
      "3\t13k\t0.37\t\t0.31\t\t0.034\t\t0.641\t\t0.30m - 1.0m / 10.8m                                                                   \n",
      "4\t17k\t0.33\t\t0.26\t\t0.045\t\t0.774\t\t0.30m - 1.4m / 10.8m                                                                   \n",
      "5\t22k\t0.30\t\t0.24\t\t0.054\t\t0.853\t\t0.30m - 1.8m / 10.9m                                                                   \n",
      "6\t26k\t0.29\t\t0.22\t\t0.029\t\t0.924\t\t0.30m - 2.1m / 10.9m                                                                   \n",
      "7\t30k\t0.28\t\t0.22\t\t0.019\t\t0.933\t\t0.30m - 2.5m / 11.0m                                                                   \n",
      "8\t35k\t0.27\t\t0.22\t\t0.020\t\t0.935\t\t0.31m - 2.9m / 11.0m                                                                   \n",
      "9\t39k\t0.27\t\t0.22\t\t0.020\t\t0.935\t\t0.30m - 3.2m / 11.3m                                                                   \n",
      "10\t44k\t0.27\t\t0.22\t\t0.033\t\t0.934\t\t0.30m - 3.6m / 11.2m                                                                  \n",
      "VAL f1\t0.05399792315680166 - (0.05399792315680166)                                                                     \n",
      "VAL loss\t0.21709673355023065                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21709673355023065\n",
      "        | \\     )|_\tf1: 0.05399792315680166\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\64 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 44.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         44                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0009953094284369654               |\n",
      "|  noam_learning_rate_warmup   |                        7942                       |\n",
      "|  noam_learning_rate_factor   |                 0.3545151498331214                |\n",
      "|          adam_beta1          |                 0.7194378607263863                |\n",
      "|          adam_beta2          |                 0.8930654821815297                |\n",
      "|           adam_eps           |                2.60819428273488e-09               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.19325671665822297                |\n",
      "|     pointwise_layer_size     |                         90                        |\n",
      "|      last_layer_dropout      |                 0.6804577093196037                |\n",
      "|   output_conv_num_filters    |                        146                        |\n",
      "|   output_conv_kernel_size    |                         10                        |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        144                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 83%|████████████████████████████████████▌       | 83/100 [8:12:47<1:35:01, 335.40s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6aaad9e2f45b4855953398bbd5622843",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.47\t\t0.43\t\t0.113\t\t0.932\t\t0.52m - 0.5m / 0.0m                                                                     \n",
      "2\t9k\t0.37\t\t0.42\t\t0.163\t\t0.919\t\t0.53m - 1.1m / 18.4m                                                                    \n",
      "3\t13k\t0.35\t\t0.40\t\t0.260\t\t0.904\t\t0.53m - 1.7m / 18.6m                                                                   \n",
      "4\t17k\t0.32\t\t0.39\t\t0.252\t\t0.867\t\t0.53m - 2.3m / 18.6m                                                                   \n",
      "5\t22k\t0.30\t\t0.37\t\t0.226\t\t0.853\t\t0.53m - 2.9m / 18.7m                                                                   \n",
      "6\t26k\t0.28\t\t0.36\t\t0.223\t\t0.835\t\t0.53m - 3.4m / 18.7m                                                                   \n",
      "7\t30k\t0.27\t\t0.37\t\t0.200\t\t0.819\t\t0.53m - 4.0m / 18.8m                                                                   \n",
      "8\t35k\t0.25\t\t0.36\t\t0.220\t\t0.825\t\t0.53m - 4.6m / 18.9m                                                                   \n",
      "VAL f1\t0.26047565118912797 - (0.26047565118912797)                                                                     \n",
      "VAL loss\t0.3578339164907282                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3578339164907282\n",
      "        | \\     )|_\tf1: 0.26047565118912797\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\65 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 58.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         58                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00031880464425246426              |\n",
      "|  noam_learning_rate_warmup   |                        5678                       |\n",
      "|  noam_learning_rate_factor   |                 0.6533362883625277                |\n",
      "|          adam_beta1          |                 0.7821838330885265                |\n",
      "|          adam_beta2          |                 0.9189118047756124                |\n",
      "|           adam_eps           |               2.9631954965110183e-08              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.14855305293264628                |\n",
      "|     pointwise_layer_size     |                        168                        |\n",
      "|      last_layer_dropout      |                 0.6172121818172589                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        153                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 84%|████████████████████████████████████▉       | 84/100 [8:17:58<1:27:30, 328.13s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4bcee4ee9794ea998f1394d0385fb68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.52\t\t0.39\t\t0.044\t\t0.610\t\t0.33m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.34\t\t0.27\t\t0.057\t\t0.908\t\t0.33m - 0.7m / 11.4m                                                                    \n",
      "3\t13k\t0.30\t\t0.26\t\t0.025\t\t0.936\t\t0.33m - 1.1m / 11.7m                                                                   \n",
      "4\t17k\t0.30\t\t0.25\t\t0.064\t\t0.933\t\t0.33m - 1.5m / 11.6m                                                                   \n",
      "5\t22k\t0.29\t\t0.25\t\t0.121\t\t0.911\t\t0.33m - 1.9m / 11.8m                                                                   \n",
      "6\t26k\t0.27\t\t0.24\t\t0.165\t\t0.897\t\t0.33m - 2.3m / 11.9m                                                                   \n",
      "7\t30k\t0.26\t\t0.23\t\t0.201\t\t0.898\t\t0.33m - 2.6m / 11.8m                                                                   \n",
      "8\t35k\t0.24\t\t0.23\t\t0.195\t\t0.883\t\t0.33m - 3.0m / 12.0m                                                                   \n",
      "9\t39k\t0.23\t\t0.23\t\t0.203\t\t0.862\t\t0.33m - 3.4m / 12.0m                                                                   \n",
      "10\t44k\t0.21\t\t0.22\t\t0.223\t\t0.865\t\t0.33m - 3.8m / 12.0m                                                                  \n",
      "11\t48k\t0.20\t\t0.23\t\t0.218\t\t0.854\t\t0.33m - 4.2m / 12.1m                                                                  \n",
      "12\t52k\t0.19\t\t0.23\t\t0.204\t\t0.850\t\t0.33m - 4.6m / 12.3m                                                                  \n",
      "13\t57k\t0.18\t\t0.23\t\t0.232\t\t0.875\t\t0.32m - 5.0m / 12.2m                                                                  \n",
      "14\t61k\t0.17\t\t0.24\t\t0.216\t\t0.858\t\t0.33m - 5.3m / 12.2m                                                                  \n",
      "15\t65k\t0.16\t\t0.24\t\t0.219\t\t0.856\t\t0.33m - 5.8m / 12.3m                                                                  \n",
      "16\t70k\t0.16\t\t0.24\t\t0.235\t\t0.863\t\t0.33m - 6.1m / 12.5m                                                                  \n",
      "17\t74k\t0.15\t\t0.25\t\t0.235\t\t0.855\t\t0.33m - 6.5m / 12.5m                                                                  \n",
      "18\t78k\t0.14\t\t0.26\t\t0.229\t\t0.867\t\t0.33m - 6.9m / 12.5m                                                                  \n",
      "19\t83k\t0.13\t\t0.26\t\t0.229\t\t0.870\t\t0.33m - 7.3m / 12.5m                                                                  \n",
      "20\t87k\t0.12\t\t0.27\t\t0.229\t\t0.869\t\t0.33m - 7.7m / 12.6m                                                                  \n",
      "21\t91k\t0.12\t\t0.27\t\t0.229\t\t0.872\t\t0.34m - 8.1m / 12.7m                                                                  \n",
      "22\t96k\t0.11\t\t0.28\t\t0.235\t\t0.877\t\t0.33m - 8.5m / 12.8m                                                                  \n",
      "VAL f1\t0.23547637490317583 - (0.23547637490317583)                                                                     \n",
      "VAL loss\t0.22412021680809988                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22412021680809988\n",
      "        | \\     )|_\tf1: 0.23547637490317583\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\66 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 30.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         30                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.08697162148978815                |\n",
      "|  noam_learning_rate_warmup   |                        5055                       |\n",
      "|  noam_learning_rate_factor   |                 1.0842961569891343                |\n",
      "|          adam_beta1          |                 0.7617345388829085                |\n",
      "|          adam_beta2          |                 0.8985239916058667                |\n",
      "|           adam_eps           |               1.3052677865331314e-09              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.27989496082753373                |\n",
      "|     pointwise_layer_size     |                        127                        |\n",
      "|      last_layer_dropout      |                 0.717556200734462                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        137                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 85%|█████████████████████████████████████▍      | 85/100 [8:26:51<1:37:21, 389.45s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4187a72bb354e82b0897f5aef70b089",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.64\t\t0.42\t\t0.000\t\t0.940\t\t0.46m - 0.5m / 0.0m                                                                     \n",
      "2\t9k\t0.52\t\t0.43\t\t0.032\t\t0.931\t\t0.46m - 1.0m / 16.1m                                                                    \n",
      "3\t13k\t0.50\t\t0.40\t\t0.199\t\t0.896\t\t0.46m - 1.5m / 16.0m                                                                   \n",
      "4\t17k\t0.45\t\t0.39\t\t0.236\t\t0.878\t\t0.48m - 2.0m / 16.4m                                                                   \n",
      "5\t22k\t0.40\t\t0.39\t\t0.258\t\t0.877\t\t0.46m - 2.5m / 16.8m                                                                   \n",
      "6\t26k\t0.38\t\t0.39\t\t0.214\t\t0.852\t\t0.46m - 3.0m / 16.4m                                                                   \n",
      "7\t30k\t0.35\t\t0.39\t\t0.222\t\t0.867\t\t0.46m - 3.6m / 16.3m                                                                   \n",
      "8\t35k\t0.32\t\t0.40\t\t0.263\t\t0.867\t\t0.46m - 4.1m / 16.6m                                                                   \n",
      "9\t39k\t0.30\t\t0.41\t\t0.227\t\t0.845\t\t0.47m - 4.6m / 16.6m                                                                   \n",
      "10\t43k\t0.28\t\t0.41\t\t0.219\t\t0.848\t\t0.46m - 5.1m / 16.8m                                                                  \n",
      "11\t48k\t0.27\t\t0.42\t\t0.223\t\t0.862\t\t0.46m - 5.6m / 16.7m                                                                  \n",
      "12\t52k\t0.25\t\t0.42\t\t0.279\t\t0.878\t\t0.46m - 6.1m / 16.8m                                                                  \n",
      "13\t56k\t0.24\t\t0.44\t\t0.234\t\t0.863\t\t0.47m - 6.7m / 16.8m                                                                  \n",
      "14\t60k\t0.22\t\t0.45\t\t0.267\t\t0.887\t\t0.46m - 7.2m / 17.0m                                                                  \n",
      "15\t65k\t0.21\t\t0.46\t\t0.257\t\t0.859\t\t0.47m - 7.7m / 16.9m                                                                  \n",
      "16\t69k\t0.20\t\t0.54\t\t0.271\t\t0.882\t\t0.46m - 8.2m / 17.1m                                                                  \n",
      "17\t73k\t0.19\t\t0.52\t\t0.247\t\t0.871\t\t0.47m - 8.7m / 17.0m                                                                  \n",
      "VAL f1\t0.27854671280276816 - (0.27854671280276816)                                                                     \n",
      "VAL loss\t0.38600194454193115                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.38600194454193115\n",
      "        | \\     )|_\tf1: 0.27854671280276816\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\67 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 11.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         11                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0005127559622462262               |\n",
      "|  noam_learning_rate_warmup   |                        8291                       |\n",
      "|  noam_learning_rate_factor   |                 0.9048478824039707                |\n",
      "|          adam_beta1          |                 0.7024423679163267                |\n",
      "|          adam_beta2          |                 0.9069569541443988                |\n",
      "|           adam_eps           |               8.371918512203036e-10               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.34993348217867637                |\n",
      "|     pointwise_layer_size     |                         79                        |\n",
      "|      last_layer_dropout      |                 0.5584844170110023                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        163                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 86%|█████████████████████████████████████▊      | 86/100 [8:36:11<1:42:49, 440.69s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "004ab84c65f24a149ab1a01cfad7c6b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.55\t\t0.92\t\t0.070\t\t0.935\t\t1.08m - 1.1m / 0.0m                                                                     \n",
      "2\t9k\t1.05\t\t0.85\t\t0.218\t\t0.903\t\t1.08m - 2.2m / 37.8m                                                                    \n",
      "3\t13k\t0.96\t\t0.81\t\t0.233\t\t0.900\t\t1.09m - 3.4m / 37.8m                                                                   \n",
      "4\t17k\t0.89\t\t0.80\t\t0.250\t\t0.892\t\t1.10m - 4.5m / 38.4m                                                                   \n",
      "5\t22k\t0.84\t\t0.90\t\t0.212\t\t0.856\t\t1.08m - 5.7m / 38.7m                                                                   \n",
      "6\t26k\t0.82\t\t0.83\t\t0.253\t\t0.869\t\t1.08m - 6.8m / 38.0m                                                                   \n",
      "7\t30k\t0.77\t\t0.85\t\t0.275\t\t0.893\t\t1.07m - 7.9m / 38.1m                                                                   \n",
      "8\t35k\t0.76\t\t0.84\t\t0.270\t\t0.894\t\t1.08m - 9.1m / 38.1m                                                                   \n",
      "9\t39k\t0.75\t\t0.86\t\t0.281\t\t0.888\t\t1.07m - 10.2m / 38.2m                                                                  \n",
      "10\t43k\t0.72\t\t0.86\t\t0.284\t\t0.878\t\t1.08m - 11.3m / 38.2m                                                                 \n",
      "11\t48k\t0.70\t\t0.93\t\t0.273\t\t0.889\t\t1.07m - 12.4m / 38.3m                                                                 \n",
      "12\t52k\t0.70\t\t0.93\t\t0.321\t\t0.906\t\t1.08m - 13.6m / 38.1m                                                                 \n",
      "13\t56k\t0.68\t\t0.89\t\t0.269\t\t0.886\t\t1.08m - 14.7m / 38.4m                                                                 \n",
      "14\t61k\t0.66\t\t0.92\t\t0.245\t\t0.896\t\t1.08m - 15.8m / 38.4m                                                                 \n",
      "15\t65k\t0.65\t\t0.99\t\t0.237\t\t0.881\t\t1.12m - 17.0m / 38.5m                                                                 \n",
      "16\t69k\t0.65\t\t0.97\t\t0.266\t\t0.888\t\t1.10m - 18.1m / 39.4m                                                                 \n",
      "17\t73k\t0.63\t\t0.93\t\t0.264\t\t0.897\t\t1.07m - 19.3m / 39.1m                                                                 \n",
      "VAL f1\t0.32124352331606215 - (0.32124352331606215)                                                                     \n",
      "VAL loss\t0.8036709433077367                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.8036709433077367\n",
      "        | \\     )|_\tf1: 0.32124352331606215\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\68 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 23.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         23                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0004357056216868457               |\n",
      "|  noam_learning_rate_warmup   |                        5312                       |\n",
      "|  noam_learning_rate_factor   |                0.46718349985858953                |\n",
      "|          adam_beta1          |                 0.7439026154221396                |\n",
      "|          adam_beta2          |                 0.9580586917265113                |\n",
      "|           adam_eps           |               1.0525220367952572e-10              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.24783712066948216                |\n",
      "|     pointwise_layer_size     |                        142                        |\n",
      "|      last_layer_dropout      |                 0.4397613879418498                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        171                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 87%|██████████████████████████████████████▎     | 87/100 [8:56:16<2:25:08, 669.88s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac7a4bd77c91445c80cb923a3e50d301",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.95\t\t0.53\t\t0.170\t\t0.924\t\t0.71m - 0.7m / 0.0m                                                                     \n",
      "2\t9k\t0.63\t\t0.50\t\t0.065\t\t0.935\t\t0.71m - 1.5m / 25.0m                                                                    \n",
      "3\t13k\t0.60\t\t0.46\t\t0.230\t\t0.918\t\t0.71m - 2.3m / 24.9m                                                                   \n",
      "4\t17k\t0.56\t\t0.45\t\t0.264\t\t0.885\t\t0.72m - 3.0m / 25.0m                                                                   \n",
      "5\t22k\t0.51\t\t0.44\t\t0.264\t\t0.880\t\t0.72m - 3.8m / 25.4m                                                                   \n",
      "6\t26k\t0.48\t\t0.46\t\t0.233\t\t0.866\t\t0.71m - 4.6m / 25.3m                                                                   \n",
      "7\t30k\t0.46\t\t0.45\t\t0.244\t\t0.873\t\t0.72m - 5.3m / 25.3m                                                                   \n",
      "8\t35k\t0.43\t\t0.49\t\t0.217\t\t0.846\t\t0.72m - 6.1m / 25.4m                                                                   \n",
      "9\t39k\t0.41\t\t0.45\t\t0.257\t\t0.874\t\t0.72m - 6.9m / 25.5m                                                                   \n",
      "10\t43k\t0.39\t\t0.47\t\t0.251\t\t0.875\t\t0.72m - 7.6m / 25.6m                                                                  \n",
      "VAL f1\t0.2643884892086331 - (0.2643884892086331)                                                                       \n",
      "VAL loss\t0.44399190847424497                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.44399190847424497\n",
      "        | \\     )|_\tf1: 0.2643884892086331\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\69 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 53.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         53                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               3.4358748287869126e-05              |\n",
      "|  noam_learning_rate_warmup   |                        6003                       |\n",
      "|  noam_learning_rate_factor   |                 1.2234946390567765                |\n",
      "|          adam_beta1          |                 0.8215211950752996                |\n",
      "|          adam_beta2          |                 0.8567956039204749                |\n",
      "|           adam_eps           |               7.425604040831573e-09               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                 0.3213950254270883                |\n",
      "|     pointwise_layer_size     |                         69                        |\n",
      "|      last_layer_dropout      |                 0.4924410305886063                |\n",
      "|   output_conv_num_filters    |                         56                        |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         7                         |\n",
      "|     output_conv_padding      |                         4                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        157                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 88%|██████████████████████████████████████▋     | 88/100 [9:04:28<2:03:19, 616.65s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a2f9f13316846ed8ea47e09ffcabd11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.39\t\t0.32\t\t0.087\t\t0.929\t\t0.39m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.32\t\t0.139\t\t0.909\t\t0.39m - 0.8m / 13.6m                                                                    \n",
      "3\t13k\t0.30\t\t0.29\t\t0.238\t\t0.898\t\t0.39m - 1.3m / 13.7m                                                                   \n",
      "4\t17k\t0.27\t\t0.28\t\t0.225\t\t0.864\t\t0.39m - 1.7m / 13.7m                                                                   \n",
      "5\t22k\t0.25\t\t0.27\t\t0.211\t\t0.860\t\t0.39m - 2.2m / 13.8m                                                                   \n",
      "6\t26k\t0.24\t\t0.28\t\t0.181\t\t0.813\t\t0.39m - 2.6m / 13.8m                                                                   \n",
      "7\t30k\t0.23\t\t0.26\t\t0.207\t\t0.828\t\t0.39m - 3.1m / 14.0m                                                                   \n",
      "8\t35k\t0.22\t\t0.26\t\t0.202\t\t0.824\t\t0.39m - 3.5m / 14.0m                                                                   \n",
      "VAL f1\t0.23751387347391786 - (0.23751387347391786)                                                                     \n",
      "VAL loss\t0.257069893603055                                                                                             \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.257069893603055\n",
      "        | \\     )|_\tf1: 0.23751387347391786\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\70 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 45.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         45                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.19806023604244763                |\n",
      "|  noam_learning_rate_warmup   |                        5048                       |\n",
      "|  noam_learning_rate_factor   |                 1.4545508482327745                |\n",
      "|          adam_beta1          |                 0.7936053122362415                |\n",
      "|          adam_beta2          |                 0.9495422221115998                |\n",
      "|           adam_eps           |               2.3715279948869957e-10              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.22111116113884272                |\n",
      "|     pointwise_layer_size     |                         59                        |\n",
      "|      last_layer_dropout      |                 0.5331486349283139                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        178                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 89%|███████████████████████████████████████▏    | 89/100 [9:08:24<1:32:05, 502.33s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "536fa9d26e404a68a06bb98b95502187",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.59\t\t0.32\t\t0.018\t\t0.927\t\t0.38m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.37\t\t0.31\t\t0.041\t\t0.917\t\t0.39m - 0.8m / 13.5m                                                                    \n",
      "3\t13k\t0.36\t\t0.29\t\t0.153\t\t0.923\t\t0.39m - 1.3m / 13.6m                                                                   \n",
      "4\t17k\t0.33\t\t0.27\t\t0.224\t\t0.893\t\t0.39m - 1.7m / 13.7m                                                                   \n",
      "5\t22k\t0.30\t\t0.26\t\t0.258\t\t0.894\t\t0.40m - 2.2m / 13.8m                                                                   \n",
      "6\t26k\t0.27\t\t0.27\t\t0.238\t\t0.875\t\t0.39m - 2.6m / 14.2m                                                                   \n",
      "7\t30k\t0.25\t\t0.27\t\t0.275\t\t0.893\t\t0.39m - 3.0m / 14.0m                                                                   \n",
      "8\t35k\t0.23\t\t0.29\t\t0.214\t\t0.848\t\t0.39m - 3.5m / 13.9m                                                                   \n",
      "9\t39k\t0.21\t\t0.30\t\t0.219\t\t0.841\t\t0.39m - 3.9m / 14.1m                                                                   \n",
      "10\t43k\t0.19\t\t0.29\t\t0.235\t\t0.858\t\t0.39m - 4.4m / 14.0m                                                                  \n",
      "11\t48k\t0.18\t\t0.30\t\t0.274\t\t0.878\t\t0.39m - 4.8m / 14.2m                                                                  \n",
      "12\t52k\t0.17\t\t0.29\t\t0.284\t\t0.894\t\t0.39m - 5.3m / 14.2m                                                                  \n",
      "13\t56k\t0.16\t\t0.31\t\t0.273\t\t0.885\t\t0.39m - 5.7m / 14.3m                                                                  \n",
      "14\t60k\t0.15\t\t0.33\t\t0.249\t\t0.864\t\t0.42m - 6.2m / 14.4m                                                                  \n",
      "15\t65k\t0.14\t\t0.33\t\t0.256\t\t0.883\t\t0.40m - 6.7m / 15.0m                                                                  \n",
      "16\t69k\t0.13\t\t0.34\t\t0.262\t\t0.877\t\t0.39m - 7.1m / 14.8m                                                                  \n",
      "17\t73k\t0.12\t\t0.34\t\t0.270\t\t0.879\t\t0.39m - 7.5m / 14.6m                                                                  \n",
      "VAL f1\t0.28351012536162007 - (0.28351012536162007)                                                                     \n",
      "VAL loss\t0.2649561577373081                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.2649561577373081\n",
      "        | \\     )|_\tf1: 0.28351012536162007\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\71 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 38.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         38                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0019200889356683869               |\n",
      "|  noam_learning_rate_warmup   |                        6487                       |\n",
      "|  noam_learning_rate_factor   |                 0.7588840047454872                |\n",
      "|          adam_beta1          |                 0.7314182256973063                |\n",
      "|          adam_beta2          |                 0.941048385580699                 |\n",
      "|           adam_eps           |               1.7143900669031543e-08              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                0.42567477809791443                |\n",
      "|     pointwise_layer_size     |                        109                        |\n",
      "|      last_layer_dropout      |                 0.6000498310907842                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        125                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 90%|███████████████████████████████████████▌    | 90/100 [9:16:31<1:22:58, 497.85s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3773fa030015465aa143368c64794441",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.65\t\t0.41\t\t0.110\t\t0.831\t\t0.44m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.44\t\t0.37\t\t0.000\t\t0.931\t\t0.45m - 0.9m / 15.6m                                                                    \n",
      "3\t13k\t0.43\t\t0.36\t\t0.024\t\t0.919\t\t0.47m - 1.5m / 15.8m                                                                   \n",
      "4\t17k\t0.42\t\t0.36\t\t0.144\t\t0.919\t\t0.48m - 2.0m / 16.6m                                                                   \n",
      "5\t22k\t0.39\t\t0.34\t\t0.154\t\t0.894\t\t0.47m - 2.5m / 16.8m                                                                   \n",
      "6\t26k\t0.37\t\t0.32\t\t0.221\t\t0.884\t\t0.48m - 3.1m / 16.7m                                                                   \n",
      "7\t30k\t0.34\t\t0.34\t\t0.181\t\t0.860\t\t0.45m - 3.6m / 17.0m                                                                   \n",
      "8\t35k\t0.33\t\t0.35\t\t0.208\t\t0.849\t\t0.44m - 4.1m / 16.2m                                                                   \n",
      "9\t39k\t0.31\t\t0.35\t\t0.192\t\t0.833\t\t0.44m - 4.6m / 15.9m                                                                   \n",
      "10\t43k\t0.30\t\t0.35\t\t0.225\t\t0.840\t\t0.44m - 5.0m / 16.1m                                                                  \n",
      "11\t48k\t0.30\t\t0.39\t\t0.175\t\t0.785\t\t0.44m - 5.6m / 16.0m                                                                  \n",
      "12\t52k\t0.29\t\t0.37\t\t0.201\t\t0.813\t\t0.44m - 6.1m / 16.3m                                                                  \n",
      "13\t56k\t0.28\t\t0.37\t\t0.198\t\t0.809\t\t0.44m - 6.6m / 16.2m                                                                  \n",
      "14\t61k\t0.27\t\t0.39\t\t0.179\t\t0.797\t\t0.44m - 7.0m / 16.3m                                                                  \n",
      "15\t65k\t0.26\t\t0.35\t\t0.242\t\t0.852\t\t0.44m - 7.5m / 16.3m                                                                  \n",
      "16\t69k\t0.26\t\t0.40\t\t0.190\t\t0.810\t\t0.44m - 8.0m / 16.4m                                                                  \n",
      "17\t74k\t0.25\t\t0.39\t\t0.221\t\t0.830\t\t0.44m - 8.5m / 16.4m                                                                  \n",
      "18\t78k\t0.24\t\t0.38\t\t0.214\t\t0.836\t\t0.44m - 9.0m / 16.5m                                                                  \n",
      "19\t82k\t0.23\t\t0.39\t\t0.240\t\t0.859\t\t0.44m - 9.5m / 16.5m                                                                  \n",
      "20\t87k\t0.23\t\t0.41\t\t0.204\t\t0.813\t\t0.44m - 10.0m / 16.6m                                                                 \n",
      "VAL f1\t0.2419716206123973 - (0.2419716206123973)                                                                       \n",
      "VAL loss\t0.3209751600410506                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.3209751600410506\n",
      "        | \\     )|_\tf1: 0.2419716206123973\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\72 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 62.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         62                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0011606762190391308               |\n",
      "|  noam_learning_rate_warmup   |                        2801                       |\n",
      "|  noam_learning_rate_factor   |                 1.872715212487615                 |\n",
      "|          adam_beta1          |                 0.7615580242038568                |\n",
      "|          adam_beta2          |                 0.8854982117731213                |\n",
      "|           adam_eps           |               3.3495406520212677e-06              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.37871665161235163                |\n",
      "|     pointwise_layer_size     |                        158                        |\n",
      "|      last_layer_dropout      |                0.36370458191421645                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        146                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 91%|████████████████████████████████████████    | 91/100 [9:26:59<1:20:30, 536.75s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "933c208bca164f44abf50445e356d916",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.38\t\t0.24\t\t0.005\t\t0.940\t\t0.28m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.29\t\t0.24\t\t0.050\t\t0.889\t\t0.28m - 0.6m / 9.9m                                                                     \n",
      "3\t13k\t0.27\t\t0.20\t\t0.194\t\t0.883\t\t0.29m - 1.0m / 10.0m                                                                   \n",
      "4\t17k\t0.24\t\t0.23\t\t0.183\t\t0.811\t\t0.28m - 1.3m / 10.1m                                                                   \n",
      "5\t22k\t0.22\t\t0.21\t\t0.193\t\t0.827\t\t0.28m - 1.6m / 10.2m                                                                   \n",
      "6\t26k\t0.20\t\t0.23\t\t0.190\t\t0.809\t\t0.29m - 2.0m / 10.2m                                                                   \n",
      "7\t30k\t0.19\t\t0.23\t\t0.192\t\t0.822\t\t0.29m - 2.3m / 10.3m                                                                   \n",
      "8\t35k\t0.18\t\t0.24\t\t0.194\t\t0.835\t\t0.29m - 2.7m / 10.4m                                                                   \n",
      "9\t39k\t0.18\t\t0.24\t\t0.221\t\t0.829\t\t0.29m - 3.0m / 10.5m                                                                   \n",
      "10\t43k\t0.17\t\t0.25\t\t0.220\t\t0.842\t\t0.29m - 3.4m / 10.5m                                                                  \n",
      "11\t48k\t0.16\t\t0.24\t\t0.238\t\t0.842\t\t0.29m - 3.7m / 10.6m                                                                  \n",
      "12\t52k\t0.16\t\t0.26\t\t0.229\t\t0.864\t\t0.29m - 4.0m / 10.7m                                                                  \n",
      "13\t56k\t0.15\t\t0.29\t\t0.235\t\t0.849\t\t0.29m - 4.4m / 10.7m                                                                  \n",
      "14\t61k\t0.15\t\t0.28\t\t0.226\t\t0.865\t\t0.29m - 4.7m / 10.7m                                                                  \n",
      "15\t65k\t0.15\t\t0.25\t\t0.208\t\t0.834\t\t0.29m - 5.1m / 10.9m                                                                  \n",
      "16\t69k\t0.14\t\t0.30\t\t0.229\t\t0.868\t\t0.29m - 5.4m / 10.8m                                                                  \n",
      "VAL f1\t0.23782234957020057 - (0.23782234957020057)                                                                     \n",
      "VAL loss\t0.20292732920697942                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.20292732920697942\n",
      "        | \\     )|_\tf1: 0.23782234957020057\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\73 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 55.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         55                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0002510937334921627               |\n",
      "|  noam_learning_rate_warmup   |                        4694                       |\n",
      "|  noam_learning_rate_factor   |                0.27654925787258755                |\n",
      "|          adam_beta1          |                 0.7109616964971731                |\n",
      "|          adam_beta2          |                 0.9649044299552045                |\n",
      "|           adam_eps           |               5.761805735224156e-08               |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.006533178559211705               |\n",
      "|     pointwise_layer_size     |                        195                        |\n",
      "|      last_layer_dropout      |                 0.3302535463234033                |\n",
      "|   output_conv_num_filters    |                        214                        |\n",
      "|   output_conv_kernel_size    |                         8                         |\n",
      "|      output_conv_stride      |                         2                         |\n",
      "|     output_conv_padding      |                         2                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        140                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 92%|████████████████████████████████████████▍   | 92/100 [9:32:44<1:03:55, 479.46s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82fa0eec06304d17a384c3cd9c4f9227",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception while training: size mismatch, m1: [11770 x 2], m2: [214 x 4] at c:\\a\\w\\1\\s\\tmp_conda_3.6_105809\\conda\\conda-bld\\pytorch_1544094150554\\work\\aten\\src\\thc\\generic/THCTensorMathBlas.cu:266\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\74 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 60.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         60                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.002955366861839995               |\n",
      "|  noam_learning_rate_warmup   |                        3974                       |\n",
      "|  noam_learning_rate_factor   |                 0.5923916788155714                |\n",
      "|          adam_beta1          |                 0.776903839086232                 |\n",
      "|          adam_beta2          |                 0.9298030129308339                |\n",
      "|           adam_eps           |               5.538297775605811e-10               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         4                         |\n",
      "|             d_k              |                         75                        |\n",
      "|             d_v              |                         75                        |\n",
      "|         dropout_rate         |                0.14616576600719336                |\n",
      "|     pointwise_layer_size     |                        130                        |\n",
      "|      last_layer_dropout      |                 0.5866298378869469                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        167                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 93%|██████████████████████████████████████████▊   | 93/100 [9:32:48<39:17, 336.78s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "476a1f957de94297a0e90e7350412774",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.55\t\t0.39\t\t0.045\t\t0.647\t\t0.35m - 0.3m / 0.0m                                                                     \n",
      "2\t9k\t0.33\t\t0.26\t\t0.058\t\t0.920\t\t0.34m - 0.7m / 12.2m                                                                    \n",
      "3\t13k\t0.29\t\t0.25\t\t0.181\t\t0.915\t\t0.35m - 1.1m / 12.1m                                                                   \n",
      "4\t17k\t0.28\t\t0.25\t\t0.117\t\t0.925\t\t0.35m - 1.6m / 12.3m                                                                   \n",
      "5\t22k\t0.27\t\t0.24\t\t0.184\t\t0.905\t\t0.35m - 2.0m / 12.4m                                                                   \n",
      "6\t26k\t0.26\t\t0.24\t\t0.209\t\t0.891\t\t0.35m - 2.4m / 12.6m                                                                   \n",
      "7\t30k\t0.24\t\t0.23\t\t0.188\t\t0.859\t\t0.35m - 2.8m / 12.7m                                                                   \n",
      "8\t35k\t0.22\t\t0.23\t\t0.182\t\t0.865\t\t0.35m - 3.2m / 12.6m                                                                   \n",
      "9\t39k\t0.21\t\t0.22\t\t0.234\t\t0.882\t\t0.35m - 3.6m / 12.7m                                                                   \n",
      "10\t43k\t0.20\t\t0.23\t\t0.226\t\t0.866\t\t0.35m - 4.0m / 12.8m                                                                  \n",
      "11\t48k\t0.18\t\t0.23\t\t0.255\t\t0.883\t\t0.35m - 4.4m / 12.8m                                                                  \n",
      "12\t52k\t0.17\t\t0.24\t\t0.226\t\t0.858\t\t0.35m - 4.8m / 12.9m                                                                  \n",
      "13\t56k\t0.16\t\t0.24\t\t0.229\t\t0.865\t\t0.35m - 5.2m / 12.9m                                                                  \n",
      "14\t60k\t0.15\t\t0.25\t\t0.247\t\t0.873\t\t0.35m - 5.6m / 13.0m                                                                  \n",
      "15\t65k\t0.14\t\t0.26\t\t0.271\t\t0.889\t\t0.35m - 6.0m / 13.0m                                                                  \n",
      "16\t69k\t0.14\t\t0.27\t\t0.230\t\t0.858\t\t0.35m - 6.4m / 13.1m                                                                  \n",
      "17\t73k\t0.13\t\t0.26\t\t0.236\t\t0.866\t\t0.35m - 6.8m / 13.1m                                                                  \n",
      "18\t78k\t0.12\t\t0.27\t\t0.233\t\t0.853\t\t0.35m - 7.2m / 13.2m                                                                  \n",
      "19\t82k\t0.11\t\t0.29\t\t0.250\t\t0.877\t\t0.35m - 7.6m / 13.3m                                                                  \n",
      "20\t86k\t0.11\t\t0.28\t\t0.241\t\t0.872\t\t0.35m - 8.0m / 13.3m                                                                  \n",
      "VAL f1\t0.27109004739336495 - (0.27109004739336495)                                                                     \n",
      "VAL loss\t0.22415755854712593                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.22415755854712593\n",
      "        | \\     )|_\tf1: 0.27109004739336495\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\75 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 35.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         35                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0001845295648456494               |\n",
      "|  noam_learning_rate_warmup   |                        3706                       |\n",
      "|  noam_learning_rate_factor   |                0.10017271749645845                |\n",
      "|          adam_beta1          |                 0.8404725038396931                |\n",
      "|          adam_beta2          |                 0.8728333668127677                |\n",
      "|           adam_eps           |               7.673387467448837e-07               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.23243450561533863                |\n",
      "|     pointwise_layer_size     |                         86                        |\n",
      "|      last_layer_dropout      |                 0.6522896696608618                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        151                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 94%|███████████████████████████████████████████▏  | 94/100 [9:41:20<38:51, 388.57s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e9e6d35848c413b87d0595f44188bee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.80\t\t0.65\t\t0.086\t\t0.494\t\t0.43m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.55\t\t0.38\t\t0.037\t\t0.931\t\t0.43m - 0.9m / 15.1m                                                                    \n",
      "3\t13k\t0.46\t\t0.37\t\t0.005\t\t0.938\t\t0.43m - 1.4m / 15.1m                                                                   \n",
      "4\t17k\t0.45\t\t0.37\t\t0.005\t\t0.938\t\t0.43m - 1.9m / 15.3m                                                                   \n",
      "5\t22k\t0.45\t\t0.37\t\t0.011\t\t0.938\t\t0.43m - 2.4m / 15.4m                                                                   \n",
      "6\t26k\t0.44\t\t0.37\t\t0.016\t\t0.937\t\t0.43m - 2.9m / 15.4m                                                                   \n",
      "VAL f1\t0.08625180897250362 - (0.08625180897250362)                                                                     \n",
      "VAL loss\t0.36710313252040316                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.36710313252040316\n",
      "        | \\     )|_\tf1: 0.08625180897250362\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\76 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 41.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         41                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00011679471204641737              |\n",
      "|  noam_learning_rate_warmup   |                        8544                       |\n",
      "|  noam_learning_rate_factor   |                 1.176367304550232                 |\n",
      "|          adam_beta1          |                 0.7424181802192132                |\n",
      "|          adam_beta2          |                 0.842462757939765                 |\n",
      "|           adam_eps           |               1.4501475193537361e-09              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.20797536964752222                |\n",
      "|     pointwise_layer_size     |                        116                        |\n",
      "|      last_layer_dropout      |                0.45684414982628735                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        112                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 95%|███████████████████████████████████████████▋  | 95/100 [9:44:49<27:56, 335.39s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7edfd5081db545499ff9b24901612935",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.67\t\t0.41\t\t0.017\t\t0.831\t\t0.45m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.41\t\t0.32\t\t0.064\t\t0.925\t\t0.44m - 1.0m / 15.7m                                                                    \n",
      "3\t13k\t0.39\t\t0.31\t\t0.060\t\t0.904\t\t0.44m - 1.5m / 15.7m                                                                   \n",
      "4\t17k\t0.37\t\t0.30\t\t0.128\t\t0.914\t\t0.43m - 1.9m / 15.7m                                                                   \n",
      "5\t22k\t0.35\t\t0.29\t\t0.168\t\t0.888\t\t0.41m - 2.4m / 15.4m                                                                   \n",
      "6\t26k\t0.33\t\t0.28\t\t0.199\t\t0.889\t\t0.42m - 2.9m / 14.8m                                                                   \n",
      "7\t30k\t0.32\t\t0.28\t\t0.252\t\t0.888\t\t0.43m - 3.4m / 15.3m                                                                   \n",
      "8\t35k\t0.29\t\t0.27\t\t0.216\t\t0.880\t\t0.41m - 3.8m / 15.6m                                                                   \n",
      "9\t39k\t0.28\t\t0.27\t\t0.228\t\t0.869\t\t0.41m - 4.3m / 14.8m                                                                   \n",
      "10\t43k\t0.27\t\t0.28\t\t0.226\t\t0.861\t\t0.41m - 4.8m / 15.1m                                                                  \n",
      "11\t48k\t0.26\t\t0.28\t\t0.227\t\t0.852\t\t0.41m - 5.3m / 15.0m                                                                  \n",
      "12\t52k\t0.24\t\t0.29\t\t0.249\t\t0.869\t\t0.42m - 5.7m / 15.1m                                                                  \n",
      "VAL f1\t0.25169409486931266 - (0.25169409486931266)                                                                     \n",
      "VAL loss\t0.27119745859285677                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.27119745859285677\n",
      "        | \\     )|_\tf1: 0.25169409486931266\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\77 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 14.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         14                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.0006738733893761705               |\n",
      "|  noam_learning_rate_warmup   |                        5492                       |\n",
      "|  noam_learning_rate_factor   |                 0.8229483440402771                |\n",
      "|          adam_beta1          |                 0.7561536103757951                |\n",
      "|          adam_beta2          |                 0.8286941348827052                |\n",
      "|           adam_eps           |               3.768345098379125e-09               |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                 0.2557548051398951                |\n",
      "|     pointwise_layer_size     |                         46                        |\n",
      "|      last_layer_dropout      |                 0.4113922409720791                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        139                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 96%|████████████████████████████████████████████▏ | 96/100 [9:51:00<23:04, 346.13s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c260289a42cf40c8b1bb5d56fe603cb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t1.23\t\t0.77\t\t0.011\t\t0.939\t\t0.86m - 0.9m / 0.0m                                                                     \n",
      "2\t9k\t0.89\t\t0.71\t\t0.229\t\t0.914\t\t0.87m - 1.8m / 30.2m                                                                    \n",
      "3\t13k\t0.79\t\t0.69\t\t0.178\t\t0.888\t\t0.86m - 2.7m / 30.5m                                                                   \n",
      "4\t17k\t0.74\t\t0.68\t\t0.247\t\t0.894\t\t0.86m - 3.6m / 30.2m                                                                   \n",
      "5\t22k\t0.69\t\t0.71\t\t0.267\t\t0.872\t\t0.86m - 4.5m / 30.2m                                                                   \n",
      "6\t26k\t0.66\t\t0.68\t\t0.284\t\t0.880\t\t0.86m - 5.4m / 30.4m                                                                   \n",
      "7\t30k\t0.63\t\t0.72\t\t0.230\t\t0.893\t\t0.86m - 6.3m / 30.5m                                                                   \n",
      "8\t35k\t0.61\t\t0.70\t\t0.297\t\t0.901\t\t0.87m - 7.3m / 30.6m                                                                   \n",
      "9\t39k\t0.58\t\t0.78\t\t0.246\t\t0.875\t\t0.86m - 8.2m / 30.7m                                                                   \n",
      "10\t43k\t0.56\t\t0.80\t\t0.236\t\t0.896\t\t0.91m - 9.1m / 30.5m                                                                  \n",
      "11\t48k\t0.54\t\t0.81\t\t0.226\t\t0.877\t\t0.94m - 10.1m / 31.9m                                                                 \n",
      "12\t52k\t0.53\t\t0.79\t\t0.244\t\t0.882\t\t0.88m - 11.1m / 32.8m                                                                 \n",
      "13\t56k\t0.52\t\t0.81\t\t0.259\t\t0.871\t\t0.86m - 12.0m / 31.3m                                                                 \n",
      "VAL f1\t0.29690721649484536 - (0.29690721649484536)                                                                     \n",
      "VAL loss\t0.6784311462016333                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.6784311462016333\n",
      "        | \\     )|_\tf1: 0.29690721649484536\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\78 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 51.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         51                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               8.470170717386564e-05               |\n",
      "|  noam_learning_rate_warmup   |                        7673                       |\n",
      "|  noam_learning_rate_factor   |                 0.3551679559482436                |\n",
      "|          adam_beta1          |                 0.7234515957463107                |\n",
      "|          adam_beta2          |                 0.9154476758808161                |\n",
      "|           adam_eps           |               4.5767812403059565e-08              |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         1                         |\n",
      "|             d_k              |                        300                        |\n",
      "|             d_v              |                        300                        |\n",
      "|         dropout_rate         |                0.29013562830467077                |\n",
      "|     pointwise_layer_size     |                        214                        |\n",
      "|      last_layer_dropout      |                0.48147157702458687                |\n",
      "|   output_conv_num_filters    |                         75                        |\n",
      "|   output_conv_kernel_size    |                         2                         |\n",
      "|      output_conv_stride      |                         4                         |\n",
      "|     output_conv_padding      |                         3                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        157                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 97%|███████████████████████████████████████████▋ | 97/100 [10:03:40<23:30, 470.12s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e88ee092bef459b92368269307f96d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.48\t\t0.44\t\t0.047\t\t0.663\t\t0.38m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.37\t\t0.37\t\t0.081\t\t0.937\t\t0.39m - 0.8m / 13.4m                                                                    \n",
      "3\t13k\t0.33\t\t0.35\t\t0.021\t\t0.940\t\t0.39m - 1.3m / 13.6m                                                                   \n",
      "4\t17k\t0.32\t\t0.34\t\t0.097\t\t0.936\t\t0.38m - 1.7m / 13.6m                                                                   \n",
      "5\t22k\t0.32\t\t0.34\t\t0.148\t\t0.932\t\t0.39m - 2.1m / 13.7m                                                                   \n",
      "6\t26k\t0.31\t\t0.33\t\t0.232\t\t0.913\t\t0.39m - 2.6m / 13.9m                                                                   \n",
      "7\t30k\t0.30\t\t0.32\t\t0.233\t\t0.891\t\t0.39m - 3.0m / 13.9m                                                                   \n",
      "8\t35k\t0.28\t\t0.31\t\t0.221\t\t0.875\t\t0.38m - 3.5m / 13.9m                                                                   \n",
      "9\t39k\t0.27\t\t0.29\t\t0.234\t\t0.877\t\t0.39m - 3.9m / 13.9m                                                                   \n",
      "10\t43k\t0.25\t\t0.29\t\t0.232\t\t0.867\t\t0.39m - 4.3m / 14.0m                                                                  \n",
      "11\t48k\t0.25\t\t0.29\t\t0.233\t\t0.858\t\t0.39m - 4.8m / 14.1m                                                                  \n",
      "12\t52k\t0.24\t\t0.28\t\t0.228\t\t0.856\t\t0.39m - 5.2m / 14.2m                                                                  \n",
      "13\t56k\t0.23\t\t0.28\t\t0.217\t\t0.852\t\t0.39m - 5.7m / 14.2m                                                                  \n",
      "14\t61k\t0.23\t\t0.28\t\t0.210\t\t0.846\t\t0.39m - 6.1m / 14.3m                                                                  \n",
      "VAL f1\t0.23431734317343172 - (0.23431734317343172)                                                                     \n",
      "VAL loss\t0.27672311371447994                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.27672311371447994\n",
      "        | \\     )|_\tf1: 0.23431734317343172\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\79 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 53.0, 'learning_rate_sche[...]lse} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         53                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               4.2049349780437656e-05              |\n",
      "|  noam_learning_rate_warmup   |                        5776                       |\n",
      "|  noam_learning_rate_factor   |                 3.9139558493241813                |\n",
      "|          adam_beta1          |                 0.7432369487168599                |\n",
      "|          adam_beta2          |                 0.8993731905679776                |\n",
      "|           adam_eps           |               1.0383946843695459e-08              |\n",
      "|      adam_weight_decay       |                       1e-08                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         4                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.10814256236961665                |\n",
      "|     pointwise_layer_size     |                         95                        |\n",
      "|      last_layer_dropout      |                 0.758030692903689                 |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        175                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 98%|████████████████████████████████████████████ | 98/100 [10:10:13<14:54, 447.18s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3e75d6522a24be3bd574ba378f40a19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.41\t\t0.27\t\t0.000\t\t0.933\t\t0.43m - 0.4m / 0.0m                                                                     \n",
      "2\t9k\t0.32\t\t0.25\t\t0.072\t\t0.899\t\t0.44m - 0.9m / 15.2m                                                                    \n",
      "3\t13k\t0.29\t\t0.23\t\t0.215\t\t0.904\t\t0.44m - 1.4m / 15.4m                                                                   \n",
      "4\t17k\t0.26\t\t0.24\t\t0.219\t\t0.866\t\t0.44m - 1.9m / 15.4m                                                                   \n",
      "5\t22k\t0.23\t\t0.23\t\t0.241\t\t0.891\t\t0.45m - 2.4m / 15.6m                                                                   \n",
      "6\t26k\t0.21\t\t0.22\t\t0.227\t\t0.887\t\t0.44m - 2.9m / 16.0m                                                                   \n",
      "7\t30k\t0.19\t\t0.24\t\t0.237\t\t0.867\t\t0.44m - 3.4m / 15.8m                                                                   \n",
      "8\t35k\t0.17\t\t0.25\t\t0.253\t\t0.871\t\t0.44m - 3.9m / 15.9m                                                                   \n",
      "9\t39k\t0.16\t\t0.26\t\t0.253\t\t0.868\t\t0.46m - 4.4m / 15.8m                                                                   \n",
      "10\t43k\t0.15\t\t0.27\t\t0.275\t\t0.891\t\t0.46m - 5.0m / 16.4m                                                                  \n",
      "11\t48k\t0.13\t\t0.28\t\t0.246\t\t0.892\t\t0.46m - 5.5m / 16.4m                                                                  \n",
      "12\t52k\t0.12\t\t0.31\t\t0.211\t\t0.858\t\t0.46m - 6.0m / 16.6m                                                                  \n",
      "13\t56k\t0.12\t\t0.33\t\t0.263\t\t0.888\t\t0.45m - 6.5m / 16.6m                                                                  \n",
      "14\t61k\t0.11\t\t0.32\t\t0.239\t\t0.877\t\t0.44m - 7.0m / 16.4m                                                                  \n",
      "15\t65k\t0.10\t\t0.33\t\t0.253\t\t0.878\t\t0.44m - 7.5m / 16.2m                                                                  \n",
      "VAL f1\t0.27454718779790277 - (0.27454718779790277)                                                                     \n",
      "VAL loss\t0.21726968153467716                                                                                           \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.21726968153467716\n",
      "        | \\     )|_\tf1: 0.27454718779790277\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "Log path is                                                                                                            \n",
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\OrganicCoarseHyperopt\\20190424\\80 \n",
      "                                                                                                                       \n",
      "\n",
      "#########################################################################\n",
      "+----------------------------------------------------------------------------------+                                   \n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 27.0, 'learning_rate_sche[...]rue} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         27                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |               0.00032363834548846033              |\n",
      "|  noam_learning_rate_warmup   |                        3272                       |\n",
      "|  noam_learning_rate_factor   |                 0.9279199830049817                |\n",
      "|          adam_beta1          |                 0.7842943647400444                |\n",
      "|          adam_beta2          |                 0.8141496966317816                |\n",
      "|           adam_eps           |               1.0244938465042967e-10              |\n",
      "|      adam_weight_decay       |                       1e-07                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         3                         |\n",
      "|           n_heads            |                         3                         |\n",
      "|             d_k              |                        100                        |\n",
      "|             d_v              |                        100                        |\n",
      "|         dropout_rate         |                0.17974199520105763                |\n",
      "|     pointwise_layer_size     |                        134                        |\n",
      "|      last_layer_dropout      |                0.42563984181321124                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        133                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n",
      " 99%|████████████████████████████████████████████▌| 99/100 [10:18:20<07:38, 459.00s/it, best loss: 0.19632605711619058]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00098da2a160404fa8d017d595e09d81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time                                                             \n",
      "1\t4k\t0.70\t\t0.46\t\t0.000\t\t0.939\t\t0.52m - 0.5m / 0.0m                                                                     \n",
      "2\t9k\t0.55\t\t0.43\t\t0.218\t\t0.916\t\t0.52m - 1.1m / 18.4m                                                                    \n",
      "3\t13k\t0.48\t\t0.43\t\t0.214\t\t0.878\t\t0.53m - 1.7m / 18.2m                                                                   \n",
      "4\t17k\t0.44\t\t0.42\t\t0.218\t\t0.854\t\t0.51m - 2.3m / 18.6m                                                                   \n",
      "5\t22k\t0.40\t\t0.41\t\t0.243\t\t0.877\t\t0.51m - 2.8m / 18.1m                                                                   \n",
      "6\t26k\t0.38\t\t0.42\t\t0.233\t\t0.860\t\t0.53m - 3.4m / 18.3m                                                                   \n",
      "7\t30k\t0.35\t\t0.43\t\t0.215\t\t0.871\t\t0.51m - 4.0m / 18.9m                                                                   \n",
      "8\t35k\t0.32\t\t0.50\t\t0.213\t\t0.856\t\t0.50m - 4.5m / 18.5m                                                                   \n",
      "9\t39k\t0.31\t\t0.46\t\t0.244\t\t0.875\t\t0.53m - 5.1m / 18.1m                                                                   \n",
      "10\t43k\t0.29\t\t0.49\t\t0.233\t\t0.867\t\t0.54m - 5.7m / 18.9m                                                                  \n",
      "11\t48k\t0.27\t\t0.50\t\t0.258\t\t0.886\t\t0.52m - 6.3m / 19.2m                                                                  \n",
      "12\t52k\t0.26\t\t0.51\t\t0.238\t\t0.873\t\t0.51m - 6.9m / 18.8m                                                                  \n",
      "13\t56k\t0.25\t\t0.52\t\t0.260\t\t0.886\t\t0.53m - 7.4m / 18.7m                                                                  \n",
      "14\t60k\t0.24\t\t0.57\t\t0.237\t\t0.881\t\t0.57m - 8.1m / 19.2m                                                                  \n",
      "15\t65k\t0.23\t\t0.49\t\t0.270\t\t0.878\t\t0.53m - 8.7m / 20.1m                                                                  \n",
      "16\t69k\t0.23\t\t0.57\t\t0.238\t\t0.870\t\t0.54m - 9.3m / 19.3m                                                                  \n",
      "17\t73k\t0.23\t\t0.55\t\t0.221\t\t0.874\t\t0.55m - 9.9m / 19.6m                                                                  \n",
      "18\t78k\t0.20\t\t0.59\t\t0.251\t\t0.879\t\t0.53m - 10.4m / 19.7m                                                                 \n",
      "19\t82k\t0.21\t\t0.66\t\t0.259\t\t0.902\t\t0.54m - 11.0m / 19.5m                                                                 \n",
      "20\t86k\t0.21\t\t0.59\t\t0.225\t\t0.888\t\t0.55m - 11.6m / 19.7m                                                                 \n",
      "VAL f1\t0.2702702702702703 - (0.2702702702702703)                                                                       \n",
      "VAL loss\t0.4099737055960544                                                                                            \n",
      "       .---.                                                                                                           \n",
      "          /     \\\n",
      "          \\.@-@./\n",
      "          /`\\_/`\\\n",
      "         //  _  \\\\\tLoss: 0.4099737055960544\n",
      "        | \\     )|_\tf1: 0.2702702702702703\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n",
      "100%|████████████████████████████████████████████| 100/100 [10:30:32<00:00, 541.07s/it, best loss: 0.19632605711619058]\n",
      "{'adam_beta1': 0.7767090225510399, 'adam_beta2': 0.9777552715687797, 'adam_eps': 1.5205703918047645e-09, 'adam_learning_rate': 0.00016840973956259016, 'adam_weight_decay': -7.0, 'batch_size': 62.0, 'clip_comments_to': 156.0, 'dropout_rate': 0.18454418705535447, 'embedding_type': 1, 'last_layer_dropout': 0.3063965836318771, 'learning_rate_scheduler': 0, 'noam_learning_rate_factor': 0.4150230798936053, 'noam_learning_rate_warmup': 3623.0, 'num_encoder_blocks': 3.0, 'num_heads': 2, 'optimizer': 0, 'output_layer': 1, 'pointwise_layer_size': 83.0, 'transformer_use_bias': 0, 'use_spell_checker': 0, 'use_stop_words': 1}\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    logger.info('Current commit: ' + utils.get_current_git_commit())\n",
    "    print('Current commit: ' + utils.get_current_git_commit())\n",
    "except Exception as err:\n",
    "    logger.exception('Could not print current commit')\n",
    "\n",
    "trials = Trials()\n",
    "try:\n",
    "\n",
    "    best = fmin(objective,\n",
    "        space=search_space,\n",
    "        algo=tpe.suggest,\n",
    "        max_evals=runs,\n",
    "        trials=trials)\n",
    "\n",
    "    print(best)\n",
    "except Exception as err:\n",
    "    logger.exception('Could not complete optimization')\n",
    "    print('Could not complete optimization. The log file provides more details.')\n",
    "\n",
    "\n",
    "path = os.path.join(os.getcwd(), 'logs', f'hp_run_{main_experiment_name}.pkl')\n",
    "with open(path, 'wb') as f:\n",
    "    pickle.dump(trials, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}