09_ElmoEmbedder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from elmoformanylangs import Embedder\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\felix\\\\OneDrive\\\\Studium\\\\Studium\\\\6. Semester\\\\MA\\\\Project\\\\ABSA-Transformer\\\\.vector_cache\\\\elmo\\\\de.model'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join(os.getcwd(), '.vector_cache', 'elmo', 'de.model')\n",
    "path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-03-18 15:18:44,327 INFO: char embedding size: 2784\n",
      "2019-03-18 15:18:45,963 INFO: word embedding size: 267883\n",
      "2019-03-18 15:18:56,079 INFO: Model(\n",
      "  (token_embedder): ConvTokenEmbedder(\n",
      "    (word_emb_layer): EmbeddingLayer(\n",
      "      (embedding): Embedding(267883, 100, padding_idx=3)\n",
      "    )\n",
      "    (char_emb_layer): EmbeddingLayer(\n",
      "      (embedding): Embedding(2784, 50, padding_idx=2781)\n",
      "    )\n",
      "    (convolutions): ModuleList(\n",
      "      (0): Conv1d(50, 32, kernel_size=(1,), stride=(1,))\n",
      "      (1): Conv1d(50, 32, kernel_size=(2,), stride=(1,))\n",
      "      (2): Conv1d(50, 64, kernel_size=(3,), stride=(1,))\n",
      "      (3): Conv1d(50, 128, kernel_size=(4,), stride=(1,))\n",
      "      (4): Conv1d(50, 256, kernel_size=(5,), stride=(1,))\n",
      "      (5): Conv1d(50, 512, kernel_size=(6,), stride=(1,))\n",
      "      (6): Conv1d(50, 1024, kernel_size=(7,), stride=(1,))\n",
      "    )\n",
      "    (highways): Highway(\n",
      "      (_layers): ModuleList(\n",
      "        (0): Linear(in_features=2048, out_features=4096, bias=True)\n",
      "        (1): Linear(in_features=2048, out_features=4096, bias=True)\n",
      "      )\n",
      "    )\n",
      "    (projection): Linear(in_features=2148, out_features=512, bias=True)\n",
      "  )\n",
      "  (encoder): ElmobiLm(\n",
      "    (forward_layer_0): LstmCellWithProjection(\n",
      "      (input_linearity): Linear(in_features=512, out_features=16384, bias=False)\n",
      "      (state_linearity): Linear(in_features=512, out_features=16384, bias=True)\n",
      "      (state_projection): Linear(in_features=4096, out_features=512, bias=False)\n",
      "    )\n",
      "    (backward_layer_0): LstmCellWithProjection(\n",
      "      (input_linearity): Linear(in_features=512, out_features=16384, bias=False)\n",
      "      (state_linearity): Linear(in_features=512, out_features=16384, bias=True)\n",
      "      (state_projection): Linear(in_features=4096, out_features=512, bias=False)\n",
      "    )\n",
      "    (forward_layer_1): LstmCellWithProjection(\n",
      "      (input_linearity): Linear(in_features=512, out_features=16384, bias=False)\n",
      "      (state_linearity): Linear(in_features=512, out_features=16384, bias=True)\n",
      "      (state_projection): Linear(in_features=4096, out_features=512, bias=False)\n",
      "    )\n",
      "    (backward_layer_1): LstmCellWithProjection(\n",
      "      (input_linearity): Linear(in_features=512, out_features=16384, bias=False)\n",
      "      (state_linearity): Linear(in_features=512, out_features=16384, bias=True)\n",
      "      (state_projection): Linear(in_features=4096, out_features=512, bias=False)\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "e = Embedder(path, 12)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-03-18 15:53:16,229 INFO: 1 batches, avg len: 7.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[-0.21167928, -0.34235975,  0.2310075 , ...,  0.42909566,\n",
       "         -0.32124928, -0.03182214],\n",
       "        [-0.12173772,  0.3038745 ,  0.07743659, ...,  0.07158012,\n",
       "          0.13374679,  0.3907275 ],\n",
       "        [-0.02208417,  0.6092438 , -0.04707598, ...,  0.10765642,\n",
       "          0.6545157 , -0.01098429],\n",
       "        [ 0.23242176,  0.36897016,  0.12116945, ..., -0.14314151,\n",
       "          0.12278528,  0.27407527],\n",
       "        [ 0.26969424, -0.01854969, -0.20829582, ...,  0.20597999,\n",
       "          0.00509887,  0.14988309]],\n",
       "\n",
       "       [[-0.22358878, -0.32304028,  0.21788923, ...,  0.46814358,\n",
       "         -0.35962287, -0.14571239],\n",
       "        [-0.11986082,  0.31139013,  0.07441226, ..., -0.0430906 ,\n",
       "         -0.12256253, -0.16584812],\n",
       "        [-0.02208438,  0.6097606 , -0.04522142, ..., -0.02053065,\n",
       "          0.47471547, -0.41620994],\n",
       "        [ 0.03423853, -0.34851435, -0.80426836, ...,  0.02990749,\n",
       "          0.8801318 , -0.5644725 ],\n",
       "        [-0.32386902,  0.24779652,  0.1218968 , ..., -0.5733752 ,\n",
       "          0.1323498 , -0.16403721]]], dtype=float32)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = [['Das', 'ist', 'ein', 'test', 'asdkajsdlaasd'], ['Das', 'ist', 'ein', 'zweiter', 'Satz']]\n",
    "embedding = np.array(e.sents2elmo(sentence))\n",
    "embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = torch.as_tensor(embedding, dtype=torch.float32, device=torch.device('cuda'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 5, 1024])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1024,)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding[0][-1].shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_dict = {'Das ist ein Satz': 1, 'Das ist noch ein Satz': 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dict['Das ist noch ein Satz']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}