cs224n-2017-lecture9-paper3.pdf

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<title>[1602.02410] Exploring the Limits of Language Modeling</title>
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
<link rel="stylesheet" type="text/css" media="screen" href="/css/arXiv.css?v=20161221" />
<meta name="citation_title" content="Exploring the Limits of Language Modeling" />
<meta name="citation_author" content="Jozefowicz, Rafal" />
<meta name="citation_author" content="Vinyals, Oriol" />
<meta name="citation_author" content="Schuster, Mike" />
<meta name="citation_author" content="Shazeer, Noam" />
<meta name="citation_author" content="Wu, Yonghui" />
<meta name="citation_date" content="2016/02/07" />
<meta name="citation_online_date" content="2016/02/11" />
<meta name="citation_pdf_url" content="http://arxiv.org/pdf/1602.02410" />
<meta name="citation_arxiv_id" content="1602.02410" />
<script src="/js/mathjaxToggle.min.js" type="text/javascript"></script>

<!-- Piwik -->
<script type="text/javascript">
var _paq = _paq || [];
_paq.push(["setDomains", ["*.arxiv.org"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function()
{ var u="//webanalytics.library.cornell.edu/"; _paq.push(['setTrackerUrl', u+'piwik.php']); _paq.push(['setSiteId', 538]); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); }
)();
</script>
<!-- End Piwik Code -->


</head>
<body class="with-cu-identity">

<noscript><img src="//webanalytics.library.cornell.edu/piwik.php?idsite=538&rec=1" style="border:0;" alt="" /></noscript>

<div id="cu-identity">
<div id="cu-logo">
<a id="insignia-link" href="http://www.cornell.edu/"><img src="/icons/cu/cul_signature_unstyled.gif" alt="Cornell University" width="283" height="76" border="0" /></a>
<div id="unit-signature-links">
<a id="cornell-link" href="http://www.cornell.edu/">Cornell University</a>
<a id="unit-link" href="http://www.library.cornell.edu/">Library</a>
</div>
</div>
<div id="support-ack">
<a href="https://confluence.cornell.edu/x/ALlRF">We gratefully acknowledge support from<br />the Simons Foundation<br /> and member institutions</a>
</div>
</div>
<div id="header" class="cs">
<h1><a href="/">arXiv.org</a> &gt; <a href="/list/cs/recent">cs</a> &gt; arXiv:1602.02410</h1>

<div id="search">
<form id="search-arxiv" method="post" action="/search">
                
<div class="wrapper-search-arxiv">
<input class="keyword-field" type="text" name="query" placeholder="Search or Article ID inside arXiv"/>

<div class="filter-field">
 <select name="searchtype">
<option value="all" selected="selected">All papers</option>
<option value="ti">Titles</option>
<option value="au">Authors</option>
<option value="abs">Abstracts</option>
<option value="ft">Full text</option>
<option value="help">Help pages</option>
</select>
</div>
<input class="btn-search-arxiv" value="" type="submit">
<div class="links">(<a href="/help">Help</a> | <a href="/find">Advanced search</a>)</div>
</div>
</form>

<form id="search-semantic" method="get" action="https://www.semanticscholar.org/search" onSubmit="_paq.push(['trackGoal', 6])">
<div class="wrapper-search-semantic">
<input class="keyword-field" type="text" name="q" placeholder="Broaden your search using"/>
<input class="btn-search-semantic" value="" type="submit">
</div>
</form>
</div>
</div> 
</div>
<div id="content">

<!--
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
         xmlns:dc="http://purl.org/dc/elements/1.1/"
         xmlns:trackback="http://madskills.com/public/xml/rss/module/trackback/">
    <rdf:Description
        rdf:about="http://arxiv.org/abs/1602.02410"
        dc:identifier="http://arxiv.org/abs/1602.02410"
        dc:title="Exploring the Limits of Language Modeling"
        trackback:ping="http://arxiv.org/trackback/1602.02410" />
    </rdf:RDF>
-->

<div id="abs">
<div class="extra-services">

<div class="full-text">
<span class="descriptor">Full-text links:</span>
<h2>Download:</h2>
<ul>
<li><a href="/pdf/1602.02410" accesskey="f">PDF</a></li>
<li><a href="/format/1602.02410">Other formats</a></li>
</ul>
<div class="abs-license">(<a href="http://arxiv.org/licenses/nonexclusive-distrib/1.0/" title="Rights to this article">license</a>)</div>
</div><!--end full-text-->

<div class="browse">
<h3>Current browse context:</h3>
<div class="current">cs.CL</div>
<div class="prevnext"><span class="arrow"><a href="http://arxiv.org/prevnext?site=arxiv.org&amp;id=1602.02410&amp;context=cs.CL&amp;function=prev" accesskey="p" title="previous in cs.CL (accesskey p)">&lt;&nbsp;prev</a></span>&nbsp;|&nbsp;<span class="arrow"><a href="http://arxiv.org/prevnext?site=arxiv.org&amp;id=1602.02410&amp;context=cs.CL&amp;function=next" accesskey="n" title="next in cs.CL (accesskey n)">next&nbsp;&gt;</a></span>
<br /></div>
<div class="list"><a href="/list/cs.CL/new">new</a>&nbsp;| <a href="/list/cs.CL/recent">recent</a>&nbsp;| <a href="/list/cs.CL/1602">1602</a></div><h3>Change to browse by:</h3><div class="switch">
<a href="/abs/1602.02410?context=cs">cs</a>
</div>

</div>
<div class="extra-ref-cite">
<h3>References &amp; Citations</h3><ul><li><a href="http://adsabs.harvard.edu/cgi-bin/bib_query?arXiv:1602.02410">NASA ADS</a></li>
</ul>

</div>
<div class="dblp">
<h3><a href="http://www.informatik.uni-trier.de/~ley/db/">DBLP</a> - CS Bibliography</h3><div class="list">
<a href="http://www.informatik.uni-trier.de/~ley/db/journals/corr/corr1602.html#JozefowiczVSSW16" title="listing on DBLP">listing</a> | <a href="http://dblp.uni-trier.de/rec/bibtex/journals/corr/JozefowiczVSSW16" title="DBLP bibtex record">bibtex</a>
</div>
<div class="list">
<a href="http://dblp.uni-trier.de/search/author?author=Rafal%20J&#xF3;zefowicz" title="DBLP author search">Rafal J&#xF3;zefowicz</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Oriol%20Vinyals" title="DBLP author search">Oriol Vinyals</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Mike%20Schuster" title="DBLP author search">Mike Schuster</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Noam%20Shazeer" title="DBLP author search">Noam Shazeer</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Yonghui%20Wu" title="DBLP author search">Yonghui Wu</a><br />
</div>

</div>
<div class="bookmarks">
<div class="what-is-this">
<h3>Bookmark</h3> (<a href="/help/social_bookmarking">what is this?</a>)
</div>
<a href="/ct?url=http%3A%2F%2Fwww.citeulike.org%2Fposturl%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410%26title%3DExploring%2520the%2520Limits%2520of%2520Language%2520Modeling%26authors%3D&amp;v=a18a52bc" title="Bookmark on CiteULike"><img src="//static.arxiv.org/icons/social/citeulike.png" alt="CiteULike logo" /></a>
<a href="/ct?url=http%3A%2F%2Fwww.bibsonomy.org%2FBibtexHandler%3FrequTask%3Dupload%26url%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410%26description%3DExploring%2520the%2520Limits%2520of%2520Language%2520Modeling&amp;v=ee940803" title="Bookmark on BibSonomy"><img src="//static.arxiv.org/icons/social/bibsonomy.png" alt="BibSonomy logo" /></a>
<a href="/ct?url=https%3A%2F%2Fwww.mendeley.com%2Fimport%2F%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410&amp;v=94204eb0" title="Bookmark on Mendeley"><img src="//static.arxiv.org/icons/social/mendeley.png" alt="Mendeley logo" /></a>
<a href="/ct?url=https%3A%2F%2Fdel.icio.us%2Fpost%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410%26description%3DExploring%2520the%2520Limits%2520of%2520Language%2520Modeling&amp;v=67f8f87f" title="Bookmark on del.icio.us"><img src="//static.arxiv.org/icons/social/delicious.png" alt="del.icio.us logo" /></a>
<a href="/ct?url=https%3A%2F%2Fdigg.com%2Fsubmit%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410%26title%3DExploring%2520the%2520Limits%2520of%2520Language%2520Modeling&amp;v=fb4905bd" title="Bookmark on Digg"><img src="//static.arxiv.org/icons/social/digg.png" alt="Digg logo" /></a>
<a href="/ct?url=https%3A%2F%2Freddit.com%2Fsubmit%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410%26title%3DExploring%2520the%2520Limits%2520of%2520Language%2520Modeling&amp;v=caac688e" title="Bookmark on Reddit"><img src="//static.arxiv.org/icons/social/reddit.png" alt="Reddit logo" /></a>
<a href="/ct?url=http%3A%2F%2Fsciencewise.info%2Fbookmarks%2Fadd%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1602.02410&amp;v=20fd6905" title="Bookmark on ScienceWISE"><img src="//static.arxiv.org/icons/social/sciencewise.png" alt="ScienceWISE logo" /></a>

</div>
</div><!--end extra-services-->

<div class="leftcolumn">
<div class="subheader">
<h1>Computer Science > Computation and Language</h1>
</div>
<h1 class="title mathjax"><span class="descriptor">Title:</span>
Exploring the Limits of Language Modeling</h1>
<div class="authors"><span class="descriptor">Authors:</span>
<a href="/find/cs/1/au:+Jozefowicz_R/0/1/0/all/0/1">Rafal Jozefowicz</a>, 
<a href="/find/cs/1/au:+Vinyals_O/0/1/0/all/0/1">Oriol Vinyals</a>, 
<a href="/find/cs/1/au:+Schuster_M/0/1/0/all/0/1">Mike Schuster</a>, 
<a href="/find/cs/1/au:+Shazeer_N/0/1/0/all/0/1">Noam Shazeer</a>, 
<a href="/find/cs/1/au:+Wu_Y/0/1/0/all/0/1">Yonghui Wu</a></div>
<div class="dateline">(Submitted on 7 Feb 2016 (<a href="/abs/1602.02410v1">v1</a>), last revised 11 Feb 2016 (this version, v2))</div>
<blockquote class="abstract mathjax">
<span class="descriptor">Abstract:</span> In this work we explore recent advances in Recurrent Neural Networks for
large scale Language Modeling, a task central to language understanding. We
extend current models to deal with two key challenges present in this task:
corpora and vocabulary sizes, and complex, long term structure of language. We
perform an exhaustive study on techniques such as character Convolutional
Neural Networks or Long-Short Term Memory, on the One Billion Word Benchmark.
Our best single model significantly improves state-of-the-art perplexity from
51.3 down to 30.0 (whilst reducing the number of parameters by a factor of 20),
while an ensemble of models sets a new record by improving perplexity from 41.0
down to 23.7. We also release these models for the NLP and ML community to
study and improve upon.
</blockquote>
<!--CONTEXT-->
<div class="metatable">
<table summary="Additional metadata">
<tr>
<td class="tablecell label">Subjects:
</td>
<td class="tablecell subjects"><span class="primary-subject">Computation and Language (cs.CL)</span></td>
</tr>
<tr>
<td class="tablecell label">
Cite&nbsp;as:
</td>
<td class="tablecell arxivid"><a href="/abs/1602.02410">arXiv:1602.02410</a> [cs.CL]</td>
</tr>
<tr>
<td class="tablecell label">&nbsp;</td>
<td class="tablecell arxividv">(or <span class="arxivid"><a href="/abs/1602.02410v2">arXiv:1602.02410v2</a> [cs.CL]</span> for this version)</td>
</tr>
</table>
</div>
<div class="submission-history">
<h2>Submission history</h2>
From: Oriol Vinyals [<a href="https://arxiv.org/show-email/1a1c9b1a/1602.02410">view email</a>]
<br />
<b><a href="/abs/1602.02410v1">[v1]</a></b> Sun, 7 Feb 2016 19:11:17 GMT  (76kb,D)<br />
<b>[v2]</b> Thu, 11 Feb 2016 23:01:48 GMT  (77kb,D)<br />
</div>
<div class="endorsers"><a href="http://arxiv.org/auth/show-endorsers/1602.02410">Which authors of this paper are endorsers?</a> | <a id="mathjax_toggle" href="javascript:setMathjaxCookie()">Disable MathJax</a> (<a href="/help/mathjax/">What is MathJax?</a>)</div><script type="text/javascript" language="javascript">mathjaxToggle();</script>
</div><!--end leftcolumn-->
</div><!--end abs-->
</div>
<div id="footer">
<p>Link back to: <a href="http://arxiv.org/">arXiv</a>, <a href="/form">form interface</a>, <a href="/help/contact">contact</a>.</p>
</div>
</body>
</html>