cs224n-2017-lecture10-paper.pdf

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<title>[1611.04558] Google&#x27;s Multilingual Neural Machine Translation System: Enabling
  Zero-Shot Translation</title>
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
<link rel="stylesheet" type="text/css" media="screen" href="/css/arXiv.css?v=20161221" />
<meta name="citation_title" content="Google&#x27;s Multilingual Neural Machine Translation System: Enabling Zero-Shot Translation" />
<meta name="citation_author" content="Johnson, Melvin" />
<meta name="citation_author" content="Schuster, Mike" />
<meta name="citation_author" content="Le, Quoc V." />
<meta name="citation_author" content="Krikun, Maxim" />
<meta name="citation_author" content="Wu, Yonghui" />
<meta name="citation_author" content="Chen, Zhifeng" />
<meta name="citation_author" content="Thorat, Nikhil" />
<meta name="citation_author" content="Vi&#xe9;gas, Fernanda" />
<meta name="citation_author" content="Wattenberg, Martin" />
<meta name="citation_author" content="Corrado, Greg" />
<meta name="citation_author" content="Hughes, Macduff" />
<meta name="citation_author" content="Dean, Jeffrey" />
<meta name="citation_date" content="2016/11/14" />
<meta name="citation_online_date" content="2016/11/14" />
<meta name="citation_pdf_url" content="http://arxiv.org/pdf/1611.04558" />
<meta name="citation_arxiv_id" content="1611.04558" />
<script src="/js/mathjaxToggle.min.js" type="text/javascript"></script>

<!-- Piwik -->
<script type="text/javascript">
var _paq = _paq || [];
_paq.push(["setDomains", ["*.arxiv.org"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function()
{ var u="//webanalytics.library.cornell.edu/"; _paq.push(['setTrackerUrl', u+'piwik.php']); _paq.push(['setSiteId', 538]); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); }
)();
</script>
<!-- End Piwik Code -->


</head>
<body class="with-cu-identity">

<noscript><img src="//webanalytics.library.cornell.edu/piwik.php?idsite=538&rec=1" style="border:0;" alt="" /></noscript>

<div id="cu-identity">
<div id="cu-logo">
<a id="insignia-link" href="http://www.cornell.edu/"><img src="/icons/cu/cul_signature_unstyled.gif" alt="Cornell University" width="283" height="76" border="0" /></a>
<div id="unit-signature-links">
<a id="cornell-link" href="http://www.cornell.edu/">Cornell University</a>
<a id="unit-link" href="http://www.library.cornell.edu/">Library</a>
</div>
</div>
<div id="support-ack">
<a href="https://confluence.cornell.edu/x/ALlRF">We gratefully acknowledge support from<br />the Simons Foundation<br /> and member institutions</a>
</div>
</div>
<div id="header" class="cs">
<h1><a href="/">arXiv.org</a> &gt; <a href="/list/cs/recent">cs</a> &gt; arXiv:1611.04558</h1>

<div id="search">
<form id="search-arxiv" method="post" action="/search">
                
<div class="wrapper-search-arxiv">
<input class="keyword-field" type="text" name="query" placeholder="Search or Article ID inside arXiv"/>

<div class="filter-field">
 <select name="searchtype">
<option value="all" selected="selected">All papers</option>
<option value="ti">Titles</option>
<option value="au">Authors</option>
<option value="abs">Abstracts</option>
<option value="ft">Full text</option>
<option value="help">Help pages</option>
</select>
</div>
<input class="btn-search-arxiv" value="" type="submit">
<div class="links">(<a href="/help">Help</a> | <a href="/find">Advanced search</a>)</div>
</div>
</form>

<form id="search-semantic" method="get" action="https://www.semanticscholar.org/search" onSubmit="_paq.push(['trackGoal', 6])">
<div class="wrapper-search-semantic">
<input class="keyword-field" type="text" name="q" placeholder="Broaden your search using"/>
<input class="btn-search-semantic" value="" type="submit">
</div>
</form>
</div>
</div> 
</div>
<div id="content">

<!--
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
         xmlns:dc="http://purl.org/dc/elements/1.1/"
         xmlns:trackback="http://madskills.com/public/xml/rss/module/trackback/">
    <rdf:Description
        rdf:about="http://arxiv.org/abs/1611.04558"
        dc:identifier="http://arxiv.org/abs/1611.04558"
        dc:title="Google's Multilingual Neural Machine Translation System: Enabling
  Zero-Shot Translation"
        trackback:ping="http://arxiv.org/trackback/1611.04558" />
    </rdf:RDF>
-->

<div id="abs">
<div class="extra-services">

<div class="full-text">
<span class="descriptor">Full-text links:</span>
<h2>Download:</h2>
<ul>
<li><a href="/pdf/1611.04558" accesskey="f">PDF</a></li>
<li><a href="/format/1611.04558">Other formats</a></li>
</ul>
<div class="abs-license">(<a href="http://arxiv.org/licenses/nonexclusive-distrib/1.0/" title="Rights to this article">license</a>)</div>
</div><!--end full-text-->

<div class="browse">
<h3>Current browse context:</h3>
<div class="current">cs.CL</div>
<div class="prevnext"><span class="arrow"><a href="http://arxiv.org/prevnext?site=arxiv.org&amp;id=1611.04558&amp;context=cs.CL&amp;function=prev" accesskey="p" title="previous in cs.CL (accesskey p)">&lt;&nbsp;prev</a></span>&nbsp;|&nbsp;<span class="arrow"><a href="http://arxiv.org/prevnext?site=arxiv.org&amp;id=1611.04558&amp;context=cs.CL&amp;function=next" accesskey="n" title="next in cs.CL (accesskey n)">next&nbsp;&gt;</a></span>
<br /></div>
<div class="list"><a href="/list/cs.CL/new">new</a>&nbsp;| <a href="/list/cs.CL/recent">recent</a>&nbsp;| <a href="/list/cs.CL/1611">1611</a></div><h3>Change to browse by:</h3><div class="switch">
<a href="/abs/1611.04558?context=cs">cs</a><br />
<span class="subclass"><a href="/abs/1611.04558?context=cs.AI">cs.AI</a></span>
</div>

</div>
<div class="extra-ref-cite">
<h3>References &amp; Citations</h3><ul><li><a href="http://adsabs.harvard.edu/cgi-bin/bib_query?arXiv:1611.04558">NASA ADS</a></li>
</ul>

</div>
<div class="extra-general">
<div class="what-is-this">
<h3><a href="/tb/1611.04558">1 blog link</a></h3> (<a href="/help/trackback/">what is this?</a>)

</div>

</div>
<div class="dblp">
<h3><a href="http://www.informatik.uni-trier.de/~ley/db/">DBLP</a> - CS Bibliography</h3><div class="list">
<a href="http://www.informatik.uni-trier.de/~ley/db/journals/corr/corr1611.html#JohnsonSLKWCTVW16" title="listing on DBLP">listing</a> | <a href="http://dblp.uni-trier.de/rec/bibtex/journals/corr/JohnsonSLKWCTVW16" title="DBLP bibtex record">bibtex</a>
</div>
<div class="list">
<a href="http://dblp.uni-trier.de/search/author?author=Melvin%20Johnson" title="DBLP author search">Melvin Johnson</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Mike%20Schuster" title="DBLP author search">Mike Schuster</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Quoc%20V.%20Le" title="DBLP author search">Quoc V. Le</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Maxim%20Krikun" title="DBLP author search">Maxim Krikun</a><br />
<a href="http://dblp.uni-trier.de/search/author?author=Yonghui%20Wu" title="DBLP author search">Yonghui Wu</a><br />
<div class="list">
 ... 
</div>
</div>

</div>
<div class="bookmarks">
<div class="what-is-this">
<h3>Bookmark</h3> (<a href="/help/social_bookmarking">what is this?</a>)
</div>
<a href="/ct?url=http%3A%2F%2Fwww.citeulike.org%2Fposturl%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558%26title%3DGoogle%2527s%2520Multilingual%2520Neural%2520Machine%2520Translation%2520System%253A%2520Enabling%250A%2520%2520Zero-Shot%2520Translation%26authors%3D&amp;v=949687d3" title="Bookmark on CiteULike"><img src="//static.arxiv.org/icons/social/citeulike.png" alt="CiteULike logo" /></a>
<a href="/ct?url=http%3A%2F%2Fwww.bibsonomy.org%2FBibtexHandler%3FrequTask%3Dupload%26url%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558%26description%3DGoogle%2527s%2520Multilingual%2520Neural%2520Machine%2520Translation%2520System%253A%2520Enabling%250A%2520%2520Zero-Shot%2520Translation&amp;v=b681b6d5" title="Bookmark on BibSonomy"><img src="//static.arxiv.org/icons/social/bibsonomy.png" alt="BibSonomy logo" /></a>
<a href="/ct?url=https%3A%2F%2Fwww.mendeley.com%2Fimport%2F%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558&amp;v=3a5f6b11" title="Bookmark on Mendeley"><img src="//static.arxiv.org/icons/social/mendeley.png" alt="Mendeley logo" /></a>
<a href="/ct?url=https%3A%2F%2Fdel.icio.us%2Fpost%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558%26description%3DGoogle%2527s%2520Multilingual%2520Neural%2520Machine%2520Translation%2520System%253A%2520Enabling%250A%2520%2520Zero-Shot%2520Translation&amp;v=43078e5b" title="Bookmark on del.icio.us"><img src="//static.arxiv.org/icons/social/delicious.png" alt="del.icio.us logo" /></a>
<a href="/ct?url=https%3A%2F%2Fdigg.com%2Fsubmit%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558%26title%3DGoogle%2527s%2520Multilingual%2520Neural%2520Machine%2520Translation%2520System%253A%2520Enabling%250A%2520%2520Zero-Shot%2520Translation&amp;v=d6505cab" title="Bookmark on Digg"><img src="//static.arxiv.org/icons/social/digg.png" alt="Digg logo" /></a>
<a href="/ct?url=https%3A%2F%2Freddit.com%2Fsubmit%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558%26title%3DGoogle%2527s%2520Multilingual%2520Neural%2520Machine%2520Translation%2520System%253A%2520Enabling%250A%2520%2520Zero-Shot%2520Translation&amp;v=f264ee9f" title="Bookmark on Reddit"><img src="//static.arxiv.org/icons/social/reddit.png" alt="Reddit logo" /></a>
<a href="/ct?url=http%3A%2F%2Fsciencewise.info%2Fbookmarks%2Fadd%3Furl%3Dhttp%3A%2F%2Farxiv.org%2Fabs%2F1611.04558&amp;v=12ed5608" title="Bookmark on ScienceWISE"><img src="//static.arxiv.org/icons/social/sciencewise.png" alt="ScienceWISE logo" /></a>

</div>
</div><!--end extra-services-->

<div class="leftcolumn">
<div class="subheader">
<h1>Computer Science > Computation and Language</h1>
</div>
<h1 class="title mathjax"><span class="descriptor">Title:</span>
Google&#x27;s Multilingual Neural Machine Translation System: Enabling  Zero-Shot Translation</h1>
<div class="authors"><span class="descriptor">Authors:</span>
<a href="/find/cs/1/au:+Johnson_M/0/1/0/all/0/1">Melvin Johnson</a>, 
<a href="/find/cs/1/au:+Schuster_M/0/1/0/all/0/1">Mike Schuster</a>, 
<a href="/find/cs/1/au:+Le_Q/0/1/0/all/0/1">Quoc V. Le</a>, 
<a href="/find/cs/1/au:+Krikun_M/0/1/0/all/0/1">Maxim Krikun</a>, 
<a href="/find/cs/1/au:+Wu_Y/0/1/0/all/0/1">Yonghui Wu</a>, 
<a href="/find/cs/1/au:+Chen_Z/0/1/0/all/0/1">Zhifeng Chen</a>, 
<a href="/find/cs/1/au:+Thorat_N/0/1/0/all/0/1">Nikhil Thorat</a>, 
<a href="/find/cs/1/au:+Viegas_F/0/1/0/all/0/1">Fernanda Vi&#xe9;gas</a>, 
<a href="/find/cs/1/au:+Wattenberg_M/0/1/0/all/0/1">Martin Wattenberg</a>, 
<a href="/find/cs/1/au:+Corrado_G/0/1/0/all/0/1">Greg Corrado</a>, 
<a href="/find/cs/1/au:+Hughes_M/0/1/0/all/0/1">Macduff Hughes</a>, 
<a href="/find/cs/1/au:+Dean_J/0/1/0/all/0/1">Jeffrey Dean</a></div>
<div class="dateline">(Submitted on 14 Nov 2016)</div>
<blockquote class="abstract mathjax">
<span class="descriptor">Abstract:</span> We propose a simple, elegant solution to use a single Neural Machine
Translation (NMT) model to translate between multiple languages. Our solution
requires no change in the model architecture from our base system but instead
introduces an artificial token at the beginning of the input sentence to
specify the required target language. The rest of the model, which includes
encoder, decoder and attention, remains unchanged and is shared across all
languages. Using a shared wordpiece vocabulary, our approach enables
Multilingual NMT using a single model without any increase in parameters, which
is significantly simpler than previous proposals for Multilingual NMT. Our
method often improves the translation quality of all involved language pairs,
even while keeping the total number of model parameters constant. On the WMT'14
benchmarks, a single multilingual model achieves comparable performance for
English$\rightarrow$French and surpasses state-of-the-art results for
English$\rightarrow$German. Similarly, a single multilingual model surpasses
state-of-the-art results for French$\rightarrow$English and
German$\rightarrow$English on WMT'14 and WMT'15 benchmarks respectively. On
production corpora, multilingual models of up to twelve language pairs allow
for better translation of many individual pairs. In addition to improving the
translation quality of language pairs that the model was trained with, our
models can also learn to perform implicit bridging between language pairs never
seen explicitly during training, showing that transfer learning and zero-shot
translation is possible for neural translation. Finally, we show analyses that
hints at a universal interlingua representation in our models and show some
interesting examples when mixing languages.
</blockquote>
<!--CONTEXT-->
<div class="metatable">
<table summary="Additional metadata">
<tr>
<td class="tablecell label">Subjects:
</td>
<td class="tablecell subjects"><span class="primary-subject">Computation and Language (cs.CL)</span>; Artificial Intelligence (cs.AI)</td>
</tr>
<tr>
<td class="tablecell label">
Cite&nbsp;as:
</td>
<td class="tablecell arxivid"><a href="/abs/1611.04558">arXiv:1611.04558</a> [cs.CL]</td>
</tr>
<tr>
<td class="tablecell label">&nbsp;</td>
<td class="tablecell arxividv">(or <span class="arxivid"><a href="/abs/1611.04558v1">arXiv:1611.04558v1</a> [cs.CL]</span> for this version)</td>
</tr>
</table>
</div>
<div class="submission-history">
<h2>Submission history</h2>
From: Mike Schuster [<a href="https://arxiv.org/show-email/7122308f/1611.04558">view email</a>]
<br />
<b>[v1]</b> Mon, 14 Nov 2016 20:24:39 GMT  (2497kb,D)<br />
</div>
<div class="endorsers"><a href="http://arxiv.org/auth/show-endorsers/1611.04558">Which authors of this paper are endorsers?</a> | <a id="mathjax_toggle" href="javascript:setMathjaxCookie()">Disable MathJax</a> (<a href="/help/mathjax/">What is MathJax?</a>)</div><script type="text/javascript" language="javascript">mathjaxToggle();</script>
</div><!--end leftcolumn-->
</div><!--end abs-->
</div>
<div id="footer">
<p>Link back to: <a href="http://arxiv.org/">arXiv</a>, <a href="/form">form interface</a>, <a href="/help/contact">contact</a>.</p>
</div>
</body>
</html>