Bug 1321290 - Migrate HTML entities to Unicode characters (#28)

Author: stasm

fluent/migrate/transforms.py

@@ -93,26 +93,19 @@ def __call__(self, ctx):
 class Source(Transform):
     """Declare the source translation to be migrated with other transforms.
 
-    When evaluated `Source` returns a simple string value.  All \\uXXXX from
-    the original translations are converted beforehand to the literal
-    characters they encode.
+    When evaluated, `Source` returns a simple string value. Escaped characters
+    are unescaped by the compare-locales parser according to the file format:
 
-    HTML entities are left unchanged for now because we can't know if they
-    should be converted to the characters they represent or not.  Consider the
-    following example in which `&` could be replaced with the literal `&`:
+      - in properties files: \\uXXXX,
+      - in DTD files: known named, decimal, and hexadecimal HTML entities.
 
-        Privacy & History
+    Consult the following files for the list of known named HTML entities:
 
-    vs. these two examples where the HTML encoding should be preserved:
-
-        Erreur !
-        Use /help <command> for more information.
+    https://github.com/python/cpython/blob/2.7/Lib/htmlentitydefs.py
+    https://github.com/python/cpython/blob/3.6/Lib/html/entities.py
 
     """
 
-    # XXX Perhaps there's a strict subset of HTML entities which must or must
-    # not be replaced?
-
     def __init__(self, path, key):
         if path.endswith('.ftl'):
             raise NotSupportedError(

tests/migrate/test_copy.py

@@ -25,11 +25,8 @@ class TestCopy(MockContext):
     def setUp(self):
         self.strings = parse(PropertiesParser, '''
             foo = Foo
-            foo.unicode.middle = Foo\\u0020Bar
             foo.unicode.begin = \\u0020Foo
             foo.unicode.end = Foo\\u0020
-
-            foo.html.entity = <⇧⌘K>
         ''')
 
     def test_copy(self):
@@ -45,19 +42,6 @@ def test_copy(self):
             ''')
         )
 
-    def test_copy_escape_unicode_middle(self):
-        msg = FTL.Message(
-            FTL.Identifier('foo-unicode-middle'),
-            value=COPY('test.properties', 'foo.unicode.middle')
-        )
-
-        self.assertEqual(
-            evaluate(self, msg).to_json(),
-            ftl_message_to_json('''
-                foo-unicode-middle = Foo Bar
-            ''')
-        )
-
     @unittest.skip('Parser/Serializer trim whitespace')
     def test_copy_escape_unicode_begin(self):
         msg = FTL.Message(
@@ -86,19 +70,6 @@ def test_copy_escape_unicode_end(self):
             ''')
         )
 
-    def test_copy_html_entity(self):
-        msg = FTL.Message(
-            FTL.Identifier('foo-html-entity'),
-            value=COPY('test.properties', 'foo.html.entity')
-        )
-
-        self.assertEqual(
-            evaluate(self, msg).to_json(),
-            ftl_message_to_json('''
-                foo-html-entity = <⇧⌘K>
-            ''')
-        )
-
 
 @unittest.skipUnless(DTDParser, 'compare-locales required')
 class TestCopyAttributes(MockContext):

tests/migrate/test_source.py

@@ -3,10 +3,16 @@
 
 import unittest
 
+try:
+    from compare_locales.parser import PropertiesParser, DTDParser
+except ImportError:
+    PropertiesParser = DTDParser = None
+
 import fluent.syntax.ast as FTL
 
 from fluent.migrate.errors import NotSupportedError
 from fluent.migrate.transforms import Source, COPY, PLURALS, REPLACE
+from fluent.migrate.util import parse
 from fluent.migrate.helpers import EXTERNAL_ARGUMENT
 
 
@@ -49,3 +55,87 @@ def test_replace(self):
                     }
                 )
             )
+
+
+class MockContext(unittest.TestCase):
+    def get_source(self, _path, key):
+        # Ignore _path (test.properties) and get translations from self.strings.
+        return self.strings[key].val
+
+
+@unittest.skipUnless(PropertiesParser, 'compare-locales required')
+class TestProperties(MockContext):
+    def setUp(self):
+        self.strings = parse(PropertiesParser, '''
+            foo = Foo
+
+            unicode-start = \\u0020Foo
+            unicode-middle = Foo\\u0020Bar
+            unicode-end = Foo\\u0020
+
+            html-entity = <⇧⌘K>
+        ''')
+
+    def test_simple_text(self):
+        source = Source('test.properties', 'foo')
+        self.assertEqual(source(self), 'Foo')
+
+    def test_escape_unicode_start(self):
+        source = Source('test.properties', 'unicode-start')
+        self.assertEqual(source(self), ' Foo')
+
+    def test_escape_unicode_middle(self):
+        source = Source('test.properties', 'unicode-middle')
+        self.assertEqual(source(self), 'Foo Bar')
+
+    def test_escape_unicode_end(self):
+        source = Source('test.properties', 'unicode-end')
+        self.assertEqual(source(self), 'Foo ')
+
+    def test_html_entity(self):
+        source = Source('test.properties', 'html-entity')
+        self.assertEqual(source(self), '<⇧⌘K>')
+
+
+@unittest.skipUnless(DTDParser, 'compare-locales required')
+class TestDTD(MockContext):
+    def setUp(self):
+        self.strings = parse(DTDParser, '''
+            <!ENTITY foo "Foo">
+
+            <!ENTITY unicodeEscape "Foo\\u0020Bar">
+
+            <!ENTITY named "&amp;">
+            <!ENTITY decimal "&#38;">
+            <!ENTITY shorthexcode "&#x26;">
+            <!ENTITY longhexcode "&#x0026;">
+            <!ENTITY unknown "&unknownEntity;">
+        ''')
+
+    def test_simple_text(self):
+        source = Source('test.dtd', 'foo')
+        self.assertEqual(source(self), 'Foo')
+
+    def test_backslash_unicode_escape(self):
+        source = Source('test.dtd', 'unicodeEscape')
+        self.assertEqual(source(self), 'Foo\\u0020Bar')
+
+    def test_named_entity(self):
+        source = Source('test.dtd', 'named')
+        self.assertEqual(source(self), '&')
+
+    def test_decimal_entity(self):
+        source = Source('test.dtd', 'decimal')
+        self.assertEqual(source(self), '&')
+
+    def test_shorthex_entity(self):
+        source = Source('test.dtd', 'shorthexcode')
+        self.assertEqual(source(self), '&')
+
+    def test_longhex_entity(self):
+        source = Source('test.dtd', 'longhexcode')
+        self.assertEqual(source(self), '&')
+
+    def test_unknown_entity(self):
+        source = Source('test.dtd', 'unknown')
+        self.assertEqual(source(self), '&unknownEntity;')