Allow all entities from HTML5 spec.

.gitignore

@@ -1,6 +1,7 @@
 build/
 dist/
 snudown.egg-info/
+src/html_entities.gperf.generated
 src/html_entities.h
 *.pyc
 *.so

setup.py

@@ -1,11 +1,13 @@
 from distutils.spawn import find_executable
+from distutils.dep_util import newer_group
 from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
 
 import re
 import os
 import subprocess
 import fnmatch
+import json
 
 def c_files_in(directory):
     paths = []
@@ -14,11 +16,55 @@ def c_files_in(directory):
         paths.append(os.path.join(directory, f))
     return paths
 
+def send_html_entities(entities_file, outfh, seen_entities):
+    # Convert entity list from HTML5 spec JSON and send it to gperf
+    with open(entities_file) as entitiesfh:
+        for entity, entityinfo in sorted(json.load(entitiesfh).items()):
+            if not entity.endswith(';'):
+                continue
+            if entity in seen_entities:
+                continue
+            seen_entities.add(entity)
+            # Some sanity checks on the codepoints
+            assert len(entityinfo['codepoints']) <= 2
+            for bad_range in [xrange(0, 8), xrange(11, 12), xrange(14, 31),
+                              xrange(55296, 57343), xrange(65534, 65535)]:
+                for codepoint in entityinfo['codepoints']:
+                    assert codepoint not in bad_range
+            representation = ",".join(str(x) for x in entityinfo['codepoints'])
+            # Output the entity
+            outfh.write(entity + ", {" + representation + "}\n")
 
-def process_gperf_file(gperf_file, output_file):
+
+def process_gperf_file(gperf_file, entities_file, output_file, force=False):
     if not find_executable("gperf"):
         raise Exception("Couldn't find `gperf`, is it installed?")
-    subprocess.check_call(["gperf", gperf_file, "--output-file=%s" % output_file])
+
+    # Do not rerun gperf if no change to input files
+    if not force and not newer_group((gperf_file, entities_file), output_file):
+        return
+
+    # Output combined gperf input file
+    gperf_temp_file = gperf_file + ".generated"
+    seen_entities = set()
+    found_separator = 0
+    with open(gperf_temp_file, 'w') as outfh:
+        with open(gperf_file) as f:
+            for line in f:
+                entity = line.strip()
+                if entity == '%%':
+                    found_separator += 1
+                    if found_separator == 2:
+                        send_html_entities(entities_file, outfh, seen_entities)
+                elif found_separator == 1:
+                    entity = entity.split()[0]
+                    seen_entities.add(entity)
+                outfh.write(line)
+            if found_separator < 2:
+                send_html_entities(entities_file, outfh, seen_entities)
+
+    subprocess.check_call(["gperf", gperf_temp_file,
+                           "--output-file", output_file])
 
 version = None
 version_re = re.compile(r'^#define\s+SNUDOWN_VERSION\s+"([^"]+)"$')
@@ -32,7 +78,8 @@ def process_gperf_file(gperf_file, output_file):
 
 class GPerfingBuildExt(build_ext):
     def run(self):
-        process_gperf_file("src/html_entities.gperf", "src/html_entities.h")
+        process_gperf_file("src/html_entities.gperf", "src/html_entities.json",
+                           "src/html_entities.h", force=self.force)
         build_ext.run(self)
 
 setup(
@@ -47,6 +94,7 @@ def run(self):
         Extension(
             name='snudown',
             sources=['snudown.c'] + c_files_in('src/') + c_files_in('html/'),
+            depends=['src/html_entities.h'],
             include_dirs=['src', 'html']
         )
     ],

src/html_entities.gperf

@@ -35,8 +35,20 @@ inline int is_valid_numeric_entity(uint32_t entity_val)
 			&& entity_val <= MAX_NUM_ENTITY_VAL);
 }
 
+/* Maximum number of codepoints for a named entity. */
+#define MAX_ENTITY_CODEPOINTS 2
 %}
+%struct-type
+%define slot-name entity
+struct html_entity {
+	const char *entity;
+	const int codepoints[MAX_ENTITY_CODEPOINTS];
+}
 %%
+# Additional entities are supported by HTML5, which are converted by
+# Snudown to numeric entities to maintain XHTML compatibility. The HTML5
+# entities are inserted by setup.py based on html_entities.json. Source:
+#   http://www.w3.org/TR/2014/REC-html5-20141028/entities.json
 &AElig;
 &Aacute;
 &Acirc;