Handle invalid Content-Type headers (#76)

When checking the `Content-Type` header (if provided) for HTML-related diffs (e.g. html_render and links), we previously treated malformed header values (e.g. `Content-Type: #<mime::nulltype:0x007f2a523499b8>`) as non-HTML content and would bail out before diffing. This wasn't intentional -- we just didn't bother to consider a completely invalid header value in the logic. Since bad values *do* occasionally occur in the wild, however, we need to handle them. This change does so by treating invalid values as if the header was not set at all.

Fixes #75.

Author: Mr0grog

docs/source/release-history.rst

@@ -2,6 +2,12 @@
 Release History
 ===============
 
+In Development
+--------------
+
+- Ignore invalid `Content-Type` headers when diffing HTML. (:issue:`75`)
+
+
 Version 0.1.2 (2021-04-01)
 -----------------------------
 

web_monitoring_diff/content_type.py

@@ -41,6 +41,14 @@
     r'text/.+'
 )))
 
+# Roughly checks whether a Content-Type header value is valid. For syntax, see:
+# - https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
+# - https://datatracker.ietf.org/doc/html/rfc6838#section-4.2
+VALID_CONTENT_TYPE_PATTERN = re.compile(
+    r'^[a-z0-9][a-z0-9!#$&^_.+-]*/[a-z0-9][a-z0-9!#$&^_.+-]*$',
+    re.IGNORECASE
+)
+
 
 def is_not_html(text, headers=None, check_options='normal'):
     """
@@ -63,7 +71,7 @@ def is_not_html(text, headers=None, check_options='normal'):
     """
     if headers and (check_options == 'normal' or check_options == 'nosniff'):
         content_type = headers.get('Content-Type', '').split(';', 1)[0].strip()
-        if content_type:
+        if content_type and VALID_CONTENT_TYPE_PATTERN.match(content_type):
             if content_type in ACCEPTABLE_CONTENT_TYPES:
                 return False
             elif not UNKNOWN_CONTENT_TYPE_PATTERN.match(content_type):

web_monitoring_diff/tests/test_html_diff_validity.py

@@ -175,6 +175,28 @@ def test_html_diff_render_should_check_content_type_header():
             a_headers={'Content-Type': 'text/html'},
             b_headers={'Content-Type': 'image/jpeg'})
 
+    with pytest.raises(UndiffableContentError):
+        html_diff_render(
+            '<p>Just a little HTML</p>',
+            'Some other text',
+            a_headers={'Content-Type': 'image/jpeg'},
+            b_headers={'Content-Type': 'text/html'})
+
+    with pytest.raises(UndiffableContentError):
+        html_diff_render(
+            '<p>Just a little HTML</p>',
+            'Some other text',
+            a_headers={'Content-Type': 'image/jpeg'},
+            b_headers={'Content-Type': 'image/jpeg'})
+
+
+def test_html_diff_render_should_not_check_content_type_header_if_header_is_malformed():
+    html_diff_render(
+        '<p>Just a little HTML</p>',
+        '<p>Just some HTML</p>',
+        a_headers={'Content-Type': '#<mime::nulltype:0x007f2a523499b8>'},
+        b_headers={'Content-Type': 'text/html'})
+
 
 def test_html_diff_render_should_not_check_content_type_header_if_content_type_options_is_nocheck():
     html_diff_render(