Merge branch 'feature/jamovi-renderer' into develop

 [SVCS-549]
 Closes: #279

Author: felliott

mfr/extensions/jamovi/README.md

@@ -0,0 +1,5 @@
+# jamovi .omv file renderer
+
+`.omv` is the native file format for the free and open source [jamovi statistical spreadsheet](https://www.jamovi.org). An `.omv` file is a 'compound' file format, containing data, analyses, and results.
+
+`.omv` files created by recent versions of jamovi contain an `index.html` file which represents the results of the analyses performed. The jamovi `.omv` file renderer extracts the contents of `index.html` from the archive and replaces image paths from the archive with equivalent data URIs. This then serves as the rendered content.

mfr/extensions/jamovi/__init__.py

@@ -0,0 +1 @@
+from .render import JamoviRenderer  # noqa

mfr/extensions/jamovi/exceptions.py

@@ -0,0 +1,45 @@
+from mfr.core.exceptions import RendererError
+
+
+class JamoviRendererError(RendererError):
+
+    def __init__(self, message, *args, **kwargs):
+        super().__init__(message, *args, renderer_class='jamovi', **kwargs)
+
+
+class JamoviVersionError(JamoviRendererError):
+    """The jamovi related errors raised from a :class:`mfr.extentions.jamovi` and relating to minimum
+    data archive version should throw or subclass JamoviVersionError.
+    """
+
+    __TYPE = 'jamovi_version'
+
+    def __init__(self, message, *args, code: int=400, created_by: str='',
+                 actual_version: str='', required_version: str='', **kwargs):
+        super().__init__(message, *args, code=code, **kwargs)
+        self.created_by = created_by
+        self.actual_version = actual_version
+        self.required_version = required_version
+        self.attr_stack.append([self.__TYPE, {
+            'created_by': self.created_by,
+            'actual_version': self.actual_version,
+            'required_version': self.required_version,
+        }])
+
+
+class JamoviFileCorruptError(JamoviRendererError):
+    """The jamovi related errors raised from a :class:`mfr.extentions.jamovi` and relating to failure
+    while consuming jamovi files should inherit from JamoviFileCorruptError
+    """
+
+    __TYPE = 'jamovi_file_corrupt'
+
+    def __init__(self, message, *args, code: int=400, corruption_type: str='',
+                 reason: str='', **kwargs):
+        super().__init__(message, *args, code=code, **kwargs)
+        self.corruption_type = corruption_type
+        self.reason = reason
+        self.attr_stack.append([self.__TYPE, {
+            'corruption_type': self.corruption_type,
+            'reason': self.reason,
+        }])

mfr/extensions/jamovi/html_processor.py

@@ -0,0 +1,71 @@
+import base64
+from html.parser import HTMLParser
+from io import StringIO
+
+
+class HTMLProcessor(HTMLParser):
+
+    # The HTMLProcessor replaces the src attribute in <image> tags with the base64 equivalent.
+    # The image content comes from the zip_file (specified with set_src_source()).
+    # It also strips <script> and <object> tags and on$foo attributes from the HTML (potential
+    # attack vectors)
+
+    FILTERED_TAGS = ['script', 'object']
+
+    def __init__(self, zip_file):
+        HTMLParser.__init__(self)
+        self._html = StringIO()  # buffer for the processed HTML
+        self._zip_file = zip_file
+
+        # used to exclude the contents of script and object tags
+        self._excl_nested_level = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag in self.FILTERED_TAGS:  # filter scripts and objects (attack vectors)
+            self._excl_nested_level += 1
+            return
+
+        self._html.write('<')
+        self._html.write(tag)
+
+        for attr in attrs:
+            if attr[0].startswith('on'):
+                # skip onclick="", on...="" attributes (attack vectors)
+                continue
+            self._html.write(' ')
+            self._html.write(attr[0])
+            if attr[1] is not None:
+                self._html.write('="')
+                if attr[0] == 'src':
+                    self._insert_data_uri(attr[1])
+                else:
+                    self._html.write(attr[1])
+
+                self._html.write('"')
+
+        self._html.write('>')
+
+    def _insert_data_uri(self, src):
+        with self._zip_file.open(src) as src_file:
+            src_data = src_file.read()
+            src_b64 = base64.b64encode(src_data)
+
+            self._html.write('data:image/png;base64,')
+            self._html.write(src_b64.decode('utf-8'))
+
+    def handle_endtag(self, tag):
+        if tag in self.FILTERED_TAGS:
+            if self._excl_nested_level > 0:
+                self._excl_nested_level -= 1
+            return
+
+        self._html.write('</')
+        self._html.write(tag)
+        self._html.write('>')
+
+    def handle_data(self, data):
+        if self._excl_nested_level == 0:
+            self._html.write(data)
+
+    def final_html(self):
+        return self._html.getvalue()

mfr/extensions/jamovi/render.py

@@ -0,0 +1,114 @@
+from distutils.version import LooseVersion
+import os
+from zipfile import ZipFile, BadZipFile
+
+from mako.lookup import TemplateLookup
+
+from mfr.core import extension
+from mfr.extensions.jamovi import exceptions as jamovi_exceptions
+from mfr.extensions.jamovi.html_processor import HTMLProcessor
+
+
+class JamoviRenderer(extension.BaseRenderer):
+
+    # Minimum data archive version supported
+    MINIMUM_VERSION = LooseVersion('1.0.2')
+
+    TEMPLATE = TemplateLookup(
+        directories=[
+            os.path.join(os.path.dirname(__file__), 'templates')
+        ]).get_template('viewer.mako')
+
+    MESSAGE_FILE_CORRUPT = 'This jamovi file is corrupt and cannot be viewed.'
+    MESSAGE_NO_PREVIEW = 'This jamovi file does not support previews.'
+
+    def render(self):
+        try:
+            with ZipFile(self.file_path) as zip_file:
+                self._check_file(zip_file)
+                body = self._render_html(zip_file, self.metadata.ext)
+                return self.TEMPLATE.render(base=self.assets_url, body=body)
+        except BadZipFile as err:
+            raise jamovi_exceptions.JamoviRendererError(
+                '{} {}.'.format(self.MESSAGE_FILE_CORRUPT, str(err)),
+                extension=self.metadata.ext,
+                corruption_type='bad_zip',
+                reason=str(err),
+            )
+
+    @property
+    def file_required(self):
+        return True
+
+    @property
+    def cache_result(self):
+        return True
+
+    def _render_html(self, zip_file, ext, *args, **kwargs):
+        index = None
+        try:
+            with zip_file.open('index.html') as index_data:
+                index = index_data.read().decode('utf-8')
+        except KeyError:
+            raise jamovi_exceptions.JamoviRendererError(
+                self.MESSAGE_NO_PREVIEW,
+            )
+
+        processor = HTMLProcessor(zip_file)
+        processor.feed(index)
+
+        return processor.final_html()
+
+    def _check_file(self, zip_file):
+        """Check if the file is OK (not corrupt)
+        :param zip_file: an opened ZipFile representing the jamovi file
+        :return: True
+        """
+        # Extract manifest file content
+        try:
+            with zip_file.open('META-INF/MANIFEST.MF') as manifest_data:
+                manifest = manifest_data.read().decode('utf-8')
+        except KeyError:
+            raise jamovi_exceptions.JamoviFileCorruptError(
+                '{} Missing META-INF/MANIFEST.MF'.format(self.MESSAGE_FILE_CORRUPT),
+                extension=self.metadata.ext,
+                corruption_type='key_error',
+                reason='zip missing ./META-INF/MANIFEST.MF',
+            )
+
+        lines = manifest.split('\n')
+
+        # Search for Data-Archive-Version
+        version_str = None
+        for line in lines:
+            key_value = line.split(':')
+            if len(key_value) == 2 and key_value[0].strip() == 'Data-Archive-Version':
+                version_str = key_value[1].strip()
+                break
+        else:
+            raise jamovi_exceptions.JamoviFileCorruptError(
+                '{} Data-Archive-Version not found.'.format(self.MESSAGE_FILE_CORRUPT),
+                extension=self.metadata.ext,
+                corruption_type='manifest_parse_error',
+                reason='Data-Archive-Version not found.',
+            )
+
+        # Check that the file is new enough (contains preview content)
+        archive_version = LooseVersion(version_str)
+        try:
+            if archive_version < self.MINIMUM_VERSION:
+                raise jamovi_exceptions.JamoviFileCorruptError(
+                    '{} Data-Archive-Version is too old.'.format(self.MESSAGE_FILE_CORRUPT),
+                    extension=self.metadata.ext,
+                    corruption_type='manifest_parse_error',
+                    reason='Data-Archive-Version not found.',
+                )
+        except TypeError:
+            raise jamovi_exceptions.JamoviFileCorruptError(
+                '{} Data-Archive-Version not parsable.'.format(self.MESSAGE_FILE_CORRUPT),
+                extension=self.metadata.ext,
+                corruption_type='manifest_parse_error',
+                reason='Data-Archive-Version ({}) not parsable.'.format(version_str),
+            )
+
+        return True

mfr/extensions/jamovi/templates/viewer.mako

@@ -0,0 +1,6 @@
+<div style="word-wrap: break-word; overflow: auto;" class="mfrViewer">
+  ${body}
+</div>
+
+<script src="/static/js/mfr.js"></script>
+<script src="/static/js/mfr.child.js"></script>

mfr/extensions/jasp/templates/viewer.mako

@@ -1,4 +1,4 @@
-<div style="word-wrap: break-word; overflow: auto" class="mfrViewer">
+<div style="word-wrap: break-word; overflow: auto;" class="mfrViewer">
 ${body}
 </div>
 

setup.py

@@ -795,6 +795,9 @@ def parse_requirements(requirements):
             #'.wmv = mfr.extensions.video:VideoRenderer',
             '.webm = mfr.extensions.video:VideoRenderer',
 
+            # jamovi
+            '.omv = mfr.extensions.jamovi:JamoviRenderer',
+
             # JASP
             '.jasp = mfr.extensions.jasp:JASPRenderer',
 

supportedextensions.md

@@ -35,6 +35,9 @@ Some file types may not be in the correct list. Please search for the file type
 ## JASP
 * .jasp
 
+## Jamovi
+* .omv
+
 ## Google Documents
 * .gdoc
 * .gsheet

tests/extensions/jamovi/__init__.py

@@ -0,0 +1 @@
+this really isn't a zip file