From 9d928351999a2f4463093d286079f1d007b8ffb7 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Thu, 9 Jan 2020 03:26:13 -0500 Subject: [PATCH 01/12] Use dominate to render HTML --- notion/renderer.py | 234 +++++++++++++++++++-------------------------- 1 file changed, 100 insertions(+), 134 deletions(-) diff --git a/notion/renderer.py b/notion/renderer.py index f6f5f53..beee4bf 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -1,8 +1,15 @@ import markdown2 import requests +import dominate +from dominate.tags import * from .block import * +class nullcontext: + def __enter__(self): + pass + def __exit__(self,a,b,c): + pass class BaseRenderer(object): @@ -10,7 +17,9 @@ def __init__(self, start_block): self.start_block = start_block def render(self): - return self.render_block(self.start_block) + with div() as d: + self.render_block(self.start_block) + return d def calculate_child_indent(self, block): if block.type == "page": @@ -18,7 +27,7 @@ def calculate_child_indent(self, block): else: return 1 - def render_block(self, block, level=0, preblock=None, postblock=None): + def render_block(self, block, level=0): assert isinstance(block, Block) type_renderer = getattr(self, "handle_" + block._type, None) if not callable(type_renderer): @@ -26,21 +35,12 @@ def render_block(self, block, level=0, preblock=None, postblock=None): type_renderer = self.handle_default else: raise Exception("No handler for block type '{}'.".format(block._type)) - pretext = type_renderer(block, level=level, preblock=preblock, postblock=postblock) - if isinstance(pretext, tuple): - pretext, posttext = pretext - else: - posttext = "" - return pretext + self.render_children(block, level=level+self.calculate_child_indent(block)) + posttext - - def render_children(self, block, level): - kids = block.children - if not kids: - return "" - text = "" - for i in range(len(kids)): - text += self.render_block(kids[i], level=level) - return text + #Render ourselves to an HTML element and then add all our children to it + selfEl = type_renderer(block, level=level) + with selfEl if isinstance(selfEl, dominate.dom_tag.dom_tag) else nullcontext(): + for child in block.children: + self.render_block(child, level=level+self.calculate_child_indent(block)) + return selfEl bookmark_template = """ @@ -81,161 +81,127 @@ def render_children(self, block, level): """ class BaseHTMLRenderer(BaseRenderer): + def handle_default(self, block, level=0): + p(block.title) - def create_opening_tag(self, tagname, attributes={}): - attrs = "".join(' {}="{}"'.format(key, val) for key, val in attributes.items()) - return "<{tagname}{attrs}>".format(tagname=tagname, attrs=attrs) - - def wrap_in_tag(self, block, tagname, fieldname="title", attributes={}): - opentag = self.create_opening_tag(tagname, attributes) - innerhtml = markdown2.markdown(getattr(block, fieldname)) - return "{opentag}{innerhtml}".format(opentag=opentag, tagname=tagname, innerhtml=innerhtml) + def handle_divider(self, block, level=0): + hr() - def left_margin_for_level(self, level): - return {"display": "margin-left: {}px;".format(level * 20)} + def handle_column_list(self, block, level=0): + return div(style='display: flex;', _class='column-list') - def handle_default(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "p", attributes=self.left_margin_for_level(level)) + def handle_column(self, block, level=0): + return div(_class='column') - def handle_divider(self, block, level=0, preblock=None, postblock=None): - return "
" + def handle_to_do(self, block, level=0): + id = f'chk_{block.id}' + input(type='checkbox', id=id, checked=block.checked, title=block.title).add( + label(_for=id)).add(br()) - def handle_column_list(self, block, level=0, preblock=None, postblock=None): - return '
', '
' + def handle_code(self, block, level=0): + code(block.title) - def handle_column(self, block, level=0, preblock=None, postblock=None): - buffer = (len(block.parent.children) - 1) * 46 - width = block.get("format.column_ratio") - return '
'.format(buffer, width), '
' + def handle_factory(self, block, level=0): + pass - def handle_to_do(self, block, level=0, preblock=None, postblock=None): - return '
'.format( - id="chk_" + block.id, - checked=" checked" if block.checked else "", - title=block.title, - ) + def handle_header(self, block, level=0): + h2(block.title) - def handle_code(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "code", attributes=self.left_margin_for_level(level)) + def handle_sub_header(self, block, level=0): + h3(block.title) - def handle_factory(self, block, level=0, preblock=None, postblock=None): - return "" + def handle_sub_sub_header(self, block, level=0): + h4(block.title) - def handle_header(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "h2", attributes=self.left_margin_for_level(level)) + def handle_page(self, block, level=0): + h1(block.title) - def handle_sub_header(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "h3", attributes=self.left_margin_for_level(level)) + def handle_bulleted_list(self, block, level=0): + ctx = next(dom_tag._with_contexts.values()) + with ctx.children[-1] if isinstance(ctx.children[-1], ul) else ul(): + li(block.title) - def handle_sub_sub_header(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "h4", attributes=self.left_margin_for_level(level)) + def handle_numbered_list(self, block, level=0): + ctx = next(dom_tag._with_contexts.values()) + with ctx.children[-1] if isinstance(ctx.children[-1], ol) else ol(): + li(block.title) - def handle_page(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "h1", attributes=self.left_margin_for_level(level)) + def handle_toggle(self, block, level=0): + details(summary(block.title)) - def handle_bulleted_list(self, block, level=0, preblock=None, postblock=None): - text = "" - if preblock is None or preblock.type != "bulleted_list": - text = self.create_opening_tag("ul", attributes=self.left_margin_for_level(level)) - text += self.wrap_in_tag(block, "li") - if postblock is None or postblock.type != "bulleted_list": - text += "" - return text + def handle_quote(self, block, level=0): + blockquote(block.title) - def handle_numbered_list(self, block, level=0, preblock=None, postblock=None): - text = "" - if preblock is None or preblock.type != "numbered_list": - text = self.create_opening_tag("ol", attributes=self.left_margin_for_level(level)) - text += self.wrap_in_tag(block, "li") - if postblock is None or postblock.type != "numbered_list": - text += "" - return text + def handle_text(self, block, level=0): + return self.handle_default(block=block, level=level) - def handle_toggle(self, block, level=0, preblock=None, postblock=None): - innerhtml = markdown2.markdown(block.title) - opentag = self.create_opening_tag("details", attributes=self.left_margin_for_level(level)) - return '{opentag}{innerhtml}'.format(opentag=opentag, innerhtml=innerhtml), '' + def handle_equation(self, block, level=0): + p(img(src=f'https://chart.googleapis.com/chart?cht=tx&chl={block.latex}')) - def handle_quote(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "blockquote", attributes=self.left_margin_for_level(level)) + def handle_embed(self, block, level=0): + iframe(src=block.display_source or block.source, frameborder=0, + sandbox='allow-scripts allow-popups allow-forms allow-same-origin', + allowfullscreen='') - def handle_text(self, block, level=0, preblock=None, postblock=None): - return self.handle_default(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_video(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_equation(self, block, level=0, preblock=None, postblock=None): - text = self.create_opening_tag("p", attributes=self.left_margin_for_level(level)) - return text + '

'.format(block.latex) + def handle_file(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_embed(self, block, level=0, preblock=None, postblock=None): - iframetag = self.create_opening_tag("iframe", attributes={ - "src": block.display_source or block.source, - "frameborder": 0, - "sandbox": "allow-scripts allow-popups allow-forms allow-same-origin", - "allowfullscreen": "", - "style": "width: {width}px; height: {height}px; border-radius: 1px;".format(width=block.width, height=block.height), - }) - return '
' + iframetag + "
" + def handle_audio(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_video(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_pdf(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_file(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_image(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_audio(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) - - def handle_pdf(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) - - def handle_image(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) - - def handle_bookmark(self, block, level=0, preblock=None, postblock=None): + def handle_bookmark(self, block, level=0): return bookmark_template.format(link=block.link, title=block.title, description=block.description, icon=block.bookmark_icon, cover=block.bookmark_cover) - def handle_link_to_collection(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "p", attributes={"href": "https://www.notion.so/" + block.id.replace("-", "")}) + def handle_link_to_collection(self, block, level=0): + a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_breadcrumb(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "p", attributes=self.left_margin_for_level(level)) + def handle_breadcrumb(self, block, level=0): + p(block.title) - def handle_collection_view(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "p", attributes={"href": "https://www.notion.so/" + block.id.replace("-", "")}) + def handle_collection_view(self, block, level=0): + a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_collection_view_page(self, block, level=0, preblock=None, postblock=None): - return self.wrap_in_tag(block, "p", attributes={"href": "https://www.notion.so/" + block.id.replace("-", "")}) + def handle_collection_view_page(self, block, level=0): + a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_framer(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_framer(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_tweet(self, block, level=0, preblock=None, postblock=None): + def handle_tweet(self, block, level=0): return requests.get("https://publish.twitter.com/oembed?url=" + block.source).json()["html"] - def handle_gist(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_gist(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_drive(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_drive(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_figma(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_figma(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_loom(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_loom(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_typeform(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_typeform(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_codepen(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_codepen(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_maps(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_maps(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_invision(self, block, level=0, preblock=None, postblock=None): - return self.handle_embed(block=block, level=level, preblock=preblock, postblock=postblock) + def handle_invision(self, block, level=0): + return self.handle_embed(block=block, level=level) - def handle_callout(self, block, level=0, preblock=None, postblock=None): + def handle_callout(self, block, level=0): return callout_template.format(icon=block.icon, title=markdown2.markdown(block.title)) - From 58be04af563fa2a4da5379db729b14bacc33684b Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 17 Jan 2020 02:30:43 -0500 Subject: [PATCH 02/12] Remove level, smaller function names for duplicates, correct audio, video, and img tags, render markdown, moved styling to stub stylesheet --- notion/renderer.py | 294 ++++++++++++++++++++++++--------------------- 1 file changed, 156 insertions(+), 138 deletions(-) diff --git a/notion/renderer.py b/notion/renderer.py index beee4bf..a4c1116 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -1,7 +1,9 @@ import markdown2 import requests import dominate +import threading from dominate.tags import * +from dominate.util import raw from .block import * @@ -17,17 +19,48 @@ def __init__(self, start_block): self.start_block = start_block def render(self): - with div() as d: - self.render_block(self.start_block) - return d + pass + + def render_block(self, block): + pass + +def renderMD(mdStr): + """ + Render the markdown string to HTML, wrapped with dominate "raw" so Dominate + renders it straight to HTML + """ + return raw(markdown2.markdown(mdStr)) - def calculate_child_indent(self, block): - if block.type == "page": - return 0 - else: - return 1 +class BaseHTMLRenderer(BaseRenderer): + """ + BaseRenderer for HTML output, uses [Dominate](https://github.com/Knio/dominate) + internally for generating HTML output + Each token rendering method should create a dominate tag and it automatically + gets added to the parent context (because of the with statement). If you return + a given tag, it will be used as the parent container for all rendered children + """ - def render_block(self, block, level=0): + def __init__(self, start_block): + self.start_block = start_block + + def render(self): + with div() as d: + self.render_block(self.start_block) + return "".join(str(d) for d in d.children) #Return the array of Dominate dom_tags as strings + + def get_parent_element(self): + """ + Returns the current parent Dominate element (uses Dominate internals) + https://github.com/Knio/dominate/issues/123 + """ + def _get_thread_context(): + context = [threading.current_thread()] + # if greenlet: + # context.append(greenlet.getcurrent()) + # return hash(tuple(context)) + return dom_tag._with_contexts[_get_thread_context()] + + def render_block(self, block): assert isinstance(block, Block) type_renderer = getattr(self, "handle_" + block._type, None) if not callable(type_renderer): @@ -35,173 +68,158 @@ def render_block(self, block, level=0): type_renderer = self.handle_default else: raise Exception("No handler for block type '{}'.".format(block._type)) - #Render ourselves to an HTML element and then add all our children to it - selfEl = type_renderer(block, level=level) - with selfEl if isinstance(selfEl, dominate.dom_tag.dom_tag) else nullcontext(): - for child in block.children: - self.render_block(child, level=level+self.calculate_child_indent(block)) - return selfEl - - -bookmark_template = """ -
-
- -
-
-
{title}
-
{description}
-
- -
{link}/div> -
-
-
-
-
-
-
-
-
-
-
-""" - -callout_template = """ -
-
-
-
-
{icon}
-
-
-
-
{title}
-
-""" - -class BaseHTMLRenderer(BaseRenderer): - def handle_default(self, block, level=0): - p(block.title) - - def handle_divider(self, block, level=0): + #Render ourselves to an HTML element which automatically gets added to the parent + #context with Dominate + selfEl = type_renderer(block) + if block.children: + #If we have children, we need to add them to ourself (if we returned a tag + #to add to) or make a new container for the children + with selfEl or div(_class='children-list'): + for child in block.children: + self.render_block(child) + + def handle_default(self, block): + p(renderMD(block.title)) + + def handle_divider(self, block): hr() - def handle_column_list(self, block, level=0): + def handle_column_list(self, block): return div(style='display: flex;', _class='column-list') - def handle_column(self, block, level=0): + def handle_column(self, block): return div(_class='column') - def handle_to_do(self, block, level=0): + def handle_to_do(self, block): id = f'chk_{block.id}' - input(type='checkbox', id=id, checked=block.checked, title=block.title).add( - label(_for=id)).add(br()) + input(label(_for=id), type='checkbox', id=id, checked=block.checked, title=block.title) - def handle_code(self, block, level=0): - code(block.title) + def handle_code(self, block): + #TODO: Do we want this to support Markdown? I think there's a notion-py + #change that might affect this... (the unstyled-title or whatever) + pre(code(block.title)) - def handle_factory(self, block, level=0): + def handle_factory(self, block): pass - def handle_header(self, block, level=0): - h2(block.title) + def handle_header(self, block): + h2(renderMD(block.title)) - def handle_sub_header(self, block, level=0): - h3(block.title) + def handle_sub_header(self, block): + h3(renderMD(block.title)) - def handle_sub_sub_header(self, block, level=0): - h4(block.title) + def handle_sub_sub_header(self, block): + h4(renderMD(block.title)) - def handle_page(self, block, level=0): - h1(block.title) + def handle_page(self, block): + h1(renderMD(block.title)) - def handle_bulleted_list(self, block, level=0): - ctx = next(dom_tag._with_contexts.values()) - with ctx.children[-1] if isinstance(ctx.children[-1], ul) else ul(): - li(block.title) + def handle_bulleted_list(self, block): + parent = self.get_parent_element() + #print(parent) + lastChild = parent.children[-1] if hasattr(parent, 'children') else None + #Open a new ul if the last child was not a ul + with lastChild if lastChild and isinstance(lastChild, ul) else ul(): + li(renderMD(block.title)) - def handle_numbered_list(self, block, level=0): - ctx = next(dom_tag._with_contexts.values()) - with ctx.children[-1] if isinstance(ctx.children[-1], ol) else ol(): - li(block.title) + def handle_numbered_list(self, block): + parent = self.get_parent_element() + #print(parent) + lastChild = parent.children[-1] if hasattr(parent, 'children') else None + #Open a new ul if the last child was not a ol + with lastChild if lastChild and isinstance(lastChild, ol) else ol(): + li(renderMD(block.title)) - def handle_toggle(self, block, level=0): - details(summary(block.title)) + def handle_toggle(self, block): + details(summary(renderMD(block.title))) - def handle_quote(self, block, level=0): - blockquote(block.title) + def handle_quote(self, block): + blockquote(renderMD(block.title)) - def handle_text(self, block, level=0): - return self.handle_default(block=block, level=level) + handle_text = handle_default - def handle_equation(self, block, level=0): + def handle_equation(self, block): p(img(src=f'https://chart.googleapis.com/chart?cht=tx&chl={block.latex}')) - def handle_embed(self, block, level=0): + def handle_embed(self, block): iframe(src=block.display_source or block.source, frameborder=0, sandbox='allow-scripts allow-popups allow-forms allow-same-origin', allowfullscreen='') - def handle_video(self, block, level=0): - return self.handle_embed(block=block, level=level) + def handle_video(self, block): + #TODO, this won't work if there's no file extension, we might have + #to query and get the MIME type... + src = block.display_source or block.source + srcType = src.split('.')[-1] + video(source(src=src, type=f"video/{srcType}"), controls=True) - def handle_file(self, block, level=0): - return self.handle_embed(block=block, level=level) + handle_file = handle_embed + handle_pdf = handle_embed - def handle_audio(self, block, level=0): - return self.handle_embed(block=block, level=level) + def handle_audio(self, block): + audio(src=block.display_source or block.source, controls=True) - def handle_pdf(self, block, level=0): - return self.handle_embed(block=block, level=level) + def handle_image(self, block): + attrs = {} + if block.caption: # Add the alt attribute if there's a caption + attrs['alt'] = block.caption + img(src=block.display_source or block.source, **attrs) - def handle_image(self, block, level=0): - return self.handle_embed(block=block, level=level) + def handle_bookmark(self, block): + #return bookmark_template.format(link=, title=block.title, description=block.description, icon=block.bookmark_icon, cover=block.bookmark_cover) + #It's just a social share card for the website we're bookmarking + return a(href="block.link") - def handle_bookmark(self, block, level=0): - return bookmark_template.format(link=block.link, title=block.title, description=block.description, icon=block.bookmark_icon, cover=block.bookmark_cover) - - def handle_link_to_collection(self, block, level=0): + def handle_link_to_collection(self, block): a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_breadcrumb(self, block, level=0): - p(block.title) + def handle_breadcrumb(self, block): + p(renderMD(block.title)) - def handle_collection_view(self, block, level=0): + def handle_collection_view(self, block): a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_collection_view_page(self, block, level=0): + def handle_collection_view_page(self, block): a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_framer(self, block, level=0): - return self.handle_embed(block=block, level=level) + handle_framer = handle_embed - def handle_tweet(self, block, level=0): + def handle_tweet(self, block): return requests.get("https://publish.twitter.com/oembed?url=" + block.source).json()["html"] - def handle_gist(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_drive(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_figma(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_loom(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_typeform(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_codepen(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_maps(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_invision(self, block, level=0): - return self.handle_embed(block=block, level=level) - - def handle_callout(self, block, level=0): - return callout_template.format(icon=block.icon, title=markdown2.markdown(block.title)) + handle_gist = handle_embed + handle_drive = handle_embed + handle_figma = handle_embed + handle_loom = handle_embed + handle_typeform = handle_embed + handle_codepen = handle_embed + handle_maps = handle_embed + handle_invision = handle_embed + + def handle_callout(self, block): + div( \ + div(block.icon, _class="icon") + div(renderMD(block.title), _class="text"), \ + _class="callout") + +#This is the minimal css stylesheet to apply to get +#decent lookint output, it won't make it look exactly like Notion.so +#but will have the same basic structure +""" +.children-list { + margin-left: cRems(20px); +} +.column-list { + display: flex; + align-items: center; + justify-content: center; +} +.callout { + display: flex; +} +.callout > .icon { + flex: 0 1 40px; +} +.callout > .text { + flex: 1 1 auto; +} +""" \ No newline at end of file From fb9e0a052bce26caa1b306ef7272c27f346b0b58 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Sat, 18 Jan 2020 18:50:37 -0500 Subject: [PATCH 03/12] Properly renders markdown now (no double p tags, no block MD tags), passes render kwargs to dominate, fixed bugs with lists, added tests --- notion/renderer.py | 249 +++++++++++++++++++++++------------------ tests/test_renderer.py | 45 ++++++++ 2 files changed, 187 insertions(+), 107 deletions(-) create mode 100644 tests/test_renderer.py diff --git a/notion/renderer.py b/notion/renderer.py index a4c1116..fc55567 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -1,4 +1,6 @@ -import markdown2 +import mistletoe +from mistletoe import block_token +from mistletoe.html_renderer import HTMLRenderer import requests import dominate import threading @@ -7,11 +9,25 @@ from .block import * -class nullcontext: - def __enter__(self): - pass - def __exit__(self,a,b,c): - pass + +class MistletoeHTMLRendererSpanTokens(HTMLRenderer): + """ + Renders Markdown to HTML without any MD block tokens (like blockquote or code) + except for the paragraph block token, because you need at least one + """ + + def __enter__(self): + ret = super().__enter__() + for tokenClsName in block_token.__all__[:-1]: #All but Paragraph token + block_token.remove_token(getattr(block_token, tokenClsName)) + return ret + # Auto resets tokens in __exit__, so no need to readd the tokens anywhere + + def render_paragraph(self, token): + """ + Only used for span tokens, so don't render out anything + """ + return self.render_inner(token) class BaseRenderer(object): @@ -27,9 +43,12 @@ def render_block(self, block): def renderMD(mdStr): """ Render the markdown string to HTML, wrapped with dominate "raw" so Dominate - renders it straight to HTML + renders it straight to HTML. """ - return raw(markdown2.markdown(mdStr)) + #[:-1] because it adds a newline for some reason + #TODO: Follow up on this and make it more robust + #https://github.com/miyuchina/mistletoe/blob/master/mistletoe/block_token.py#L138-L152 + return raw(mistletoe.markdown(mdStr, MistletoeHTMLRendererSpanTokens)[:-1]) class BaseHTMLRenderer(BaseRenderer): """ @@ -41,163 +60,179 @@ class BaseHTMLRenderer(BaseRenderer): """ def __init__(self, start_block): + self._render_stack = [] self.start_block = start_block - def render(self): - with div() as d: - self.render_block(self.start_block) - return "".join(str(d) for d in d.children) #Return the array of Dominate dom_tags as strings + def render(self, **kwargs): + """ + Renders the HTML, kwargs takes kwargs for Dominate's render() function + https://github.com/Knio/dominate#rendering - def get_parent_element(self): + These can be: + `pretty` - Whether or not to be pretty + `indent` - Indent character to use + `xhtml` - Whether or not to use XHTML instead of HTML (
instead of
) """ - Returns the current parent Dominate element (uses Dominate internals) - https://github.com/Knio/dominate/issues/123 + els = self.render_block(self.start_block) + return "".join(el.render(**kwargs) for el in els) + + def get_previous_sibling_el(self): + """ + Gets the previous sibling element in the rendered HTML tree """ - def _get_thread_context(): - context = [threading.current_thread()] - # if greenlet: - # context.append(greenlet.getcurrent()) - # return hash(tuple(context)) - return dom_tag._with_contexts[_get_thread_context()] + current_parent = self._render_stack[-1] + if not current_parent.children: + return None #No children + return current_parent.children[-1] def render_block(self, block): assert isinstance(block, Block) - type_renderer = getattr(self, "handle_" + block._type, None) + type_renderer = getattr(self, "render_" + block._type, None) if not callable(type_renderer): - if hasattr(self, "handle_default"): - type_renderer = self.handle_default + if hasattr(self, "render_default"): + type_renderer = self.render_default else: raise Exception("No handler for block type '{}'.".format(block._type)) - #Render ourselves to an HTML element which automatically gets added to the parent - #context with Dominate + #Render ourselves to a Dominate HTML element selfEl = type_renderer(block) - if block.children: - #If we have children, we need to add them to ourself (if we returned a tag - #to add to) or make a new container for the children - with selfEl or div(_class='children-list'): - for child in block.children: - self.render_block(child) - - def handle_default(self, block): - p(renderMD(block.title)) - - def handle_divider(self, block): - hr() - - def handle_column_list(self, block): - return div(style='display: flex;', _class='column-list') - - def handle_column(self, block): - return div(_class='column') - - def handle_to_do(self, block): + if not block.children: + #No children, return early + return [selfEl] + + #If children, render them inside of us or inside a container + selfIsContainerEl = 'data_is_container' in selfEl + containerEl = selfEl if selfIsContainerEl else div(_class='children-list') + retList = [selfEl] + if not selfIsContainerEl: + retList.append(containerEl) + self._render_stack.append(containerEl) + for child in block.children: + for childEl in self.render_block(child): + if childEl: #Might return None if pass or if no extra element to add + containerEl.add(childEl) + self._render_stack.pop() + return retList + + def render_default(self, block): + return p(renderMD(block.title)) + + def render_divider(self, block): + return hr() + + def render_column_list(self, block): + return div(style='display: flex;', _class='column-list', data_is_container='true') + + def render_column(self, block): + return div(_class='column', data_is_container='true') + + def render_to_do(self, block): id = f'chk_{block.id}' - input(label(_for=id), type='checkbox', id=id, checked=block.checked, title=block.title) + return input(label(_for=id), type='checkbox', id=id, checked=block.checked, title=block.title) - def handle_code(self, block): + def render_code(self, block): #TODO: Do we want this to support Markdown? I think there's a notion-py #change that might affect this... (the unstyled-title or whatever) - pre(code(block.title)) + return pre(code(block.title)) - def handle_factory(self, block): + def render_factory(self, block): pass - def handle_header(self, block): - h2(renderMD(block.title)) + def render_header(self, block): + return h2(renderMD(block.title)) - def handle_sub_header(self, block): - h3(renderMD(block.title)) + def render_sub_header(self, block): + return h3(renderMD(block.title)) - def handle_sub_sub_header(self, block): - h4(renderMD(block.title)) + def render_sub_sub_header(self, block): + return h4(renderMD(block.title)) - def handle_page(self, block): - h1(renderMD(block.title)) + def render_page(self, block): + return h1(renderMD(block.title)) - def handle_bulleted_list(self, block): - parent = self.get_parent_element() - #print(parent) - lastChild = parent.children[-1] if hasattr(parent, 'children') else None + def render_bulleted_list(self, block): + previousSibling = self.get_previous_sibling_el() + previousSiblingIsUl = previousSibling and isinstance(previousSibling, ul) #Open a new ul if the last child was not a ul - with lastChild if lastChild and isinstance(lastChild, ul) else ul(): + with previousSibling if previousSiblingIsUl else ul() as ret: li(renderMD(block.title)) + return None if previousSiblingIsUl else ret - def handle_numbered_list(self, block): - parent = self.get_parent_element() - #print(parent) - lastChild = parent.children[-1] if hasattr(parent, 'children') else None + def render_numbered_list(self, block): + previousSibling = self.get_previous_sibling_el() + previousSiblingIsOl = previousSibling and isinstance(previousSibling, ol) #Open a new ul if the last child was not a ol - with lastChild if lastChild and isinstance(lastChild, ol) else ol(): + with previousSibling if previousSiblingIsOl else ol() as ret: li(renderMD(block.title)) + return None if previousSiblingIsOl else ret - def handle_toggle(self, block): - details(summary(renderMD(block.title))) + def render_toggle(self, block): + return details(summary(renderMD(block.title))) - def handle_quote(self, block): - blockquote(renderMD(block.title)) + def render_quote(self, block): + return blockquote(renderMD(block.title)) - handle_text = handle_default + render_text = render_default - def handle_equation(self, block): - p(img(src=f'https://chart.googleapis.com/chart?cht=tx&chl={block.latex}')) + def render_equation(self, block): + return p(img(src=f'https://chart.googleapis.com/chart?cht=tx&chl={block.latex}')) - def handle_embed(self, block): - iframe(src=block.display_source or block.source, frameborder=0, + def render_embed(self, block): + return iframe(src=block.display_source or block.source, frameborder=0, sandbox='allow-scripts allow-popups allow-forms allow-same-origin', allowfullscreen='') - def handle_video(self, block): + def render_video(self, block): #TODO, this won't work if there's no file extension, we might have #to query and get the MIME type... src = block.display_source or block.source srcType = src.split('.')[-1] - video(source(src=src, type=f"video/{srcType}"), controls=True) + return video(source(src=src, type=f"video/{srcType}"), controls=True) - handle_file = handle_embed - handle_pdf = handle_embed + render_file = render_embed + render_pdf = render_embed - def handle_audio(self, block): - audio(src=block.display_source or block.source, controls=True) + def render_audio(self, block): + return audio(src=block.display_source or block.source, controls=True) - def handle_image(self, block): + def render_image(self, block): attrs = {} if block.caption: # Add the alt attribute if there's a caption attrs['alt'] = block.caption - img(src=block.display_source or block.source, **attrs) + return img(src=block.display_source or block.source, **attrs) - def handle_bookmark(self, block): + def render_bookmark(self, block): #return bookmark_template.format(link=, title=block.title, description=block.description, icon=block.bookmark_icon, cover=block.bookmark_cover) #It's just a social share card for the website we're bookmarking return a(href="block.link") - def handle_link_to_collection(self, block): - a(href=f'https://www.notion.so/{block.id.replace("-", "")}') + def render_link_to_collection(self, block): + return a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_breadcrumb(self, block): - p(renderMD(block.title)) + def render_breadcrumb(self, block): + return p(renderMD(block.title)) - def handle_collection_view(self, block): - a(href=f'https://www.notion.so/{block.id.replace("-", "")}') + def render_collection_view(self, block): + return a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - def handle_collection_view_page(self, block): - a(href=f'https://www.notion.so/{block.id.replace("-", "")}') + def render_collection_view_page(self, block): + return a(href=f'https://www.notion.so/{block.id.replace("-", "")}') - handle_framer = handle_embed + render_framer = render_embed - def handle_tweet(self, block): + def render_tweet(self, block): return requests.get("https://publish.twitter.com/oembed?url=" + block.source).json()["html"] - handle_gist = handle_embed - handle_drive = handle_embed - handle_figma = handle_embed - handle_loom = handle_embed - handle_typeform = handle_embed - handle_codepen = handle_embed - handle_maps = handle_embed - handle_invision = handle_embed - - def handle_callout(self, block): - div( \ + render_gist = render_embed + render_drive = render_embed + render_figma = render_embed + render_loom = render_embed + render_typeform = render_embed + render_codepen = render_embed + render_maps = render_embed + render_invision = render_embed + + def render_callout(self, block): + return div( \ div(block.icon, _class="icon") + div(renderMD(block.title), _class="text"), \ _class="callout") diff --git a/tests/test_renderer.py b/tests/test_renderer.py new file mode 100644 index 0000000..12c0243 --- /dev/null +++ b/tests/test_renderer.py @@ -0,0 +1,45 @@ +''' +Tests for notion-py renderer +''' +import pytest +from functools import partial +from notion.renderer import BaseHTMLRenderer +from notion.block import TextBlock, BulletedListBlock, PageBlock +from unittest.mock import Mock + +def BlockMock(blockType, inputDict, children=[]): + mock = Mock(spec=blockType) + mock._type = blockType._type + mock.__dict__.update(inputDict) + mock.children = children + return mock + +for blockCls in [TextBlock, BulletedListBlock, PageBlock]: + globals()["Mock" + blockCls.__name__] = partial(BlockMock, blockCls) + + +def test_TextBlock(): + '''it renders a TextBlock''' + #arrange + block = MockTextBlock({ 'title': 'Hold up, lemme test this block...' }) + + #act + output = BaseHTMLRenderer(block).render(pretty=False) + + #assert + assert output == '

Hold up, lemme test this block...

' + +def test_BulletedListBlock(): + '''it renders a TextBlock''' + #arrange + block = MockPageBlock({ 'title': 'Test Page' }, [ + MockBulletedListBlock({ 'title': ':3' }), + MockBulletedListBlock({ 'title': ':F' }), + MockBulletedListBlock({ 'title': '>:D'}) + ]) + + #act + output = BaseHTMLRenderer(block).render(pretty=False) + + #assert + assert output == '

Test Page

  • :3
  • :F
  • >:D
' \ No newline at end of file From 1cf253c7515c7d4d9ae7ceb16f7c10bb90909e11 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Sat, 18 Jan 2020 18:55:32 -0500 Subject: [PATCH 04/12] Added two more renderer tests --- tests/test_renderer.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 12c0243..a25cdce 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -4,7 +4,8 @@ import pytest from functools import partial from notion.renderer import BaseHTMLRenderer -from notion.block import TextBlock, BulletedListBlock, PageBlock +from notion.block import TextBlock, BulletedListBlock, PageBlock, NumberedListBlock, \ + ImageBlock from unittest.mock import Mock def BlockMock(blockType, inputDict, children=[]): @@ -14,7 +15,8 @@ def BlockMock(blockType, inputDict, children=[]): mock.children = children return mock -for blockCls in [TextBlock, BulletedListBlock, PageBlock]: +for blockCls in [TextBlock, BulletedListBlock, PageBlock, NumberedListBlock, \ + ImageBlock]: globals()["Mock" + blockCls.__name__] = partial(BlockMock, blockCls) @@ -42,4 +44,34 @@ def test_BulletedListBlock(): output = BaseHTMLRenderer(block).render(pretty=False) #assert - assert output == '

Test Page

  • :3
  • :F
  • >:D
' \ No newline at end of file + assert output == '

Test Page

  • :3
  • :F
  • >:D
' + +def test_NumberedListBlock(): + '''it renders a TextBlock''' + #arrange + block = MockPageBlock({ 'title': 'Test Page' }, [ + MockNumberedListBlock({ 'title': ':3' }), + MockNumberedListBlock({ 'title': ':F' }), + MockNumberedListBlock({ 'title': '>:D'}) + ]) + + #act + output = BaseHTMLRenderer(block).render(pretty=False) + + #assert + assert output == '

Test Page

  1. :3
  2. :F
  3. >:D
' + +def test_ImageBlock(): + '''it renders a TextBlock''' + #arrange + block = MockImageBlock({ + 'caption': 'Its a me! Placeholderio', + 'display_source': 'https://via.placeholder.com/20x20', + 'source': 'https://via.placeholder.com/20x20' + }) + + #act + output = BaseHTMLRenderer(block).render(pretty=False) + + #assert + assert output == 'Its a me! Placeholderio' \ No newline at end of file From b8386656fb3eb7f28da36059edc91ebd38ee80b8 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Sat, 18 Jan 2020 19:32:56 -0500 Subject: [PATCH 05/12] Fixed and added tests for ColumnListBlock and ColumnBlock --- notion/renderer.py | 17 ++++++++++------- tests/test_renderer.py | 34 ++++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/notion/renderer.py b/notion/renderer.py index fc55567..3d854a9 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -80,10 +80,9 @@ def get_previous_sibling_el(self): """ Gets the previous sibling element in the rendered HTML tree """ - current_parent = self._render_stack[-1] - if not current_parent.children: - return None #No children - return current_parent.children[-1] + if not self._render_stack or not self._render_stack[-1].children: + return None #Nothing on stack or no children, so no previous sibling + return self._render_stack[-1].children[-1] def render_block(self, block): assert isinstance(block, Block) @@ -100,7 +99,11 @@ def render_block(self, block): return [selfEl] #If children, render them inside of us or inside a container - selfIsContainerEl = 'data_is_container' in selfEl + #NOTE: If you don't use selfEl.attribute, it doesn't work due to not fully + #implementing the dict syntax on selfEl... :/ + selfIsContainerEl = 'data-is-container' in selfEl.attributes + if selfIsContainerEl: + del selfEl['data-is-container'] containerEl = selfEl if selfIsContainerEl else div(_class='children-list') retList = [selfEl] if not selfIsContainerEl: @@ -120,10 +123,10 @@ def render_divider(self, block): return hr() def render_column_list(self, block): - return div(style='display: flex;', _class='column-list', data_is_container='true') + return div(style='display: flex;', _class='column-list', data_is_container=True) def render_column(self, block): - return div(_class='column', data_is_container='true') + return div(_class='column', data_is_container=True) def render_to_do(self, block): id = f'chk_{block.id}' diff --git a/tests/test_renderer.py b/tests/test_renderer.py index a25cdce..92ffae2 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -5,7 +5,7 @@ from functools import partial from notion.renderer import BaseHTMLRenderer from notion.block import TextBlock, BulletedListBlock, PageBlock, NumberedListBlock, \ - ImageBlock + ImageBlock, ColumnBlock, ColumnListBlock from unittest.mock import Mock def BlockMock(blockType, inputDict, children=[]): @@ -16,7 +16,7 @@ def BlockMock(blockType, inputDict, children=[]): return mock for blockCls in [TextBlock, BulletedListBlock, PageBlock, NumberedListBlock, \ - ImageBlock]: + ImageBlock, ColumnBlock, ColumnListBlock]: globals()["Mock" + blockCls.__name__] = partial(BlockMock, blockCls) @@ -32,7 +32,7 @@ def test_TextBlock(): assert output == '

Hold up, lemme test this block...

' def test_BulletedListBlock(): - '''it renders a TextBlock''' + '''it renders BulletedListBlocks''' #arrange block = MockPageBlock({ 'title': 'Test Page' }, [ MockBulletedListBlock({ 'title': ':3' }), @@ -47,7 +47,7 @@ def test_BulletedListBlock(): assert output == '

Test Page

  • :3
  • :F
  • >:D
' def test_NumberedListBlock(): - '''it renders a TextBlock''' + '''it renders NumberedListBlocks''' #arrange block = MockPageBlock({ 'title': 'Test Page' }, [ MockNumberedListBlock({ 'title': ':3' }), @@ -62,7 +62,7 @@ def test_NumberedListBlock(): assert output == '

Test Page

  1. :3
  2. :F
  3. >:D
' def test_ImageBlock(): - '''it renders a TextBlock''' + '''it renders an ImageBlock''' #arrange block = MockImageBlock({ 'caption': 'Its a me! Placeholderio', @@ -74,4 +74,26 @@ def test_ImageBlock(): output = BaseHTMLRenderer(block).render(pretty=False) #assert - assert output == 'Its a me! Placeholderio' \ No newline at end of file + assert output == 'Its a me! Placeholderio' + +def test_ColumnList(): + '''it renders a ColumnList''' + #arrange + block = MockColumnListBlock({},[ + MockColumnBlock({},[ + MockTextBlock({ 'title': 'Whats wrong Jimmykun?' }) + ]), + MockColumnBlock({},[ + MockTextBlock({ 'title': 'Could it be that youre' }), + MockTextBlock({ 'title': 'craving my, c r o i s s a n t?' }), + ]) + ]) + + #act + output = BaseHTMLRenderer(block).render(pretty=False) + + #assert + assert output == '
' + \ + '

Whats wrong Jimmykun?

' + \ + '

Could it be that youre

craving my, c r o i s s a n t?

' \ + '
' \ No newline at end of file From 25cde863f7907f45b8bef77ff66c00bfd3f9a103 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 24 Jan 2020 18:50:59 -0500 Subject: [PATCH 06/12] Big refactor, added a bunch of options for controlling diving into pages and links, started writing the collection view rendering --- notion/renderer.py | 246 +++++++++++++++++++++++++++-------------- tests/test_renderer.py | 45 ++++++-- 2 files changed, 197 insertions(+), 94 deletions(-) diff --git a/notion/renderer.py b/notion/renderer.py index 3d854a9..d396f0f 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -1,16 +1,48 @@ import mistletoe from mistletoe import block_token -from mistletoe.html_renderer import HTMLRenderer +from mistletoe.html_renderer import HTMLRenderer as MistletoeHTMLRenderer import requests import dominate -import threading from dominate.tags import * from dominate.util import raw +from more_itertools import flatten from .block import * +from .collection import CollectionRowBlock +#This is the minimal css stylesheet to apply to get +#decent lookint output, it won't make it look exactly like Notion.so +#but will have the same basic structure +HTMLRendererStyles = """ + +""" -class MistletoeHTMLRendererSpanTokens(HTMLRenderer): +class MistletoeHTMLRendererSpanTokens(MistletoeHTMLRenderer): """ Renders Markdown to HTML without any MD block tokens (like blockquote or code) except for the paragraph block token, because you need at least one @@ -50,6 +82,10 @@ def renderMD(mdStr): #https://github.com/miyuchina/mistletoe/blob/master/mistletoe/block_token.py#L138-L152 return raw(mistletoe.markdown(mdStr, MistletoeHTMLRendererSpanTokens)[:-1]) +def handles_children_rendering(func): + setattr(func, 'handles_children_rendering', True) + return func + class BaseHTMLRenderer(BaseRenderer): """ BaseRenderer for HTML output, uses [Dominate](https://github.com/Knio/dominate) @@ -59,9 +95,20 @@ class BaseHTMLRenderer(BaseRenderer): a given tag, it will be used as the parent container for all rendered children """ - def __init__(self, start_block): - self._render_stack = [] + def __init__(self, start_block, follow_links=False, follow_pages=True, + follow_table_pages=True, with_styles=False): + """ + start_block The root block to render from + follow_links Whether to follow "Links to pages" + """ + self.exclude_ids = [] #TODO: Add option for this self.start_block = start_block + self.follow_links = follow_links + self.follow_pages = follow_pages + self.follow_table_pages = follow_table_pages + self.with_styles = with_styles + + self._render_stack = [] def render(self, **kwargs): """ @@ -74,17 +121,30 @@ def render(self, **kwargs): `xhtml` - Whether or not to use XHTML instead of HTML (
instead of
) """ els = self.render_block(self.start_block) - return "".join(el.render(**kwargs) for el in els) + return (HTMLRendererStyles if self.with_styles else "") + \ + "".join(el.render(**kwargs) for el in els) + + def get_parent_el(self): + """ + Gets the current parent off the render stack + """ + if not self._render_stack: + return None + return self._render_stack[-1] def get_previous_sibling_el(self): """ Gets the previous sibling element in the rendered HTML tree """ - if not self._render_stack or not self._render_stack[-1].children: - return None #Nothing on stack or no children, so no previous sibling - return self._render_stack[-1].children[-1] + parentEl = self.get_parent_el() + if not parentEl or not parentEl.children: + return None #No parent or no siblings + return parentEl.children[-1] def render_block(self, block): + if block.id in self.exclude_ids: + return [] #don't render this block + assert isinstance(block, Block) type_renderer = getattr(self, "render_" + block._type, None) if not callable(type_renderer): @@ -92,137 +152,164 @@ def render_block(self, block): type_renderer = self.render_default else: raise Exception("No handler for block type '{}'.".format(block._type)) + class_function = getattr(self.__class__, type_renderer.__name__) + #Render ourselves to a Dominate HTML element - selfEl = type_renderer(block) + els = type_renderer(block) #Returns a list of elements + + # If the function handled the children (using the flag on the function) then + # don't render them out using the default append method + return els if hasattr(class_function, 'handles_children_rendering') else \ + els + self.render_block_children_into(block) + + def render_block_children_into(self, block, containerEl=None): if not block.children: - #No children, return early - return [selfEl] - - #If children, render them inside of us or inside a container - #NOTE: If you don't use selfEl.attribute, it doesn't work due to not fully - #implementing the dict syntax on selfEl... :/ - selfIsContainerEl = 'data-is-container' in selfEl.attributes - if selfIsContainerEl: - del selfEl['data-is-container'] - containerEl = selfEl if selfIsContainerEl else div(_class='children-list') - retList = [selfEl] - if not selfIsContainerEl: - retList.append(containerEl) + return [] + if containerEl is None: + containerEl = div(_class='children-list') self._render_stack.append(containerEl) - for child in block.children: - for childEl in self.render_block(child): - if childEl: #Might return None if pass or if no extra element to add - containerEl.add(childEl) + for block in block.children: + els = self.render_block(block) + containerEl.add(els) self._render_stack.pop() - return retList + return [containerEl] + + # == Conversions for rendering notion-py block types to elemenets == + # Each function should return a list containing dominate tags + # Marking a function with handles_children_rendering means it handles rendering + # it's own `.children` and doesn't need to perform the default rendering def render_default(self, block): - return p(renderMD(block.title)) + return [p(renderMD(block.title))] def render_divider(self, block): - return hr() + return [hr()] + @handles_children_rendering def render_column_list(self, block): - return div(style='display: flex;', _class='column-list', data_is_container=True) + return self.render_block_children_into(block, div(style='display: flex;', _class='column-list')) + @handles_children_rendering def render_column(self, block): - return div(_class='column', data_is_container=True) + return self.render_block_children_into(block, div(_class='column')) def render_to_do(self, block): id = f'chk_{block.id}' - return input(label(_for=id), type='checkbox', id=id, checked=block.checked, title=block.title) + return [input( \ + label(_for=id), \ + type='checkbox', id=id, checked=block.checked, title=block.title)] def render_code(self, block): #TODO: Do we want this to support Markdown? I think there's a notion-py #change that might affect this... (the unstyled-title or whatever) - return pre(code(block.title)) + return [pre(code(block.title))] def render_factory(self, block): - pass + return [] def render_header(self, block): - return h2(renderMD(block.title)) + return [h2(renderMD(block.title))] def render_sub_header(self, block): - return h3(renderMD(block.title)) + return [h3(renderMD(block.title))] def render_sub_sub_header(self, block): - return h4(renderMD(block.title)) + return [h4(renderMD(block.title))] + @handles_children_rendering def render_page(self, block): - return h1(renderMD(block.title)) - + if block.parent.id != block.get()['parent_id']: + #A link is a PageBlock where the parent id doesn't equal the _actual_ parent id + #of the block + pageEl = h1(renderMD(block.title)) #TODO: Make this an too? + if not self.follow_links: + return [pageEl] #Don't render children + else: #A normal PageBlock + pageEl = h1(renderMD(block.title)) + if not self.follow_pages and self._render_stack: + return [pageEl] + + #If no early out, render the children with the pageEl + return [pageEl] + self.render_block_children_into(block) + + @handles_children_rendering def render_bulleted_list(self, block): previousSibling = self.get_previous_sibling_el() previousSiblingIsUl = previousSibling and isinstance(previousSibling, ul) - #Open a new ul if the last child was not a ul - with previousSibling if previousSiblingIsUl else ul() as ret: - li(renderMD(block.title)) - return None if previousSiblingIsUl else ret + containerEl = previousSibling if previousSiblingIsUl else ul() #Make a new ul if there's no previous ul + blockEl = li(renderMD(block.title)) + containerEl.add(blockEl) #Render out ourself into the stack + self.render_block_children_into(block, containerEl) + return [] if containerEl.parent else [containerEl] #Only return if it's not in the rendered output yet + + @handles_children_rendering def render_numbered_list(self, block): previousSibling = self.get_previous_sibling_el() previousSiblingIsOl = previousSibling and isinstance(previousSibling, ol) - #Open a new ul if the last child was not a ol - with previousSibling if previousSiblingIsOl else ol() as ret: - li(renderMD(block.title)) - return None if previousSiblingIsOl else ret + containerEl = previousSibling if previousSiblingIsOl else ol() #Make a new ol if there's no previous ol + + blockEl = li(renderMD(block.title)) + containerEl.add(blockEl) #Render out ourself into the stack + self.render_block_children_into(block, containerEl) + return [] if containerEl.parent else [containerEl] #Only return if it's not in the rendered output yet def render_toggle(self, block): - return details(summary(renderMD(block.title))) + return [details(summary(renderMD(block.title)))] def render_quote(self, block): - return blockquote(renderMD(block.title)) + return [blockquote(renderMD(block.title))] render_text = render_default def render_equation(self, block): - return p(img(src=f'https://chart.googleapis.com/chart?cht=tx&chl={block.latex}')) + return [p(img(src=f'https://chart.googleapis.com/chart?cht=tx&chl={block.latex}'))] def render_embed(self, block): - return iframe(src=block.display_source or block.source, frameborder=0, + return [iframe(src=block.display_source or block.source, frameborder=0, sandbox='allow-scripts allow-popups allow-forms allow-same-origin', - allowfullscreen='') + allowfullscreen='')] def render_video(self, block): #TODO, this won't work if there's no file extension, we might have #to query and get the MIME type... src = block.display_source or block.source srcType = src.split('.')[-1] - return video(source(src=src, type=f"video/{srcType}"), controls=True) + return [video(source(src=src, type=f"video/{srcType}"), controls=True)] render_file = render_embed render_pdf = render_embed def render_audio(self, block): - return audio(src=block.display_source or block.source, controls=True) + return [audio(src=block.display_source or block.source, controls=True)] def render_image(self, block): attrs = {} if block.caption: # Add the alt attribute if there's a caption attrs['alt'] = block.caption - return img(src=block.display_source or block.source, **attrs) + return [img(src=block.display_source or block.source, **attrs)] def render_bookmark(self, block): #return bookmark_template.format(link=, title=block.title, description=block.description, icon=block.bookmark_icon, cover=block.bookmark_cover) - #It's just a social share card for the website we're bookmarking - return a(href="block.link") + #TODO: It's just a social share card for the website we're bookmarking + return [a(href="block.link")] def render_link_to_collection(self, block): - return a(href=f'https://www.notion.so/{block.id.replace("-", "")}') + return [a(href=f'https://www.notion.so/{block.id.replace("-", "")}')] def render_breadcrumb(self, block): - return p(renderMD(block.title)) + return [p(renderMD(block.title))] def render_collection_view(self, block): - return a(href=f'https://www.notion.so/{block.id.replace("-", "")}') + return [a(href=f'https://www.notion.so/{block.id.replace("-", "")}')] def render_collection_view_page(self, block): - return a(href=f'https://www.notion.so/{block.id.replace("-", "")}') + return [a(href=f'https://www.notion.so/{block.id.replace("-", "")}')] render_framer = render_embed def render_tweet(self, block): + #TODO: Convert to a list or something return requests.get("https://publish.twitter.com/oembed?url=" + block.source).json()["html"] render_gist = render_embed @@ -235,29 +322,16 @@ def render_tweet(self, block): render_invision = render_embed def render_callout(self, block): - return div( \ + return [div( \ div(block.icon, _class="icon") + div(renderMD(block.title), _class="text"), \ - _class="callout") + _class="callout")] -#This is the minimal css stylesheet to apply to get -#decent lookint output, it won't make it look exactly like Notion.so -#but will have the same basic structure -""" -.children-list { - margin-left: cRems(20px); -} -.column-list { - display: flex; - align-items: center; - justify-content: center; -} -.callout { - display: flex; -} -.callout > .icon { - flex: 0 1 40px; -} -.callout > .text { - flex: 1 1 auto; -} -""" \ No newline at end of file + def render_collection_view(self, block): + #Render out the table itself + #TODO + + #Render out all the embedded PageBlocks + if not self.follow_table_pages: + return [] #Don't render out any of the internal pages + + return [h2(block.title)] + list(flatten(self.render_block(block) for block in block.collection.get_rows())) \ No newline at end of file diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 92ffae2..a2024d4 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -1,25 +1,54 @@ ''' Tests for notion-py renderer ''' -import pytest +import uuid from functools import partial +import pytest from notion.renderer import BaseHTMLRenderer from notion.block import TextBlock, BulletedListBlock, PageBlock, NumberedListBlock, \ ImageBlock, ColumnBlock, ColumnListBlock -from unittest.mock import Mock +from unittest.mock import Mock, PropertyMock + +def MockSpace(pages=[]): + #TODO: Doesn't operate at all like *Block types... + spaceMock = Mock() + spaceMock.pages = pages + spaceMock.id = uuid.uuid4() + for page in pages: + type(page).parent = PropertyMock(return_value = spaceMock) + return spaceMock +testSpace = MockSpace() def BlockMock(blockType, inputDict, children=[]): - mock = Mock(spec=blockType) - mock._type = blockType._type - mock.__dict__.update(inputDict) - mock.children = children - return mock + global testSpace + + blockMock = Mock(spec=blockType) + blockMock._type = blockType._type + blockMock.__dict__.update(inputDict) + blockMock.id = uuid.uuid4() + blockMock.get = Mock(return_value={}) + blockMock.children = children + if issubclass(blockType, PageBlock): + #PageBlocks always need a parent, might be overwritten later + type(blockMock).parent = PropertyMock(return_value = testSpace) + blockMock.get = Mock(return_value={ + 'parent_id': testSpace.id + }) + + #Setup children references if passed + for child in children: + #Can't set a mock on a property of a mock in a circular relationship + #or it messes up so use PropertyMock + type(child).parent = PropertyMock(return_value = blockMock) + child.get = Mock(return_value = { + 'parent_id': blockMock.id + }) + return blockMock for blockCls in [TextBlock, BulletedListBlock, PageBlock, NumberedListBlock, \ ImageBlock, ColumnBlock, ColumnListBlock]: globals()["Mock" + blockCls.__name__] = partial(BlockMock, blockCls) - def test_TextBlock(): '''it renders a TextBlock''' #arrange From 22661d07b81381fe89257dcdedf8af7acacf2040 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 24 Jan 2020 18:51:15 -0500 Subject: [PATCH 07/12] Ignore pytest folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0fc065b..5fd9ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ build/ dist/ *.egg-info *_testing.py +.pytest_cache # pipenv .env From a9ce831ea1acb310175de1a0aca5ad4e96a9012e Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 24 Jan 2020 18:54:37 -0500 Subject: [PATCH 08/12] Added test for nested lists --- renderthing.py | 7 +++++++ tests/test_renderer.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 renderthing.py diff --git a/renderthing.py b/renderthing.py new file mode 100644 index 0000000..15563f4 --- /dev/null +++ b/renderthing.py @@ -0,0 +1,7 @@ +import subprocess +from notion.client import NotionClient +from notion.renderer import BaseHTMLRenderer + +cl = NotionClient("46a40b826ac2a89b1089342be5090a8f5cc0863b93dee048cc2fe42eb462e5a277451ce92ab951f215b85a6dd61e177633c5b1ea6e6af48a25ad6dda0d87c0965cf351f5944fb4a2cdaae42ddb13") +b = cl.get_block("https://www.notion.so/sourceequine/P-Source-Equine-Prototype-Scope-410bde26aa1a495c9c4e15dea45ad5f9") +subprocess.run("clip", universal_newlines=True, input=BaseHTMLRenderer(b, follow_pages=False).render()) \ No newline at end of file diff --git a/tests/test_renderer.py b/tests/test_renderer.py index a2024d4..a2e64ae 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -75,6 +75,21 @@ def test_BulletedListBlock(): #assert assert output == '

Test Page

  • :3
  • :F
  • >:D
' +def test_BulletedListBlockNested(): + '''it renders BulletedListBlocks''' + #arrange + block = MockPageBlock({ 'title': 'Test Page' }, [ + MockBulletedListBlock({ 'title': 'owo' }, [ + MockBulletedListBlock({ 'title': 'OwO' }) + ]) + ]) + + #act + output = BaseHTMLRenderer(block).render(pretty=False) + + #assert + assert output == '

Test Page

  • owo
    • OwO
' + def test_NumberedListBlock(): '''it renders NumberedListBlocks''' #arrange From 5db47109014e5937888348b164d306082470e10b Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 24 Jan 2020 20:24:24 -0500 Subject: [PATCH 09/12] Removed autolinking, fixed issues with rendering CollectionRowBlocks --- notion/renderer.py | 85 ++++++++++++++++++++++++++-------------------- renderthing.py | 2 +- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/notion/renderer.py b/notion/renderer.py index d396f0f..17df688 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -1,5 +1,5 @@ import mistletoe -from mistletoe import block_token +from mistletoe import block_token, span_token from mistletoe.html_renderer import HTMLRenderer as MistletoeHTMLRenderer import requests import dominate @@ -8,7 +8,7 @@ from more_itertools import flatten from .block import * -from .collection import CollectionRowBlock +from .collection import Collection #This is the minimal css stylesheet to apply to get #decent lookint output, it won't make it look exactly like Notion.so @@ -52,6 +52,7 @@ def __enter__(self): ret = super().__enter__() for tokenClsName in block_token.__all__[:-1]: #All but Paragraph token block_token.remove_token(getattr(block_token, tokenClsName)) + span_token.remove_token(span_token.AutoLink) #don't autolink urls in markdown return ret # Auto resets tokens in __exit__, so no need to readd the tokens anywhere @@ -82,6 +83,12 @@ def renderMD(mdStr): #https://github.com/miyuchina/mistletoe/blob/master/mistletoe/block_token.py#L138-L152 return raw(mistletoe.markdown(mdStr, MistletoeHTMLRendererSpanTokens)[:-1]) +def href_for_block(block): + """ + Gets the href for a given block + """ + return f'https://www.notion.so/{block.id.replace("-", "")}' + def handles_children_rendering(func): setattr(func, 'handles_children_rendering', True) return func @@ -95,17 +102,17 @@ class BaseHTMLRenderer(BaseRenderer): a given tag, it will be used as the parent container for all rendered children """ - def __init__(self, start_block, follow_links=False, follow_pages=True, - follow_table_pages=True, with_styles=False): + def __init__(self, start_block, render_linked_pages=False, render_sub_pages=True, + render_table_pages_after_table=False, with_styles=False): """ start_block The root block to render from follow_links Whether to follow "Links to pages" """ self.exclude_ids = [] #TODO: Add option for this self.start_block = start_block - self.follow_links = follow_links - self.follow_pages = follow_pages - self.follow_table_pages = follow_table_pages + self.render_linked_pages = render_linked_pages + self.render_sub_pages = render_sub_pages + self.render_table_pages_after_table = render_table_pages_after_table self.with_styles = with_styles self._render_stack = [] @@ -157,18 +164,19 @@ def render_block(self, block): #Render ourselves to a Dominate HTML element els = type_renderer(block) #Returns a list of elements - # If the function handled the children (using the flag on the function) then - # don't render them out using the default append method - return els if hasattr(class_function, 'handles_children_rendering') else \ - els + self.render_block_children_into(block) + #If the block has no children, or the called function handles the child + #rendering itself, don't render the children + if not block.children or hasattr(class_function, 'handles_children_rendering'): + return els - def render_block_children_into(self, block, containerEl=None): - if not block.children: - return [] - if containerEl is None: + #Otherwise, render and use the default append as a children-list + return els + self.render_blocks_into(block.children, None) + + def render_blocks_into(self, blocks, containerEl=None): + if containerEl is None: #Default behavior is to add a container for the children containerEl = div(_class='children-list') self._render_stack.append(containerEl) - for block in block.children: + for block in blocks: els = self.render_block(block) containerEl.add(els) self._render_stack.pop() @@ -187,11 +195,11 @@ def render_divider(self, block): @handles_children_rendering def render_column_list(self, block): - return self.render_block_children_into(block, div(style='display: flex;', _class='column-list')) + return self.render_blocks_into(block.children, div(style='display: flex;', _class='column-list')) @handles_children_rendering def render_column(self, block): - return self.render_block_children_into(block, div(_class='column')) + return self.render_blocks_into(block.children, div(_class='column')) def render_to_do(self, block): id = f'chk_{block.id}' @@ -218,19 +226,26 @@ def render_sub_sub_header(self, block): @handles_children_rendering def render_page(self, block): - if block.parent.id != block.get()['parent_id']: + #TODO: I would use isinstance(xxx, CollectionRowBlock) here but it's buggy + #https://github.com/jamalex/notion-py/issues/103 + if isinstance(block.parent, Collection): #If it's a child of a collection (CollectionRowBlock) + if not self.render_table_pages_after_table: + return [] + return [h3(renderMD(block.title))] + self.render_blocks_into(block.children) + elif block.parent.id != block.get()['parent_id']: #A link is a PageBlock where the parent id doesn't equal the _actual_ parent id #of the block - pageEl = h1(renderMD(block.title)) #TODO: Make this an
too? - if not self.follow_links: - return [pageEl] #Don't render children + if not self.render_linked_pages: + #Render only the link, none of the content in the link + return [a(h4(renderMD(block.title)), href=href_for_block(block))] else: #A normal PageBlock - pageEl = h1(renderMD(block.title)) - if not self.follow_pages and self._render_stack: - return [pageEl] + if not self.render_sub_pages and self._render_stack: + return [h4(renderMD(block.title))] #Subpages when not rendering them render like in Notion, as a simple heading - #If no early out, render the children with the pageEl - return [pageEl] + self.render_block_children_into(block) + #Otherwise, render a page normally in it's entirety + #TODO: This should probably not use a "children-list" but we need to refactor + #the _render_stack to make that work... + return [h1(renderMD(block.title))] + self.render_blocks_into(block.children) @handles_children_rendering def render_bulleted_list(self, block): @@ -240,7 +255,7 @@ def render_bulleted_list(self, block): blockEl = li(renderMD(block.title)) containerEl.add(blockEl) #Render out ourself into the stack - self.render_block_children_into(block, containerEl) + self.render_blocks_into(block.children, containerEl) return [] if containerEl.parent else [containerEl] #Only return if it's not in the rendered output yet @handles_children_rendering @@ -251,7 +266,7 @@ def render_numbered_list(self, block): blockEl = li(renderMD(block.title)) containerEl.add(blockEl) #Render out ourself into the stack - self.render_block_children_into(block, containerEl) + self.render_blocks_into(block.children, containerEl) return [] if containerEl.parent else [containerEl] #Only return if it's not in the rendered output yet def render_toggle(self, block): @@ -295,16 +310,14 @@ def render_bookmark(self, block): return [a(href="block.link")] def render_link_to_collection(self, block): - return [a(href=f'https://www.notion.so/{block.id.replace("-", "")}')] + return [a(renderMD(block.title), href=href_for_block(block))] def render_breadcrumb(self, block): return [p(renderMD(block.title))] - def render_collection_view(self, block): - return [a(href=f'https://www.notion.so/{block.id.replace("-", "")}')] - def render_collection_view_page(self, block): - return [a(href=f'https://www.notion.so/{block.id.replace("-", "")}')] + print("TEST") + return [a(renderMD(block.title), href=href_for_block(block))] render_framer = render_embed @@ -331,7 +344,7 @@ def render_collection_view(self, block): #TODO #Render out all the embedded PageBlocks - if not self.follow_table_pages: + if not self.render_table_pages_after_table: return [] #Don't render out any of the internal pages - return [h2(block.title)] + list(flatten(self.render_block(block) for block in block.collection.get_rows())) \ No newline at end of file + return [h2(block.title)] + self.render_blocks_into(block.collection.get_rows()) \ No newline at end of file diff --git a/renderthing.py b/renderthing.py index 15563f4..a3ba1e3 100644 --- a/renderthing.py +++ b/renderthing.py @@ -4,4 +4,4 @@ cl = NotionClient("46a40b826ac2a89b1089342be5090a8f5cc0863b93dee048cc2fe42eb462e5a277451ce92ab951f215b85a6dd61e177633c5b1ea6e6af48a25ad6dda0d87c0965cf351f5944fb4a2cdaae42ddb13") b = cl.get_block("https://www.notion.so/sourceequine/P-Source-Equine-Prototype-Scope-410bde26aa1a495c9c4e15dea45ad5f9") -subprocess.run("clip", universal_newlines=True, input=BaseHTMLRenderer(b, follow_pages=False).render()) \ No newline at end of file +subprocess.run("clip", universal_newlines=True, input=BaseHTMLRenderer(b, render_sub_pages=False, with_styles=True, render_table_pages_after_table=True).render()) \ No newline at end of file From 4db725c430d0b3887323b7cd830776fac2bf8dcf Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 24 Jan 2020 20:30:41 -0500 Subject: [PATCH 10/12] Remove extraneous file --- renderthing.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 renderthing.py diff --git a/renderthing.py b/renderthing.py deleted file mode 100644 index a3ba1e3..0000000 --- a/renderthing.py +++ /dev/null @@ -1,7 +0,0 @@ -import subprocess -from notion.client import NotionClient -from notion.renderer import BaseHTMLRenderer - -cl = NotionClient("46a40b826ac2a89b1089342be5090a8f5cc0863b93dee048cc2fe42eb462e5a277451ce92ab951f215b85a6dd61e177633c5b1ea6e6af48a25ad6dda0d87c0965cf351f5944fb4a2cdaae42ddb13") -b = cl.get_block("https://www.notion.so/sourceequine/P-Source-Equine-Prototype-Scope-410bde26aa1a495c9c4e15dea45ad5f9") -subprocess.run("clip", universal_newlines=True, input=BaseHTMLRenderer(b, render_sub_pages=False, with_styles=True, render_table_pages_after_table=True).render()) \ No newline at end of file From 3efe1ad598cc8b813071110d6fa8a12f930cd259 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Sun, 16 Feb 2020 02:00:41 -0500 Subject: [PATCH 11/12] Can process strings in the heirarchy now --- notion/renderer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/notion/renderer.py b/notion/renderer.py index 17df688..8a20bca 100644 --- a/notion/renderer.py +++ b/notion/renderer.py @@ -5,6 +5,7 @@ import dominate from dominate.tags import * from dominate.util import raw +from dominate.dom_tag import dom_tag from more_itertools import flatten from .block import * @@ -128,8 +129,9 @@ def render(self, **kwargs): `xhtml` - Whether or not to use XHTML instead of HTML (
instead of
) """ els = self.render_block(self.start_block) + #Strings render as themselves, DOMinate tags user the passed kwargs return (HTMLRendererStyles if self.with_styles else "") + \ - "".join(el.render(**kwargs) for el in els) + "".join(el.render(**kwargs) if isinstance(el, dom_tag) else el for el in els) def get_parent_el(self): """ @@ -183,7 +185,7 @@ def render_blocks_into(self, blocks, containerEl=None): return [containerEl] # == Conversions for rendering notion-py block types to elemenets == - # Each function should return a list containing dominate tags + # Each function should return a list containing dominate tags or a string of HTML # Marking a function with handles_children_rendering means it handles rendering # it's own `.children` and doesn't need to perform the default rendering From d538bcbda12191f49cbc87bec9841b37a02d8a56 Mon Sep 17 00:00:00 2001 From: Cobertos / Peter Date: Fri, 10 Apr 2020 06:13:40 -0400 Subject: [PATCH 12/12] Add mistletoe dependency --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c25cb42..3a5b1de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ tzlocal python-slugify dictdiffer cached-property -markdown2 \ No newline at end of file +markdown2 +mistletoe \ No newline at end of file