Add renderer for Sphinx .rst files, for user manual previews

Ref infrastructure/blender-projects-platform#51

Pull Request: https://projects.blender.org/infrastructure/gitea-custom/pulls/2

sphinx/README.md

@@ -0,0 +1,28 @@
+# Sphinx RST to HTML Preview
+
+Command for generating previews of RST files on projects.blender.org.
+
+The template is adapted from the Blender manual to support the same extensions.
+
+### Deployment
+
+Install dependencies.
+
+    pip3 install -r requirements.txt
+
+Add to Gitea app.ini.
+
+    [markup.restructuredtext]
+    ENABLED = true
+    FILE_EXTENSIONS = .rst
+    RENDER_COMMAND = "timeout 30s ./custom/sphinx/sphinx_to_html.py --user sphinx --user-work-dir /path/to/dir"
+    IS_INPUT_FILE = true
+
+    [repository.editor]
+    LINE_WRAP_EXTENSIONS = .txt,.md,.markdown,.mdown,.mkd,.rst
+    PREVIEWABLE_FILE_MODES = markdown,restructuredtext
+
+The `sphinx` user is required for sandboxing of sphinx-build which we do not
+assume to be secure. The work directory should be writable by both the gitea
+user and sphinx user, with the sphinx user having as little access as possible
+to other directories.

sphinx/requirements.txt

@@ -0,0 +1 @@
+sphinx==6.1.3

sphinx/sphinx_to_html.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python3
+
+import argparse
+import html
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+parser = argparse.ArgumentParser(prog="sphinx_to_html")
+parser.add_argument("filename_rst", help="Input .rst file")
+parser.add_argument("--user", help="Run sphinx as another user", type=str)
+parser.add_argument("--user-work-dir", help="Do work in specified folder accessible by user", type=str)
+args = parser.parse_args()
+
+base_url = "https://projects.blender.org"
+local_url = "http://localhost:3000"
+placeholder_url = "https://placeholder.org"
+
+# Gitea sets this environment variable with the URL prefix for the current file.
+gitea_prefix = os.environ.get("GITEA_PREFIX_SRC", "")
+if gitea_prefix.startswith(base_url):
+    gitea_prefix = gitea_prefix[len(base_url):]
+if gitea_prefix.startswith(local_url):
+    gitea_prefix = gitea_prefix[len(local_url):]
+
+if len(gitea_prefix):
+    org, repo, view, ref, branch = gitea_prefix.strip('/').split('/')[:5]
+
+    doc_url = f"{base_url}/{org}/{repo}/{view}/{ref}/{branch}"
+    image_url = f"{base_url}/{org}/{repo}/raw/{ref}/{branch}"
+else:
+    doc_url = ""
+    image_url = ""
+
+# Set up temporary directory with sphinx configuration.
+with tempfile.TemporaryDirectory(dir=args.user_work_dir) as tmp_dir:
+    work_dir = pathlib.Path(tmp_dir) / "work"
+
+    script_dir = pathlib.Path(__file__).parent.resolve()
+    shutil.copytree(script_dir / "template", work_dir)
+    page_filepath = work_dir / "contents.rst"
+    shutil.copyfile(args.filename_rst, page_filepath)
+
+    page_contents = page_filepath.read_text()
+
+    # Turn links into external links since internal links are not found and stripped.
+    def doc_link(matchobj):
+        return f"`{matchobj.group(1)}<{doc_url}/{matchobj.group(2).strip('/')}.rst>`_"
+    def ref_link(matchobj):
+        return f"`{matchobj.group(1)} <{placeholder_url}>`_"
+    def term_link(matchobj):
+        return f"`{matchobj.group(1)} <{placeholder_url}>`_"
+    def figure_link(matchobj):
+        return f"figure:: {image_url}/{matchobj.group(1).strip('/')}"
+    def image_link(matchobj):
+        return f"image:: {image_url}/{matchobj.group(1).strip('/')}"
+
+    page_contents = re.sub(":doc:`(.*)<(.+)>`", doc_link, page_contents)
+    page_contents = re.sub(":ref:`(.+)<(.+)>`", ref_link, page_contents)
+    page_contents = re.sub(":ref:`(.+)`", ref_link, page_contents)
+    page_contents = re.sub(":term:`(.+)`", term_link, page_contents)
+    page_contents = re.sub("figure:: (.+)", figure_link, page_contents)
+    page_contents = re.sub("image:: (.+)", image_link, page_contents)
+
+    # Disable include directives and raw for security. They are already disabled
+    # by docutils.py, this is just to be extra careful.
+    def include_directive(matchobj):
+        return f"warning:: include directives disabled: {html.escape(matchobj.group(1))}"
+    def raw_directive(matchobj):
+        return f"warning:: raw disabled: {html.escape(matchobj.group(1))}"
+    page_contents = re.sub("literalinclude::.*", include_directive, page_contents)
+    page_contents = re.sub("include::(.*)", include_directive, page_contents)
+    page_contents = re.sub("raw::(.*)", raw_directive, page_contents)
+
+    page_filepath.write_text(page_contents)
+
+    # Debug processed RST
+    # print(html.escape(page_contents).replace('\n', '<br/>\n'))
+    # sys.exit(0)
+
+    # Run sphinx-build.
+    out_dir = work_dir / "out"
+    out_filepath = out_dir / "contents.html"
+
+    sphinx_cmd = ["sphinx-build", "-b", "html", work_dir, out_dir]
+    if args.user:
+        result = subprocess.run(sphinx_cmd, capture_output=True, user=args.user)
+    else:
+        result = subprocess.run(sphinx_cmd, capture_output=True)
+
+    # Output errors.
+    error = result.stderr.decode("utf-8", "ignore").strip()
+    if len(error):
+        error = error.replace(str(page_filepath) + ":", "")
+        error = html.escape(error)
+        print("<h2>Sphinx Warnings</h2>\n")
+        print(f"<pre>{error}</pre>")
+        print("<p>Note the preview is not accurate and warnings may not indicate real issues.</p>")
+
+    # Output contents of body.
+    if result.returncode == 0 and out_filepath.is_file():
+        contents = out_filepath.read_text()
+        body = contents.split("<body>")[1].split("</body>")[0]
+        body = body.replace(placeholder_url, "#")
+        body = body.replace('href="http', 'target="_blank" href="http')
+        print(body)

sphinx/template/conf.py

@@ -0,0 +1,22 @@
+# Configuration file for RST preview.
+
+import os
+import sys
+
+# Extensions
+sys.path.append(os.path.abspath('exts'))
+extensions = [
+    'reference',
+    'peertube',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+]
+
+# Project
+project = 'projects.blender.org'
+root_doc = 'contents'
+
+# Theme: epub hides all navigation, sidebars, footers.
+html_theme = 'epub'
+html_permalinks = False

sphinx/template/docutils.conf

@@ -0,0 +1,6 @@
+# Disable file inclusion and raw HTML for security.
+# https://docutils.sourceforge.io/docs/howto/security.html
+[parsers]
+[restructuredtext parser]
+file-insertion-enabled:false
+raw-enabled:false

sphinx/template/exts/peertube.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import re
+from docutils import nodes
+from docutils.parsers.rst import directives, Directive
+from sphinx.environment import BuildEnvironment
+from sphinx.locale import __
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_size(d, key):
+    if key not in d:
+        return None
+    m = re.match(r"(\d+)(|%|px)$", d[key])
+    if not m:
+        raise ValueError("invalid size %r" % d[key])
+    return int(m.group(1)), m.group(2) or "px"
+
+
+def css(d):
+    return "; ".join(sorted("%s: %s" % kv for kv in d.items()))
+
+
+class peertube(nodes.General, nodes.Element):
+    pass
+
+
+def visit_peertube_node(self, node):
+    instance = node["instance"]
+    aspect = node["aspect"]
+    width = node["width"]
+    height = node["height"]
+
+    if not (self.config.peertube_instance or instance):
+        logger.warning(__("No peertube instance defined"))
+        return
+
+    if instance is None:
+        instance = self.config.peertube_instance
+
+    if aspect is None:
+        aspect = 16, 9
+
+    div_style = {}
+    if (height is None) and (width is not None) and (width[1] == "%"):
+        div_style = {
+            "padding-bottom": "%f%%" % (width[0] * aspect[1] / aspect[0]),
+            "width": "%d%s" % width,
+            "position": "relative",
+        }
+        style = {
+            "position": "absolute",
+            "top": "0",
+            "left": "0",
+            "width": "100%",
+            "height": "100%",
+        }
+        attrs = {
+            "src": instance + "videos/embed/%s" % node["id"],
+            "style": css(style),
+        }
+    else:
+        if width is None:
+            if height is None:
+                width = 560, "px"
+            else:
+                width = height[0] * aspect[0] / aspect[1], "px"
+        if height is None:
+            height = width[0] * aspect[1] / aspect[0], "px"
+        style = {
+            "width": "%d%s" % width,
+            "height": "%d%s" % (height[0], height[1]),
+        }
+        attrs = {
+            "src": instance + "videos/embed/%s" % node["id"],
+            "style": css(style),
+        }
+    attrs["allowfullscreen"] = "true"
+    div_attrs = {
+        "CLASS": "peertube_wrapper",
+        "style": css(div_style),
+    }
+    self.body.append(self.starttag(node, "div", **div_attrs))
+    self.body.append(self.starttag(node, "iframe", **attrs))
+    self.body.append("</iframe></div>")
+
+
+def depart_peertube_node(self, node):
+    pass
+
+
+def visit_peertube_node_latex(self, node):
+    instance = node["instance"]
+
+    if not (self.config.peertube_instance or instance):
+        logger.warning(__("No peertube instance defined"))
+        return
+
+    if instance is None:
+        instance = self.config.peertube_instance
+
+    self.body.append(
+        r'\begin{quote}\begin{center}\fbox{\url{' +
+        instance +
+        r'videos/watch/%s}}\end{center}\end{quote}' %
+        node['id'])
+
+
+class PeerTube(Directive):
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {
+        "instance": directives.unchanged,
+        "width": directives.unchanged,
+        "height": directives.unchanged,
+        "aspect": directives.unchanged,
+    }
+
+    def run(self):
+        instance = self.options.get("instance")
+        if "aspect" in self.options:
+            aspect = self.options.get("aspect")
+            m = re.match(r"(\d+):(\d+)", aspect)
+            if m is None:
+                raise ValueError("invalid aspect ratio %r" % aspect)
+            aspect = tuple(int(x) for x in m.groups())
+        else:
+            aspect = None
+        width = get_size(self.options, "width")
+        height = get_size(self.options, "height")
+        return [peertube(id=self.arguments[0], instance=instance, aspect=aspect, width=width, height=height)]
+
+
+def unsupported_visit_peertube(self, node):
+    self.builder.warn('PeerTube: unsupported output format (node skipped)')
+    raise nodes.SkipNode
+
+
+_NODE_VISITORS = {
+    'html': (visit_peertube_node, depart_peertube_node),
+    'latex': (visit_peertube_node_latex, depart_peertube_node),
+    'man': (unsupported_visit_peertube, None),
+    'texinfo': (unsupported_visit_peertube, None),
+    'text': (unsupported_visit_peertube, None)
+}
+
+
+def setup(app):
+    app.add_node(peertube, **_NODE_VISITORS)
+    app.add_directive("peertube", PeerTube)
+    app.add_config_value('peertube_instance', "", True, [str])
+    return {
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinx/template/exts/reference.py

@@ -0,0 +1,68 @@
+
+from docutils import nodes
+from docutils.nodes import Element, Node
+from docutils.parsers.rst import directives
+from docutils.parsers.rst.directives.admonitions import BaseAdmonition
+
+from sphinx.locale import _
+from sphinx.util.docutils import SphinxDirective
+
+
+class refbox(nodes.Admonition, nodes.Element):
+    pass
+
+
+def visit_refbox_node(self, node):
+    self.visit_admonition(node)
+
+
+def depart_refbox_node(self, node):
+    self.depart_admonition(node)
+
+
+class ReferenceDirective(BaseAdmonition, SphinxDirective):
+    node_class = refbox
+    has_content = True
+    required_arguments = 0
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {
+        'class': directives.class_option,
+        'name': directives.unchanged,
+    }
+
+    def run(self):
+        if not self.options.get('class'):
+            self.options['class'] = ['refbox']
+
+        (reference,) = super().run()
+        if isinstance(reference, nodes.system_message):
+            return [reference]
+        elif isinstance(reference, refbox):
+            reference.insert(0, nodes.title(text=_('Reference')))
+            reference['docname'] = self.env.docname
+            self.add_name(reference)
+            self.set_source_info(reference)
+            self.state.document.note_explicit_target(reference)
+            return [reference]
+        else:
+            raise RuntimeError  # never reached here
+
+
+def setup(app):
+    app.add_node(
+        refbox,
+        html=(visit_refbox_node, depart_refbox_node),
+        latex=(visit_refbox_node, depart_refbox_node),
+        text=(visit_refbox_node, depart_refbox_node),
+        man=(visit_refbox_node, depart_refbox_node),
+        texinfo=(visit_refbox_node, depart_refbox_node),
+    )
+
+    app.add_directive('reference', ReferenceDirective)
+
+    return {
+        'version': '1.0',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }