From 06140348a614f8220f73309bf6a06e67c375129c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Tue, 20 Mar 2018 16:01:57 +0100 Subject: [PATCH] first step to #42, #111, custom export to rst --- .../rst_notebooks/notebook_with_svg.ipynb | 128 ++++++++++++++ .../ut_helpgen/test_notebooks_exporter.py | 65 +++++++ .../helpgen/notebook_exporter.py | 164 ++++++++++++++++++ .../helpgen/process_notebook_api.py | 94 +++++----- .../helpgen/process_notebooks.py | 6 +- 5 files changed, 404 insertions(+), 53 deletions(-) create mode 100644 _unittests/ut_helpgen/data/rst_notebooks/notebook_with_svg.ipynb create mode 100644 _unittests/ut_helpgen/test_notebooks_exporter.py create mode 100644 src/pyquickhelper/helpgen/notebook_exporter.py diff --git a/_unittests/ut_helpgen/data/rst_notebooks/notebook_with_svg.ipynb b/_unittests/ut_helpgen/data/rst_notebooks/notebook_with_svg.ipynb new file mode 100644 index 000000000..e526e7a9b --- /dev/null +++ b/_unittests/ut_helpgen/data/rst_notebooks/notebook_with_svg.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Notebook with SVG and javascript\n", + "\n", + "SVG in a notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\t\n", + "\t\t\n", + "\t\t\n", + "\t\n", + "\n", + "\n", + "\n", + "\n", + "x\n", + "0\n", + "50\n", + "100\n", + "150\n", + "200\n", + "250\n", + "\n", + "\n", + "y\n", + "0\n", + "50\n", + "100\n", + "150\n", + "200\n", + "250\n", + "\n", + "\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.core.display import SVG\n", + "SVG(url=\"https://upload.wikimedia.org/wikipedia/commons/1/1a/SVG_example_markup_grid.svg\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from jyquickhelper import RenderJsDot\n", + "RenderJsDot('digraph{ a-> b; a-> c -> d;}')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/_unittests/ut_helpgen/test_notebooks_exporter.py b/_unittests/ut_helpgen/test_notebooks_exporter.py new file mode 100644 index 000000000..1ea4da32a --- /dev/null +++ b/_unittests/ut_helpgen/test_notebooks_exporter.py @@ -0,0 +1,65 @@ +""" +@brief test log(time=6s) +@author Xavier Dupre +""" + +import sys +import os +import unittest + +try: + import src +except ImportError: + path = os.path.normpath( + os.path.abspath( + os.path.join( + os.path.split(__file__)[0], + "..", + ".."))) + if path not in sys.path: + sys.path.append(path) + import src + +from src.pyquickhelper.loghelper import fLOG +from src.pyquickhelper.helpgen import process_notebooks +from src.pyquickhelper.pycode import get_temp_folder, ExtTestCase + + +if sys.version_info[0] == 2: + from codecs import open + + +class TestNoteBooksExporter(ExtTestCase): + + def test_notebook_rst_svg(self): + fLOG( + __file__, + self._testMethodName, + OutputPrint=__name__ == "__main__") + if sys.version_info[0] == 2: + # does not work on Python 2 + return + temp = get_temp_folder(__file__, "temp_nb_rst_svg") + nbs = [os.path.normpath(os.path.join( + temp, '..', "data", "rst_notebooks", "notebook_with_svg.ipynb"))] + formats = ["rst"] + + res = process_notebooks(nbs, temp, temp, formats=formats, fLOG=fLOG) + name = res[0][0] + with open(name, 'r', encoding='utf-8') as f: + content = f.read() + self.assertIn('SVG in a notebook.', content) + self.assertIn('.. image::', content) + + nb = 0 + for line in content.split('\n'): + if '.. image::' in line: + name = line.replace('.. image::', '').strip(' \r\t') + dest = os.path.join(temp, name) + self.assertExists(dest) + nb += 1 + self.assertGreater(nb, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/pyquickhelper/helpgen/notebook_exporter.py b/src/pyquickhelper/helpgen/notebook_exporter.py new file mode 100644 index 000000000..affed4824 --- /dev/null +++ b/src/pyquickhelper/helpgen/notebook_exporter.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +""" +@file +@brief Customer notebook exporters. + +.. versionadded:: 1.7 +""" +import os +from traitlets import default +from traitlets.config import Config +from nbconvert.exporters import RSTExporter + + +rst_template = """ +{%- extends 'display_priority.tpl' -%} + + +{% block in_prompt %} +{% endblock in_prompt %} + +{% block output_prompt %} +{% endblock output_prompt %} + +{% block input scoped%} +{%- if cell.source.strip() -%} +{{".. code:: "-}} +{%- if 'magics_language' in cell.metadata -%} + {{ cell.metadata.magics_language}} +{%- elif 'pygments_lexer' in nb.metadata.get('language_info', {}) -%} + {{ nb.metadata.language_info.pygments_lexer }} +{%- elif 'name' in nb.metadata.get('language_info', {}) -%} + {{ nb.metadata.language_info.name }} +{%- endif %} + +{{ cell.source | indent}} +{% endif -%} +{% endblock input %} + +{% block error %} +:: + +{{ super() }} +{% endblock error %} + +{% block traceback_line %} +{{ line | indent | strip_ansi }} +{% endblock traceback_line %} + +{% block execute_result %} +{% block data_priority scoped %} +{{ super() }} +{% endblock %} +{% endblock execute_result %} + +{% block stream %} +.. parsed-literal:: + +{{ output.text | indent }} +{% endblock stream %} + +{% block data_svg %} +.. image:: {{ output.metadata.filenames['image/svg+xml'] | urlencode }} +{% endblock data_svg %} + +{% block data_png %} +.. image:: {{ output.metadata.filenames['image/png'] | urlencode }} +{%- set width=output | get_metadata('width', 'image/png') -%} +{%- if width is not none %} + :width: {{ width }}px +{%- endif %} +{%- set height=output | get_metadata('height', 'image/png') -%} +{%- if height is not none %} + :height: {{ height }}px +{%- endif %} +{% endblock data_png %} + +{% block data_jpg %} +.. image:: {{ output.metadata.filenames['image/jpeg'] | urlencode }} +{%- set width=output | get_metadata('width', 'image/jpeg') -%} +{%- if width is not none %} + :width: {{ width }}px +{%- endif %} +{%- set height=output | get_metadata('height', 'image/jpeg') -%} +{%- if height is not none %} + :height: {{ height }}px +{%- endif %} +{% endblock data_jpg %} + +{% block data_markdown %} +{{ output.data['text/markdown'] | convert_pandoc("markdown", "rst") }} +{% endblock data_markdown %} + +{% block data_latex %} +.. math:: + +{{ output.data['text/latex'] | strip_dollars | indent }} +{% endblock data_latex %} + +{% block data_text scoped %} +.. parsed-literal:: + +{{ output.data['text/plain'] | indent }} +{% endblock data_text %} + +{% block data_html scoped %} +.. raw:: html + +{{ output.data['text/html'] | indent }} +{% endblock data_html %} + +{% block markdowncell scoped %} +{{ cell.source | convert_pandoc("markdown", "rst") }} +{% endblock markdowncell %} + +{%- block rawcell scoped -%} +{%- if cell.metadata.get('raw_mimetype', '').lower() in resources.get('raw_mimetypes', ['']) %} +{{cell.source}} +{% endif -%} +{%- endblock rawcell -%} + +{% block headingcell scoped %} +{{ ("#" * cell.level + cell.source) | replace('\n', ' ') | convert_pandoc("markdown", "rst") }} +{% endblock headingcell %} + +{% block unknowncell scoped %} +unknown type {{cell.type}} +{% endblock unknowncell %} +""" + + +class UpgradedRSTExporter(RSTExporter): + """ + Exports :epkg:`rst` documents. + Overwrites `RSTExporter `_. + """ + + @default('template_file') + def _template_file_default(self): + raise RuntimeError("This should not be called.") + + def _load_template(self): + """ + Overwrites the method in base class. + """ + template_file = 'rst.tpl' + self.log.debug("Attempting to load template %s", template_file) + self.log.debug(" template_path: %s", + os.pathsep.join(self.template_path)) + content = self.environment.get_template(template_file) + # content is of type jinja2.environment.Template + return content + + @property + def default_config(self): + c = Config({ + 'ExtractOutputPreprocessor': { + 'enabled': True + }, + 'HighlightMagicsPreprocessor': { + 'enabled': True + }, + }) + c.merge(super(UpgradedRSTExporter, self).default_config) + return c diff --git a/src/pyquickhelper/helpgen/process_notebook_api.py b/src/pyquickhelper/helpgen/process_notebook_api.py index 41dcad8f8..268cb31b7 100644 --- a/src/pyquickhelper/helpgen/process_notebook_api.py +++ b/src/pyquickhelper/helpgen/process_notebook_api.py @@ -17,75 +17,58 @@ from io import StringIO -def get_exporter(format): +def get_exporter(format, add_writer=False): """ Returns the :epkg:`IPython` exporter associated to a format. @param format string (see below) + @param add_writer add writer as well @return class Available formats: *slides*, *pdf*, *latex*, *markdown*, *html*, *rst*, *python*, *notebook*, *template*. + + ..versionchanged:: 1.7 + Add parameter *add_writer*. """ if format == "python": - try: - from nbconvert import PythonExporter - except ImportError: - from IPython.nbconvert import PythonExporter - return PythonExporter + from nbconvert import PythonExporter + exp = PythonExporter elif format == "slides": - try: - from nbconvert import SlidesExporter - except ImportError: - from IPython.nbconvert import SlidesExporter - return SlidesExporter + from nbconvert import SlidesExporter + exp = SlidesExporter elif format == "html": - try: - from nbconvert import HTMLExporter - except ImportError: - from IPython.nbconvert import HTMLExporter - return HTMLExporter + from nbconvert import HTMLExporter + exp = HTMLExporter elif format == "pdf": - try: - from nbconvert import PDFExporter - except ImportError: - from IPython.nbconvert import PDFExporter - return PDFExporter + from nbconvert import PDFExporter + exp = PDFExporter elif format == "template": - try: - from nbconvert import TemplateExporter - except ImportError: - from IPython.nbconvert import TemplateExporter - return TemplateExporter + from nbconvert import TemplateExporter + exp = TemplateExporter elif format == "markdown": - try: - from nbconvert import MarkdownExporter - except ImportError: - from IPython.nbconvert import MarkdownExporter - return MarkdownExporter + from nbconvert import MarkdownExporter + exp = MarkdownExporter elif format == "notebook": - try: - from nbconvert import NotebookExporter - except ImportError: - from IPython.nbconvert import NotebookExporter - return NotebookExporter + from nbconvert import NotebookExporter + exp = NotebookExporter elif format == "rst": - try: - from nbconvert import RSTExporter - except ImportError: - from IPython.nbconvert import RSTExporter - return RSTExporter + from .notebook_exporter import UpgradedRSTExporter + exp = UpgradedRSTExporter elif format == "lagex": - try: - from nbconvert import LatexExporter - except ImportError: - from IPython.nbconvert import LatexExporter - return LatexExporter + from nbconvert import LatexExporter + exp = LatexExporter else: form = "slides, pdf, latex, markdown, html, rst, python, notebook, template" raise ValueError( "unexpected format: {0}, it should be in:\n{1}".format(format, form)) + if add_writer: + from nbconvert.writers import FilesWriter + return exp, FilesWriter + else: + return exp + def nb2slides(nb_file, outfile, add_tag=True): """ @@ -234,13 +217,14 @@ def nb2html(nb_file, outfile, exc=True): return res -def nb2rst(nb_file, outfile, exc=True): +def nb2rst(nb_file, outfile, exc=True, post_process=True): """ Converts a notebook into RST. @param nb_file notebook file or a stream or a @see fn read_nb @param outfile output file (a string) @param exc raises an exception (True) or a warning (False) + @param post_process calls @see fn post_process_rst_output @return impacted files .. versionadded:: 1.5 @@ -254,14 +238,20 @@ def nb2rst(nb_file, outfile, exc=True): nbr = read_nb(nb_file, kernel=False) nb = nbr.nb - exporter = get_exporter("rst")() + exp_class, writer_class = get_exporter("rst", add_writer=True) + exporter = exp_class() + writer = writer_class() source, meta = exporter.from_notebook_node(nb) - with open(outfile, 'w+', encoding="utf8") as fh: - fh.writelines(source) + name, ext = os.path.splitext(outfile) + if ext != '.rst': + raise ValueError("'{0}' should have extension '.rst'".format(outfile)) + writer.build_directory = os.path.dirname(outfile) + writer.write(source, meta, notebook_name=name) # post_processing - post_process_rst_output(outfile, False, False, - False, False, False, exc=exc) + if post_process: + post_process_rst_output(outfile, False, False, + False, False, False, exc=exc) res = [outfile] return res diff --git a/src/pyquickhelper/helpgen/process_notebooks.py b/src/pyquickhelper/helpgen/process_notebooks.py index 1d934bc4d..87622ba02 100644 --- a/src/pyquickhelper/helpgen/process_notebooks.py +++ b/src/pyquickhelper/helpgen/process_notebooks.py @@ -17,6 +17,7 @@ from .helpgen_exceptions import NotebookConvertError from .install_js_dep import install_javascript_tools from .style_css_template import THUMBNAIL_TEMPLATE, THUMBNAIL_TEMPLATE_TABLE +from .process_notebook_api import nb2rst from ..loghelper.flog import run_cmd, fLOG, noLOG from ..ipythonhelper import read_nb, notebook_coverage, badge_notebook_coverage from ..pandashelper import df2rst @@ -424,7 +425,10 @@ def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_pat # nbconvert is messing up with static variables in sphinx or # docutils if format is slides, not sure about the others - if nbconvert_main != fnbcexe or format not in {"slides", "latex", "pdf"}: + if format in {'rst'}: + nb2rst(notebook, outputfile, post_process=False) + err = "" + elif nbconvert_main != fnbcexe or format not in {"slides", "latex", "pdf"}: out, err = _process_notebooks_in_private( fnbcexe, list_args, options_args) else: