1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """rest publishing functions

    19 

    20 contains some functions and setup of docutils for cubicweb. Provides the

    21 following ReST directives:

    22 

    23 * `eid`, create link to entity in the repository by their eid

    24 

    25 * `card`, create link to card entity in the repository by their wikiid

    26   (proposing to create it when the refered card doesn't exist yet)

    27 

    28 * `winclude`, reference to a web documentation file (in wdoc/ directories)

    29 

    30 * `sourcecode` (if pygments is installed), source code colorization

    31 

    32 * `rql-table`, create a table from a RQL query

    33 

    34 """

    35 __docformat__ = "restructuredtext en"

    36 

    37 import sys

    38 from itertools import chain

    39 from logging import getLogger

    40 from os.path import join

    41 

    42 from six import text_type

    43 from six.moves.urllib.parse import urlsplit

    44 

    45 from docutils import statemachine, nodes, utils, io

    46 from docutils.core import Publisher

    47 from docutils.parsers.rst import Parser, states, directives, Directive

    48 from docutils.parsers.rst.roles import register_canonical_role, set_classes

    49 

    50 from logilab.mtconverter import ESC_UCAR_TABLE, ESC_CAR_TABLE, xml_escape

    51 

    52 from cubicweb import UnknownEid

    53 from cubicweb.ext.html4zope import Writer

    54 

    55 from cubicweb.web.views import vid_from_rset  # XXX better not to import c.w.views here...

    56 

    57 # We provide our own parser as an attempt to get rid of

    58 # state machine reinstanciation

    59 

    60 import re

    61 # compile states.Body patterns

    62 for k, v in states.Body.patterns.items():

    63     if isinstance(v, str):

    64         states.Body.patterns[k] = re.compile(v)

    65 

    66 # register ReStructured Text mimetype / extensions

    67 import mimetypes

    68 mimetypes.add_type('text/rest', '.rest')

    69 mimetypes.add_type('text/rest', '.rst')

    70 

    71 

    72 LOGGER = getLogger('cubicweb.rest')

    73 

    74 

    75 def eid_reference_role(role, rawtext, text, lineno, inliner,

    76                        options={}, content=[]):

    77     try:

    78         try:

    79             eid_num, rest = text.split(u':', 1)

    80         except ValueError:

    81             eid_num, rest = text, '#'+text

    82         eid_num = int(eid_num)

    83         if eid_num < 0:

    84             raise ValueError

    85     except ValueError:

    86         msg = inliner.reporter.error(

    87             'EID number must be a positive number; "%s" is invalid.'

    88             % text, line=lineno)

    89         prb = inliner.problematic(rawtext, rawtext, msg)

    90         return [prb], [msg]

    91     # Base URL mainly used by inliner.pep_reference; so this is correct:

    92     context = inliner.document.settings.context

    93     try:

    94         refedentity = context._cw.entity_from_eid(eid_num)

    95     except UnknownEid:

    96         ref = '#'

    97         rest += u' ' + context._cw._('(UNEXISTANT EID)')

    98     else:

    99         ref = refedentity.absolute_url()

   100     set_classes(options)

   101     return [nodes.reference(rawtext, utils.unescape(rest), refuri=ref,

   102                             **options)], []

   103 

   104 

   105 def rql_role(role, rawtext, text, lineno, inliner, options={}, content=[]):

   106     """``:rql:`<rql-expr>``` or ``:rql:`<rql-expr>:<vid>```

   107 

   108     Example: ``:rql:`Any X,Y WHERE X is CWUser, X login Y:table```

   109 

   110     Replace the directive with the output of applying the view to the resultset

   111     returned by the query.

   112 

   113     "X eid %(userid)s" can be used in the RQL query for this query will be

   114     executed with the argument {'userid': _cw.user.eid}.

   115     """

   116     _cw = inliner.document.settings.context._cw

   117     text = text.strip()

   118     if ':' in text:

   119         rql, vid = text.rsplit(u':', 1)

   120         rql = rql.strip()

   121     else:

   122         rql, vid = text, None

   123     _cw.ensure_ro_rql(rql)

   124     try:

   125         rset = _cw.execute(rql, {'userid': _cw.user.eid})

   126         if rset:

   127             if vid is None:

   128                 vid = vid_from_rset(_cw, rset, _cw.vreg.schema)

   129         else:

   130             vid = 'noresult'

   131         view = _cw.vreg['views'].select(vid, _cw, rset=rset)

   132         content = view.render()

   133     except Exception as exc:

   134         content = 'an error occurred while interpreting this rql directive: %r' % exc

   135     set_classes(options)

   136     return [nodes.raw('', content, format='html')], []

   137 

   138 

   139 def bookmark_role(role, rawtext, text, lineno, inliner, options={}, content=[]):

   140     """``:bookmark:`<bookmark-eid>``` or ``:bookmark:`<eid>:<vid>```

   141 

   142     Example: ``:bookmark:`1234:table```

   143 

   144     Replace the directive with the output of applying the view to the resultset

   145     returned by the query stored in the bookmark. By default, the view is the one

   146     stored in the bookmark, but it can be overridden by the directive as in the

   147     example above.

   148 

   149     "X eid %(userid)s" can be used in the RQL query stored in the Bookmark, for

   150     this query will be executed with the argument {'userid': _cw.user.eid}.

   151     """

   152     _cw = inliner.document.settings.context._cw

   153     text = text.strip()

   154     try:

   155         if ':' in text:

   156             eid, vid = text.rsplit(u':', 1)

   157             eid = int(eid)

   158         else:

   159             eid, vid = int(text), None

   160     except ValueError:

   161         msg = inliner.reporter.error(

   162             'EID number must be a positive number; "%s" is invalid.'

   163             % text, line=lineno)

   164         prb = inliner.problematic(rawtext, rawtext, msg)

   165         return [prb], [msg]

   166     try:

   167         bookmark = _cw.entity_from_eid(eid)

   168     except UnknownEid:

   169         msg = inliner.reporter.error('Unknown EID %s.' % text, line=lineno)

   170         prb = inliner.problematic(rawtext, rawtext, msg)

   171         return [prb], [msg]

   172     try:

   173         params = dict(_cw.url_parse_qsl(urlsplit(bookmark.path).query))

   174         rql = params['rql']

   175         if vid is None:

   176             vid = params.get('vid')

   177     except (ValueError, KeyError) as exc:

   178         msg = inliner.reporter.error('Could not parse bookmark path %s [%s].'

   179                                      % (bookmark.path, exc), line=lineno)

   180         prb = inliner.problematic(rawtext, rawtext, msg)

   181         return [prb], [msg]

   182     try:

   183         rset = _cw.execute(rql, {'userid': _cw.user.eid})

   184         if rset:

   185             if vid is None:

   186                 vid = vid_from_rset(_cw, rset, _cw.vreg.schema)

   187         else:

   188             vid = 'noresult'

   189         view = _cw.vreg['views'].select(vid, _cw, rset=rset)

   190         content = view.render()

   191     except Exception as exc:

   192         content = 'An error occurred while interpreting directive bookmark: %r' % exc

   193     set_classes(options)

   194     return [nodes.raw('', content, format='html')], []

   195 

   196 

   197 def winclude_directive(name, arguments, options, content, lineno,

   198                        content_offset, block_text, state, state_machine):

   199     """Include a reST file as part of the content of this reST file.

   200 

   201     same as standard include directive but using config.locate_doc_resource to

   202     get actual file to include.

   203 

   204     Most part of this implementation is copied from `include` directive defined

   205     in `docutils.parsers.rst.directives.misc`

   206     """

   207     context = state.document.settings.context

   208     cw = context._cw

   209     source = state_machine.input_lines.source(

   210         lineno - state_machine.input_offset - 1)

   211     #source_dir = os.path.dirname(os.path.abspath(source))

   212     fid = arguments[0]

   213     for lang in chain((cw.lang, cw.vreg.property_value('ui.language')),

   214                       cw.vreg.config.available_languages()):

   215         rid = '%s_%s.rst' % (fid, lang)

   216         resourcedir = cw.vreg.config.locate_doc_file(rid)

   217         if resourcedir:

   218             break

   219     else:

   220         severe = state_machine.reporter.severe(

   221               'Problems with "%s" directive path:\nno resource matching %s.'

   222               % (name, fid),

   223               nodes.literal_block(block_text, block_text), line=lineno)

   224         return [severe]

   225     path = join(resourcedir, rid)

   226     encoding = options.get('encoding', state.document.settings.input_encoding)

   227     try:

   228         state.document.settings.record_dependencies.add(path)

   229         include_file = io.FileInput(

   230             source_path=path, encoding=encoding,

   231             error_handler=state.document.settings.input_encoding_error_handler,

   232             handle_io_errors=None)

   233     except IOError as error:

   234         severe = state_machine.reporter.severe(

   235               'Problems with "%s" directive path:\n%s: %s.'

   236               % (name, error.__class__.__name__, error),

   237               nodes.literal_block(block_text, block_text), line=lineno)

   238         return [severe]

   239     try:

   240         include_text = include_file.read()

   241     except UnicodeError as error:

   242         severe = state_machine.reporter.severe(

   243               'Problem with "%s" directive:\n%s: %s'

   244               % (name, error.__class__.__name__, error),

   245               nodes.literal_block(block_text, block_text), line=lineno)

   246         return [severe]

   247     if 'literal' in options:

   248         literal_block = nodes.literal_block(include_text, include_text,

   249                                             source=path)

   250         literal_block.line = 1

   251         return literal_block

   252     else:

   253         include_lines = statemachine.string2lines(include_text,

   254                                                   convert_whitespace=1)

   255         state_machine.insert_input(include_lines, path)

   256         return []

   257 

   258 winclude_directive.arguments = (1, 0, 1)

   259 winclude_directive.options = {'literal': directives.flag,

   260                               'encoding': directives.encoding}

   261 

   262 

   263 class RQLTableDirective(Directive):

   264     """rql-table directive

   265 

   266     Example:

   267 

   268         .. rql-table::

   269            :vid: mytable

   270            :headers: , , progress

   271            :colvids: 2=progress

   272 

   273             Any X,U,X WHERE X is Project, X url U

   274 

   275     All fields but the RQL string are optionnal. The ``:headers:`` option can

   276     contain empty column names.

   277     """

   278 

   279     required_arguments = 0

   280     optional_arguments = 0

   281     has_content= True

   282     final_argument_whitespace = True

   283     option_spec = {'vid': directives.unchanged,

   284                    'headers': directives.unchanged,

   285                    'colvids': directives.unchanged}

   286 

   287     def run(self):

   288         errid = "rql-table directive"

   289         self.assert_has_content()

   290         if self.arguments:

   291             raise self.warning('%s does not accept arguments' % errid)

   292         rql = ' '.join([l.strip() for l in self.content])

   293         _cw = self.state.document.settings.context._cw

   294         _cw.ensure_ro_rql(rql)

   295         try:

   296             rset = _cw.execute(rql)

   297         except Exception as exc:

   298             raise self.error("fail to execute RQL query in %s: %r" %

   299                              (errid, exc))

   300         if not rset:

   301             raise self.warning("empty result set")

   302         vid = self.options.get('vid', 'table')

   303         try:

   304             view = _cw.vreg['views'].select(vid, _cw, rset=rset)

   305         except Exception as exc:

   306             raise self.error("fail to select '%s' view in %s: %r" %

   307                              (vid, errid, exc))

   308         headers = None

   309         if 'headers' in self.options:

   310             headers = [h.strip() for h in self.options['headers'].split(',')]

   311             while headers.count(''):

   312                 headers[headers.index('')] = None

   313             if len(headers) != len(rset[0]):

   314                 raise self.error("the number of 'headers' does not match the "

   315                                  "number of columns in %s" % errid)

   316         cellvids = None

   317         if 'colvids' in self.options:

   318             cellvids = {}

   319             for f in self.options['colvids'].split(','):

   320                 try:

   321                     idx, vid = f.strip().split('=')

   322                 except ValueError:

   323                     raise self.error("malformatted 'colvids' option in %s" %

   324                                      errid)

   325                 cellvids[int(idx.strip())] = vid.strip()

   326         try:

   327             content = view.render(headers=headers, cellvids=cellvids)

   328         except Exception as exc:

   329             raise self.error("Error rendering %s (%s)" % (errid, exc))

   330         return [nodes.raw('', content, format='html')]

   331 

   332 

   333 try:

   334     from pygments import highlight

   335     from pygments.lexers import get_lexer_by_name

   336     from pygments.formatters.html import HtmlFormatter

   337 except ImportError:

   338     pygments_directive = None

   339 else:

   340     _PYGMENTS_FORMATTER = HtmlFormatter()

   341 

   342     def pygments_directive(name, arguments, options, content, lineno,

   343                            content_offset, block_text, state, state_machine):

   344         try:

   345             lexer = get_lexer_by_name(arguments[0])

   346         except ValueError:

   347             # no lexer found

   348             lexer = get_lexer_by_name('text')

   349         parsed = highlight(u'\n'.join(content), lexer, _PYGMENTS_FORMATTER)

   350         # don't fail if no context set on the sourcecode directive

   351         try:

   352             context = state.document.settings.context

   353             context._cw.add_css('pygments.css')

   354         except AttributeError:

   355             # used outside cubicweb XXX use hasattr instead

   356             pass

   357         return [nodes.raw('', parsed, format='html')]

   358 

   359     pygments_directive.arguments = (1, 0, 1)

   360     pygments_directive.content = 1

   361 

   362 

   363 class CubicWebReSTParser(Parser):

   364     """The (customized) reStructuredText parser."""

   365 

   366     def __init__(self):

   367         self.initial_state = 'Body'

   368         self.state_classes = states.state_classes

   369         self.inliner = states.Inliner()

   370         self.statemachine = states.RSTStateMachine(

   371               state_classes=self.state_classes,

   372               initial_state=self.initial_state,

   373               debug=0)

   374 

   375     def parse(self, inputstring, document):

   376         """Parse `inputstring` and populate `document`, a document tree."""

   377         self.setup_parse(inputstring, document)

   378         inputlines = statemachine.string2lines(inputstring,

   379                                                convert_whitespace=1)

   380         self.statemachine.run(inputlines, document, inliner=self.inliner)

   381         self.finish_parse()

   382 

   383 

   384 # XXX docutils keep a ref on context, can't find a correct way to remove it

   385 class CWReSTPublisher(Publisher):

   386     def __init__(self, context, settings, **kwargs):

   387         Publisher.__init__(self, **kwargs)

   388         self.set_components('standalone', 'restructuredtext', 'pseudoxml')

   389         self.process_programmatic_settings(None, settings, None)

   390         self.settings.context = context

   391 

   392 

   393 def rest_publish(context, data):

   394     """publish a string formatted as ReStructured Text to HTML

   395 

   396     :type context: a cubicweb application object

   397 

   398     :type data: str

   399     :param data: some ReST text

   400 

   401     :rtype: unicode

   402     :return:

   403       the data formatted as HTML or the original data if an error occurred

   404     """

   405     req = context._cw

   406     if isinstance(data, text_type):

   407         encoding = 'unicode'

   408         # remove unprintable characters unauthorized in xml

   409         data = data.translate(ESC_UCAR_TABLE)

   410     else:

   411         encoding = req.encoding

   412         # remove unprintable characters unauthorized in xml

   413         data = data.translate(ESC_CAR_TABLE)

   414     settings = {'input_encoding': encoding, 'output_encoding': 'unicode',

   415                 'warning_stream': False,

   416                 'traceback': True, # don't sys.exit

   417                 'stylesheet': None, # don't try to embed stylesheet (may cause

   418                                     # obscure bug due to docutils computing

   419                                     # relative path according to the directory

   420                                     # used *at import time*

   421                 # dunno what's the max, severe is 4, and we never want a crash

   422                 # (though try/except may be a better option...). May be the

   423                 # above traceback option will avoid this?

   424                 'halt_level': 10,

   425                 # disable stupid switch to colspan=2 if field name is above a size limit

   426                 'field_name_limit': sys.maxsize,

   427                 }

   428     if context:

   429         if hasattr(req, 'url'):

   430             base_url = req.url()

   431         elif hasattr(context, 'absolute_url'):

   432             base_url = context.absolute_url()

   433         else:

   434             base_url = req.base_url()

   435     else:

   436         base_url = None

   437     try:

   438         pub = CWReSTPublisher(context, settings,

   439                               parser=CubicWebReSTParser(),

   440                               writer=Writer(base_url=base_url),

   441                               source_class=io.StringInput,

   442                               destination_class=io.StringOutput)

   443         pub.set_source(data)

   444         pub.set_destination()

   445         res = pub.publish(enable_exit_status=None)

   446         # necessary for proper garbage collection, else a ref is kept somewhere in docutils...

   447         del pub.settings.context

   448         return res

   449     except BaseException:

   450         LOGGER.exception('error while publishing ReST text')

   451         if not isinstance(data, text_type):

   452             data = text_type(data, encoding, 'replace')

   453         return xml_escape(req._('error while publishing ReST text')

   454                            + '\n\n' + data)

   455 

   456 

   457 _INITIALIZED = False

   458 def cw_rest_init():

   459     global _INITIALIZED

   460     if _INITIALIZED:

   461         return

   462     _INITIALIZED = True

   463     register_canonical_role('eid', eid_reference_role)

   464     register_canonical_role('rql', rql_role)

   465     register_canonical_role('bookmark', bookmark_role)

   466     directives.register_directive('winclude', winclude_directive)

   467     if pygments_directive is not None:

   468         directives.register_directive('sourcecode', pygments_directive)

   469     directives.register_directive('rql-table', RQLTableDirective)