mttransforms.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Mon, 29 Mar 2010 13:34:06 +0200
branchstable
changeset 5060 ee3b856e1406
parent 4252 6c4f109c2b03
child 5421 8167de96c523
permissions -rw-r--r--
[repo] optimize massive insertion/deletion by using the new set_operation function Idea is that on massive insertion, cost of handling the list of operation become non negligeable, so we should minimize the number of operations in that list. The set_operation function ease usage of operation associated to data in session.transaction_data, and we only add the operation when data set isn't initialized yet, else we simply add data to the set. The operation then simply process accumulated data.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     1
"""mime type transformation engine for cubicweb, based on mtconverter
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     2
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     3
:organization: Logilab
4212
ab6573088b4a update copyright: welcome 2010
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 2459
diff changeset
     4
:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     5
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
1977
606923dff11b big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1584
diff changeset
     6
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     7
"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     8
__docformat__ = "restructuredtext en"
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
     9
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    10
from logilab import mtconverter
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    11
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    12
from logilab.mtconverter.engine import TransformEngine
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    13
from logilab.mtconverter.transform import Transform
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    14
from logilab.mtconverter import (register_base_transforms,
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
    15
                                 register_pil_transforms,
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    16
                                 register_pygments_transforms)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    17
2817
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    18
from cubicweb.utils import UStringIO
4023
eae23c40627a drop common subpackage
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3418
diff changeset
    19
from cubicweb.uilib import rest_publish, html_publish
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    20
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    21
HTML_MIMETYPES = ('text/html', 'text/xhtml', 'application/xhtml+xml')
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    22
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    23
# CubicWeb specific transformations
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    24
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    25
class rest_to_html(Transform):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    26
    inputs = ('text/rest', 'text/x-rst')
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    27
    output = 'text/html'
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    28
    def _convert(self, trdata):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    29
        return rest_publish(trdata.appobject, trdata.decode())
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    30
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    31
class html_to_html(Transform):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    32
    inputs = HTML_MIMETYPES
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    33
    output = 'text/html'
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    34
    def _convert(self, trdata):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    35
        return html_publish(trdata.appobject, trdata.data)
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
    36
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    37
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    38
# Instantiate and configure the transformation engine
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    39
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    40
mtconverter.UNICODE_POLICY = 'replace'
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    41
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    42
ENGINE = TransformEngine()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    43
ENGINE.add_transform(rest_to_html())
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    44
ENGINE.add_transform(html_to_html())
704
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    45
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    46
try:
2817
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    47
    from cubicweb.ext.tal import CubicWebContext, compile_template
704
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    48
except ImportError:
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    49
    HAS_TAL = False
2459
d088d0ff48a1 move RichString and co to yams, keeping only a small monkeypatch for cw-page-template here
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
    50
    from cubicweb import schema
d088d0ff48a1 move RichString and co to yams, keeping only a small monkeypatch for cw-page-template here
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
    51
    schema.NEED_PERM_FORMATS.remove('text/cubicweb-page-template')
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
    52
704
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    53
else:
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    54
    HAS_TAL = True
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
    55
704
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    56
    class ept_to_html(Transform):
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    57
        inputs = ('text/cubicweb-page-template',)
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    58
        output = 'text/html'
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    59
        output_encoding = 'utf-8'
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    60
        def _convert(self, trdata):
2817
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    61
            context = CubicWebContext()
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    62
            appobject = trdata.appobject
3418
7b49fa7e942d [api] use _cw, cw_row, cw_col, cw_rset etc.
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 2817
diff changeset
    63
            context.update({'self': appobject, 'rset': appobject.cw_rset,
7b49fa7e942d [api] use _cw, cw_row, cw_col, cw_rset etc.
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 2817
diff changeset
    64
                            'req': appobject._cw,
7b49fa7e942d [api] use _cw, cw_row, cw_col, cw_rset etc.
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 2817
diff changeset
    65
                            '_' : appobject._cw._,
7b49fa7e942d [api] use _cw, cw_row, cw_col, cw_rset etc.
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 2817
diff changeset
    66
                            'user': appobject._cw.user})
2817
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    67
            output = UStringIO()
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    68
            template = compile_template(trdata.encode(self.output_encoding))
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    69
            template.expand(context, output)
0313eecc707e entity has no more tal_render method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2459
diff changeset
    70
            return output.getvalue()
704
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    71
0c2c8f0a6ded new ext package for modules depending on an option third party package
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    72
    ENGINE.add_transform(ept_to_html())
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    73
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    74
if register_pil_transforms(ENGINE, verb=False):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    75
    HAS_PIL_TRANSFORMS = True
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    76
else:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    77
    HAS_PIL_TRANSFORMS = False
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
    78
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    79
try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    80
    from logilab.mtconverter.transforms import pygmentstransforms
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    81
    for mt in ('text/plain',) + HTML_MIMETYPES:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    82
        try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    83
            pygmentstransforms.mimetypes.remove(mt)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    84
        except ValueError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    85
            continue
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    86
    register_pygments_transforms(ENGINE, verb=False)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    87
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    88
    def patch_convert(cls):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    89
        def _convert(self, trdata, origconvert=cls._convert):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    90
            try:
3418
7b49fa7e942d [api] use _cw, cw_row, cw_col, cw_rset etc.
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 2817
diff changeset
    91
                trdata.appobject._cw.add_css('pygments.css')
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    92
            except AttributeError: # session has no add_css, only http request
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    93
                pass
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    94
            return origconvert(self, trdata)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    95
        cls._convert = _convert
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    96
    patch_convert(pygmentstransforms.PygmentsHTMLTransform)
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
    97
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    98
    HAS_PYGMENTS_TRANSFORMS = True
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    99
except ImportError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   100
    HAS_PYGMENTS_TRANSFORMS = False
1584
323eed40a2b7 delete-trailing-whitespaces
sylvain.thenault@logilab.fr
parents: 1132
diff changeset
   101
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   102
register_base_transforms(ENGINE, verb=False)