1 # copyright 2010-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """Specific views for data sources and related entities (eg CWSource,

    19 CWSourceHostConfig, CWSourceSchemaConfig).

    20 """

    21 

    22 __docformat__ = "restructuredtext en"

    23 from cubicweb import _

    24 

    25 import logging

    26 from itertools import repeat

    27 

    28 from six.moves import range

    29 

    30 from logilab.mtconverter import xml_escape

    31 from logilab.common.decorators import cachedproperty

    32 

    33 from cubicweb import Unauthorized, tags

    34 from cubicweb.utils import make_uid

    35 from cubicweb.predicates import (is_instance, score_entity, has_related_entities,

    36                                  match_user_groups, match_kwargs, match_view, one_line_rset)

    37 from cubicweb.view import EntityView, StartupView

    38 from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, display_name

    39 from cubicweb.web import Redirect, formwidgets as wdgs, facet, action

    40 from cubicweb.web.views import add_etype_button

    41 from cubicweb.web.views import (uicfg, tabs, actions, ibreadcrumbs, navigation,

    42                                 tableview, pyviews)

    43 

    44 

    45 _abaa = uicfg.actionbox_appearsin_addmenu

    46 # there are explicit 'add' buttons for those

    47 _abaa.tag_object_of(('CWSourceSchemaConfig', 'cw_schema', '*'), False)

    48 _abaa.tag_object_of(('CWSourceSchemaConfig', 'cw_for_source', '*'), False)

    49 _abaa.tag_object_of(('CWSourceSchemaConfig', 'cw_host_config_of', '*'), False)

    50 _abaa.tag_object_of(('CWDataImport', 'cw_import_of', '*'), False)

    51 

    52 _afs = uicfg.autoform_section

    53 _afs.tag_attribute(('CWSource', 'latest_retrieval'), 'main', 'hidden')

    54 _afs.tag_attribute(('CWSource', 'in_synchronization'), 'main', 'hidden')

    55 _afs.tag_object_of(('*', 'cw_for_source', 'CWSource'), 'main', 'hidden')

    56 

    57 _affk = uicfg.autoform_field_kwargs

    58 _affk.tag_attribute(('CWSource', 'parser'), {'widget': wdgs.TextInput})

    59 

    60 # source primary views #########################################################

    61 

    62 _pvs = uicfg.primaryview_section

    63 _pvs.tag_attribute(('CWSource', 'name'), 'hidden')

    64 _pvs.tag_object_of(('*', 'cw_for_source', 'CWSource'), 'hidden')

    65 _pvs.tag_object_of(('*', 'cw_host_config_of', 'CWSource'), 'hidden')

    66 

    67 _pvdc = uicfg.primaryview_display_ctrl

    68 _pvdc.tag_attribute(('CWSource', 'type'), {'vid': 'attribute'})# disable reledit

    69 

    70 _rc = uicfg.reledit_ctrl

    71 _rc.tag_attribute(('CWSource', 'config'), {'rvid': 'verbatimattr'})

    72 _rc.tag_attribute(('CWSourceHostConfig', 'config'), {'rvid': 'verbatimattr'})

    73 _rc.tag_attribute(('CWSourceSchemaConfig', 'options'), {'rvid': 'verbatimattr'})

    74 

    75 

    76 class CWSourcePrimaryView(tabs.TabbedPrimaryView):

    77     __select__ = is_instance('CWSource')

    78     tabs = [_('cwsource-main'), _('cwsource-mapping'), _('cwsource-imports')]

    79     default_tab = 'cwsource-main'

    80 

    81 

    82 class CWSourceMainTab(tabs.PrimaryTab):

    83     __regid__ = 'cwsource-main'

    84     __select__ = is_instance('CWSource')

    85 

    86     def render_entity_attributes(self, entity):

    87         super(CWSourceMainTab, self).render_entity_attributes(entity)

    88         self.w(add_etype_button(self._cw, 'CWSourceHostConfig',

    89                                 __linkto='cw_host_config_of:%s:subject' % entity.eid,

    90                                 __redirectpath=entity.rest_path()))

    91         try:

    92             hostconfig = self._cw.execute(

    93                 'Any X, XC, XH WHERE X cw_host_config_of S, S eid %(s)s, '

    94                 'X config XC, X match_host XH', {'s': entity.eid})

    95         except Unauthorized:

    96             pass

    97         else:

    98             if hostconfig:

    99                 self.w(u'<h3>%s</h3>' % self._cw._('CWSourceHostConfig_plural'))

   100                 self._cw.view('table', hostconfig, w=self.w,

   101                               displaycols=list(range(2)),

   102                               cellvids={1: 'editable-final'})

   103 

   104 

   105 MAPPED_SOURCE_TYPES = set( ('datafeed',) )

   106 

   107 class CWSourceMappingTab(EntityView):

   108     __regid__ = 'cwsource-mapping'

   109     __select__ = (is_instance('CWSource')

   110                   & match_user_groups('managers')

   111                   & score_entity(lambda x:x.type in MAPPED_SOURCE_TYPES))

   112 

   113     def entity_call(self, entity):

   114         _ = self._cw._

   115         self.w('<h3>%s</h3>' % _('Entity and relation supported by this source'))

   116         self.w(add_etype_button(self._cw, 'CWSourceSchemaConfig',

   117                                 __linkto='cw_for_source:%s:subject' % entity.eid))

   118         self.w(u'<div class="clear"></div>')

   119         rset = self._cw.execute(

   120             'Any X, SCH, XO ORDERBY ET WHERE X options XO, X cw_for_source S, S eid %(s)s, '

   121             'X cw_schema SCH, SCH is ET', {'s': entity.eid})

   122         self.wview('table', rset, 'noresult')

   123         checker = MappingChecker(entity)

   124         checker.check()

   125         if (checker.errors or checker.warnings or checker.infos):

   126             self.w('<h2>%s</h2>' % _('Detected problems'))

   127             errors = zip(repeat(_('error')), checker.errors)

   128             warnings = zip(repeat(_('warning')), checker.warnings)

   129             infos = zip(repeat(_('warning')), checker.infos)

   130             self.wview('pyvaltable', pyvalue=errors + warnings + infos)

   131 

   132 

   133 class MappingChecker(object):

   134     def __init__(self, cwsource):

   135         self.cwsource = cwsource

   136         self.errors = []

   137         self.warnings = []

   138         self.infos = []

   139         self.schema = cwsource._cw.vreg.schema

   140 

   141     def init(self):

   142         # supported entity types

   143         self.sentities = set()

   144         # supported relations

   145         self.srelations = {}

   146         # avoid duplicated messages

   147         self.seen = set()

   148         # first get mapping as dict/sets

   149         for schemacfg in self.cwsource.reverse_cw_for_source:

   150             self.init_schemacfg(schemacfg)

   151 

   152     def init_schemacfg(self, schemacfg):

   153         cwerschema = schemacfg.schema

   154         if cwerschema.__regid__ == 'CWEType':

   155             self.sentities.add(cwerschema.name)

   156         elif cwerschema.__regid__ == 'CWRType':

   157             assert not cwerschema.name in self.srelations

   158             self.srelations[cwerschema.name] = None

   159         else: # CWAttribute/CWRelation

   160             self.srelations.setdefault(cwerschema.rtype.name, []).append(

   161                 (cwerschema.stype.name, cwerschema.otype.name) )

   162             self.sentities.add(cwerschema.stype.name)

   163             self.sentities.add(cwerschema.otype.name)

   164 

   165     def check(self):

   166         self.init()

   167         error = self.errors.append

   168         warning = self.warnings.append

   169         info = self.infos.append

   170         for etype in self.sentities:

   171             eschema = self.schema[etype]

   172             for rschema, ttypes, role in eschema.relation_definitions():

   173                 if rschema in META_RTYPES:

   174                     continue

   175                 ttypes = [ttype for ttype in ttypes if ttype in self.sentities]

   176                 if not rschema in self.srelations:

   177                     for ttype in ttypes:

   178                         rdef = rschema.role_rdef(etype, ttype, role)

   179                         self.seen.add(rdef)

   180                         if rdef.role_cardinality(role) in '1+':

   181                             error(_('relation %(type)s with %(etype)s as %(role)s '

   182                                     'and target type %(target)s is mandatory but '

   183                                     'not supported') %

   184                                   {'rtype': rschema, 'etype': etype, 'role': role,

   185                                    'target': ttype})

   186                         elif ttype in self.sentities:

   187                             warning(_('%s could be supported') % rdef)

   188                 elif not ttypes:

   189                     warning(_('relation %(rtype)s with %(etype)s as %(role)s is '

   190                               'supported but no target type supported') %

   191                             {'rtype': rschema, 'role': role, 'etype': etype})

   192         for rtype, rdefs in self.srelations.items():

   193             if rdefs is None:

   194                 rschema = self.schema[rtype]

   195                 for subj, obj in rschema.rdefs:

   196                     if subj in self.sentities and obj in self.sentities:

   197                         break

   198                 else:

   199                     error(_('relation %s is supported but none of its definitions '

   200                             'matches supported entities') % rtype)

   201         self.custom_check()

   202 

   203     def custom_check(self):

   204         pass

   205 

   206 

   207 

   208 class CWSourceImportsTab(EntityView):

   209     __regid__ = 'cwsource-imports'

   210     __select__ = (is_instance('CWSource')

   211                   & has_related_entities('cw_import_of', 'object'))

   212 

   213     def entity_call(self, entity):

   214         rset = self._cw.execute('Any X, XST, XET, XS ORDERBY XST DESC WHERE '

   215                                 'X cw_import_of S, S eid %(s)s, X status XS, '

   216                                 'X start_timestamp XST, X end_timestamp XET',

   217                                 {'s': entity.eid})

   218         self._cw.view('cw.imports-table', rset, w=self.w)

   219 

   220 

   221 class CWImportsTable(tableview.EntityTableView):

   222     __regid__ = 'cw.imports-table'

   223     __select__ = is_instance('CWDataImport')

   224     columns = ['import', 'start_timestamp', 'end_timestamp']

   225     column_renderers = {'import': tableview.MainEntityColRenderer()}

   226     layout_args = {'display_filter': 'top'}

   227 

   228 

   229 class CWSourceSyncAction(action.Action):

   230     __regid__ = 'cw.source-sync'

   231     __select__ = (action.Action.__select__ & match_user_groups('managers')

   232                   & one_line_rset() & is_instance('CWSource')

   233                   & score_entity(lambda x: x.name != 'system'))

   234 

   235     title = _('synchronize')

   236     category = 'mainactions'

   237     order = 20

   238 

   239     def url(self):

   240         entity = self.cw_rset.get_entity(self.cw_row or 0, self.cw_col or 0)

   241         return entity.absolute_url(vid=self.__regid__)

   242 

   243 

   244 class CWSourceSyncView(EntityView):

   245     __regid__ = 'cw.source-sync'

   246     __select__ = (match_user_groups('managers')

   247                   & one_line_rset() & is_instance('CWSource')

   248                   & score_entity(lambda x: x.name != 'system'))

   249 

   250     title = _('synchronize')

   251 

   252     def entity_call(self, entity):

   253         self._cw.call_service('source-sync', source_eid=entity.eid)

   254         msg = self._cw._('Source has been synchronized')

   255         url = entity.absolute_url(tab='cwsource-imports', __message=msg)

   256         raise Redirect(url)

   257 

   258 

   259 

   260 

   261 # sources management view ######################################################

   262 

   263 class ManageSourcesAction(actions.ManagersAction):

   264     __regid__ = 'cwsource'

   265     title = _('data sources')

   266     category = 'manage'

   267     order = 100

   268 

   269 

   270 class CWSourcesManagementView(StartupView):

   271     __regid__ = 'cw.sources-management'

   272     rql = ('Any S,ST,SP,SD,SN ORDERBY SN WHERE S is CWSource, S name SN, S type ST, '

   273            'S latest_retrieval SD, S parser SP')

   274     title = _('data sources management')

   275 

   276     def call(self, **kwargs):

   277         self.w('<h1>%s</h1>' % self._cw._(self.title))

   278         self.w(add_etype_button(self._cw, 'CWSource'))

   279         self.w(u'<div class="clear"></div>')

   280         self.wview('cw.sources-table', self._cw.execute(self.rql))

   281 

   282 

   283 class CWSourcesTable(tableview.EntityTableView):

   284     __regid__ = 'cw.sources-table'

   285     __select__ = is_instance('CWSource')

   286     columns = ['source', 'type', 'parser', 'latest_retrieval', 'latest_import']

   287 

   288     class LatestImportColRenderer(tableview.EntityTableColRenderer):

   289         def render_cell(self, w, rownum):

   290             entity = self.entity(rownum)

   291             rset = self._cw.execute('Any X,XS,XST ORDERBY XST DESC LIMIT 1 WHERE '

   292                                     'X cw_import_of S, S eid %(s)s, X status XS, '

   293                                     'X start_timestamp XST', {'s': entity.eid})

   294             if rset:

   295                 self._cw.view('incontext', rset, row=0, w=w)

   296             else:

   297                 w(self.empty_cell_content)

   298 

   299     column_renderers = {

   300         'source': tableview.MainEntityColRenderer(),

   301         'latest_import': LatestImportColRenderer(header=_('latest import'),

   302                                                  sortable=False)

   303         }

   304 

   305 # datafeed source import #######################################################

   306 

   307 REVERSE_SEVERITIES = {

   308     logging.DEBUG :   _('DEBUG'),

   309     logging.INFO :    _('INFO'),

   310     logging.WARNING : _('WARNING'),

   311     logging.ERROR :   _('ERROR'),

   312     logging.FATAL :   _('FATAL')

   313 }

   314 

   315 

   316 def log_to_table(req, rawdata):

   317     data = []

   318     for msg_idx, msg in enumerate(rawdata.split('<br/>')):

   319         record = msg.strip()

   320         if not record:

   321             continue

   322         try:

   323             severity, url, line, msg = record.split('\t', 3)

   324         except ValueError:

   325             req.warning('badly formated log %s' % record)

   326             url = line = u''

   327             severity = logging.DEBUG

   328             msg = record

   329         data.append( (severity, url, line, msg) )

   330     return data

   331 

   332 

   333 class LogTableLayout(tableview.TableLayout):

   334     __select__ = match_view('cw.log.table')

   335     needs_js = tableview.TableLayout.needs_js + ('cubicweb.log.js',)

   336     needs_css = tableview.TableLayout.needs_css + ('cubicweb.log.css',)

   337     columns_css = {

   338         0: 'logSeverity',

   339         1: 'logPath',

   340         2: 'logLine',

   341         3: 'logMsg',

   342         }

   343 

   344     def render_table(self, w, actions, paginate):

   345         default_level = self.view.cw_extra_kwargs['default_level']

   346         if default_level != 'Debug':

   347             self._cw.add_onload('$("select.logFilter").val("%s").change();'

   348                            % self._cw.form.get('logLevel', default_level))

   349         w(u'\n<form action="#"><fieldset>')

   350         w(u'<label>%s</label>' % self._cw._(u'Message threshold'))

   351         w(u'<select class="log_filter" onchange="filterLog(\'%s\', this.options[this.selectedIndex].value)">'

   352           % self.view.domid)

   353         for level in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'FATAL'):

   354             w('<option value="%s">%s</option>' % (level.capitalize(),

   355                                                   self._cw._(level)))

   356         w(u'</select>')

   357         w(u'</fieldset></form>')

   358         super(LogTableLayout, self).render_table(w, actions, paginate)

   359 

   360     def table_attributes(self):

   361         attrs = super(LogTableLayout, self).table_attributes()

   362         attrs['id'] = 'table'+self.view.domid

   363         return attrs

   364 

   365     def row_attributes(self, rownum):

   366         attrs = super(LogTableLayout, self).row_attributes(rownum)

   367         attrs['id'] = 'log_msg_%i' % rownum

   368         severityname = REVERSE_SEVERITIES[int(self.view.pyvalue[rownum][0])]

   369         attrs['class'] = 'log%s' % severityname.capitalize()

   370         return attrs

   371 

   372     def cell_attributes(self, rownum, colnum, colid):

   373         attrs = super(LogTableLayout, self).cell_attributes(rownum, colnum, colid)

   374         attrs['class'] = self.columns_css[colnum]

   375         return attrs

   376 

   377 

   378 class LogTable(pyviews.PyValTableView):

   379     __regid__ = 'cw.log.table'

   380     headers = [_('severity'), _('url'), _('line'), _('message')]

   381 

   382     @cachedproperty

   383     def domid(self):

   384         return make_uid('logTable')

   385 

   386     class SeverityRenderer(pyviews.PyValTableColRenderer):

   387         def render_cell(self, w, rownum):

   388             severity = self.data[rownum][0]

   389             w(u'<a class="internallink" href="javascript:;" title="%(title)s" '

   390               u'''onclick="document.location.hash='%(msg_id)s';">&#182;</a>'''

   391               u' %(severity)s' % {

   392                 'severity': self._cw._(REVERSE_SEVERITIES[int(severity)]),

   393                 'title': self._cw._('permalink to this message'),

   394                 'msg_id': 'log_msg_%i' % rownum,

   395             })

   396         def sortvalue(self, rownum):

   397             return int(self.data[rownum][0])

   398 

   399     class URLRenderer(pyviews.PyValTableColRenderer):

   400         def render_cell(self, w, rownum):

   401             url = self.data[rownum][1]

   402             if url and url.startswith('http'):

   403                 url = tags.a(url, href=url)

   404             w(url or u' ')

   405 

   406     class LineRenderer(pyviews.PyValTableColRenderer):

   407         def render_cell(self, w, rownum):

   408             line = self.data[rownum][2]

   409             w(line or u' ')

   410 

   411     class MessageRenderer(pyviews.PyValTableColRenderer):

   412         snip_over = 7

   413         def render_cell(self, w, rownum):

   414             msg = self.data[rownum][3]

   415             lines = msg.splitlines()

   416             if len(lines) <= self.snip_over:

   417                 w(u'<pre class="rawtext">%s</pre>' % msg)

   418             else:

   419                 # The make_uid argument has no specific meaning here.

   420                 div_snip_id = make_uid(u'log_snip_')

   421                 div_full_id = make_uid(u'log_full_')

   422                 divs_id = (div_snip_id, div_full_id)

   423                 snip = u'\n'.join((lines[0], lines[1],

   424                                    u'  ...',

   425                                    u'    %i more lines [double click to expand]' % (len(lines)-4),

   426                                    u'  ...',

   427                                    lines[-2], lines[-1]))

   428                 divs = (

   429                         (div_snip_id, snip, u'expand', "class='collapsed'"),

   430                         (div_full_id, msg,  u'collapse', "class='hidden'")

   431                 )

   432                 for div_id, content, button, h_class in divs:

   433                     text = self._cw._(button)

   434                     js = u"toggleVisibility('%s'); toggleVisibility('%s');" % divs_id

   435                     w(u'<div id="%s" %s>' % (div_id, h_class))

   436                     w(u'<pre class="raw_test" ondblclick="javascript: %s" '

   437                       u'title="%s" style="display: block;">' % (js, text))

   438                     w(content)

   439                     w(u'</pre>')

   440                     w(u'</div>')

   441 

   442     column_renderers = {0: SeverityRenderer(),

   443                         1: URLRenderer(sortable=False),

   444                         2: LineRenderer(sortable=False),

   445                         3: MessageRenderer(sortable=False),

   446                         }

   447 

   448 

   449 class DataFeedSourceDataImport(EntityView):

   450     __select__ = EntityView.__select__ & match_kwargs('rtype')

   451     __regid__ = 'cw.formated_log'

   452 

   453     def cell_call(self, row, col, rtype, loglevel='Info', **kwargs):

   454         if 'dispctrl' in self.cw_extra_kwargs:

   455             loglevel = self.cw_extra_kwargs['dispctrl'].get('loglevel', loglevel)

   456         entity = self.cw_rset.get_entity(row, col)

   457         value = getattr(entity, rtype)

   458         if value:

   459             self._cw.view('cw.log.table', pyvalue=log_to_table(self._cw, value),

   460                           default_level=loglevel, w=self.w)

   461         else:

   462             self.w(self._cw._('no log to display'))

   463 

   464 

   465 _pvs.tag_attribute(('CWDataImport', 'log'), 'relations')

   466 _pvdc.tag_attribute(('CWDataImport', 'log'), {'vid': 'cw.formated_log'})

   467 _pvs.tag_subject_of(('CWDataImport', 'cw_import_of', '*'), 'hidden') # in breadcrumbs

   468 _pvs.tag_object_of(('*', 'cw_import_of', 'CWSource'), 'hidden') # in dedicated tab

   469 

   470 

   471 class CWDataImportIPrevNextAdapter(navigation.IPrevNextAdapter):

   472     __select__ = is_instance('CWDataImport')

   473 

   474     def next_entity(self):

   475         if self.entity.start_timestamp is not None:

   476             # add NOT X eid %(e)s because > may not be enough

   477             rset = self._cw.execute(

   478                 'Any X,XSTS ORDERBY 2 LIMIT 1 WHERE X is CWDataImport, '

   479                 'X cw_import_of S, S eid %(s)s, NOT X eid %(e)s, '

   480                 'X start_timestamp XSTS, X start_timestamp > %(sts)s',

   481                 {'sts': self.entity.start_timestamp,

   482                  'e': self.entity.eid,

   483                  's': self.entity.cwsource.eid})

   484             if rset:

   485                 return rset.get_entity(0, 0)

   486 

   487     def previous_entity(self):

   488         if self.entity.start_timestamp is not None:

   489             # add NOT X eid %(e)s because < may not be enough

   490             rset = self._cw.execute(

   491                 'Any X,XSTS ORDERBY 2 DESC LIMIT 1 WHERE X is CWDataImport, '

   492                 'X cw_import_of S, S eid %(s)s, NOT X eid %(e)s, '

   493                 'X start_timestamp XSTS, X start_timestamp < %(sts)s',

   494                 {'sts': self.entity.start_timestamp,

   495                  'e': self.entity.eid,

   496                  's': self.entity.cwsource.eid})

   497             if rset:

   498                 return rset.get_entity(0, 0)

   499 

   500 class CWDataImportStatusFacet(facet.AttributeFacet):

   501     __regid__ = 'datafeed.dataimport.status'

   502     __select__ = is_instance('CWDataImport')

   503     rtype = 'status'

   504 

   505 

   506 # breadcrumbs configuration ####################################################

   507 

   508 class CWsourceConfigIBreadCrumbsAdapter(ibreadcrumbs.IBreadCrumbsAdapter):

   509     __select__ = is_instance('CWSourceHostConfig', 'CWSourceSchemaConfig')

   510     def parent_entity(self):

   511         return self.entity.cwsource

   512 

   513 class CWDataImportIBreadCrumbsAdapter(ibreadcrumbs.IBreadCrumbsAdapter):

   514     __select__ = is_instance('CWDataImport')

   515     def parent_entity(self):

   516         return self.entity.cw_import_of[0]