diff -r 058bb3dc685f -r 0b59724cb3f2 web/views/cwsources.py --- a/web/views/cwsources.py Mon Jan 04 18:40:30 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,516 +0,0 @@ -# copyright 2010-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr -# -# This file is part of CubicWeb. -# -# CubicWeb is free software: you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 2.1 of the License, or (at your option) -# any later version. -# -# CubicWeb is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with CubicWeb. If not, see . -"""Specific views for data sources and related entities (eg CWSource, -CWSourceHostConfig, CWSourceSchemaConfig). -""" - -__docformat__ = "restructuredtext en" -from cubicweb import _ - -import logging -from itertools import repeat - -from six.moves import range - -from logilab.mtconverter import xml_escape -from logilab.common.decorators import cachedproperty - -from cubicweb import Unauthorized, tags -from cubicweb.utils import make_uid -from cubicweb.predicates import (is_instance, score_entity, has_related_entities, - match_user_groups, match_kwargs, match_view, one_line_rset) -from cubicweb.view import EntityView, StartupView -from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, display_name -from cubicweb.web import Redirect, formwidgets as wdgs, facet, action -from cubicweb.web.views import add_etype_button -from cubicweb.web.views import (uicfg, tabs, actions, ibreadcrumbs, navigation, - tableview, pyviews) - - -_abaa = uicfg.actionbox_appearsin_addmenu -# there are explicit 'add' buttons for those -_abaa.tag_object_of(('CWSourceSchemaConfig', 'cw_schema', '*'), False) -_abaa.tag_object_of(('CWSourceSchemaConfig', 'cw_for_source', '*'), False) -_abaa.tag_object_of(('CWSourceSchemaConfig', 'cw_host_config_of', '*'), False) -_abaa.tag_object_of(('CWDataImport', 'cw_import_of', '*'), False) - -_afs = uicfg.autoform_section -_afs.tag_attribute(('CWSource', 'latest_retrieval'), 'main', 'hidden') -_afs.tag_attribute(('CWSource', 'in_synchronization'), 'main', 'hidden') -_afs.tag_object_of(('*', 'cw_for_source', 'CWSource'), 'main', 'hidden') - -_affk = uicfg.autoform_field_kwargs -_affk.tag_attribute(('CWSource', 'parser'), {'widget': wdgs.TextInput}) - -# source primary views ######################################################### - -_pvs = uicfg.primaryview_section -_pvs.tag_attribute(('CWSource', 'name'), 'hidden') -_pvs.tag_object_of(('*', 'cw_for_source', 'CWSource'), 'hidden') -_pvs.tag_object_of(('*', 'cw_host_config_of', 'CWSource'), 'hidden') - -_pvdc = uicfg.primaryview_display_ctrl -_pvdc.tag_attribute(('CWSource', 'type'), {'vid': 'attribute'})# disable reledit - -_rc = uicfg.reledit_ctrl -_rc.tag_attribute(('CWSource', 'config'), {'rvid': 'verbatimattr'}) -_rc.tag_attribute(('CWSourceHostConfig', 'config'), {'rvid': 'verbatimattr'}) -_rc.tag_attribute(('CWSourceSchemaConfig', 'options'), {'rvid': 'verbatimattr'}) - - -class CWSourcePrimaryView(tabs.TabbedPrimaryView): - __select__ = is_instance('CWSource') - tabs = [_('cwsource-main'), _('cwsource-mapping'), _('cwsource-imports')] - default_tab = 'cwsource-main' - - -class CWSourceMainTab(tabs.PrimaryTab): - __regid__ = 'cwsource-main' - __select__ = is_instance('CWSource') - - def render_entity_attributes(self, entity): - super(CWSourceMainTab, self).render_entity_attributes(entity) - self.w(add_etype_button(self._cw, 'CWSourceHostConfig', - __linkto='cw_host_config_of:%s:subject' % entity.eid, - __redirectpath=entity.rest_path())) - try: - hostconfig = self._cw.execute( - 'Any X, XC, XH WHERE X cw_host_config_of S, S eid %(s)s, ' - 'X config XC, X match_host XH', {'s': entity.eid}) - except Unauthorized: - pass - else: - if hostconfig: - self.w(u'

%s

' % self._cw._('CWSourceHostConfig_plural')) - self._cw.view('table', hostconfig, w=self.w, - displaycols=list(range(2)), - cellvids={1: 'editable-final'}) - - -MAPPED_SOURCE_TYPES = set( ('datafeed',) ) - -class CWSourceMappingTab(EntityView): - __regid__ = 'cwsource-mapping' - __select__ = (is_instance('CWSource') - & match_user_groups('managers') - & score_entity(lambda x:x.type in MAPPED_SOURCE_TYPES)) - - def entity_call(self, entity): - _ = self._cw._ - self.w('

%s

' % _('Entity and relation supported by this source')) - self.w(add_etype_button(self._cw, 'CWSourceSchemaConfig', - __linkto='cw_for_source:%s:subject' % entity.eid)) - self.w(u'
') - rset = self._cw.execute( - 'Any X, SCH, XO ORDERBY ET WHERE X options XO, X cw_for_source S, S eid %(s)s, ' - 'X cw_schema SCH, SCH is ET', {'s': entity.eid}) - self.wview('table', rset, 'noresult') - checker = MappingChecker(entity) - checker.check() - if (checker.errors or checker.warnings or checker.infos): - self.w('

%s

' % _('Detected problems')) - errors = zip(repeat(_('error')), checker.errors) - warnings = zip(repeat(_('warning')), checker.warnings) - infos = zip(repeat(_('warning')), checker.infos) - self.wview('pyvaltable', pyvalue=errors + warnings + infos) - - -class MappingChecker(object): - def __init__(self, cwsource): - self.cwsource = cwsource - self.errors = [] - self.warnings = [] - self.infos = [] - self.schema = cwsource._cw.vreg.schema - - def init(self): - # supported entity types - self.sentities = set() - # supported relations - self.srelations = {} - # avoid duplicated messages - self.seen = set() - # first get mapping as dict/sets - for schemacfg in self.cwsource.reverse_cw_for_source: - self.init_schemacfg(schemacfg) - - def init_schemacfg(self, schemacfg): - cwerschema = schemacfg.schema - if cwerschema.__regid__ == 'CWEType': - self.sentities.add(cwerschema.name) - elif cwerschema.__regid__ == 'CWRType': - assert not cwerschema.name in self.srelations - self.srelations[cwerschema.name] = None - else: # CWAttribute/CWRelation - self.srelations.setdefault(cwerschema.rtype.name, []).append( - (cwerschema.stype.name, cwerschema.otype.name) ) - self.sentities.add(cwerschema.stype.name) - self.sentities.add(cwerschema.otype.name) - - def check(self): - self.init() - error = self.errors.append - warning = self.warnings.append - info = self.infos.append - for etype in self.sentities: - eschema = self.schema[etype] - for rschema, ttypes, role in eschema.relation_definitions(): - if rschema in META_RTYPES: - continue - ttypes = [ttype for ttype in ttypes if ttype in self.sentities] - if not rschema in self.srelations: - for ttype in ttypes: - rdef = rschema.role_rdef(etype, ttype, role) - self.seen.add(rdef) - if rdef.role_cardinality(role) in '1+': - error(_('relation %(type)s with %(etype)s as %(role)s ' - 'and target type %(target)s is mandatory but ' - 'not supported') % - {'rtype': rschema, 'etype': etype, 'role': role, - 'target': ttype}) - elif ttype in self.sentities: - warning(_('%s could be supported') % rdef) - elif not ttypes: - warning(_('relation %(rtype)s with %(etype)s as %(role)s is ' - 'supported but no target type supported') % - {'rtype': rschema, 'role': role, 'etype': etype}) - for rtype, rdefs in self.srelations.items(): - if rdefs is None: - rschema = self.schema[rtype] - for subj, obj in rschema.rdefs: - if subj in self.sentities and obj in self.sentities: - break - else: - error(_('relation %s is supported but none of its definitions ' - 'matches supported entities') % rtype) - self.custom_check() - - def custom_check(self): - pass - - - -class CWSourceImportsTab(EntityView): - __regid__ = 'cwsource-imports' - __select__ = (is_instance('CWSource') - & has_related_entities('cw_import_of', 'object')) - - def entity_call(self, entity): - rset = self._cw.execute('Any X, XST, XET, XS ORDERBY XST DESC WHERE ' - 'X cw_import_of S, S eid %(s)s, X status XS, ' - 'X start_timestamp XST, X end_timestamp XET', - {'s': entity.eid}) - self._cw.view('cw.imports-table', rset, w=self.w) - - -class CWImportsTable(tableview.EntityTableView): - __regid__ = 'cw.imports-table' - __select__ = is_instance('CWDataImport') - columns = ['import', 'start_timestamp', 'end_timestamp'] - column_renderers = {'import': tableview.MainEntityColRenderer()} - layout_args = {'display_filter': 'top'} - - -class CWSourceSyncAction(action.Action): - __regid__ = 'cw.source-sync' - __select__ = (action.Action.__select__ & match_user_groups('managers') - & one_line_rset() & is_instance('CWSource') - & score_entity(lambda x: x.name != 'system')) - - title = _('synchronize') - category = 'mainactions' - order = 20 - - def url(self): - entity = self.cw_rset.get_entity(self.cw_row or 0, self.cw_col or 0) - return entity.absolute_url(vid=self.__regid__) - - -class CWSourceSyncView(EntityView): - __regid__ = 'cw.source-sync' - __select__ = (match_user_groups('managers') - & one_line_rset() & is_instance('CWSource') - & score_entity(lambda x: x.name != 'system')) - - title = _('synchronize') - - def entity_call(self, entity): - self._cw.call_service('source-sync', source_eid=entity.eid) - msg = self._cw._('Source has been synchronized') - url = entity.absolute_url(tab='cwsource-imports', __message=msg) - raise Redirect(url) - - - - -# sources management view ###################################################### - -class ManageSourcesAction(actions.ManagersAction): - __regid__ = 'cwsource' - title = _('data sources') - category = 'manage' - order = 100 - - -class CWSourcesManagementView(StartupView): - __regid__ = 'cw.sources-management' - rql = ('Any S,ST,SP,SD,SN ORDERBY SN WHERE S is CWSource, S name SN, S type ST, ' - 'S latest_retrieval SD, S parser SP') - title = _('data sources management') - - def call(self, **kwargs): - self.w('

%s

' % self._cw._(self.title)) - self.w(add_etype_button(self._cw, 'CWSource')) - self.w(u'
') - self.wview('cw.sources-table', self._cw.execute(self.rql)) - - -class CWSourcesTable(tableview.EntityTableView): - __regid__ = 'cw.sources-table' - __select__ = is_instance('CWSource') - columns = ['source', 'type', 'parser', 'latest_retrieval', 'latest_import'] - - class LatestImportColRenderer(tableview.EntityTableColRenderer): - def render_cell(self, w, rownum): - entity = self.entity(rownum) - rset = self._cw.execute('Any X,XS,XST ORDERBY XST DESC LIMIT 1 WHERE ' - 'X cw_import_of S, S eid %(s)s, X status XS, ' - 'X start_timestamp XST', {'s': entity.eid}) - if rset: - self._cw.view('incontext', rset, row=0, w=w) - else: - w(self.empty_cell_content) - - column_renderers = { - 'source': tableview.MainEntityColRenderer(), - 'latest_import': LatestImportColRenderer(header=_('latest import'), - sortable=False) - } - -# datafeed source import ####################################################### - -REVERSE_SEVERITIES = { - logging.DEBUG : _('DEBUG'), - logging.INFO : _('INFO'), - logging.WARNING : _('WARNING'), - logging.ERROR : _('ERROR'), - logging.FATAL : _('FATAL') -} - - -def log_to_table(req, rawdata): - data = [] - for msg_idx, msg in enumerate(rawdata.split('
')): - record = msg.strip() - if not record: - continue - try: - severity, url, line, msg = record.split('\t', 3) - except ValueError: - req.warning('badly formated log %s' % record) - url = line = u'' - severity = logging.DEBUG - msg = record - data.append( (severity, url, line, msg) ) - return data - - -class LogTableLayout(tableview.TableLayout): - __select__ = match_view('cw.log.table') - needs_js = tableview.TableLayout.needs_js + ('cubicweb.log.js',) - needs_css = tableview.TableLayout.needs_css + ('cubicweb.log.css',) - columns_css = { - 0: 'logSeverity', - 1: 'logPath', - 2: 'logLine', - 3: 'logMsg', - } - - def render_table(self, w, actions, paginate): - default_level = self.view.cw_extra_kwargs['default_level'] - if default_level != 'Debug': - self._cw.add_onload('$("select.logFilter").val("%s").change();' - % self._cw.form.get('logLevel', default_level)) - w(u'\n
') - w(u'' % self._cw._(u'Message threshold')) - w(u'') - w(u'
') - super(LogTableLayout, self).render_table(w, actions, paginate) - - def table_attributes(self): - attrs = super(LogTableLayout, self).table_attributes() - attrs['id'] = 'table'+self.view.domid - return attrs - - def row_attributes(self, rownum): - attrs = super(LogTableLayout, self).row_attributes(rownum) - attrs['id'] = 'log_msg_%i' % rownum - severityname = REVERSE_SEVERITIES[int(self.view.pyvalue[rownum][0])] - attrs['class'] = 'log%s' % severityname.capitalize() - return attrs - - def cell_attributes(self, rownum, colnum, colid): - attrs = super(LogTableLayout, self).cell_attributes(rownum, colnum, colid) - attrs['class'] = self.columns_css[colnum] - return attrs - - -class LogTable(pyviews.PyValTableView): - __regid__ = 'cw.log.table' - headers = [_('severity'), _('url'), _('line'), _('message')] - - @cachedproperty - def domid(self): - return make_uid('logTable') - - class SeverityRenderer(pyviews.PyValTableColRenderer): - def render_cell(self, w, rownum): - severity = self.data[rownum][0] - w(u'''' - u' %(severity)s' % { - 'severity': self._cw._(REVERSE_SEVERITIES[int(severity)]), - 'title': self._cw._('permalink to this message'), - 'msg_id': 'log_msg_%i' % rownum, - }) - def sortvalue(self, rownum): - return int(self.data[rownum][0]) - - class URLRenderer(pyviews.PyValTableColRenderer): - def render_cell(self, w, rownum): - url = self.data[rownum][1] - if url and url.startswith('http'): - url = tags.a(url, href=url) - w(url or u' ') - - class LineRenderer(pyviews.PyValTableColRenderer): - def render_cell(self, w, rownum): - line = self.data[rownum][2] - w(line or u' ') - - class MessageRenderer(pyviews.PyValTableColRenderer): - snip_over = 7 - def render_cell(self, w, rownum): - msg = self.data[rownum][3] - lines = msg.splitlines() - if len(lines) <= self.snip_over: - w(u'
%s
' % msg) - else: - # The make_uid argument has no specific meaning here. - div_snip_id = make_uid(u'log_snip_') - div_full_id = make_uid(u'log_full_') - divs_id = (div_snip_id, div_full_id) - snip = u'\n'.join((lines[0], lines[1], - u' ...', - u' %i more lines [double click to expand]' % (len(lines)-4), - u' ...', - lines[-2], lines[-1])) - divs = ( - (div_snip_id, snip, u'expand', "class='collapsed'"), - (div_full_id, msg, u'collapse', "class='hidden'") - ) - for div_id, content, button, h_class in divs: - text = self._cw._(button) - js = u"toggleVisibility('%s'); toggleVisibility('%s');" % divs_id - w(u'
' % (div_id, h_class)) - w(u'
' % (js, text))
-                    w(content)
-                    w(u'
') - w(u'
') - - column_renderers = {0: SeverityRenderer(), - 1: URLRenderer(sortable=False), - 2: LineRenderer(sortable=False), - 3: MessageRenderer(sortable=False), - } - - -class DataFeedSourceDataImport(EntityView): - __select__ = EntityView.__select__ & match_kwargs('rtype') - __regid__ = 'cw.formated_log' - - def cell_call(self, row, col, rtype, loglevel='Info', **kwargs): - if 'dispctrl' in self.cw_extra_kwargs: - loglevel = self.cw_extra_kwargs['dispctrl'].get('loglevel', loglevel) - entity = self.cw_rset.get_entity(row, col) - value = getattr(entity, rtype) - if value: - self._cw.view('cw.log.table', pyvalue=log_to_table(self._cw, value), - default_level=loglevel, w=self.w) - else: - self.w(self._cw._('no log to display')) - - -_pvs.tag_attribute(('CWDataImport', 'log'), 'relations') -_pvdc.tag_attribute(('CWDataImport', 'log'), {'vid': 'cw.formated_log'}) -_pvs.tag_subject_of(('CWDataImport', 'cw_import_of', '*'), 'hidden') # in breadcrumbs -_pvs.tag_object_of(('*', 'cw_import_of', 'CWSource'), 'hidden') # in dedicated tab - - -class CWDataImportIPrevNextAdapter(navigation.IPrevNextAdapter): - __select__ = is_instance('CWDataImport') - - def next_entity(self): - if self.entity.start_timestamp is not None: - # add NOT X eid %(e)s because > may not be enough - rset = self._cw.execute( - 'Any X,XSTS ORDERBY 2 LIMIT 1 WHERE X is CWDataImport, ' - 'X cw_import_of S, S eid %(s)s, NOT X eid %(e)s, ' - 'X start_timestamp XSTS, X start_timestamp > %(sts)s', - {'sts': self.entity.start_timestamp, - 'e': self.entity.eid, - 's': self.entity.cwsource.eid}) - if rset: - return rset.get_entity(0, 0) - - def previous_entity(self): - if self.entity.start_timestamp is not None: - # add NOT X eid %(e)s because < may not be enough - rset = self._cw.execute( - 'Any X,XSTS ORDERBY 2 DESC LIMIT 1 WHERE X is CWDataImport, ' - 'X cw_import_of S, S eid %(s)s, NOT X eid %(e)s, ' - 'X start_timestamp XSTS, X start_timestamp < %(sts)s', - {'sts': self.entity.start_timestamp, - 'e': self.entity.eid, - 's': self.entity.cwsource.eid}) - if rset: - return rset.get_entity(0, 0) - -class CWDataImportStatusFacet(facet.AttributeFacet): - __regid__ = 'datafeed.dataimport.status' - __select__ = is_instance('CWDataImport') - rtype = 'status' - - -# breadcrumbs configuration #################################################### - -class CWsourceConfigIBreadCrumbsAdapter(ibreadcrumbs.IBreadCrumbsAdapter): - __select__ = is_instance('CWSourceHostConfig', 'CWSourceSchemaConfig') - def parent_entity(self): - return self.entity.cwsource - -class CWDataImportIBreadCrumbsAdapter(ibreadcrumbs.IBreadCrumbsAdapter): - __select__ = is_instance('CWDataImport') - def parent_entity(self): - return self.entity.cw_import_of[0]