# HG changeset patch # User Pierre-Yves David # Date 1366024681 -7200 # Node ID be91151107f6594e7681b6b32fc07243912a795c # Parent 738f97bc3e1983003cf0b72de204828be1f57a4c# Parent 7920f439f383200b7a7832dded3aeb03fc516cf3 merge 3.16.x fix diff -r 7920f439f383 -r be91151107f6 __init__.py --- a/__init__.py Fri Apr 12 11:39:10 2013 +0200 +++ b/__init__.py Mon Apr 15 13:18:01 2013 +0200 @@ -38,10 +38,10 @@ import sys, os, logging from StringIO import StringIO +from logilab.common.deprecation import deprecated from logilab.common.logging_ext import set_log_methods from yams.constraints import BASE_CONVERTERS - if os.environ.get('APYCOT_ROOT'): logging.basicConfig(level=logging.CRITICAL) else: @@ -57,8 +57,9 @@ from logilab.common.registry import ObjectNotFound, NoSelectableObject, RegistryNotFound # convert eid to the right type, raise ValueError if it's not a valid eid -typed_eid = int - +@deprecated('[3.17] typed_eid() was removed. replace it with int() when needed.') +def typed_eid(eid): + return int(eid) #def log_thread(f, w, a): # print f.f_code.co_filename, f.f_code.co_name diff -r 7920f439f383 -r be91151107f6 __pkginfo__.py --- a/__pkginfo__.py Fri Apr 12 11:39:10 2013 +0200 +++ b/__pkginfo__.py Mon Apr 15 13:18:01 2013 +0200 @@ -43,7 +43,7 @@ 'logilab-common': '>= 0.59.0', 'logilab-mtconverter': '>= 0.8.0', 'rql': '>= 0.31.2', - 'yams': '>= 0.36.0', + 'yams': '>= 0.37.0', #gettext # for xgettext, msgcat, etc... # web dependancies 'simplejson': '>= 2.0.9', diff -r 7920f439f383 -r be91151107f6 cwconfig.py --- a/cwconfig.py Fri Apr 12 11:39:10 2013 +0200 +++ b/cwconfig.py Mon Apr 15 13:18:01 2013 +0200 @@ -565,19 +565,27 @@ todo.append(depcube) return cubes - @classmethod - def reorder_cubes(cls, cubes): + def reorder_cubes(self, cubes): """reorder cubes from the top level cubes to inner dependencies cubes """ from logilab.common.graph import ordered_nodes, UnorderableGraph + # See help string for 'ui-cube' in web/webconfig.py for the reasons + # behind this hack. + uicube = self.get('ui-cube', None) graph = {} + if uicube: + graph[uicube] = set() for cube in cubes: cube = CW_MIGRATION_MAP.get(cube, cube) - graph[cube] = set(dep for dep in cls.cube_dependencies(cube) + graph[cube] = set(dep for dep in self.cube_dependencies(cube) if dep in cubes) - graph[cube] |= set(dep for dep in cls.cube_recommends(cube) + graph[cube] |= set(dep for dep in self.cube_recommends(cube) if dep in cubes) + if uicube and cube != uicube \ + and cube not in self.cube_dependencies(uicube) \ + and cube not in self.cube_recommends(uicube): + graph[cube].add(uicube) try: return ordered_nodes(graph) except UnorderableGraph as ex: diff -r 7920f439f383 -r be91151107f6 cwctl.py --- a/cwctl.py Fri Apr 12 11:39:10 2013 +0200 +++ b/cwctl.py Mon Apr 15 13:18:01 2013 +0200 @@ -205,9 +205,12 @@ class ListCommand(Command): """List configurations, cubes and instances. - list available configurations, installed cubes, and registered instances + List available configurations, installed cubes, and registered instances. + + If given, the optional argument allows to restrict listing only a category of items. """ name = 'list' + arguments = '[all|cubes|configurations|instances]' options = ( ('verbose', {'short': 'v', 'action' : 'store_true', @@ -216,92 +219,107 @@ def run(self, args): """run the command with its specific arguments""" - if args: + if not args: + mode = 'all' + elif len(args) == 1: + mode = args[0] + else: raise BadCommandUsage('Too many arguments') + from cubicweb.migration import ConfigurationProblem - print 'CubicWeb %s (%s mode)' % (cwcfg.cubicweb_version(), cwcfg.mode) - print - print 'Available configurations:' - for config in CONFIGURATIONS: - print '*', config.name - for line in config.__doc__.splitlines(): - line = line.strip() - if not line: - continue - print ' ', line - print - cfgpb = ConfigurationProblem(cwcfg) - try: - cubesdir = pathsep.join(cwcfg.cubes_search_path()) - namesize = max(len(x) for x in cwcfg.available_cubes()) - except ConfigurationError as ex: - print 'No cubes available:', ex - except ValueError: - print 'No cubes available in %s' % cubesdir - else: - print 'Available cubes (%s):' % cubesdir - for cube in cwcfg.available_cubes(): - try: - tinfo = cwcfg.cube_pkginfo(cube) - tversion = tinfo.version - cfgpb.add_cube(cube, tversion) - except (ConfigurationError, AttributeError) as ex: - tinfo = None - tversion = '[missing cube information: %s]' % ex - print '* %s %s' % (cube.ljust(namesize), tversion) - if self.config.verbose: - if tinfo: - descr = getattr(tinfo, 'description', '') - if not descr: - descr = getattr(tinfo, 'short_desc', '') + + if mode == 'all': + print 'CubicWeb %s (%s mode)' % (cwcfg.cubicweb_version(), cwcfg.mode) + print + + if mode in ('all', 'config', 'configurations'): + print 'Available configurations:' + for config in CONFIGURATIONS: + print '*', config.name + for line in config.__doc__.splitlines(): + line = line.strip() + if not line: + continue + print ' ', line + print + + if mode in ('all', 'cubes'): + cfgpb = ConfigurationProblem(cwcfg) + try: + cubesdir = pathsep.join(cwcfg.cubes_search_path()) + namesize = max(len(x) for x in cwcfg.available_cubes()) + except ConfigurationError as ex: + print 'No cubes available:', ex + except ValueError: + print 'No cubes available in %s' % cubesdir + else: + print 'Available cubes (%s):' % cubesdir + for cube in cwcfg.available_cubes(): + try: + tinfo = cwcfg.cube_pkginfo(cube) + tversion = tinfo.version + cfgpb.add_cube(cube, tversion) + except (ConfigurationError, AttributeError) as ex: + tinfo = None + tversion = '[missing cube information: %s]' % ex + print '* %s %s' % (cube.ljust(namesize), tversion) + if self.config.verbose: + if tinfo: + descr = getattr(tinfo, 'description', '') + if not descr: + descr = getattr(tinfo, 'short_desc', '') + if descr: + warn('[3.8] short_desc is deprecated, update %s' + ' pkginfo' % cube, DeprecationWarning) + else: + descr = tinfo.__doc__ if descr: - warn('[3.8] short_desc is deprecated, update %s' - ' pkginfo' % cube, DeprecationWarning) - else: - descr = tinfo.__doc__ - if descr: - print ' '+ ' \n'.join(descr.splitlines()) - modes = detect_available_modes(cwcfg.cube_dir(cube)) - print ' available modes: %s' % ', '.join(modes) - print - try: - regdir = cwcfg.instances_dir() - except ConfigurationError as ex: - print 'No instance available:', ex + print ' '+ ' \n'.join(descr.splitlines()) + modes = detect_available_modes(cwcfg.cube_dir(cube)) + print ' available modes: %s' % ', '.join(modes) print - return - instances = list_instances(regdir) - if instances: - print 'Available instances (%s):' % regdir - for appid in instances: - modes = cwcfg.possible_configurations(appid) - if not modes: - print '* %s (BROKEN instance, no configuration found)' % appid - continue - print '* %s (%s)' % (appid, ', '.join(modes)) - try: - config = cwcfg.config_for(appid, modes[0]) - except Exception as exc: - print ' (BROKEN instance, %s)' % exc - continue - else: - print 'No instance available in %s' % regdir - print - # configuration management problem solving - cfgpb.solve() - if cfgpb.warnings: - print 'Warnings:\n', '\n'.join('* '+txt for txt in cfgpb.warnings) - if cfgpb.errors: - print 'Errors:' - for op, cube, version, src in cfgpb.errors: - if op == 'add': - print '* cube', cube, - if version: - print ' version', version, - print 'is not installed, but required by %s' % src - else: - print '* cube %s version %s is installed, but version %s is required by %s' % ( - cube, cfgpb.cubes[cube], version, src) + + if mode in ('all', 'instances'): + try: + regdir = cwcfg.instances_dir() + except ConfigurationError as ex: + print 'No instance available:', ex + print + return + instances = list_instances(regdir) + if instances: + print 'Available instances (%s):' % regdir + for appid in instances: + modes = cwcfg.possible_configurations(appid) + if not modes: + print '* %s (BROKEN instance, no configuration found)' % appid + continue + print '* %s (%s)' % (appid, ', '.join(modes)) + try: + config = cwcfg.config_for(appid, modes[0]) + except Exception as exc: + print ' (BROKEN instance, %s)' % exc + continue + else: + print 'No instance available in %s' % regdir + print + + if mode == 'all': + # configuration management problem solving + cfgpb.solve() + if cfgpb.warnings: + print 'Warnings:\n', '\n'.join('* '+txt for txt in cfgpb.warnings) + if cfgpb.errors: + print 'Errors:' + for op, cube, version, src in cfgpb.errors: + if op == 'add': + print '* cube', cube, + if version: + print ' version', version, + print 'is not installed, but required by %s' % src + else: + print '* cube %s version %s is installed, but version %s is required by %s' % ( + cube, cfgpb.cubes[cube], version, src) def check_options_consistency(config): if config.automatic and config.config_level > 0: @@ -347,6 +365,12 @@ ' "list" command. Default to "all-in-one", e.g. an installation ' 'embedding both the RQL repository and the web server.', }), + ('no-post-create', + {'short': 'P', + 'action': 'store_true', + 'default': False, + 'help': 'do not run post-create tasks (database creation, etc.)', + }), ) def run(self, args): @@ -415,7 +439,8 @@ print 'set %s as owner of the data directory' % config['uid'] chown(config.appdatahome, config['uid']) print '\n-> creation done for %s\n' % repr(config.apphome)[1:-1] - helper.postcreate(self.config.automatic) + if not self.config.no_post_create: + helper.postcreate(self.config.automatic) def _handle_win32(self, config, appid): if sys.platform != 'win32': diff -r 7920f439f383 -r be91151107f6 dataimport.py --- a/dataimport.py Fri Apr 12 11:39:10 2013 +0200 +++ b/dataimport.py Mon Apr 15 13:18:01 2013 +0200 @@ -72,6 +72,7 @@ import traceback import cPickle import os.path as osp +import inspect from collections import defaultdict from contextlib import contextmanager from copy import copy @@ -323,7 +324,6 @@ return [(k, len(v)) for k, v in buckets.items() if k is not None and len(v) > 1] - # sql generator utility functions ############################################# @@ -431,7 +431,14 @@ # If an error is raised, do not continue. formatted_row = [] for col in columns: - value = row[col] + if isinstance(row, dict): + value = row.get(col) + elif isinstance(row, (tuple, list)): + value = row[col] + else: + raise ValueError("Input data of improper type: %s; " + "expected tuple, list or dict." + % type(row).__name__) if value is None: value = 'NULL' elif isinstance(value, (long, int, float)): @@ -506,7 +513,7 @@ item['eid'] = data['eid'] return item - def relate(self, eid_from, rtype, eid_to, inlined=False): + def relate(self, eid_from, rtype, eid_to, **kwargs): """Add new relation""" relation = eid_from, rtype, eid_to self.relations.add(relation) @@ -523,6 +530,18 @@ """ pass + def flush(self): + """The method is provided so that all stores share a common API. + It just tries to call the commit method. + """ + print 'starting flush' + try: + self.commit() + except: + print 'failed to flush' + else: + print 'flush done' + def rql(self, *args): if self._rql is not None: return self._rql(*args) @@ -538,76 +557,6 @@ def nb_inserted_relations(self): return len(self.relations) - @deprecated("[3.7] index support will disappear") - def build_index(self, name, type, func=None, can_be_empty=False): - """build internal index for further search""" - index = {} - if func is None or not callable(func): - func = lambda x: x['eid'] - for eid in self.types[type]: - index.setdefault(func(self.eids[eid]), []).append(eid) - if not can_be_empty: - assert index, "new index '%s' cannot be empty" % name - self.indexes[name] = index - - @deprecated("[3.7] index support will disappear") - def build_rqlindex(self, name, type, key, rql, rql_params=False, - func=None, can_be_empty=False): - """build an index by rql query - - rql should return eid in first column - ctl.store.build_index('index_name', 'users', 'login', 'Any U WHERE U is CWUser') - """ - self.types[type] = [] - rset = self.rql(rql, rql_params or {}) - if not can_be_empty: - assert rset, "new index type '%s' cannot be empty (0 record found)" % type - for entity in rset.entities(): - getattr(entity, key) # autopopulate entity with key attribute - self.eids[entity.eid] = dict(entity) - if entity.eid not in self.types[type]: - self.types[type].append(entity.eid) - - # Build index with specified key - func = lambda x: x[key] - self.build_index(name, type, func, can_be_empty=can_be_empty) - - @deprecated("[3.7] index support will disappear") - def fetch(self, name, key, unique=False, decorator=None): - """index fetcher method - - decorator is a callable method or an iterator of callable methods (usually a lambda function) - decorator=lambda x: x[:1] (first value is returned) - decorator=lambda x: x.lower (lowercased value is returned) - - decorator is handy when you want to improve index keys but without - changing the original field - - Same check functions can be reused here. - """ - eids = self.indexes[name].get(key, []) - if decorator is not None: - if not hasattr(decorator, '__iter__'): - decorator = (decorator,) - for f in decorator: - eids = f(eids) - if unique: - assert len(eids) == 1, u'expected a single one value for key "%s" in index "%s". Got %i' % (key, name, len(eids)) - eids = eids[0] - return eids - - @deprecated("[3.7] index support will disappear") - def find(self, type, key, value): - for idx in self.types[type]: - item = self.items[idx] - if item[key] == value: - yield item - - @deprecated("[3.7] checkpoint() deprecated. use commit() instead") - def checkpoint(self): - self.commit() - - class RQLObjectStore(ObjectStore): """ObjectStore that works with an actual RQL repository (production mode)""" _rql = None # bw compat @@ -630,10 +579,6 @@ self.session = session self._commit = commit or session.commit - @deprecated("[3.7] checkpoint() deprecated. use commit() instead") - def checkpoint(self): - self.commit() - def commit(self): txuuid = self._commit() self.session.set_cnxset() @@ -657,9 +602,9 @@ for k in item) return self.rql(query, item)[0][0] - def relate(self, eid_from, rtype, eid_to, inlined=False): + def relate(self, eid_from, rtype, eid_to, **kwargs): eid_from, rtype, eid_to = super(RQLObjectStore, self).relate( - eid_from, rtype, eid_to) + eid_from, rtype, eid_to, **kwargs) self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, {'x': int(eid_from), 'y': int(eid_to)}) @@ -809,8 +754,8 @@ self._nb_inserted_relations = 0 self.rql = session.execute # deactivate security - session.set_read_security(False) - session.set_write_security(False) + session.read_security = False + session.write_security = False def create_entity(self, etype, **kwargs): for k, v in kwargs.iteritems(): @@ -825,20 +770,23 @@ session = self.session self.source.add_entity(session, entity) self.source.add_info(session, entity, self.source, None, complete=False) + kwargs = dict() + if inspect.getargspec(self.add_relation).keywords: + kwargs['subjtype'] = entity.__regid__ for rtype, targeteids in rels.iteritems(): # targeteids may be a single eid or a list of eids inlined = self.rschema(rtype).inlined try: for targeteid in targeteids: self.add_relation(session, entity.eid, rtype, targeteid, - inlined) + inlined, **kwargs) except TypeError: self.add_relation(session, entity.eid, rtype, targeteids, - inlined) + inlined, **kwargs) self._nb_inserted_entities += 1 return entity - def relate(self, eid_from, rtype, eid_to): + def relate(self, eid_from, rtype, eid_to, **kwargs): assert not rtype.startswith('reverse_') self.add_relation(self.session, eid_from, rtype, eid_to, self.rschema(rtype).inlined) @@ -962,12 +910,12 @@ """Flush data to the database""" self.source.flush() - def relate(self, subj_eid, rtype, obj_eid, subjtype=None): + def relate(self, subj_eid, rtype, obj_eid, **kwargs): if subj_eid is None or obj_eid is None: return # XXX Could subjtype be inferred ? self.source.add_relation(self.session, subj_eid, rtype, obj_eid, - self.rschema(rtype).inlined, subjtype) + self.rschema(rtype).inlined, **kwargs) def drop_indexes(self, etype): """Drop indexes for a given entity type""" @@ -1081,18 +1029,20 @@ encoding=self.dbencoding) except: print 'failed to flush' + else: + print 'flush done' finally: _entities_sql.clear() _relations_sql.clear() _insertdicts.clear() _inlined_relations_sql.clear() - print 'flush done' def add_relation(self, session, subject, rtype, object, - inlined=False, subjtype=None): + inlined=False, **kwargs): if inlined: _sql = self._sql.inlined_relations data = {'cw_eid': subject, SQL_PREFIX + rtype: object} + subjtype = kwargs.get('subjtype') if subjtype is None: # Try to infer it targets = [t.type for t in @@ -1102,7 +1052,9 @@ else: raise ValueError('You should give the subject etype for ' 'inlined relation %s' - ', as it cannot be inferred' % rtype) + ', as it cannot be inferred: ' + 'this type is given as keyword argument ' + '``subjtype``'% rtype) statement = self.sqlgen.update(SQL_PREFIX + subjtype, data, ['cw_eid']) else: diff -r 7920f439f383 -r be91151107f6 debian/control --- a/debian/control Fri Apr 12 11:39:10 2013 +0200 +++ b/debian/control Mon Apr 15 13:18:01 2013 +0200 @@ -16,7 +16,7 @@ python-unittest2, python-logilab-mtconverter, python-rql, - python-yams, + python-yams (>= 0.37), python-lxml, Standards-Version: 3.9.1 Homepage: http://www.cubicweb.org diff -r 7920f439f383 -r be91151107f6 devtools/fake.py --- a/devtools/fake.py Fri Apr 12 11:39:10 2013 +0200 +++ b/devtools/fake.py Mon Apr 15 13:18:01 2013 +0200 @@ -163,10 +163,6 @@ # for use with enabled_security context manager read_security = write_security = True - def init_security(self, *args): - return None, None - def reset_security(self, *args): - return class FakeRepo(object): querier = None diff -r 7920f439f383 -r be91151107f6 devtools/repotest.py --- a/devtools/repotest.py Fri Apr 12 11:39:10 2013 +0200 +++ b/devtools/repotest.py Mon Apr 15 13:18:01 2013 +0200 @@ -262,8 +262,8 @@ u = self.repo._build_user(self.session, self.session.user.eid) u._groups = set(groups) s = Session(u, self.repo) - s._threaddata.cnxset = self.cnxset - s._threaddata.ctx_count = 1 + s._tx.cnxset = self.cnxset + s._tx.ctx_count = 1 # register session to ensure it gets closed self._dumb_sessions.append(s) return s @@ -311,7 +311,8 @@ del self.repo.sources_by_uri[source.uri] undo_monkey_patch() for session in self._dumb_sessions: - session._threaddata.cnxset = None + if session._tx.cnxset is not None: + session._tx.cnxset = None session.close() def _prepare_plan(self, rql, kwargs=None): diff -r 7920f439f383 -r be91151107f6 doc/3.17.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/3.17.rst Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,40 @@ +What's new in CubicWeb 3.17? +============================ + +New functionalities +-------------------- + +* add a command to compare db schema and file system schema + (see `#464991 `_) + +* Add CubicWebRequestBase.content with the content of the HTTP request (see #2742453) + (see `#2742453 `_) + +* Add directive bookmark to ReST rendering + (see `#2545595 `_) + + +API changes +----------- + +* drop typed_eid() in favour of int() (see `#2742462 `_) + +* The SIOC views and adapters have been removed from CubicWeb and moved to the + `sioc` cube. + +* The web page embedding views and adapters have been removed from CubicWeb and + moved to the `embed` cube. + +* The email sending views and controllers have been removed from CubicWeb and + moved to the `massmailing` cube. + + + +Deprecated Code Drops +---------------------- + +* The progress views and adapters have been removed from CubicWeb. These + classes were deprecated since 3.14.0. They are still available in the + `iprogress` cube. + +* API deprecated since 3.7 have been dropped. diff -r 7920f439f383 -r be91151107f6 doc/book/en/devrepo/repo/sessions.rst --- a/doc/book/en/devrepo/repo/sessions.rst Fri Apr 12 11:39:10 2013 +0200 +++ b/doc/book/en/devrepo/repo/sessions.rst Mon Apr 15 13:18:01 2013 +0200 @@ -199,3 +199,8 @@ if hasattr(req.cnx, 'foo_user') and req.foo_user: return 1 return 0 + +Full API Session +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: cubicweb.server.session.Session diff -r 7920f439f383 -r be91151107f6 doc/book/en/intro/concepts.rst --- a/doc/book/en/intro/concepts.rst Fri Apr 12 11:39:10 2013 +0200 +++ b/doc/book/en/intro/concepts.rst Mon Apr 15 13:18:01 2013 +0200 @@ -29,8 +29,7 @@ .. note:: - The command :command:`cubicweb-ctl list` displays the list of cubes - installed on your system. + The command :command:`cubicweb-ctl list` displays the list of available cubes. .. _`CubicWeb.org Forge`: http://www.cubicweb.org/project/ .. _`cubicweb-blog`: http://www.cubicweb.org/project/cubicweb-blog @@ -89,7 +88,7 @@ state of an object changes. See :ref:`HookIntro` below. .. [1] not to be confused with a Mercurial repository or a Debian repository. -.. _`Python Remote Objects`: http://pyro.sourceforge.net/ +.. _`Python Remote Objects`: http://pythonhosted.org/Pyro4/ .. _WebEngineIntro: diff -r 7920f439f383 -r be91151107f6 doc/book/en/intro/history.rst --- a/doc/book/en/intro/history.rst Fri Apr 12 11:39:10 2013 +0200 +++ b/doc/book/en/intro/history.rst Mon Apr 15 13:18:01 2013 +0200 @@ -28,5 +28,5 @@ and energy originally put in the design of the framework. -.. _Narval: http://www.logilab.org/project/narval +.. _Narval: http://www.logilab.org/project/narval-moved .. _Logilab: http://www.logilab.fr/ diff -r 7920f439f383 -r be91151107f6 doc/book/en/tutorials/base/conclusion.rst --- a/doc/book/en/tutorials/base/conclusion.rst Fri Apr 12 11:39:10 2013 +0200 +++ b/doc/book/en/tutorials/base/conclusion.rst Mon Apr 15 13:18:01 2013 +0200 @@ -3,7 +3,7 @@ What's next? ------------ -In this tutorial, we have seen have you can, right after the installation of +In this tutorial, we have seen that you can, right after the installation of |cubicweb|, build a web application in a few minutes by defining a data model as assembling cubes. You get a working application that you can then customize there and there while keeping something that works. This is important in agile diff -r 7920f439f383 -r be91151107f6 doc/book/en/tutorials/index.rst --- a/doc/book/en/tutorials/index.rst Fri Apr 12 11:39:10 2013 +0200 +++ b/doc/book/en/tutorials/index.rst Mon Apr 15 13:18:01 2013 +0200 @@ -18,3 +18,4 @@ base/index advanced/index tools/windmill.rst + textreports/index diff -r 7920f439f383 -r be91151107f6 doc/book/en/tutorials/textreports/index.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/book/en/tutorials/textreports/index.rst Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,13 @@ +.. -*- coding: utf-8 -*- + +Writing text reports with RestructuredText +========================================== + +|cubicweb| offers several text formats for the RichString type used in schemas, +including restructuredtext. + +Three additional restructuredtext roles are defined by |cubicweb|: + +.. autodocfunction:: cubicweb.ext.rest.eid_reference_role +.. autodocfunction:: cubicweb.ext.rest.rql_role +.. autodocfunction:: cubicweb.ext.rest.bookmark_role diff -r 7920f439f383 -r be91151107f6 doc/tools/pyjsrest.py --- a/doc/tools/pyjsrest.py Fri Apr 12 11:39:10 2013 +0200 +++ b/doc/tools/pyjsrest.py Mon Apr 15 13:18:01 2013 +0200 @@ -136,7 +136,6 @@ 'cubicweb.preferences', 'cubicweb.edition', 'cubicweb.reledit', - 'cubicweb.iprogress', 'cubicweb.rhythm', 'cubicweb.gmap', 'cubicweb.timeline-ext', diff -r 7920f439f383 -r be91151107f6 doc/tutorials/dataimport/data_import_tutorial.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/tutorials/dataimport/data_import_tutorial.rst Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,646 @@ +Importing relational data into a CubicWeb instance +================================================== + +Introduction +~~~~~~~~~~~~ + +This tutorial explains how to import data from an external source (e.g. a collection of files) +into a CubicWeb cube instance. + +First, once we know the format of the data we wish to import, we devise a +*data model*, that is, a CubicWeb (Yams) schema which reflects the way the data +is structured. This schema is implemented in the ``schema.py`` file. +In this tutorial, we will describe such a schema for a particular data set, +the Diseasome data (see below). + +Once the schema is defined, we create a cube and an instance. +The cube is a specification of an application, whereas an instance +is the application per se. + +Once the schema is defined and the instance is created, the import can be performed, via +the following steps: + +1. Build a custom parser for the data to be imported. Thus, one obtains a Python + memory representation of the data. + +2. Map the parsed data to the data model defined in ``schema.py``. + +3. Perform the actual import of the data. This comes down to "populating" + the data model with the memory representation obtained at 1, according to + the mapping defined at 2. + +This tutorial illustrates all the above steps in the context of relational data +stored in the RDF format. + +More specifically, we describe the import of Diseasome_ RDF/OWL data. + +.. _Diseasome: http://datahub.io/dataset/fu-berlin-diseasome + +Building a data model +~~~~~~~~~~~~~~~~~~~~~ + +The first thing to do when using CubicWeb for creating an application from scratch +is to devise a *data model*, that is, a relational representation of the problem to be +modeled or of the structure of the data to be imported. + +In such a schema, we define +an entity type (``EntityType`` objects) for each type of entity to import. Each such type +has several attributes. If the attributes are of known CubicWeb (Yams) types, viz. numbers, +strings or characters, then they are defined as attributes, as e.g. ``attribute = Int()`` +for an attribute named ``attribute`` which is an integer. + +Each such type also has a set of +relations, which are defined like the attributes, except that they represent, in fact, +relations between the entities of the type under discussion and the objects of a type which +is specified in the relation definition. + +For example, for the Diseasome data, we have two types of entities, genes and diseases. +Thus, we create two classes which inherit from ``EntityType``:: + + class Disease(EntityType): + # Corresponds to http://www.w3.org/2000/01/rdf-schema#label + label = String(maxsize=512, fulltextindexed=True) + ... + + #Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/diseasome/associatedGene + associated_genes = SubjectRelation('Gene', cardinality='**') + ... + + #Corresponds to 'http://www4.wiwiss.fu-berlin.de/diseasome/resource/diseasome/chromosomalLocation' + chromosomal_location = SubjectRelation('ExternalUri', cardinality='?*', inlined=True) + + + class Gene(EntityType): + ... + +In this schema, there are attributes whose values are numbers or strings. Thus, they are +defined by using the CubicWeb / Yams primitive types, e.g., ``label = String(maxsize=12)``. +These types can have several constraints or attributes, such as ``maxsize``. +There are also relations, either between the entity types themselves, or between them +and a CubicWeb type, ``ExternalUri``. The latter defines a class of URI objects in +CubicWeb. For instance, the ``chromosomal_location`` attribute is a relation between +a ``Disease`` entity and an ``ExternalUri`` entity. The relation is marked by the CubicWeb / +Yams ``SubjectRelation`` method. The latter can have several optional keyword arguments, such as +``cardinality`` which specifies the number of subjects and objects related by the relation type +specified. For example, the ``'?*'`` cardinality in the ``chromosomal_relation`` relation type says +that zero or more ``Disease`` entities are related to zero or one ``ExternalUri`` entities. +In other words, a ``Disease`` entity is related to at most one ``ExternalUri`` entity via the +``chromosomal_location`` relation type, and that we can have zero or more ``Disease`` entities in the +data base. +For a relation between the entity types themselves, the ``associated_genes`` between a ``Disease`` +entity and a ``Gene`` entity is defined, so that any number of ``Gene`` entities can be associated +to a ``Disease``, and there can be any number of ``Disease`` s if a ``Gene`` exists. + +Of course, before being able to use the CubicWeb / Yams built-in objects, we need to import them:: + + + from yams.buildobjs import EntityType, SubjectRelation, String, Int + from cubicweb.schemas.base import ExternalUri + +Building a custom data parser +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The data we wish to import is structured in the RDF format, +as a text file containing a set of lines. +On each line, there are three fields. +The first two fields are URIs ("Universal Resource Identifiers"). +The third field is either an URI or a string. Each field bares a particular meaning: + +- the leftmost field is an URI that holds the entity to be imported. + Note that the entities defined in the data model (i.e., in ``schema.py``) should + correspond to the entities whose URIs are specified in the import file. + +- the middle field is an URI that holds a relation whose subject is the entity + defined by the leftmost field. Note that this should also correspond + to the definitions in the data model. + +- the rightmost field is either an URI or a string. When this field is an URI, + it gives the object of the relation defined by the middle field. + When the rightmost field is a string, the middle field is interpreted as an attribute + of the subject (introduced by the leftmost field) and the rightmost field is + interpreted as the value of the attribute. + +Note however that some attributes (i.e. relations whose objects are strings) +have their objects defined as strings followed by ``^^`` and by another URI; +we ignore this part. + +Let us show some examples: + +- of line holding an attribute definition: + `` + "CYP17A1" .`` + The line contains the definition of the ``label`` attribute of an + entity of type ``gene``. The value of ``label`` is '``CYP17A1``'. + +- of line holding a relation definition: + `` + + .`` + The line contains the definition of the ``associatedGene`` relation between + a ``disease`` subject entity identified by ``1`` and a ``gene`` object + entity defined by ``HADH2``. + +Thus, for parsing the data, we can (:note: see the ``diseasome_parser`` module): + +1. define a couple of regular expressions for parsing the two kinds of lines, + ``RE_ATTS`` for parsing the attribute definitions, and ``RE_RELS`` for parsing + the relation definitions. + +2. define a function that iterates through the lines of the file and retrieves + (``yield`` s) a (subject, relation, object) tuple for each line. + We called it ``_retrieve_structure`` in the ``diseasome_parser`` module. + The function needs the file name and the types for which information + should be retrieved. + +Alternatively, instead of hand-making the parser, one could use the RDF parser provided +in the ``dataio`` cube. + +.. XXX To further study and detail the ``dataio`` cube usage. + +Once we get to have the (subject, relation, object) triples, we need to map them into +the data model. + + +Mapping the data to the schema +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the case of diseasome data, we can just define two dictionaries for mapping +the names of the relations as extracted by the parser, to the names of the relations +as defined in the ``schema.py`` data model. In the ``diseasome_parser`` module +they are called ``MAPPING_ATTS`` and ``MAPPING_RELS``. +Given that the relation and attribute names are given in CamelCase in the original data, +mappings are necessary if we follow the PEP08 when naming the attributes in the data model. +For example, the RDF relation ``chromosomalLocation`` is mapped into the schema relation +``chromosomal_location``. + +Once these mappings have been defined, we just iterate over the (subject, relation, object) +tuples provided by the parser and we extract the entities, with their attributes and relations. +For each entity, we thus have a dictionary with two keys, ``attributes`` and ``relations``. +The value associated to the ``attributes`` key is a dictionary containing (attribute: value) +pairs, where "value" is a string, plus the ``cwuri`` key / attribute holding the URI of +the entity itself. +The value associated to the ``relations`` key is a dictionary containing (relation: value) +pairs, where "value" is an URI. +This is implemented in the ``entities_from_rdf`` interface function of the module +``diseasome_parser``. This function provides an iterator on the dictionaries containing +the ``attributes`` and ``relations`` keys for all entities. + +However, this is a simple case. In real life, things can get much more complicated, and the +mapping can be far from trivial, especially when several data sources (which can follow +different formatting and even structuring conventions) must be mapped into the same data model. + +Importing the data +~~~~~~~~~~~~~~~~~~ + +The data import code should be placed in a Python module. Let us call it +``diseasome_import.py``. Then, this module should be called via +``cubicweb-ctl``, as follows:: + + cubicweb-ctl shell diseasome_import.py -- + +In the import module, we should use a *store* for doing the import. +A store is an object which provides three kinds of methods for +importing data: + +- a method for importing the entities, along with the values + of their attributes. +- a method for importing the relations between the entities. +- a method for committing the imports to the database. + +In CubicWeb, we have four stores: + +1. ``ObjectStore`` base class for the stores in CubicWeb. + It only provides a skeleton for all other stores and + provides the means for creating the memory structures + (dictionaries) that hold the entities and the relations + between them. + +2. ``RQLObjectStore``: store which uses the RQL language for performing + database insertions and updates. It relies on all the CubicWeb hooks + machinery, especially for dealing with security issues (database access + permissions). + +2. ``NoHookRQLObjectStore``: store which uses the RQL language for + performing database insertions and updates, but for which + all hooks are deactivated. This implies that + certain checks with respect to the CubicWeb / Yams schema + (data model) are not performed. However, all SQL queries + obtained from the RQL ones are executed in a sequential + manner, one query per inserted entity. + +4. ``SQLGenObjectStore``: store which uses the SQL language directly. + It inserts entities either sequentially, by executing an SQL query + for each entity, or directly by using one PostGRES ``COPY FROM`` + query for a set of similarly structured entities. + +For really massive imports (millions or billions of entities), there +is a cube ``dataio`` which contains another store, called +``MassiveObjectStore``. This store is similar to ``SQLGenObjectStore``, +except that anything related to CubicWeb is bypassed. That is, even the +CubicWeb EID entity identifiers are not handled. This store is the fastest, +but has a slightly different API from the other four stores mentioned above. +Moreover, it has an important limitation, in that it doesn't insert inlined [#]_ +relations in the database. + +.. [#] An inlined relation is a relation defined in the schema + with the keyword argument ``inlined=True``. Such a relation + is inserted in the database as an attribute of the entity + whose subject it is. + +In the following section we will see how to import data by using the stores +in CubicWeb's ``dataimport`` module. + +Using the stores in ``dataimport`` +++++++++++++++++++++++++++++++++++ + +``ObjectStore`` is seldom used in real life for importing data, since it is +only the base store for the other stores and it doesn't perform an actual +import of the data. Nevertheless, the other three stores, which import data, +are based on ``ObjectStore`` and provide the same API. + +All three stores ``RQLObjectStore``, ``NoHookRQLObjectStore`` and +``SQLGenObjectStore`` provide exactly the same API for importing data, that is +entities and relations, in an SQL database. + +Before using a store, one must import the ``dataimport`` module and then initialize +the store, with the current ``session`` as a parameter:: + + import cubicweb.dataimport as cwdi + ... + + store = cwdi.RQLObjectStore(session) + +Each such store provides three methods for data import: + +#. ``create_entity(Etype, **attributes)``, which allows us to add + an entity of the Yams type ``Etype`` to the database. This entity's attributes + are specified in the ``attributes`` dictionary. The method returns the entity + created in the database. For example, we add two entities, + a person, of ``Person`` type, and a location, of ``Location`` type:: + + person = store.create_entity('Person', name='Toto', age='18', height='190') + + location = store.create_entity('Location', town='Paris', arrondissement='13') + +#. ``relate(subject_eid, r_type, object_eid)``, which allows us to add a relation + of the Yams type ``r_type`` to the database. The relation's subject is an entity + whose EID is ``subject_eid``; its object is another entity, whose EID is + ``object_eid``. For example [#]_:: + + store.relate(person.eid(), 'lives_in', location.eid(), **kwargs) + + ``kwargs`` is only used by the ``SQLGenObjectStore``'s ``relate`` method and is here + to allow us to specify the type of the subject of the relation, when the relation is + defined as inlined in the schema. + +.. [#] The ``eid`` method of an entity defined via ``create_entity`` returns + the entity identifier as assigned by CubicWeb when creating the entity. + This only works for entities defined via the stores in the CubicWeb's + ``dataimport`` module. + + The keyword argument that is understood by ``SQLGenObjectStore`` is called + ``subjtype`` and holds the type of the subject entity. For the example considered here, + this comes to having [#]_:: + + store.relate(person.eid(), 'lives_in', location.eid(), subjtype=person.dc_type()) + + If ``subjtype`` is not specified, then the store tries to infer the type of the subject. + However, this doesn't always work, e.g. when there are several possible subject types + for a given relation type. + +.. [#] The ``dc_type`` method of an entity defined via ``create_entity`` returns + the type of the entity just created. This only works for entities defined via + the stores in the CubicWeb's ``dataimport`` module. In the example considered + here, ``person.dc_type()`` returns ``'Person'``. + + All the other stores but ``SQLGenObjectStore`` ignore the ``kwargs`` parameters. + +#. ``flush()``, which allows us to perform the actual commit into the database, along + with some cleanup operations. Ideally, this method should be called as often as + possible, that is after each insertion in the database, so that database sessions + are kept as atomic as possible. In practice, we usually call this method twice: + first, after all the entities have been created, second, after all relations have + been created. + + Note however that before each commit the database insertions + have to be consistent with the schema. Thus, if, for instance, + an entity has an attribute defined through a relation (viz. + a ``SubjectRelation``) with a ``"1"`` or ``"+"`` object + cardinality, we have to create the entity under discussion, + the object entity of the relation under discussion, and the + relation itself, before committing the additions to the database. + + The ``flush`` method is simply called as:: + + store.flush(). + + +Using the ``MassiveObjectStore`` in the ``dataio`` cube ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +This store, available in the ``dataio`` cube, allows us to +fully dispense with the CubicWeb import mechanisms and hence +to interact directly with the database server, via SQL queries. + +Moreover, these queries rely on PostGreSQL's ``COPY FROM`` instruction +to create several entities in a single query. This brings tremendous +performance improvements with respect to the RQL-based data insertion +procedures. + +However, the API of this store is slightly different from the API of +the stores in CubicWeb's ``dataimport`` module. + +Before using the store, one has to import the ``dataio`` cube's +``dataimport`` module, then initialize the store by giving it the +``session`` parameter:: + + from cubes.dataio import dataimport as mcwdi + ... + + store = mcwdi.MassiveObjectStore(session) + +The ``MassiveObjectStore`` provides six methods for inserting data +into the database: + +#. ``init_rtype_table(SubjEtype, r_type, ObjEtype)``, which specifies the + creation of the tables associated to the relation types in the database. + Each such table has three column, the type of the subject entity, the + type of the relation (that is, the name of the attribute in the subject + entity which is defined via the relation), and the type of the object + entity. For example:: + + store.init_rtype_table('Person', 'lives_in', 'Location') + + Please note that these tables can be created before the entities, since + they only specify their types, not their unique identifiers. + +#. ``create_entity(Etype, **attributes)``, which allows us to add new entities, + whose attributes are given in the ``attributes`` dictionary. + Please note however that, by default, this method does *not* return + the created entity. The method is called, for example, as in:: + + store.create_entity('Person', name='Toto', age='18', height='190', + uri='http://link/to/person/toto_18_190') + store.create_entity('Location', town='Paris', arrondissement='13', + uri='http://link/to/location/paris_13') + + In order to be able to link these entities via the relations when needed, + we must provide ourselves a means for uniquely identifying the entities. + In general, this is done via URIs, stored in attributes like ``uri`` or + ``cwuri``. The name of the attribute is irrelevant as long as its value is + unique for each entity. + +#. ``relate_by_iid(subject_iid, r_type, object_iid)`` allows us to actually + relate the entities uniquely identified by ``subject_iid`` and + ``object_iid`` via a relation of type ``r_type``. For example:: + + store.relate_by_iid('http://link/to/person/toto_18_190', + 'lives_in', + 'http://link/to/location/paris_13') + + Please note that this method does *not* work for inlined relations! + +#. ``convert_relations(SubjEtype, r_type, ObjEtype, subj_iid_attribute, + obj_iid_attribute)`` + allows us to actually insert + the relations in the database. At one call of this method, one inserts + all the relations of type ``rtype`` between entities of given types. + ``subj_iid_attribute`` and ``object_iid_attribute`` are the names + of the attributes which store the unique identifiers of the entities, + as assigned by the user. These names can be identical, as long as + their values are unique. For example, for inserting all relations + of type ``lives_in`` between ``People`` and ``Location`` entities, + we write:: + + store.convert_relations('Person', 'lives_in', 'Location', 'uri', 'uri') + +#. ``flush()`` performs the actual commit in the database. It only needs + to be called after ``create_entity`` and ``relate_by_iid`` calls. + Please note that ``relate_by_iid`` does *not* perform insertions into + the database, hence calling ``flush()`` for it would have no effect. + +#. ``cleanup()`` performs database cleanups, by removing temporary tables. + It should only be called at the end of the import. + + + +.. XXX to add smth on the store's parameter initialization. + + + +Application to the Diseasome data ++++++++++++++++++++++++++++++++++ + +Import setup +############ + +We define an import function, ``diseasome_import``, which does basically four things: + +#. creates and initializes the store to be used, via a line such as:: + + store = cwdi.SQLGenObjectStore(session) + + where ``cwdi`` is the imported ``cubicweb.dataimport`` or + ``cubes.dataio.dataimport``. + +#. calls the diseasome parser, that is, the ``entities_from_rdf`` function in the + ``diseasome_parser`` module and iterates on its result, in a line such as:: + + for entity, relations in parser.entities_from_rdf(filename, ('gene', 'disease')): + + where ``parser`` is the imported ``diseasome_parser`` module, and ``filename`` is the + name of the file containing the data (with its path), e.g. ``../data/diseasome_dump.nt``. + +#. creates the entities to be inserted in the database; for Diseasome, there are two + kinds of entities: + + #. entities defined in the data model, viz. ``Gene`` and ``Disease`` in our case. + #. entities which are built in CubicWeb / Yams, viz. ``ExternalUri`` which define + URIs. + + As we are working with RDF data, each entity is defined through a series of URIs. Hence, + each "relational attribute" [#]_ of an entity is defined via an URI, that is, in CubicWeb + terms, via an ``ExternalUri`` entity. The entities are created, in the loop presented above, + as such:: + + ent = store.create_entity(etype, **entity) + + where ``etype`` is the appropriate entity type, either ``Gene`` or ``Disease``. + +.. [#] By "relational attribute" we denote an attribute (of an entity) which + is defined through a relation, e.g. the ``chromosomal_location`` attribute + of ``Disease`` entities, which is defined through a relation between a + ``Disease`` and an ``ExternalUri``. + + The ``ExternalUri`` entities are as many as URIs in the data file. For them, we define a unique + attribute, ``uri``, which holds the URI under discussion:: + + extu = store.create_entity('ExternalUri', uri="http://path/of/the/uri") + +#. creates the relations between the entities. We have relations between: + + #. entities defined in the schema, e.g. between ``Disease`` and ``Gene`` + entities, such as the ``associated_genes`` relation defined for + ``Disease`` entities. + #. entities defined in the schema and ``ExternalUri`` entities, such as ``gene_id``. + + The way relations are added to the database depends on the store: + + - for the stores in the CubicWeb ``dataimport`` module, we only use + ``store.relate``, in + another loop, on the relations (that is, a + loop inside the preceding one, mentioned at step 2):: + + for rtype, rels in relations.iteritems(): + ... + + store.relate(ent.eid(), rtype, extu.eid(), **kwargs) + + where ``kwargs`` is a dictionary designed to accommodate the need for specifying + the type of the subject entity of the relation, when the relation is inlined and + ``SQLGenObjectStore`` is used. For example:: + + ... + store.relate(ent.eid(), 'chromosomal_location', extu.eid(), subjtype='Disease') + + - for the ``MassiveObjectStore`` in the ``dataio`` cube's ``dataimport`` module, + the relations are created in three steps: + + #. first, a table is created for each relation type, as in:: + + ... + store.init_rtype_table(ent.dc_type(), rtype, extu.dc_type()) + + which comes down to lines such as:: + + store.init_rtype_table('Disease', 'associated_genes', 'Gene') + store.init_rtype_table('Gene', 'gene_id', 'ExternalUri') + + #. second, the URI of each entity will be used as its identifier, in the + ``relate_by_iid`` method, such as:: + + disease_uri = 'http://www4.wiwiss.fu-berlin.de/diseasome/resource/diseases/3' + gene_uri = '. + +"""This module imports the Diseasome data into a CubicWeb instance. +""" + +# Python imports +import sys +import argparse + +# Logilab import, for timing +from logilab.common.decorators import timed + +# CubicWeb imports +import cubicweb.dataimport as cwdi +from cubes.dataio import dataimport as mcwdi + +# Diseasome parser import +import diseasome_parser as parser + +def _is_of_class(instance, class_name): + """Helper function to determine whether an instance is + of a specified class or not. + Returns a True if this is the case and False otherwise. + """ + if instance.__class__.__name__ == class_name: + return True + else: + return False + +@timed +def diseasome_import(session, file_name, store): + """Main function for importing Diseasome data. + + It uses the Diseasome data parser to get the contents of the + data from a file, then uses a store for importing the data + into a CubicWeb instance. + + >>> diseasome_import(session, 'file_name', Store) + + """ + exturis = dict(session.execute('Any U, X WHERE X is ExternalUri, X uri U')) + uri_to_eid = {} + uri_to_etype = {} + all_relations = {} + etypes = {('http://www4.wiwiss.fu-berlin.de/' + 'diseasome/resource/diseasome/genes'): 'Gene', + ('http://www4.wiwiss.fu-berlin.de/' + 'diseasome/resource/diseasome/diseases'): 'Disease'} + # Read the parsed data + for entity, relations in parser.entities_from_rdf(file_name, + ('gene', 'disease')): + uri = entity.get('cwuri', None) + types = list(relations.get('types', [])) + if not types: + continue + etype = etypes.get(types[0]) + if not etype: + sys.stderr.write('Entity type %s not recognized.', types[0]) + sys.stderr.flush() + if _is_of_class(store, 'MassiveObjectStore'): + for relation in (set(relations).intersection(('classes', + 'possible_drugs', 'omim', 'omim_page', + 'chromosomal_location', 'same_as', 'gene_id', + 'hgnc_id', 'hgnc_page'))): + store.init_rtype_table(etype, relation, 'ExternalUri') + for relation in set(relations).intersection(('subtype_of',)): + store.init_rtype_table(etype, relation, 'Disease') + for relation in set(relations).intersection(('associated_genes',)): + store.init_rtype_table(etype, relation, 'Gene') + # Create the entities + ent = store.create_entity(etype, **entity) + if not _is_of_class(store, 'MassiveObjectStore'): + uri_to_eid[uri] = ent.eid + uri_to_etype[uri] = ent.dc_type() + else: + uri_to_eid[uri] = uri + uri_to_etype[uri] = etype + # Store relations for after + all_relations[uri] = relations + # Perform a first commit, of the entities + store.flush() + kwargs = {} + for uri, relations in all_relations.iteritems(): + from_eid = uri_to_eid.get(uri) + # ``subjtype`` should be initialized if ``SQLGenObjectStore`` is used + # and there are inlined relations in the schema. + # If ``subjtype`` is not given, while ``SQLGenObjectStore`` is used + # and there are inlined relations in the schema, the store + # tries to infer the type of the subject, but this does not always + # work, e.g. when there are several object types for the relation. + # ``subjtype`` is ignored for other stores, or if there are no + # inlined relations in the schema. + kwargs['subjtype'] = uri_to_etype.get(uri) + if not from_eid: + continue + for rtype, rels in relations.iteritems(): + if rtype in ('classes', 'possible_drugs', 'omim', 'omim_page', + 'chromosomal_location', 'same_as', 'gene_id', + 'hgnc_id', 'hgnc_page'): + for rel in list(rels): + if rel not in exturis: + # Create the "ExternalUri" entities, which are the + # objects of the relations + extu = store.create_entity('ExternalUri', uri=rel) + if not _is_of_class(store, 'MassiveObjectStore'): + rel_eid = extu.eid + else: + # For the "MassiveObjectStore", the EIDs are + # in fact the URIs. + rel_eid = rel + exturis[rel] = rel_eid + else: + rel_eid = exturis[rel] + # Create the relations that have "ExternalUri"s as objects + if not _is_of_class(store, 'MassiveObjectStore'): + store.relate(from_eid, rtype, rel_eid, **kwargs) + else: + store.relate_by_iid(from_eid, rtype, rel_eid) + elif rtype in ('subtype_of', 'associated_genes'): + for rel in list(rels): + to_eid = uri_to_eid.get(rel) + if to_eid: + # Create relations that have objects of other type + # than "ExternalUri" + if not _is_of_class(store, 'MassiveObjectStore'): + store.relate(from_eid, rtype, to_eid, **kwargs) + else: + store.relate_by_iid(from_eid, rtype, to_eid) + else: + sys.stderr.write('Missing entity with URI %s ' + 'for relation %s' % (rel, rtype)) + sys.stderr.flush() + # Perform a second commit, of the "ExternalUri" entities. + # when the stores in the CubicWeb ``dataimport`` module are used, + # relations are also committed. + store.flush() + # If the ``MassiveObjectStore`` is used, then entity and relation metadata + # are pushed as well. By metadata we mean information on the creation + # time and author. + if _is_of_class(store, 'MassiveObjectStore'): + store.flush_meta_data() + for relation in ('classes', 'possible_drugs', 'omim', 'omim_page', + 'chromosomal_location', 'same_as'): + # Afterwards, relations are actually created in the database. + store.convert_relations('Disease', relation, 'ExternalUri', + 'cwuri', 'uri') + store.convert_relations('Disease', 'subtype_of', 'Disease', + 'cwuri', 'cwuri') + store.convert_relations('Disease', 'associated_genes', 'Gene', + 'cwuri', 'cwuri') + for relation in ('gene_id', 'hgnc_id', 'hgnc_page', 'same_as'): + store.convert_relations('Gene', relation, 'ExternalUri', + 'cwuri', 'uri') + # Clean up temporary tables in the database + store.cleanup() + +if __name__ == '__main__': + # Change sys.argv so that ``cubicweb-ctl shell`` can work out the options + # we give to our ``diseasome_import.py`` script. + sys.argv = [arg for + arg in sys.argv[sys.argv.index("--") - 1:] if arg != "--"] + PARSER = argparse.ArgumentParser(description="Import Diseasome data") + PARSER.add_argument("-df", "--datafile", type=str, + help="RDF data file name") + PARSER.add_argument("-st", "--store", type=str, + default="RQLObjectStore", + help="data import store") + ARGS = PARSER.parse_args() + if ARGS.datafile: + FILENAME = ARGS.datafile + if ARGS.store in (st + "ObjectStore" for + st in ("RQL", "NoHookRQL", "SQLGen")): + IMPORT_STORE = getattr(cwdi, ARGS.store)(session) + elif ARGS.store == "MassiveObjectStore": + IMPORT_STORE = mcwdi.MassiveObjectStore(session) + else: + sys.exit("Import store unknown") + diseasome_import(session, FILENAME, IMPORT_STORE) + else: + sys.exit("Data file not found or not specified") diff -r 7920f439f383 -r be91151107f6 doc/tutorials/dataimport/diseasome_parser.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/tutorials/dataimport/diseasome_parser.py Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +""" +Diseasome data import module. +Its interface is the ``entities_from_rdf`` function. +""" + +import re +RE_RELS = re.compile(r'^<(.*?)>\s<(.*?)>\s<(.*?)>\s*\.') +RE_ATTS = re.compile(r'^<(.*?)>\s<(.*?)>\s"(.*)"(\^\^<(.*?)>|)\s*\.') + +MAPPING_ATTS = {'bio2rdfSymbol': 'bio2rdf_symbol', + 'label': 'label', + 'name': 'name', + 'classDegree': 'class_degree', + 'degree': 'degree', + 'size': 'size'} + +MAPPING_RELS = {'geneId': 'gene_id', + 'hgncId': 'hgnc_id', + 'hgncIdPage': 'hgnc_page', + 'sameAs': 'same_as', + 'class': 'classes', + 'diseaseSubtypeOf': 'subtype_of', + 'associatedGene': 'associated_genes', + 'possibleDrug': 'possible_drugs', + 'type': 'types', + 'omim': 'omim', + 'omimPage': 'omim_page', + 'chromosomalLocation': 'chromosomal_location'} + +def _retrieve_reltype(uri): + """ + Retrieve a relation type from an URI. + + Internal function which takes an URI containing a relation type as input + and returns the name of the relation. + If no URI string is given, then the function returns None. + """ + if uri: + return uri.rsplit('/', 1)[-1].rsplit('#', 1)[-1] + +def _retrieve_etype(tri_uri): + """ + Retrieve entity type from a triple of URIs. + + Internal function whith takes a tuple of three URIs as input + and returns the type of the entity, as obtained from the + first member of the tuple. + """ + if tri_uri: + return tri_uri.split('> <')[0].rsplit('/', 2)[-2].rstrip('s') + +def _retrieve_structure(filename, etypes): + """ + Retrieve a (subject, relation, object) tuples iterator from a file. + + Internal function which takes as input a file name and a tuple of + entity types, and returns an iterator of (subject, relation, object) + tuples. + """ + with open(filename) as fil: + for line in fil: + if _retrieve_etype(line) not in etypes: + continue + match = RE_RELS.match(line) + if not match: + match = RE_ATTS.match(line) + subj = match.group(1) + relation = _retrieve_reltype(match.group(2)) + obj = match.group(3) + yield subj, relation, obj + +def entities_from_rdf(filename, etypes): + """ + Return entities from an RDF file. + + Module interface function which takes as input a file name and + a tuple of entity types, and returns an iterator on the + attributes and relations of each entity. The attributes + and relations are retrieved as dictionaries. + + >>> for entities, relations in entities_from_rdf('data_file', + ('type_1', 'type_2')): + ... + """ + entities = {} + for subj, rel, obj in _retrieve_structure(filename, etypes): + entities.setdefault(subj, {}) + entities[subj].setdefault('attributes', {}) + entities[subj].setdefault('relations', {}) + entities[subj]['attributes'].setdefault('cwuri', unicode(subj)) + if rel in MAPPING_ATTS: + entities[subj]['attributes'].setdefault(MAPPING_ATTS[rel], + unicode(obj)) + if rel in MAPPING_RELS: + entities[subj]['relations'].setdefault(MAPPING_RELS[rel], set()) + entities[subj]['relations'][MAPPING_RELS[rel]].add(unicode(obj)) + return ((ent.get('attributes'), ent.get('relations')) + for ent in entities.itervalues()) diff -r 7920f439f383 -r be91151107f6 doc/tutorials/dataimport/schema.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/tutorials/dataimport/schema.py Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +# copyright 2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr -- mailto:contact@logilab.fr +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with this program. If not, see . + +"""cubicweb-diseasome schema""" + +from yams.buildobjs import EntityType, SubjectRelation, String, Int +from cubicweb.schemas.base import ExternalUri + + +class Disease(EntityType): + """Disease entity definition. + + A Disease entity is characterized by several attributes which are + defined by URIs: + + - a name, which we define as a CubicWeb / Yams String object + - a label, also defined as a Yams String + - a class degree, defined as a Yams Int (that is, an integer) + - a degree, also defined as a Yams Int + - size, also defined as an Int + - classes, defined as a set containing zero, one or several objects + identified by their URIs, that is, objects of type ``ExternalUri`` + - subtype_of, defined as a set containing zero, one or several + objects of type ``Disease`` + - associated_genes, defined as a set containing zero, one or several + objects of type ``Gene``, that is, of genes associated to the + disease + - possible_drugs, defined as a set containing zero, one or several + objects, identified by their URIs, that is, of type ``ExternalUri`` + - omim and omim_page are identifiers in the OMIM (Online Mendelian + Inheritance in Man) database, which contains an inventory of "human + genes and genetic phenotypes" + (see http://http://www.ncbi.nlm.nih.gov/omim). Given that a disease + only has unique omim and omim_page identifiers, when it has them, + these attributes have been defined through relations such that + for each disease there is at most one omim and one omim_page. + Each such identifier is defined through an URI, that is, through + an ``ExternalUri`` entity. + That is, these relations are of cardinality "?*". For optimization + purposes, one might be tempted to defined them as inlined, by setting + the ``inlined`` keyword argument to ``True``. + - chromosomal_location is also defined through a relation of + cardinality "?*", since any disease has at most one chromosomal + location associated to it. + - same_as is also defined through an URI, and hence through a + relation having ``ExternalUri`` entities as objects. + + For more information on this data set and the data set itself, + please consult http://datahub.io/dataset/fu-berlin-diseasome. + """ + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/name + name = String(maxsize=256, fulltextindexed=True) + # Corresponds to http://www.w3.org/2000/01/rdf-schema#label + label = String(maxsize=512, fulltextindexed=True) + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/classDegree + class_degree = Int() + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/degree + degree = Int() + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/size + size = Int() + #Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/class + classes = SubjectRelation('ExternalUri', cardinality='**') + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/diseaseSubtypeOf + subtype_of = SubjectRelation('Disease', cardinality='**') + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/associatedGene + associated_genes = SubjectRelation('Gene', cardinality='**') + #Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/possibleDrug + possible_drugs = SubjectRelation('ExternalUri', cardinality='**') + #Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/omim + omim = SubjectRelation('ExternalUri', cardinality='?*', inlined=True) + #Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/omimPage + omim_page = SubjectRelation('ExternalUri', cardinality='?*', inlined=True) + #Corresponds to 'http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/chromosomalLocation' + chromosomal_location = SubjectRelation('ExternalUri', cardinality='?*', + inlined=True) + #Corresponds to http://www.w3.org/2002/07/owl#sameAs + same_as = SubjectRelation('ExternalUri', cardinality='**') + + +class Gene(EntityType): + """Gene entity defintion. + + A gene is characterized by the following attributes: + + - label, defined through a Yams String. + - bio2rdf_symbol, also defined as a Yams String, since it is + just an identifier. + - gene_id is an URI identifying a gene, hence it is defined + as a relation with an ``ExternalUri`` object. + - a pair of unique identifiers in the HUGO Gene Nomenclature + Committee (http://http://www.genenames.org/). They are defined + as ``ExternalUri`` entities as well. + - same_as is also defined through an URI, and hence through a + relation having ``ExternalUri`` entities as objects. + """ + # Corresponds to http://www.w3.org/2000/01/rdf-schema#label + label = String(maxsize=512, fulltextindexed=True) + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/geneId + gene_id = SubjectRelation('ExternalUri', cardinality='**') + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/hgncId + hgnc_id = SubjectRelation('ExternalUri', cardinality='**') + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/hgncIdPage + hgnc_page = SubjectRelation('ExternalUri', cardinality='**') + # Corresponds to http://www4.wiwiss.fu-berlin.de/diseasome/resource/ + # diseasome/bio2rdfSymbol + bio2rdf_symbol = String(maxsize=64, fulltextindexed=True) + #Corresponds to http://www.w3.org/2002/07/owl#sameAs + same_as = SubjectRelation('ExternalUri', cardinality='**') diff -r 7920f439f383 -r be91151107f6 entities/__init__.py --- a/entities/__init__.py Fri Apr 12 11:39:10 2013 +0200 +++ b/entities/__init__.py Mon Apr 15 13:18:01 2013 +0200 @@ -24,7 +24,7 @@ from logilab.common.deprecation import deprecated from logilab.common.decorators import cached -from cubicweb import Unauthorized, typed_eid +from cubicweb import Unauthorized from cubicweb.entity import Entity diff -r 7920f439f383 -r be91151107f6 entities/adapters.py --- a/entities/adapters.py Fri Apr 12 11:39:10 2013 +0200 +++ b/entities/adapters.py Mon Apr 15 13:18:01 2013 +0200 @@ -404,117 +404,3 @@ "%(cls)s is deprecated") % {'cls': cls.__name__} warn(msg, DeprecationWarning, stacklevel=2) return type.__call__(cls, *args, **kwargs) - - -class IProgressAdapter(view.EntityAdapter): - """something that has a cost, a state and a progression. - - You should at least override progress_info an in_progress methods on - concrete implementations. - """ - __metaclass__ = adapter_deprecated - __deprecation_warning__ = '[3.14] IProgressAdapter has been moved to iprogress cube' - __needs_bw_compat__ = True - __regid__ = 'IProgress' - __select__ = implements(IProgress, warn=False) # XXX for bw compat, should be abstract - - @property - @view.implements_adapter_compat('IProgress') - def cost(self): - """the total cost""" - return self.progress_info()['estimated'] - - @property - @view.implements_adapter_compat('IProgress') - def revised_cost(self): - return self.progress_info().get('estimatedcorrected', self.cost) - - @property - @view.implements_adapter_compat('IProgress') - def done(self): - """what is already done""" - return self.progress_info()['done'] - - @property - @view.implements_adapter_compat('IProgress') - def todo(self): - """what remains to be done""" - return self.progress_info()['todo'] - - @view.implements_adapter_compat('IProgress') - def progress_info(self): - """returns a dictionary describing progress/estimated cost of the - version. - - - mandatory keys are (''estimated', 'done', 'todo') - - - optional keys are ('notestimated', 'notestimatedcorrected', - 'estimatedcorrected') - - 'noestimated' and 'notestimatedcorrected' should default to 0 - 'estimatedcorrected' should default to 'estimated' - """ - raise NotImplementedError - - @view.implements_adapter_compat('IProgress') - def finished(self): - """returns True if status is finished""" - return not self.in_progress() - - @view.implements_adapter_compat('IProgress') - def in_progress(self): - """returns True if status is not finished""" - raise NotImplementedError - - @view.implements_adapter_compat('IProgress') - def progress(self): - """returns the % progress of the task item""" - try: - return 100. * self.done / self.revised_cost - except ZeroDivisionError: - # total cost is 0 : if everything was estimated, task is completed - if self.progress_info().get('notestimated'): - return 0. - return 100 - - @view.implements_adapter_compat('IProgress') - def progress_class(self): - return '' - - -class IMileStoneAdapter(IProgressAdapter): - __metaclass__ = adapter_deprecated - __deprecation_warning__ = '[3.14] IMileStoneAdapter has been moved to iprogress cube' - __needs_bw_compat__ = True - __regid__ = 'IMileStone' - __select__ = implements(IMileStone, warn=False) # XXX for bw compat, should be abstract - - parent_type = None # specify main task's type - - @view.implements_adapter_compat('IMileStone') - def get_main_task(self): - """returns the main ITask entity""" - raise NotImplementedError - - @view.implements_adapter_compat('IMileStone') - def initial_prevision_date(self): - """returns the initial expected end of the milestone""" - raise NotImplementedError - - @view.implements_adapter_compat('IMileStone') - def eta_date(self): - """returns expected date of completion based on what remains - to be done - """ - raise NotImplementedError - - @view.implements_adapter_compat('IMileStone') - def completion_date(self): - """returns date on which the subtask has been completed""" - raise NotImplementedError - - @view.implements_adapter_compat('IMileStone') - def contractors(self): - """returns the list of persons supposed to work on this task""" - raise NotImplementedError - diff -r 7920f439f383 -r be91151107f6 entity.py --- a/entity.py Fri Apr 12 11:39:10 2013 +0200 +++ b/entity.py Mon Apr 15 13:18:01 2013 +0200 @@ -33,11 +33,10 @@ from rql.nodes import (Not, VariableRef, Constant, make_relation, Relation as RqlRelation) -from cubicweb import Unauthorized, typed_eid, neg_role +from cubicweb import Unauthorized, neg_role from cubicweb.utils import support_args from cubicweb.rset import ResultSet from cubicweb.appobject import AppObject -from cubicweb.req import _check_cw_unsafe from cubicweb.schema import (RQLVocabularyConstraint, RQLConstraint, GeneratedConstraint) from cubicweb.rqlrewrite import RQLRewriter @@ -627,7 +626,7 @@ meaning that the entity has to be created """ try: - typed_eid(self.eid) + int(self.eid) return True except (ValueError, TypeError): return False @@ -1287,7 +1286,6 @@ an entity or eid, a list of entities or eids, or None (meaning that all relations of the given type from or to this object should be deleted). """ - _check_cw_unsafe(kwargs) assert kwargs assert self.cw_is_saved(), "should not call set_attributes while entity "\ "hasn't been saved yet" @@ -1397,10 +1395,6 @@ @deprecated('[3.10] use entity.cw_attr_cache[attr]') def __getitem__(self, key): - if key == 'eid': - warn('[3.7] entity["eid"] is deprecated, use entity.eid instead', - DeprecationWarning, stacklevel=2) - return self.eid return self.cw_attr_cache[key] @deprecated('[3.10] use entity.cw_attr_cache.get(attr[, default])') @@ -1424,15 +1418,10 @@ the attribute to skip_security since we don't want to check security for such attributes set by hooks. """ - if attr == 'eid': - warn('[3.7] entity["eid"] = value is deprecated, use entity.eid = value instead', - DeprecationWarning, stacklevel=2) - self.eid = value - else: - try: - self.cw_edited[attr] = value - except AttributeError: - self.cw_attr_cache[attr] = value + try: + self.cw_edited[attr] = value + except AttributeError: + self.cw_attr_cache[attr] = value @deprecated('[3.10] use del entity.cw_edited[attr]') def __delitem__(self, attr): diff -r 7920f439f383 -r be91151107f6 etwist/request.py --- a/etwist/request.py Fri Apr 12 11:39:10 2013 +0200 +++ b/etwist/request.py Mon Apr 15 13:18:01 2013 +0200 @@ -39,6 +39,7 @@ self.form[key] = (name, stream) else: self.form[key] = (unicode(name, self.encoding), stream) + self.content = self._twreq.content # stream def http_method(self): """returns 'POST', 'GET', 'HEAD', etc.""" diff -r 7920f439f383 -r be91151107f6 etwist/test/data/views.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/etwist/test/data/views.py Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,29 @@ +# copyright 2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see . +"""only for unit tests !""" + +from cubicweb.view import View +from cubicweb.predicates import match_http_method + +class PutView(View): + __regid__ = 'put' + __select__ = match_http_method('PUT') + binary = True + + def call(self): + self.w(self._cw.content.read()) diff -r 7920f439f383 -r be91151107f6 etwist/test/unittest_server.py --- a/etwist/test/unittest_server.py Fri Apr 12 11:39:10 2013 +0200 +++ b/etwist/test/unittest_server.py Mon Apr 15 13:18:01 2013 +0200 @@ -19,6 +19,7 @@ import os, os.path as osp, glob from cubicweb.devtools.testlib import CubicWebTC +from cubicweb.devtools.httptest import CubicWebServerTC from cubicweb.etwist.server import host_prefixed_baseurl @@ -53,6 +54,13 @@ self._check('http://localhost:8080/hg/', 'code.cubicweb.org', 'http://localhost:8080/hg/') + +class ETwistHTTPTC(CubicWebServerTC): + def test_put_content(self): + body = 'hop' + response = self.web_request('?vid=put', method='PUT', body=body) + self.assertEqual(body, response.body) + if __name__ == '__main__': from logilab.common.testlib import unittest_main unittest_main() diff -r 7920f439f383 -r be91151107f6 etwist/twconfig.py --- a/etwist/twconfig.py Fri Apr 12 11:39:10 2013 +0200 +++ b/etwist/twconfig.py Mon Apr 15 13:18:01 2013 +0200 @@ -103,8 +103,8 @@ return join(self.apphome, '%s-%s.py' % (self.appid, self.name)) def default_base_url(self): - from socket import gethostname - return 'http://%s:%s/' % (self['host'] or gethostname(), self['port'] or 8080) + from socket import getfqdn + return 'http://%s:%s/' % (self['host'] or getfqdn(), self['port'] or 8080) CONFIGURATIONS.append(TwistedConfiguration) diff -r 7920f439f383 -r be91151107f6 ext/rest.py --- a/ext/rest.py Fri Apr 12 11:39:10 2013 +0200 +++ b/ext/rest.py Mon Apr 15 13:18:01 2013 +0200 @@ -36,6 +36,7 @@ from itertools import chain from logging import getLogger from os.path import join +from urlparse import urlsplit from docutils import statemachine, nodes, utils, io from docutils.core import Publisher @@ -128,6 +129,63 @@ set_classes(options) return [nodes.raw('', content, format='html')], [] +def bookmark_role(role, rawtext, text, lineno, inliner, options={}, content=[]): + """:bookmark:`` or :bookmark:`:` + + Example: :bookmark:`1234:table` + + Replace the directive with the output of applying the view to the resultset + returned by the query stored in the bookmark. By default, the view is the one + stored in the bookmark, but it can be overridden by the directive as in the + example above. + + "X eid %(userid)s" can be used in the RQL query stored in the Bookmark, for + this query will be executed with the argument {'userid': _cw.user.eid}. + """ + _cw = inliner.document.settings.context._cw + text = text.strip() + try: + if ':' in text: + eid, vid = text.rsplit(u':', 1) + eid = int(eid) + else: + eid, vid = int(text), None + except ValueError: + msg = inliner.reporter.error( + 'EID number must be a positive number; "%s" is invalid.' + % text, line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + try: + bookmark = _cw.entity_from_eid(eid) + except UnknownEid: + msg = inliner.reporter.error('Unknown EID %s.' % text, line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + try: + params = dict(_cw.url_parse_qsl(urlsplit(bookmark.path).query)) + rql = params['rql'] + if vid is None: + vid = params.get('vid') + except (ValueError, KeyError), exc: + msg = inliner.reporter.error('Could not parse bookmark path %s [%s].' + % (bookmark.path, exc), line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + try: + rset = _cw.execute(rql, {'userid': _cw.user.eid}) + if rset: + if vid is None: + vid = vid_from_rset(_cw, rset, _cw.vreg.schema) + else: + vid = 'noresult' + view = _cw.vreg['views'].select(vid, _cw, rset=rset) + content = view.render() + except Exception, exc: + content = 'An error occured while interpreting directive bookmark: %r' % exc + set_classes(options) + return [nodes.raw('', content, format='html')], [] + def winclude_directive(name, arguments, options, content, lineno, content_offset, block_text, state, state_machine): """Include a reST file as part of the content of this reST file. @@ -323,6 +381,7 @@ _INITIALIZED = True register_canonical_role('eid', eid_reference_role) register_canonical_role('rql', rql_role) + register_canonical_role('bookmark', bookmark_role) directives.register_directive('winclude', winclude_directive) if pygments_directive is not None: directives.register_directive('sourcecode', pygments_directive) diff -r 7920f439f383 -r be91151107f6 ext/test/unittest_rest.py --- a/ext/test/unittest_rest.py Fri Apr 12 11:39:10 2013 +0200 +++ b/ext/test/unittest_rest.py Mon Apr 15 13:18:01 2013 +0200 @@ -75,5 +75,12 @@ out = rest_publish(context, ':rql:`Any X WHERE X is CWUser`') self.assertEqual(out, u'

CWUser_plural

admin
anon

\n') + def test_bookmark_role(self): + context = self.context() + rset = self.execute('INSERT Bookmark X: X title "hello", X path "/view?rql=Any X WHERE X is CWUser"') + eid = rset[0][0] + out = rest_publish(context, ':bookmark:`%s`' % eid) + self.assertEqual(out, u'

CWUser_plural

admin
anon

\n') + if __name__ == '__main__': unittest_main() diff -r 7920f439f383 -r be91151107f6 interfaces.py --- a/interfaces.py Fri Apr 12 11:39:10 2013 +0200 +++ b/interfaces.py Mon Apr 15 13:18:01 2013 +0200 @@ -148,34 +148,6 @@ def marker_icon(self): """returns the icon that should be used as the marker""" -# XXX deprecates in favor of ISIOCItemAdapter -class ISiocItem(Interface): - """interface for entities which may be represented as an ISIOC item""" - - def isioc_content(self): - """return item's content""" - - def isioc_container(self): - """return container entity""" - - def isioc_type(self): - """return container type (post, BlogPost, MailMessage)""" - - def isioc_replies(self): - """return replies items""" - - def isioc_topics(self): - """return topics items""" - -# XXX deprecates in favor of ISIOCContainerAdapter -class ISiocContainer(Interface): - """interface for entities which may be represented as an ISIOC container""" - - def isioc_type(self): - """return container type (forum, Weblog, MailingList)""" - - def isioc_items(self): - """return contained items""" # XXX deprecates in favor of IEmailableAdapter class IFeed(Interface): diff -r 7920f439f383 -r be91151107f6 misc/migration/3.3.5_Any.py --- a/misc/migration/3.3.5_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.3.5_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,8 +1,1 @@ -# some entities have been added before schema entities, fix the 'is' and -# 'is_instance_of' relations -for rtype in ('is', 'is_instance_of'): - sql('INSERT INTO %s_relation ' - 'SELECT X.eid, ET.cw_eid FROM entities as X, cw_CWEType as ET ' - 'WHERE X.type=ET.cw_name AND NOT EXISTS(' - ' SELECT 1 from is_relation ' - ' WHERE eid_from=X.eid AND eid_to=ET.cw_eid)' % rtype) +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.4.0_Any.py --- a/misc/migration/3.4.0_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.4.0_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,2 +1,1 @@ -drop_attribute('CWEType', 'meta') -drop_attribute('CWRType', 'meta') +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.4.0_common.py --- a/misc/migration/3.4.0_common.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.4.0_common.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,6 +1,1 @@ -from os.path import join -from cubicweb.toolsutils import create_dir - -option_renamed('pyro-application-id', 'pyro-instance-id') - -create_dir(join(config.appdatahome, 'backup')) +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.4.3_Any.py --- a/misc/migration/3.4.3_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.4.3_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,2 +1,1 @@ -# sync and restart to make sure cwuri does not appear in forms -sync_schema_props_perms() +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.5.0_Any.py --- a/misc/migration/3.5.0_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.5.0_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,10 +1,1 @@ -add_relation_type('prefered_form') - -rql('SET X prefered_form Y WHERE Y canonical TRUE, X identical_to Y') -commit() - -drop_attribute('EmailAddress', 'canonical') -drop_relation_definition('EmailAddress', 'identical_to', 'EmailAddress') - -if 'see_also' in schema: - sync_schema_props_perms('see_also', syncprops=False, syncrdefs=False) +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.5.10_Any.py --- a/misc/migration/3.5.10_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.5.10_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,5 +1,1 @@ -sync_schema_props_perms('state_of') -sync_schema_props_perms('transition_of') -for etype in ('State', 'BaseTransition', 'Transition', 'WorkflowTransition'): - sync_schema_props_perms((etype, 'name', 'String')) - +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.5.3_Any.py --- a/misc/migration/3.5.3_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.5.3_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,7 +1,1 @@ -# type attribute might already be there if migrating from -# version < 3.5 to version >= 3.5.3, BaseTransition being added -# in bootstrap_migration -if versions_map['cubicweb'][0] >= (3, 5, 0): - add_attribute('BaseTransition', 'type') - sync_schema_props_perms('state_of') - sync_schema_props_perms('transition_of') +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/3.6.1_Any.py --- a/misc/migration/3.6.1_Any.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/3.6.1_Any.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,2 +1,1 @@ -sync_schema_props_perms(syncprops=False) -sync_schema_props_perms('destination_state', syncperms=False) +raise NotImplementedError("Cannot migrate such an old version. Use intermediate Cubiweb version (try 3.16.x)") diff -r 7920f439f383 -r be91151107f6 misc/migration/bootstrapmigration_repository.py --- a/misc/migration/bootstrapmigration_repository.py Fri Apr 12 11:39:10 2013 +0200 +++ b/misc/migration/bootstrapmigration_repository.py Mon Apr 15 13:18:01 2013 +0200 @@ -34,6 +34,26 @@ ss.execschemarql(rql, rdef, ss.rdef2rql(rdef, CSTRMAP, groupmap=None)) commit(ask_confirm=False) +if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0): + try: + add_cube('sioc', update_database=False) + except ImportError: + if not confirm('In cubicweb 3.17 sioc views have been moved to the sioc ' + 'cube, which is not installed. Continue anyway?'): + raise + try: + add_cube('embed', update_database=False) + except ImportError: + if not confirm('In cubicweb 3.17 embedding views have been moved to the embed ' + 'cube, which is not installed. Continue anyway?'): + raise + try: + add_cube('geocoding', update_database=False) + except ImportError: + if not confirm('In cubicweb 3.17 geocoding views have been moved to the geocoding ' + 'cube, which is not installed. Continue anyway?'): + raise + if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1): sql('ALTER TABLE entities ADD asource VARCHAR(64)') sql('UPDATE entities SET asource=cw_name ' diff -r 7920f439f383 -r be91151107f6 predicates.py --- a/predicates.py Fri Apr 12 11:39:10 2013 +0200 +++ b/predicates.py Mon Apr 15 13:18:01 2013 +0200 @@ -1489,6 +1489,15 @@ return frozenset(req.form) +class match_http_method(ExpectedValuePredicate): + """Return non-zero score if one of the HTTP methods specified as + initializer arguments is the HTTP method of the request (GET, POST, ...). + """ + + def __call__(self, cls, req, **kwargs): + return int(req.http_method() in self.expected) + + class match_edited_type(ExpectedValuePredicate): """return non-zero if main edited entity type is the one specified as initializer argument, or is among initializer arguments if `mode` == 'any'. diff -r 7920f439f383 -r be91151107f6 req.py --- a/req.py Fri Apr 12 11:39:10 2013 +0200 +++ b/req.py Mon Apr 15 13:18:01 2013 +0200 @@ -29,7 +29,7 @@ from logilab.common.deprecation import deprecated from logilab.common.date import ustrftime, strptime, todate, todatetime -from cubicweb import Unauthorized, NoSelectableObject, typed_eid, uilib +from cubicweb import Unauthorized, NoSelectableObject, uilib from cubicweb.rset import ResultSet ONESECOND = timedelta(0, 1, 0) @@ -38,12 +38,6 @@ class FindEntityError(Exception): """raised when find_one_entity() can not return one and only one entity""" -def _check_cw_unsafe(kwargs): - if kwargs.pop('_cw_unsafe', False): - warn('[3.7] _cw_unsafe argument is deprecated, now unsafe by ' - 'default, control it using cw_[read|write]_security.', - DeprecationWarning, stacklevel=3) - class Cache(dict): def __init__(self): super(Cache, self).__init__() @@ -74,6 +68,7 @@ # cache result of execution for (rql expr / eids), # should be emptied on commit/rollback of the server session / web # connection + self.user = None self.local_perm_cache = {} self._ = unicode @@ -114,7 +109,7 @@ (we have the eid, we can suppose it exists and user has access to the entity) """ - eid = typed_eid(eid) + eid = int(eid) if etype is None: etype = self.describe(eid)[0] rset = ResultSet([(eid,)], 'Any X WHERE X eid %(x)s', {'x': eid}, @@ -154,7 +149,6 @@ ... works_for=c) """ - _check_cw_unsafe(kwargs) cls = self.vreg['etypes'].etype_class(etype) return cls.cw_instantiate(self.execute, **kwargs) diff -r 7920f439f383 -r be91151107f6 rqlrewrite.py --- a/rqlrewrite.py Fri Apr 12 11:39:10 2013 +0200 +++ b/rqlrewrite.py Mon Apr 15 13:18:01 2013 +0200 @@ -30,7 +30,7 @@ from logilab.common import tempattr from logilab.common.graph import has_path -from cubicweb import Unauthorized, typed_eid +from cubicweb import Unauthorized def add_types_restriction(schema, rqlst, newroot=None, solutions=None): @@ -220,7 +220,7 @@ vi = {} self.varinfos.append(vi) try: - vi['const'] = typed_eid(selectvar) + vi['const'] = int(selectvar) vi['rhs_rels'] = vi['lhs_rels'] = {} except ValueError: try: diff -r 7920f439f383 -r be91151107f6 selectors.py --- a/selectors.py Fri Apr 12 11:39:10 2013 +0200 +++ b/selectors.py Mon Apr 15 13:18:01 2013 +0200 @@ -35,8 +35,6 @@ EClassSelector = class_renamed('EClassSelector', EClassPredicate) EntitySelector = class_renamed('EntitySelector', EntityPredicate) -# XXX pre 3.7? bw compat - class on_transition(is_in_state): """Return 1 if entity is in one of the transitions given as argument list diff -r 7920f439f383 -r be91151107f6 server/migractions.py --- a/server/migractions.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/migractions.py Mon Apr 15 13:18:01 2013 +0200 @@ -300,8 +300,8 @@ if self.config is not None: session = self.repo._get_session(self.cnx.sessionid) if session.cnxset is None: - session.set_read_security(False) - session.set_write_security(False) + session.read_security = False + session.write_security = False session.set_cnxset() return session # no access to session on remote instance @@ -1515,14 +1515,6 @@ if commit: self.commit() - @deprecated("[3.7] use session.disable_hook_categories('integrity')") - def cmd_deactivate_verification_hooks(self): - self.session.disable_hook_categories('integrity') - - @deprecated("[3.7] use session.enable_hook_categories('integrity')") - def cmd_reactivate_verification_hooks(self): - self.session.enable_hook_categories('integrity') - @deprecated("[3.15] use rename_relation_type(oldname, newname)") def cmd_rename_relation(self, oldname, newname, commit=True): self.cmd_rename_relation_type(oldname, newname, commit) diff -r 7920f439f383 -r be91151107f6 server/querier.py --- a/server/querier.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/querier.py Mon Apr 15 13:18:01 2013 +0200 @@ -31,7 +31,7 @@ from yams import BASE_TYPES from cubicweb import ValidationError, Unauthorized, QueryError, UnknownEid -from cubicweb import Binary, server, typed_eid +from cubicweb import Binary, server from cubicweb.rset import ResultSet from cubicweb.utils import QueryCache, RepeatList @@ -392,7 +392,7 @@ for var in rqlst.defined_vars.itervalues(): if var.stinfo['constnode'] is not None: eid = var.stinfo['constnode'].eval(self.args) - varkwargs[var.name] = typed_eid(eid) + varkwargs[var.name] = int(eid) # dictionary of variables restricted for security reason localchecks = {} restricted_vars = set() @@ -564,11 +564,11 @@ for subj, rtype, obj in self.relation_defs(): # if a string is given into args instead of an int, we get it here if isinstance(subj, basestring): - subj = typed_eid(subj) + subj = int(subj) elif not isinstance(subj, (int, long)): subj = subj.entity.eid if isinstance(obj, basestring): - obj = typed_eid(obj) + obj = int(obj) elif not isinstance(obj, (int, long)): obj = obj.entity.eid if repo.schema.rschema(rtype).inlined: diff -r 7920f439f383 -r be91151107f6 server/repository.py --- a/server/repository.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/repository.py Mon Apr 15 13:18:01 2013 +0200 @@ -50,7 +50,7 @@ UnknownEid, AuthenticationError, ExecutionError, ETypeNotSupportedBySources, MultiSourcesError, BadConnectionId, Unauthorized, ValidationError, - RepositoryError, UniqueTogetherError, typed_eid, onevent) + RepositoryError, UniqueTogetherError, onevent) from cubicweb import cwvreg, schema, server from cubicweb.server import ShuttingDown, utils, hook, pool, querier, sources from cubicweb.server.session import Session, InternalSession, InternalManager @@ -844,7 +844,7 @@ self.debug('begin commit for session %s', sessionid) try: session = self._get_session(sessionid) - session.set_tx_data(txid) + session.set_tx(txid) return session.commit() except (ValidationError, Unauthorized): raise @@ -857,7 +857,7 @@ self.debug('begin rollback for session %s', sessionid) try: session = self._get_session(sessionid) - session.set_tx_data(txid) + session.set_tx(txid) session.rollback() except Exception: self.exception('unexpected error') @@ -1014,7 +1014,7 @@ except KeyError: raise BadConnectionId('No such session %s' % sessionid) if setcnxset: - session.set_tx_data(txid) # must be done before set_cnxset + session.set_tx(txid) # must be done before set_cnxset session.set_cnxset() return session @@ -1027,7 +1027,7 @@ uri)` for the entity of the given `eid` """ try: - eid = typed_eid(eid) + eid = int(eid) except ValueError: raise UnknownEid(eid) try: @@ -1055,7 +1055,7 @@ rqlcache = self.querier._rql_cache for eid in eids: try: - etype, uri, extid, auri = etcache.pop(typed_eid(eid)) # may be a string in some cases + etype, uri, extid, auri = etcache.pop(int(eid)) # may be a string in some cases rqlcache.pop( ('%s X WHERE X eid %s' % (etype, eid),), None) extidcache.pop((extid, uri), None) except KeyError: @@ -1084,7 +1084,7 @@ key, args[key])) cachekey.append(etype) # ensure eid is correctly typed in args - args[key] = typed_eid(args[key]) + args[key] = int(args[key]) return tuple(cachekey) def eid2extid(self, source, eid, session=None): @@ -1177,7 +1177,7 @@ hook.CleanupDeletedEidsCacheOp.get_instance(session).add_data(entity.eid) self.system_source.delete_info_multi(session, [entity], uri) if source.should_call_hooks: - session._threaddata.pending_operations = pending_operations + session._tx.pending_operations = pending_operations raise def add_info(self, session, entity, source, extid=None, complete=True): diff -r 7920f439f383 -r be91151107f6 server/schemaserial.py --- a/server/schemaserial.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/schemaserial.py Mon Apr 15 13:18:01 2013 +0200 @@ -26,7 +26,7 @@ from yams import BadSchemaDefinition, schema as schemamod, buildobjs as ybo -from cubicweb import CW_SOFTWARE_ROOT, typed_eid +from cubicweb import CW_SOFTWARE_ROOT from cubicweb.schema import (CONSTRAINTS, ETYPE_NAME_MAP, VIRTUAL_RTYPES, PURE_VIRTUAL_RTYPES) from cubicweb.server import sqlutils @@ -58,7 +58,7 @@ if not value: continue try: - eid = typed_eid(value) + eid = int(value) except ValueError: print 'eid should be an integer' continue diff -r 7920f439f383 -r be91151107f6 server/serverctl.py --- a/server/serverctl.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/serverctl.py Mon Apr 15 13:18:01 2013 +0200 @@ -39,6 +39,7 @@ from cubicweb.server.serverconfig import ( USER_OPTIONS, ServerConfiguration, SourceConfiguration, ask_source_config, generate_source_config) +from yams.diff import schema_diff # utility functions ########################################################### @@ -1065,12 +1066,31 @@ if val: print key, ':', val +class SchemaDiffCommand(Command): + """ generate a diff between schema and fsschema description + + the name of a diff tool to compare the two generated file + + """ + name = 'schema-diff' + arguments = ' ' + min_args = max_args = 2 + + def run(self, args): + appid = args.pop(0) + diff_tool = args.pop(0) + config = ServerConfiguration.config_for(appid) + repo, cnx = repo_cnx(config) + session = repo._get_session(cnx.sessionid, setcnxset=True) + fsschema = config.load_schema(expand_cubes=True) + schema_diff(repo.schema, fsschema, diff_tool) + for cmdclass in (CreateInstanceDBCommand, InitInstanceCommand, GrantUserOnInstanceCommand, ResetAdminPasswordCommand, StartRepositoryCommand, DBDumpCommand, DBRestoreCommand, DBCopyCommand, AddSourceCommand, CheckRepositoryCommand, RebuildFTICommand, - SynchronizeInstanceSchemaCommand, SynchronizeSourceCommand + SynchronizeInstanceSchemaCommand, SynchronizeSourceCommand, SchemaDiffCommand, ): CWCTL.register(cmdclass) diff -r 7920f439f383 -r be91151107f6 server/session.py --- a/server/session.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/session.py Mon Apr 15 13:18:01 2013 +0200 @@ -48,28 +48,28 @@ @objectify_predicate def is_user_session(cls, req, **kwargs): - """repository side only predicate returning 1 if the session is a regular - user session and not an internal session - """ + """return 1 when session is not internal. + + This predicate can only be used repository side only. """ return not req.is_internal_session @objectify_predicate def is_internal_session(cls, req, **kwargs): - """repository side only predicate returning 1 if the session is not a regular - user session but an internal session - """ + """return 1 when session is not internal. + + This predicate can only be used repository side only. """ return req.is_internal_session @objectify_predicate def repairing(cls, req, **kwargs): - """repository side only predicate returning 1 if the session is not a regular - user session but an internal session - """ + """return 1 when repository is running in repair mode""" return req.vreg.config.repairing class transaction(object): - """context manager to enter a transaction for a session: when exiting the + """Ensure that the transaction is either commited or rollbacked at exit + + Context manager to enter a transaction for a session: when exiting the `with` block on exception, call `session.rollback()`, else call `session.commit()` on normal exit """ @@ -87,8 +87,11 @@ else: self.session.commit(free_cnxset=self.free_cnxset) +@deprecated('[3.17] use .allow/deny_all_hooks_but instead') +def hooks_control(obj, *args, **kwargs): + return obj.hooks_control(*args, **kwargs) -class hooks_control(object): +class _hooks_control(object): """context manager to control activated hooks categories. If mode is session.`HOOKS_DENY_ALL`, given hooks categories will @@ -99,10 +102,10 @@ .. sourcecode:: python - with hooks_control(self.session, self.session.HOOKS_ALLOW_ALL, 'integrity'): + with _hooks_control(self.session, self.session.HOOKS_ALLOW_ALL, 'integrity'): # ... do stuff with all but 'integrity' hooks activated - with hooks_control(self.session, self.session.HOOKS_DENY_ALL, 'integrity'): + with _hooks_control(self.session, self.session.HOOKS_DENY_ALL, 'integrity'): # ... do stuff with none but 'integrity' hooks activated This is an internal api, you should rather use @@ -111,45 +114,612 @@ methods. """ def __init__(self, session, mode, *categories): + assert mode in (HOOKS_ALLOW_ALL, HOOKS_DENY_ALL) self.session = session + self.tx = session._tx self.mode = mode self.categories = categories + self.oldmode = None + self.changes = () def __enter__(self): - self.oldmode, self.changes = self.session.init_hooks_mode_categories( - self.mode, self.categories) + self.oldmode = self.tx.hooks_mode + self.tx.hooks_mode = self.mode + if self.mode is HOOKS_DENY_ALL: + self.changes = self.tx.enable_hook_categories(*self.categories) + else: + self.changes = self.tx.disable_hook_categories(*self.categories) + self.tx.ctx_count += 1 def __exit__(self, exctype, exc, traceback): - self.session.reset_hooks_mode_categories(self.oldmode, self.mode, self.changes) - + self.tx.ctx_count -= 1 + if self.tx.ctx_count == 0: + self.session._clear_thread_storage(self.tx) + else: + try: + if self.categories: + if self.mode is HOOKS_DENY_ALL: + self.tx.disable_hook_categories(*self.categories) + else: + self.tx.enable_hook_categories(*self.categories) + finally: + self.tx.hooks_mode = self.oldmode -class security_enabled(object): - """context manager to control security w/ session.execute, since by - default security is disabled on queries executed on the repository +@deprecated('[3.17] use .security_enabled instead') +def security_enabled(obj, *args, **kwargs): + return obj.security_enabled(*args, **kwargs) + +class _security_enabled(object): + """context manager to control security w/ session.execute, + + By default security is disabled on queries executed on the repository side. """ def __init__(self, session, read=None, write=None): self.session = session + self.tx = session._tx self.read = read self.write = write + self.oldread = None + self.oldwrite = None def __enter__(self): - self.oldread, self.oldwrite = self.session.init_security( - self.read, self.write) + if self.read is None: + self.oldread = None + else: + self.oldread = self.tx.read_security + self.tx.read_security = self.read + if self.write is None: + self.oldwrite = None + else: + self.oldwrite = self.tx.write_security + self.tx.write_security = self.write + self.tx.ctx_count += 1 def __exit__(self, exctype, exc, traceback): - self.session.reset_security(self.oldread, self.oldwrite) + self.tx.ctx_count -= 1 + if self.tx.ctx_count == 0: + self.session._clear_thread_storage(self.tx) + else: + if self.oldread is not None: + self.tx.read_security = self.oldread + if self.oldwrite is not None: + self.tx.write_security = self.oldwrite + +HOOKS_ALLOW_ALL = object() +HOOKS_DENY_ALL = object() +DEFAULT_SECURITY = object() # evaluated to true by design + +class SessionClosedError(RuntimeError): + pass + +class CnxSetTracker(object): + """Keep track of which transaction use which cnxset. + + There should be one of this object per session plus one another for + internal session. + + Session object are responsible of creating their CnxSetTracker object. + + Transaction should use the :meth:`record` and :meth:`forget` to inform the + tracker of cnxset they have acquired. + + .. automethod:: cubicweb.server.session.CnxSetTracker.record + .. automethod:: cubicweb.server.session.CnxSetTracker.forget + + Session use the :meth:`close` and :meth:`wait` method when closing. + + .. automethod:: cubicweb.server.session.CnxSetTracker.close + .. automethod:: cubicweb.server.session.CnxSetTracker.wait + + This object itself is threadsafe. It also requires caller to acquired its + lock in some situation. + """ + + def __init__(self): + self._active = True + self._condition = threading.Condition() + self._record = {} + + def __enter__(self): + self._condition.__enter__() + + def __exit__(self, *args): + self._condition.__exit__(*args) + + def record(self, txid, cnxset): + """Inform the tracker that a txid have acquired a cnxset + + This methode is to be used by Transaction object. + + This method fails when: + - The txid already have a recorded cnxset. + - The tracker is not active anymore. + + Notes about the caller: + (1) It is responsible for retrieving a cnxset. + (2) It must be prepared to release the cnxset if the + `cnxsettracker.forget` call fails. + (3) It should acquire the tracker lock until the very end of the operation. + (4) However It take care to lock the CnxSetTracker object after having + retrieved the cnxset to prevent deadlock. + + A typical usage look like:: + + cnxset = repo._get_cnxset() # (1) + try: + with cnxset_tracker: # (3) and (4) + cnxset_tracker.record(caller.id, cnxset) + # (3') operation ends when caller is in expected state only + caller.cnxset = cnxset + except Exception: + repo._free_cnxset(cnxset) # (2) + raise + """ + # dubious since the caller is suppose to have acquired it anyway. + with self._condition: + if not self._active: + raise SessionClosedError('Closed') + old = self._record.get(txid) + if old is not None: + raise ValueError('"%s" already have a cnx_set (%r)' + % (txid, old)) + self._record[txid] = cnxset + + def forget(self, txid, cnxset): + """Inform the tracker that a txid have release a cnxset + + This methode is to be used by Transaction object. + + This method fails when: + - The cnxset for the txid does not match the recorded one. + + Notes about the caller: + (1) It is responsible for releasing the cnxset. + (2) It should acquire the tracker lock during the operation to ensure + the internal tracker state is always accurate regarding its own state. + + A typical usage look like:: + + cnxset = caller.cnxset + try: + with cnxset_tracker: + # (2) you can not have caller.cnxset out of sync with + # cnxset_tracker state while unlocked + caller.cnxset = None + cnxset_tracker.forget(caller.id, cnxset) + finally: + cnxset = repo._free_cnxset(cnxset) # (1) + """ + with self._condition: + old = self._record.get(txid, None) + if old is not cnxset: + raise ValueError('recorded cnxset for "%s" mismatch: %r != %r' + % (txid, old, cnxset)) + self._record.pop(txid) + self._condition.notify_all() + + def close(self): + """Marks the tracker as inactive. + + This methode is to be used by Session object. + + Inactive tracker does not accept new record anymore. + """ + with self._condition: + self._active = False + + def wait(self, timeout=10): + """Wait for all recorded cnxset to be released + + This methode is to be used by Session object. + + returns a tuple of transaction id that remains open. + """ + with self._condition: + if self._active: + raise RuntimeError('Cannot wait on active tracker.' + ' Call tracker.close() first') + while self._record and timeout > 0: + start = time() + self._condition.wait(timeout) + timeout -= time() - start + return tuple(self._record) + +class Transaction(object): + """Repository Transaction + + Holds all transaction related data + + Database connections resource: + + :attr:`running_dbapi_query`, boolean flag telling if the executing query + is coming from a dbapi connection or is a query from within the repository + + :attr:`cnxset`, the connections set to use to execute queries on sources. + If the transaction is read only, the connection set may be freed between + actual query. This allows multiple transaction with a reasonable low + connection set pool size. control mechanism is detailed below + + .. automethod:: cubicweb.server.session.Transaction.set_cnxset + .. automethod:: cubicweb.server.session.Transaction.free_cnxset + + :attr:`mode`, string telling the connections set handling mode, may be one + of 'read' (connections set may be freed), 'write' (some write was done in + the connections set, it can't be freed before end of the transaction), + 'transaction' (we want to keep the connections set during all the + transaction, with or without writing) + + Internal transaction data: + + :attr:`data`,is a dictionary containing some shared data + cleared at the end of the transaction. Hooks and operations may put + arbitrary data in there, and this may also be used as a communication + channel between the client and the repository. + + :attr:`pending_operations`, ordered list of operations to be processed on + commit/rollback + + :attr:`commit_state`, describing the transaction commit state, may be one + of None (not yet committing), 'precommit' (calling precommit event on + operations), 'postcommit' (calling postcommit event on operations), + 'uncommitable' (some :exc:`ValidationError` or :exc:`Unauthorized` error + has been raised during the transaction and so it must be rollbacked). + + Hooks controls: + + :attr:`hooks_mode`, may be either `HOOKS_ALLOW_ALL` or `HOOKS_DENY_ALL`. + + :attr:`enabled_hook_cats`, when :attr:`hooks_mode` is + `HOOKS_DENY_ALL`, this set contains hooks categories that are enabled. + + :attr:`disabled_hook_cats`, when :attr:`hooks_mode` is + `HOOKS_ALLOW_ALL`, this set contains hooks categories that are disabled. + + Security level Management: + + :attr:`read_security` and :attr:`write_security`, boolean flags telling if + read/write security is currently activated. + + """ + + def __init__(self, txid, session, rewriter): + #: transaction unique id + self.transactionid = txid + #: reentrance handling + self.ctx_count = 0 + + #: server.Repository object + self.repo = session.repo + self.vreg = self.repo.vreg + + #: connection handling mode + self.mode = session.default_mode + #: connection set used to execute queries on sources + self._cnxset = None + #: CnxSetTracker used to report cnxset usage + self._cnxset_tracker = session._cnxset_tracker + #: is this transaction from a client or internal to the repo + self.running_dbapi_query = True + + #: dict containing arbitrary data cleared at the end of the transaction + self.data = {} + #: ordered list of operations to be processed on commit/rollback + self.pending_operations = [] + #: (None, 'precommit', 'postcommit', 'uncommitable') + self.commit_state = None + + ### hook control attribute + self.hooks_mode = HOOKS_ALLOW_ALL + self.disabled_hook_cats = set() + self.enabled_hook_cats = set() + self.pruned_hooks_cache = {} + + + ### security control attributes + self._read_security = DEFAULT_SECURITY # handled by a property + self.write_security = DEFAULT_SECURITY + + # undo control + config = session.repo.config + if config.creating or config.repairing or session.is_internal_session: + self.undo_actions = False + else: + self.undo_actions = config['undo-enabled'] + + # RQLRewriter are not thread safe + self._rewriter = rewriter + + @property + def transaction_data(self): + return self.data -class TransactionData(object): - def __init__(self, txid): - self.transactionid = txid - self.ctx_count = 0 + def clear(self): + """reset internal data""" + self.data = {} + #: ordered list of operations to be processed on commit/rollback + self.pending_operations = [] + #: (None, 'precommit', 'postcommit', 'uncommitable') + self.commit_state = None + self.pruned_hooks_cache = {} + # Connection Set Management ############################################### + @property + def cnxset(self): + return self._cnxset + + @cnxset.setter + def cnxset(self, new_cnxset): + with self._cnxset_tracker: + old_cnxset = self._cnxset + if new_cnxset is old_cnxset: + return #nothing to do + if old_cnxset is not None: + self._cnxset = None + self.ctx_count -= 1 + self._cnxset_tracker.forget(self.transactionid, old_cnxset) + if new_cnxset is not None: + self._cnxset_tracker.record(self.transactionid, new_cnxset) + self._cnxset = new_cnxset + self.ctx_count += 1 + + def set_cnxset(self): + """the transaction need a connections set to execute some queries""" + if self.cnxset is None: + cnxset = self.repo._get_cnxset() + try: + self.cnxset = cnxset + try: + cnxset.cnxset_set() + except: + self.cnxset = None + raise + except: + self.repo._free_cnxset(cnxset) + raise + return self.cnxset + + def free_cnxset(self, ignoremode=False): + """the transaction is no longer using its connections set, at least for some time""" + # cnxset may be none if no operation has been done since last commit + # or rollback + cnxset = self.cnxset + if cnxset is not None and (ignoremode or self.mode == 'read'): + try: + self.cnxset = None + finally: + cnxset.cnxset_freed() + self.repo._free_cnxset(cnxset) + + + # Entity cache management ################################################# + # + # The transaction entity cache as held in tx.data it is removed at end the + # end of the transaction (commit and rollback) + # + # XXX transaction level caching may be a pb with multiple repository + # instances, but 1. this is probably not the only one :$ and 2. it may be + # an acceptable risk. Anyway we could activate it or not according to a + # configuration option + + def set_entity_cache(self, entity): + """Add `entity` to the transaction entity cache""" + ecache = self.data.setdefault('ecache', {}) + ecache.setdefault(entity.eid, entity) + + def entity_cache(self, eid): + """get cache entity for `eid`""" + return self.data['ecache'][eid] + + def cached_entities(self): + """return the whole entity cache""" + return self.data.get('ecache', {}).values() + + def drop_entity_cache(self, eid=None): + """drop entity from the cache + + If eid is None, the whole cache is dropped""" + if eid is None: + self.data.pop('ecache', None) + else: + del self.data['ecache'][eid] + + # Tracking of entity added of removed in the transaction ################## + # + # Those are function to allows cheap call from client in other process. + + def deleted_in_transaction(self, eid): + """return True if the entity of the given eid is being deleted in the + current transaction + """ + return eid in self.data.get('pendingeids', ()) + + def added_in_transaction(self, eid): + """return True if the entity of the given eid is being created in the + current transaction + """ + return eid in self.data.get('neweids', ()) + + # Operation management #################################################### + + def add_operation(self, operation, index=None): + """add an operation to be executed at the end of the transaction""" + if index is None: + self.pending_operations.append(operation) + else: + self.pending_operations.insert(index, operation) + + # Hooks control ########################################################### + + def disable_hook_categories(self, *categories): + """disable the given hook categories: + + - on HOOKS_DENY_ALL mode, ensure those categories are not enabled + - on HOOKS_ALLOW_ALL mode, ensure those categories are disabled + """ + changes = set() + self.pruned_hooks_cache.clear() + categories = set(categories) + if self.hooks_mode is HOOKS_DENY_ALL: + enabledcats = self.enabled_hook_cats + changes = enabledcats & categories + enabledcats -= changes # changes is small hence faster + else: + disabledcats = self.disabled_hook_cats + changes = categories - disabledcats + disabledcats |= changes # changes is small hence faster + return tuple(changes) + + def enable_hook_categories(self, *categories): + """enable the given hook categories: + + - on HOOKS_DENY_ALL mode, ensure those categories are enabled + - on HOOKS_ALLOW_ALL mode, ensure those categories are not disabled + """ + changes = set() + self.pruned_hooks_cache.clear() + categories = set(categories) + if self.hooks_mode is HOOKS_DENY_ALL: + enabledcats = self.enabled_hook_cats + changes = categories - enabledcats + enabledcats |= changes # changes is small hence faster + else: + disabledcats = self.disabled_hook_cats + changes = disabledcats & categories + disabledcats -= changes # changes is small hence faster + return tuple(changes) + + def is_hook_category_activated(self, category): + """return a boolean telling if the given category is currently activated + or not + """ + if self.hooks_mode is HOOKS_DENY_ALL: + return category in self.enabled_hook_cats + return category not in self.disabled_hook_cats + + def is_hook_activated(self, hook): + """return a boolean telling if the given hook class is currently + activated or not + """ + return self.is_hook_category_activated(hook.category) + + # Security management ##################################################### + @property + def read_security(self): + return self._read_security + + @read_security.setter + def read_security(self, activated): + oldmode = self._read_security + self._read_security = activated + # running_dbapi_query used to detect hooks triggered by a 'dbapi' query + # (eg not issued on the session). This is tricky since we the execution + # model of a (write) user query is: + # + # repository.execute (security enabled) + # \-> querier.execute + # \-> repo.glob_xxx (add/update/delete entity/relation) + # \-> deactivate security before calling hooks + # \-> WE WANT TO CHECK QUERY NATURE HERE + # \-> potentially, other calls to querier.execute + # + # so we can't rely on simply checking session.read_security, but + # recalling the first transition from DEFAULT_SECURITY to something + # else (False actually) is not perfect but should be enough + # + # also reset running_dbapi_query to true when we go back to + # DEFAULT_SECURITY + self.running_dbapi_query = (oldmode is DEFAULT_SECURITY + or activated is DEFAULT_SECURITY) + + # undo support ############################################################ + + def ertype_supports_undo(self, ertype): + return self.undo_actions and ertype not in NO_UNDO_TYPES + + def transaction_uuid(self, set=True): + uuid = self.data.get('tx_uuid') + if set and uuid is None: + raise KeyError + return uuid + + def transaction_inc_action_counter(self): + num = self.data.setdefault('tx_action_count', 0) + 1 + self.data['tx_action_count'] = num + return num + # db-api like interface ################################################### + + def source_defs(self): + return self.repo.source_defs() + + def describe(self, eid, asdict=False): + """return a tuple (type, sourceuri, extid) for the entity with id """ + metas = self.repo.type_and_source_from_eid(eid, self) + if asdict: + return dict(zip(('type', 'source', 'extid', 'asource'), metas)) + # XXX :-1 for cw compat, use asdict=True for full information + return metas[:-1] + + + def source_from_eid(self, eid): + """return the source where the entity with id is located""" + return self.repo.source_from_eid(eid, self) + + # resource accessors ###################################################### + + def system_sql(self, sql, args=None, rollback_on_failure=True): + """return a sql cursor on the system database""" + if sql.split(None, 1)[0].upper() != 'SELECT': + self.mode = 'write' + source = self.cnxset.source('system') + try: + return source.doexec(self, sql, args, rollback=rollback_on_failure) + except (source.OperationalError, source.InterfaceError): + if not rollback_on_failure: + raise + source.warning("trying to reconnect") + self.cnxset.reconnect(source) + return source.doexec(self, sql, args, rollback=rollback_on_failure) + + def rtype_eids_rdef(self, rtype, eidfrom, eidto): + # use type_and_source_from_eid instead of type_from_eid for optimization + # (avoid two extra methods call) + subjtype = self.repo.type_and_source_from_eid(eidfrom, self)[0] + objtype = self.repo.type_and_source_from_eid(eidto, self)[0] + return self.vreg.schema.rschema(rtype).rdefs[(subjtype, objtype)] + + +def tx_attr(attr_name, writable=False): + """return a property to forward attribute access to transaction. + + This is to be used by session""" + args = {} + def attr_from_tx(session): + return getattr(session._tx, attr_name) + args['fget'] = attr_from_tx + if writable: + def write_attr(session, value): + return setattr(session._tx, attr_name, value) + args['fset'] = write_attr + return property(**args) + +def tx_meth(meth_name): + """return a function forwarding calls to transaction. + + This is to be used by session""" + def meth_from_tx(session, *args, **kwargs): + return getattr(session._tx, meth_name)(*args, **kwargs) + return meth_from_tx class Session(RequestSessionBase): - """Repository usersession, tie a session id, user, connections set and - other session data all together. + """Repository user session + + This tie all together: + * session id, + * user, + * connections set, + * other session data. About session storage / transactions ------------------------------------ @@ -161,20 +731,17 @@ :attr:`data` is a dictionary containing shared data, used to communicate extra information between the client and the repository - :attr:`_tx_data` is a dictionary of :class:`TransactionData` instance, one + :attr:`_txs` is a dictionary of :class:`TransactionData` instance, one for each running transaction. The key is the transaction id. By default the transaction id is the thread name but it can be otherwise (per dbapi cursor for instance, or per thread name *from another process*). - :attr:`__threaddata` is a thread local storage whose `txdata` attribute - refers to the proper instance of :class:`TransactionData` according to the + :attr:`__threaddata` is a thread local storage whose `tx` attribute + refers to the proper instance of :class:`Transaction` according to the transaction. - :attr:`_threads_in_transaction` is a set of (thread, connections set) - referencing threads that currently hold a connections set for the session. - - You should not have to use neither :attr:`_txdata` nor :attr:`__threaddata`, - simply access transaction data transparently through the :attr:`_threaddata` + You should not have to use neither :attr:`_tx` nor :attr:`__threaddata`, + simply access transaction data transparently through the :attr:`_tx` property. Also, you usually don't have to access it directly since current transaction's data may be accessed/modified through properties / methods: @@ -184,11 +751,24 @@ this may also be used as a communication channel between the client and the repository. + .. automethod:: cubicweb.server.session.Session.get_shared_data + .. automethod:: cubicweb.server.session.Session.set_shared_data + .. automethod:: cubicweb.server.session.Session.added_in_transaction + .. automethod:: cubicweb.server.session.Session.deleted_in_transaction + + Transaction state information: + + :attr:`running_dbapi_query`, boolean flag telling if the executing query + is coming from a dbapi connection or is a query from within the repository + :attr:`cnxset`, the connections set to use to execute queries on sources. During a transaction, the connection set may be freed so that is may be used by another session as long as no writing is done. This means we can have multiple sessions with a reasonably low connections set pool size. + .. automethod:: cubicweb.server.session.set_cnxset + .. automethod:: cubicweb.server.session.free_cnxset + :attr:`mode`, string telling the connections set handling mode, may be one of 'read' (connections set may be freed), 'write' (some write was done in the connections set, it can't be freed before end of the transaction), @@ -204,9 +784,20 @@ 'uncommitable' (some :exc:`ValidationError` or :exc:`Unauthorized` error has been raised during the transaction and so it must be rollbacked). + .. automethod:: cubicweb.server.session.Session.commit + .. automethod:: cubicweb.server.session.Session.rollback + .. automethod:: cubicweb.server.session.Session.close + .. automethod:: cubicweb.server.session.Session.closed + + Security level Management: + :attr:`read_security` and :attr:`write_security`, boolean flags telling if read/write security is currently activated. + .. automethod:: cubicweb.server.session.Session.security_enabled + + Hooks Management: + :attr:`hooks_mode`, may be either `HOOKS_ALLOW_ALL` or `HOOKS_DENY_ALL`. :attr:`enabled_hook_categories`, when :attr:`hooks_mode` is @@ -215,12 +806,23 @@ :attr:`disabled_hook_categories`, when :attr:`hooks_mode` is `HOOKS_ALLOW_ALL`, this set contains hooks categories that are disabled. + .. automethod:: cubicweb.server.session.Session.deny_all_hooks_but + .. automethod:: cubicweb.server.session.Session.allow_all_hooks_but + .. automethod:: cubicweb.server.session.Session.is_hook_category_activated + .. automethod:: cubicweb.server.session.Session.is_hook_activated - :attr:`running_dbapi_query`, boolean flag telling if the executing query - is coming from a dbapi connection or is a query from within the repository + Data manipulation: + + .. automethod:: cubicweb.server.session.Session.add_relation + .. automethod:: cubicweb.server.session.Session.add_relations + .. automethod:: cubicweb.server.session.Session.delete_relation - .. automethod:: cubicweb.server.session.deny_all_hooks_but - .. automethod:: cubicweb.server.session.all_all_hooks_but + Other: + + .. automethod:: cubicweb.server.session.Session.call_service + + + """ is_request = False is_internal_session = False @@ -232,11 +834,6 @@ self.repo = repo self.timestamp = time() self.default_mode = 'read' - # undo support - if repo.config.creating or repo.config.repairing or self.is_internal_session: - self.undo_actions = False - else: - self.undo_actions = repo.config['undo-enabled'] # short cut to querier .execute method self._execute = repo.querier.execute # shared data, used to communicate extra information between the client @@ -244,17 +841,52 @@ self.data = {} # i18n initialization self.set_language(user.prefered_language()) - # internals - self._tx_data = {} + ### internals + # Transaction of this section + self._txs = {} + # Data local to the thread self.__threaddata = threading.local() - self._threads_in_transaction = set() + self._cnxset_tracker = CnxSetTracker() self._closed = False - self._closed_lock = threading.Lock() + self._lock = threading.RLock() def __unicode__(self): return '' % ( unicode(self.user.login), self.id, id(self)) + def get_tx(self, txid): + """return the transaction attached to this session + + Transaction is created if necessary""" + with self._lock: # no transaction exist with the same id + try: + tx = self._txs[txid] + except KeyError: + rewriter = RQLRewriter(self) + tx = Transaction(txid, self, rewriter) + self._txs[txid] = tx + return tx + + def set_tx(self, txid=None): + """set the default transaction of the current thread to + + Transaction is created if necessary""" + if txid is None: + txid = threading.currentThread().getName() + self.__threaddata.tx = self.get_tx(txid) + + @property + def _tx(self): + """default transaction for current session in current thread""" + try: + return self.__threaddata.tx + except AttributeError: + self.set_tx() + return self.__threaddata.tx + + def get_option_value(self, option, foreid=None): + return self.repo.get_option_value(option, foreid) + def transaction(self, free_cnxset=True): """return context manager to enter a transaction for the session: when exiting the `with` block on exception, call `session.rollback()`, else @@ -265,40 +897,19 @@ """ return transaction(self, free_cnxset) - def set_tx_data(self, txid=None): - if txid is None: - txid = threading.currentThread().getName() - try: - self.__threaddata.txdata = self._tx_data[txid] - except KeyError: - self.__threaddata.txdata = self._tx_data[txid] = TransactionData(txid) - - @property - def _threaddata(self): - try: - return self.__threaddata.txdata - except AttributeError: - self.set_tx_data() - return self.__threaddata.txdata - - def get_option_value(self, option, foreid=None): - return self.repo.get_option_value(option, foreid) def hijack_user(self, user): """return a fake request/session using specified user""" session = Session(user, self.repo) - threaddata = session._threaddata - threaddata.cnxset = self.cnxset - # we attributed a connections set, need to update ctx_count else it will be freed - # while undesired - threaddata.ctx_count = 1 + tx = session._tx + tx.cnxset = self.cnxset # share pending_operations, else operation added in the hi-jacked # session such as SendMailOp won't ever be processed - threaddata.pending_operations = self.pending_operations - # everything in transaction_data should be copied back but the entity + tx.pending_operations = self.pending_operations + # everything in tx.data should be copied back but the entity # type cache we don't want to avoid security pb - threaddata.transaction_data = self.transaction_data.copy() - threaddata.transaction_data.pop('ecache', None) + tx.data = self._tx.data.copy() + tx.data.pop('ecache', None) return session def add_relation(self, fromeid, rtype, toeid): @@ -323,7 +934,7 @@ ''' edited_entities = {} relations_dict = {} - with security_enabled(self, False, False): + with self.security_enabled(False, False): for rtype, eids in relations: if self.vreg.schema[rtype].inlined: for fromeid, toeid in eids: @@ -352,7 +963,7 @@ You may use this in hooks when you know both eids of the relation you want to delete. """ - with security_enabled(self, False, False): + with self.security_enabled(False, False): if self.vreg.schema[rtype].inlined: entity = self.entity_from_eid(fromeid) entity.cw_attr_cache[rtype] = None @@ -429,266 +1040,37 @@ # resource accessors ###################################################### - def system_sql(self, sql, args=None, rollback_on_failure=True): - """return a sql cursor on the system database""" - if sql.split(None, 1)[0].upper() != 'SELECT': - self.mode = 'write' - source = self.cnxset.source('system') - try: - return source.doexec(self, sql, args, rollback=rollback_on_failure) - except (source.OperationalError, source.InterfaceError): - if not rollback_on_failure: - raise - source.warning("trying to reconnect") - self.cnxset.reconnect(source) - return source.doexec(self, sql, args, rollback=rollback_on_failure) - - def deleted_in_transaction(self, eid): - """return True if the entity of the given eid is being deleted in the - current transaction - """ - return eid in self.transaction_data.get('pendingeids', ()) - - def added_in_transaction(self, eid): - """return True if the entity of the given eid is being created in the - current transaction - """ - return eid in self.transaction_data.get('neweids', ()) - - def rtype_eids_rdef(self, rtype, eidfrom, eidto): - # use type_and_source_from_eid instead of type_from_eid for optimization - # (avoid two extra methods call) - subjtype = self.repo.type_and_source_from_eid(eidfrom, self)[0] - objtype = self.repo.type_and_source_from_eid(eidto, self)[0] - return self.vreg.schema.rschema(rtype).rdefs[(subjtype, objtype)] + system_sql = tx_meth('system_sql') + deleted_in_transaction = tx_meth('deleted_in_transaction') + added_in_transaction = tx_meth('added_in_transaction') + rtype_eids_rdef = tx_meth('rtype_eids_rdef') # security control ######################################################### - DEFAULT_SECURITY = object() # evaluated to true by design def security_enabled(self, read=None, write=None): - return security_enabled(self, read=read, write=write) - - def init_security(self, read, write): - if read is None: - oldread = None - else: - oldread = self.set_read_security(read) - if write is None: - oldwrite = None - else: - oldwrite = self.set_write_security(write) - self._threaddata.ctx_count += 1 - return oldread, oldwrite - - def reset_security(self, read, write): - txstore = self._threaddata - txstore.ctx_count -= 1 - if txstore.ctx_count == 0: - self._clear_thread_storage(txstore) - else: - if read is not None: - self.set_read_security(read) - if write is not None: - self.set_write_security(write) - - @property - def read_security(self): - """return a boolean telling if read security is activated or not""" - txstore = self._threaddata - if txstore is None: - return self.DEFAULT_SECURITY - try: - return txstore.read_security - except AttributeError: - txstore.read_security = self.DEFAULT_SECURITY - return txstore.read_security - - def set_read_security(self, activated): - """[de]activate read security, returning the previous value set for - later restoration. + return _security_enabled(self, read=read, write=write) - you should usually use the `security_enabled` context manager instead - of this to change security settings. - """ - txstore = self._threaddata - if txstore is None: - return self.DEFAULT_SECURITY - oldmode = getattr(txstore, 'read_security', self.DEFAULT_SECURITY) - txstore.read_security = activated - # dbapi_query used to detect hooks triggered by a 'dbapi' query (eg not - # issued on the session). This is tricky since we the execution model of - # a (write) user query is: - # - # repository.execute (security enabled) - # \-> querier.execute - # \-> repo.glob_xxx (add/update/delete entity/relation) - # \-> deactivate security before calling hooks - # \-> WE WANT TO CHECK QUERY NATURE HERE - # \-> potentially, other calls to querier.execute - # - # so we can't rely on simply checking session.read_security, but - # recalling the first transition from DEFAULT_SECURITY to something - # else (False actually) is not perfect but should be enough - # - # also reset dbapi_query to true when we go back to DEFAULT_SECURITY - txstore.dbapi_query = (oldmode is self.DEFAULT_SECURITY - or activated is self.DEFAULT_SECURITY) - return oldmode - - @property - def write_security(self): - """return a boolean telling if write security is activated or not""" - txstore = self._threaddata - if txstore is None: - return self.DEFAULT_SECURITY - try: - return txstore.write_security - except AttributeError: - txstore.write_security = self.DEFAULT_SECURITY - return txstore.write_security - - def set_write_security(self, activated): - """[de]activate write security, returning the previous value set for - later restoration. - - you should usually use the `security_enabled` context manager instead - of this to change security settings. - """ - txstore = self._threaddata - if txstore is None: - return self.DEFAULT_SECURITY - oldmode = getattr(txstore, 'write_security', self.DEFAULT_SECURITY) - txstore.write_security = activated - return oldmode - - @property - def running_dbapi_query(self): - """return a boolean telling if it's triggered by a db-api query or by - a session query. - - To be used in hooks, else may have a wrong value. - """ - return getattr(self._threaddata, 'dbapi_query', True) + read_security = tx_attr('read_security', writable=True) + write_security = tx_attr('write_security', writable=True) + running_dbapi_query = tx_attr('running_dbapi_query') # hooks activation control ################################################# # all hooks should be activated during normal execution - HOOKS_ALLOW_ALL = object() - HOOKS_DENY_ALL = object() - def allow_all_hooks_but(self, *categories): - return hooks_control(self, self.HOOKS_ALLOW_ALL, *categories) + return _hooks_control(self, HOOKS_ALLOW_ALL, *categories) def deny_all_hooks_but(self, *categories): - return hooks_control(self, self.HOOKS_DENY_ALL, *categories) - - @property - def hooks_mode(self): - return getattr(self._threaddata, 'hooks_mode', self.HOOKS_ALLOW_ALL) - - def set_hooks_mode(self, mode): - assert mode is self.HOOKS_ALLOW_ALL or mode is self.HOOKS_DENY_ALL - oldmode = getattr(self._threaddata, 'hooks_mode', self.HOOKS_ALLOW_ALL) - self._threaddata.hooks_mode = mode - return oldmode - - def init_hooks_mode_categories(self, mode, categories): - oldmode = self.set_hooks_mode(mode) - if mode is self.HOOKS_DENY_ALL: - changes = self.enable_hook_categories(*categories) - else: - changes = self.disable_hook_categories(*categories) - self._threaddata.ctx_count += 1 - return oldmode, changes + return _hooks_control(self, HOOKS_DENY_ALL, *categories) - def reset_hooks_mode_categories(self, oldmode, mode, categories): - txstore = self._threaddata - txstore.ctx_count -= 1 - if txstore.ctx_count == 0: - self._clear_thread_storage(txstore) - else: - try: - if categories: - if mode is self.HOOKS_DENY_ALL: - return self.disable_hook_categories(*categories) - else: - return self.enable_hook_categories(*categories) - finally: - self.set_hooks_mode(oldmode) - - @property - def disabled_hook_categories(self): - try: - return getattr(self._threaddata, 'disabled_hook_cats') - except AttributeError: - cats = self._threaddata.disabled_hook_cats = set() - return cats - - @property - def enabled_hook_categories(self): - try: - return getattr(self._threaddata, 'enabled_hook_cats') - except AttributeError: - cats = self._threaddata.enabled_hook_cats = set() - return cats + hooks_mode = tx_attr('hooks_mode') - def disable_hook_categories(self, *categories): - """disable the given hook categories: - - - on HOOKS_DENY_ALL mode, ensure those categories are not enabled - - on HOOKS_ALLOW_ALL mode, ensure those categories are disabled - """ - changes = set() - self.pruned_hooks_cache.clear() - if self.hooks_mode is self.HOOKS_DENY_ALL: - enabledcats = self.enabled_hook_categories - for category in categories: - if category in enabledcats: - enabledcats.remove(category) - changes.add(category) - else: - disabledcats = self.disabled_hook_categories - for category in categories: - if category not in disabledcats: - disabledcats.add(category) - changes.add(category) - return tuple(changes) - - def enable_hook_categories(self, *categories): - """enable the given hook categories: - - - on HOOKS_DENY_ALL mode, ensure those categories are enabled - - on HOOKS_ALLOW_ALL mode, ensure those categories are not disabled - """ - changes = set() - self.pruned_hooks_cache.clear() - if self.hooks_mode is self.HOOKS_DENY_ALL: - enabledcats = self.enabled_hook_categories - for category in categories: - if category not in enabledcats: - enabledcats.add(category) - changes.add(category) - else: - disabledcats = self.disabled_hook_categories - for category in categories: - if category in disabledcats: - disabledcats.remove(category) - changes.add(category) - return tuple(changes) - - def is_hook_category_activated(self, category): - """return a boolean telling if the given category is currently activated - or not - """ - if self.hooks_mode is self.HOOKS_DENY_ALL: - return category in self.enabled_hook_categories - return category not in self.disabled_hook_categories - - def is_hook_activated(self, hook): - """return a boolean telling if the given hook class is currently - activated or not - """ - return self.is_hook_category_activated(hook.category) + disabled_hook_categories = tx_attr('disabled_hook_cats') + enabled_hook_categories = tx_attr('enabled_hook_cats') + disable_hook_categories = tx_meth('disable_hook_categories') + enable_hook_categories = tx_meth('enable_hook_categories') + is_hook_category_activated = tx_meth('is_hook_category_activated') + is_hook_activated = tx_meth('is_hook_activated') # connection management ################################################### @@ -712,19 +1094,8 @@ else: # mode == 'write' self.default_mode = 'read' - def get_mode(self): - return getattr(self._threaddata, 'mode', self.default_mode) - def set_mode(self, value): - self._threaddata.mode = value - mode = property(get_mode, set_mode, - doc='transaction mode (read/write/transaction), resetted to' - ' default_mode on commit / rollback') - - def get_commit_state(self): - return getattr(self._threaddata, 'commit_state', None) - def set_commit_state(self, value): - self._threaddata.commit_state = value - commit_state = property(get_commit_state, set_commit_state) + mode = tx_attr('mode', writable=True) + commit_state = tx_attr('commit_state', writable=True) @property def cnxset(self): @@ -732,65 +1103,28 @@ if self._closed: self.free_cnxset(True) raise Exception('try to access connections set on a closed session %s' % self.id) - return getattr(self._threaddata, 'cnxset', None) + return self._tx.cnxset def set_cnxset(self): """the session need a connections set to execute some queries""" - with self._closed_lock: + with self._lock: # can probably be removed if self._closed: self.free_cnxset(True) raise Exception('try to set connections set on a closed session %s' % self.id) - if self.cnxset is None: - # get connections set first to avoid race-condition - self._threaddata.cnxset = cnxset = self.repo._get_cnxset() - self._threaddata.ctx_count += 1 - try: - cnxset.cnxset_set() - except Exception: - self._threaddata.cnxset = None - self.repo._free_cnxset(cnxset) - raise - self._threads_in_transaction.add( - (threading.currentThread(), cnxset) ) - return self._threaddata.cnxset - - def _free_thread_cnxset(self, thread, cnxset, force_close=False): - try: - self._threads_in_transaction.remove( (thread, cnxset) ) - except KeyError: - # race condition on cnxset freeing (freed by commit or rollback vs - # close) - pass - else: - if force_close: - cnxset.reconnect() - else: - cnxset.cnxset_freed() - # free cnxset once everything is done to avoid race-condition - self.repo._free_cnxset(cnxset) - - def free_cnxset(self, ignoremode=False): - """the session is no longer using its connections set, at least for some time""" - # cnxset may be none if no operation has been done since last commit - # or rollback - cnxset = getattr(self._threaddata, 'cnxset', None) - if cnxset is not None and (ignoremode or self.mode == 'read'): - # even in read mode, we must release the current transaction - self._free_thread_cnxset(threading.currentThread(), cnxset) - del self._threaddata.cnxset - self._threaddata.ctx_count -= 1 + return self._tx.set_cnxset() + free_cnxset = tx_meth('free_cnxset') def _touch(self): """update latest session usage timestamp and reset mode to read""" self.timestamp = time() - self.local_perm_cache.clear() # XXX simply move in transaction_data, no? + self.local_perm_cache.clear() # XXX simply move in tx.data, no? # shared data handling ################################################### def get_shared_data(self, key, default=None, pop=False, txdata=False): """return value associated to `key` in session data""" if txdata: - data = self.transaction_data + data = self._tx.data else: data = self.data if pop: @@ -801,7 +1135,7 @@ def set_shared_data(self, key, value, txdata=False): """set value associated to `key` in session data""" if txdata: - self.transaction_data[key] = value + self._tx.data[key] = value else: self.data[key] = value @@ -819,28 +1153,10 @@ """return a rql cursor""" return self - def set_entity_cache(self, entity): - # XXX session level caching may be a pb with multiple repository - # instances, but 1. this is probably not the only one :$ and 2. it - # may be an acceptable risk. Anyway we could activate it or not - # according to a configuration option - try: - self.transaction_data['ecache'].setdefault(entity.eid, entity) - except KeyError: - self.transaction_data['ecache'] = ecache = {} - ecache[entity.eid] = entity - - def entity_cache(self, eid): - return self.transaction_data['ecache'][eid] - - def cached_entities(self): - return self.transaction_data.get('ecache', {}).values() - - def drop_entity_cache(self, eid=None): - if eid is None: - self.transaction_data.pop('ecache', None) - else: - del self.transaction_data['ecache'][eid] + set_entity_cache = tx_meth('set_entity_cache') + entity_cache = tx_meth('entity_cache') + cache_entities = tx_meth('cached_entities') + drop_entity_cache = tx_meth('drop_entity_cache') def from_controller(self): """return the id (string) of the controller issuing the request (no @@ -848,22 +1164,10 @@ """ return 'view' - def source_defs(self): - return self.repo.source_defs() + source_defs = tx_meth('source_defs') + describe = tx_meth('describe') + source_from_eid = tx_meth('source_from_eid') - def describe(self, eid, asdict=False): - """return a tuple (type, sourceuri, extid) for the entity with id """ - metas = self.repo.type_and_source_from_eid(eid, self) - if asdict: - return dict(zip(('type', 'source', 'extid', 'asource'), metas)) - # XXX :-1 for cw compat, use asdict=True for full information - return metas[:-1] - - # db-api like interface ################################################### - - def source_from_eid(self, eid): - """return the source where the entity with id is located""" - return self.repo.source_from_eid(eid, self) def execute(self, rql, kwargs=None, eid_key=None, build_descr=True): """db-api like method directly linked to the querier execute method. @@ -884,34 +1188,29 @@ by _touch """ try: - txstore = self.__threaddata.txdata + tx = self.__threaddata.tx except AttributeError: pass else: if free_cnxset: self.free_cnxset() - if txstore.ctx_count == 0: - self._clear_thread_storage(txstore) + if tx.ctx_count == 0: + self._clear_thread_storage(tx) else: - self._clear_tx_storage(txstore) + self._clear_tx_storage(tx) else: - self._clear_tx_storage(txstore) + self._clear_tx_storage(tx) - def _clear_thread_storage(self, txstore): - self._tx_data.pop(txstore.transactionid, None) + def _clear_thread_storage(self, tx): + self._txs.pop(tx.transactionid, None) try: - del self.__threaddata.txdata + del self.__threaddata.tx except AttributeError: pass - def _clear_tx_storage(self, txstore): - for name in ('commit_state', 'transaction_data', - 'pending_operations', '_rewriter', - 'pruned_hooks_cache'): - try: - delattr(txstore, name) - except AttributeError: - continue + def _clear_tx_storage(self, tx): + tx.clear() + tx._rewriter = RQLRewriter(self) def commit(self, free_cnxset=True, reset_pool=None): """commit the current session's transaction""" @@ -937,7 +1236,7 @@ debug = server.DEBUG & server.DBG_OPS try: # by default, operations are executed with security turned off - with security_enabled(self, False, False): + with self.security_enabled(False, False): processed = [] self.commit_state = 'precommit' if debug: @@ -1008,7 +1307,7 @@ DeprecationWarning, stacklevel=2) free_cnxset = reset_pool # don't use self.cnxset, rollback may be called with _closed == True - cnxset = getattr(self._threaddata, 'cnxset', None) + cnxset = self._tx.cnxset if cnxset is None: self._clear_thread_data() self._touch() @@ -1016,7 +1315,7 @@ return try: # by default, operations are executed with security turned off - with security_enabled(self, False, False): + with self.security_enabled(False, False): while self.pending_operations: try: operation = self.pending_operations.pop(0) @@ -1033,97 +1332,65 @@ self._clear_thread_data(free_cnxset) def close(self): - """do not close connections set on session close, since they are shared now""" - with self._closed_lock: + # do not close connections set on session close, since they are shared now + tracker = self._cnxset_tracker + with self._lock: self._closed = True - # copy since _threads_in_transaction maybe modified while waiting - for thread, cnxset in self._threads_in_transaction.copy(): - if thread is threading.currentThread(): - continue - self.info('waiting for thread %s', thread) - # do this loop/break instead of a simple join(10) in case thread is - # the main thread (in which case it will be removed from - # self._threads_in_transaction but still be alive...) - for i in xrange(10): - thread.join(1) - if not (thread.isAlive() and - (thread, cnxset) in self._threads_in_transaction): - break - else: - self.error('thread %s still alive after 10 seconds, will close ' - 'session anyway', thread) - self._free_thread_cnxset(thread, cnxset, force_close=True) + tracker.close() self.rollback() + self.info('waiting for open transaction of session: %s', self) + timeout = 10 + pendings = tracker.wait(timeout) + if pendings: + self.error('%i transaction still alive after 10 seconds, will close ' + 'session anyway', len(pendings)) + for txid in pendings: + tx = self._txs.get(txid) + if tx is not None: + # drop tx.cnxset + with tracker: + try: + cnxset = tx.cnxset + if cnxset is None: + continue + tx.cnxset = None + except RuntimeError: + msg = 'issue while force free of cnxset in %s' + self.error(msg, tx) + # cnxset.reconnect() do an hard reset of the cnxset + # it force it to be freed + cnxset.reconnect() + self.repo._free_cnxset(cnxset) del self.__threaddata - del self._tx_data + del self._txs @property def closed(self): - return not hasattr(self, '_tx_data') + return not hasattr(self, '_txs') # transaction data/operations management ################################## - @property - def transaction_data(self): - try: - return self._threaddata.transaction_data - except AttributeError: - self._threaddata.transaction_data = {} - return self._threaddata.transaction_data - - @property - def pending_operations(self): - try: - return self._threaddata.pending_operations - except AttributeError: - self._threaddata.pending_operations = [] - return self._threaddata.pending_operations - - @property - def pruned_hooks_cache(self): - try: - return self._threaddata.pruned_hooks_cache - except AttributeError: - self._threaddata.pruned_hooks_cache = {} - return self._threaddata.pruned_hooks_cache - - def add_operation(self, operation, index=None): - """add an operation""" - if index is None: - self.pending_operations.append(operation) - else: - self.pending_operations.insert(index, operation) + transaction_data = tx_attr('data') + pending_operations = tx_attr('pending_operations') + pruned_hooks_cache = tx_attr('pruned_hooks_cache') + add_operation = tx_meth('add_operation') # undo support ############################################################ - def ertype_supports_undo(self, ertype): - return self.undo_actions and ertype not in NO_UNDO_TYPES + ertype_supports_undo = tx_meth('ertype_supports_undo') + transaction_inc_action_counter = tx_meth('transaction_inc_action_counter') def transaction_uuid(self, set=True): try: - return self.transaction_data['tx_uuid'] + return self._tx.transaction_uuid(set=set) except KeyError: - if not set: - return - self.transaction_data['tx_uuid'] = uuid = uuid4().hex + self._tx.data['tx_uuid'] = uuid = uuid4().hex self.repo.system_source.start_undoable_transaction(self, uuid) return uuid - def transaction_inc_action_counter(self): - num = self.transaction_data.setdefault('tx_action_count', 0) + 1 - self.transaction_data['tx_action_count'] = num - return num - # querier helpers ######################################################### - @property - def rql_rewriter(self): - # in thread local storage since the rewriter isn't thread safe - try: - return self._threaddata._rewriter - except AttributeError: - self._threaddata._rewriter = RQLRewriter(self) - return self._threaddata._rewriter + rql_rewriter = tx_attr('_rewriter') # deprecated ############################################################### @@ -1144,32 +1411,15 @@ def reset_pool(self): return self.free_cnxset() - @deprecated("[3.7] execute is now unsafe by default in hooks/operation. You" - " can also control security with the security_enabled context " - "manager") - def unsafe_execute(self, rql, kwargs=None, eid_key=None, build_descr=True, - propagate=False): - """like .execute but with security checking disabled (this method is - internal to the server, it's not part of the db-api) - """ - with security_enabled(self, read=False, write=False): - return self.execute(rql, kwargs, eid_key, build_descr) - - @property - @deprecated("[3.7] is_super_session is deprecated, test " - "session.read_security and or session.write_security") - def is_super_session(self): - return not self.read_security or not self.write_security - - @deprecated("[3.7] session is actual session") - def actual_session(self): - """return the original parent session if any, else self""" - return self - # these are overridden by set_log_methods below # only defining here to prevent pylint from complaining info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None +Session.HOOKS_ALLOW_ALL = HOOKS_ALLOW_ALL +Session.HOOKS_DENY_ALL = HOOKS_DENY_ALL +Session.DEFAULT_SECURITY = DEFAULT_SECURITY + + class InternalSession(Session): """special session created internaly by the repository""" @@ -1195,7 +1445,7 @@ if self.repo.shutting_down: self.free_cnxset(True) raise ShuttingDown('repository is shutting down') - return getattr(self._threaddata, 'cnxset', None) + return self._tx.cnxset class InternalManager(object): diff -r 7920f439f383 -r be91151107f6 server/ssplanner.py --- a/server/ssplanner.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/ssplanner.py Mon Apr 15 13:18:01 2013 +0200 @@ -22,7 +22,7 @@ from rql.stmts import Union, Select from rql.nodes import Constant, Relation -from cubicweb import QueryError, typed_eid +from cubicweb import QueryError from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.rqlrewrite import add_types_restriction from cubicweb.server.edition import EditedEntity @@ -84,7 +84,7 @@ and rel.children[1].operator == '='): lhs, rhs = rel.get_variable_parts() if isinstance(rhs, Constant): - eid = typed_eid(rhs.eval(plan.args)) + eid = int(rhs.eval(plan.args)) # check read permission here since it may not be done by # the generated select substep if not emited (eg nothing # to be selected) @@ -521,7 +521,7 @@ """execute this step""" results = self.execute_child() if results: - todelete = frozenset(typed_eid(eid) for eid, in results) + todelete = frozenset(int(eid) for eid, in results) session = self.plan.session session.repo.glob_delete_entities(session, todelete) return results @@ -567,7 +567,7 @@ lhsval = _handle_relterm(lhsinfo, row, newrow) rhsval = _handle_relterm(rhsinfo, row, newrow) if rschema.final or rschema.inlined: - eid = typed_eid(lhsval) + eid = int(lhsval) try: edited = edefs[eid] except KeyError: diff -r 7920f439f383 -r be91151107f6 server/test/unittest_ldapsource.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/test/unittest_ldapsource.py Mon Apr 15 13:18:01 2013 +0200 @@ -0,0 +1,632 @@ +# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see . +"""cubicweb.server.sources.ldapusers unit and functional tests""" + +import os +import shutil +import time +from os.path import join, exists +import subprocess +import tempfile + +from logilab.common.testlib import TestCase, unittest_main, mock_object, Tags + +from cubicweb import AuthenticationError +from cubicweb.devtools.testlib import CubicWebTC +from cubicweb.devtools.repotest import RQLGeneratorTC +from cubicweb.devtools.httptest import get_available_port +from cubicweb.devtools import get_test_db_handler + +from cubicweb.server.sources.ldapuser import GlobTrFunc, UnknownEid, RQL2LDAPFilter + +CONFIG = u'user-base-dn=ou=People,dc=cubicweb,dc=test' +URL = None + +def create_slapd_configuration(cls): + global URL + slapddir = tempfile.mkdtemp('cw-unittest-ldap') + config = cls.config + slapdconf = join(config.apphome, "slapd.conf") + confin = file(join(config.apphome, "slapd.conf.in")).read() + confstream = file(slapdconf, 'w') + confstream.write(confin % {'apphome': config.apphome, 'testdir': slapddir}) + confstream.close() + # fill ldap server with some data + ldiffile = join(config.apphome, "ldap_test.ldif") + config.info('Initing ldap database') + cmdline = "/usr/sbin/slapadd -f %s -l %s -c" % (slapdconf, ldiffile) + subprocess.call(cmdline, shell=True) + + #ldapuri = 'ldapi://' + join(basedir, "ldapi").replace('/', '%2f') + port = get_available_port(xrange(9000, 9100)) + host = 'localhost:%s' % port + ldapuri = 'ldap://%s' % host + cmdline = ["/usr/sbin/slapd", "-f", slapdconf, "-h", ldapuri, "-d", "0"] + config.info('Starting slapd:', ' '.join(cmdline)) + cls.slapd_process = subprocess.Popen(cmdline) + time.sleep(0.2) + if cls.slapd_process.poll() is None: + config.info('slapd started with pid %s' % cls.slapd_process.pid) + else: + raise EnvironmentError('Cannot start slapd with cmdline="%s" (from directory "%s")' % + (" ".join(cmdline), os.getcwd())) + URL = u'ldap://%s' % host + return slapddir + +def terminate_slapd(cls): + config = cls.config + if cls.slapd_process and cls.slapd_process.returncode is None: + config.info('terminating slapd') + if hasattr(cls.slapd_process, 'terminate'): + cls.slapd_process.terminate() + else: + import os, signal + os.kill(cls.slapd_process.pid, signal.SIGTERM) + cls.slapd_process.wait() + config.info('DONE') + +class LDAPTestBase(CubicWebTC): + loglevel = 'ERROR' + + @classmethod + def setUpClass(cls): + from cubicweb.cwctl import init_cmdline_log_threshold + init_cmdline_log_threshold(cls.config, cls.loglevel) + cls._tmpdir = create_slapd_configuration(cls) + + @classmethod + def tearDownClass(cls): + terminate_slapd(cls) + try: + shutil.rmtree(cls._tmpdir) + except: + pass + +class CheckWrongGroup(LDAPTestBase): + + def test_wrong_group(self): + self.session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', + url=URL, config=CONFIG) + self.commit() + with self.session.repo.internal_session(safe=True) as session: + source = self.session.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0,0) + config = source.repo_source.check_config(source) + # inject a bogus group here, along with at least a valid one + config['user-default-group'] = ('thisgroupdoesnotexists','users') + source.repo_source.update_config(source, config) + session.commit(free_cnxset=False) + # here we emitted an error log entry + stats = source.repo_source.pull_data(session, force=True, raise_on_error=True) + session.commit() + +class DeleteStuffFromLDAPFeedSourceTC(LDAPTestBase): + test_db_id = 'ldap-feed' + + @classmethod + def pre_setup_database(cls, session, config): + session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', + url=URL, config=CONFIG) + session.commit() + with session.repo.internal_session(safe=True) as isession: + lfsource = isession.repo.sources_by_uri['ldapuser'] + stats = lfsource.pull_data(isession, force=True, raise_on_error=True) + + def _pull(self): + with self.session.repo.internal_session() as isession: + lfsource = isession.repo.sources_by_uri['ldapuser'] + stats = lfsource.pull_data(isession, force=True, raise_on_error=True) + isession.commit() + + def test_a_filter_inactivate(self): + """ filtered out people should be deactivated, unable to authenticate """ + source = self.session.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0,0) + config = source.repo_source.check_config(source) + # filter with adim's phone number + config['user-filter'] = u'(%s=%s)' % ('telephoneNumber', '109') + source.repo_source.update_config(source, config) + self.commit() + self._pull() + self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') + self.assertEqual(self.execute('Any N WHERE U login "syt", ' + 'U in_state S, S name N').rows[0][0], + 'deactivated') + self.assertEqual(self.execute('Any N WHERE U login "adim", ' + 'U in_state S, S name N').rows[0][0], + 'activated') + # unfilter, syt should be activated again + config['user-filter'] = u'' + source.repo_source.update_config(source, config) + self.commit() + self._pull() + self.assertEqual(self.execute('Any N WHERE U login "syt", ' + 'U in_state S, S name N').rows[0][0], + 'activated') + self.assertEqual(self.execute('Any N WHERE U login "adim", ' + 'U in_state S, S name N').rows[0][0], + 'activated') + + def test_delete(self): + """ delete syt, pull, check deactivation, repull, + readd syt, pull, check activation + """ + uri = self.repo.sources_by_uri['ldapuser'].urls[0] + deletecmd = ("ldapdelete -H %s 'uid=syt,ou=People,dc=cubicweb,dc=test' " + "-v -x -D cn=admin,dc=cubicweb,dc=test -w'cw'" % uri) + os.system(deletecmd) + self._pull() + self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') + self.assertEqual(self.execute('Any N WHERE U login "syt", ' + 'U in_state S, S name N').rows[0][0], + 'deactivated') + # check that it doesn't choke + self._pull() + # reset the fscking ldap thing + self.tearDownClass() + self.setUpClass() + self._pull() + self.assertEqual(self.execute('Any N WHERE U login "syt", ' + 'U in_state S, S name N').rows[0][0], + 'activated') + # test reactivating the user isn't enough to authenticate, as the native source + # refuse to authenticate user from other sources + os.system(deletecmd) + self._pull() + user = self.execute('CWUser U WHERE U login "syt"').get_entity(0, 0) + user.cw_adapt_to('IWorkflowable').fire_transition('activate') + self.commit() + self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') + +class LDAPFeedSourceTC(LDAPTestBase): + test_db_id = 'ldap-feed' + + @classmethod + def pre_setup_database(cls, session, config): + session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', + url=URL, config=CONFIG) + session.commit() + isession = session.repo.internal_session(safe=True) + lfsource = isession.repo.sources_by_uri['ldapuser'] + stats = lfsource.pull_data(isession, force=True, raise_on_error=True) + + def setUp(self): + super(LDAPFeedSourceTC, self).setUp() + # ldap source url in the database may use a different port as the one + # just attributed + lfsource = self.repo.sources_by_uri['ldapuser'] + lfsource.urls = [URL] + + def assertMetadata(self, entity): + self.assertTrue(entity.creation_date) + self.assertTrue(entity.modification_date) + + def test_authenticate(self): + source = self.repo.sources_by_uri['ldapuser'] + self.session.set_cnxset() + # ensure we won't be logged against + self.assertRaises(AuthenticationError, + source.authenticate, self.session, 'toto', 'toto') + self.assertTrue(source.authenticate(self.session, 'syt', 'syt')) + self.assertTrue(self.repo.connect('syt', password='syt')) + + def test_base(self): + # check a known one + rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) + e = rset.get_entity(0, 0) + self.assertEqual(e.login, 'syt') + e.complete() + self.assertMetadata(e) + self.assertEqual(e.firstname, None) + self.assertEqual(e.surname, None) + self.assertEqual(e.in_group[0].name, 'users') + self.assertEqual(e.owned_by[0].login, 'syt') + self.assertEqual(e.created_by, ()) + self.assertEqual(e.primary_email[0].address, 'Sylvain Thenault') + # email content should be indexed on the user + rset = self.sexecute('CWUser X WHERE X has_text "thenault"') + self.assertEqual(rset.rows, [[e.eid]]) + + def test_copy_to_system_source(self): + source = self.repo.sources_by_uri['ldapuser'] + eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + self.sexecute('SET X cw_source S WHERE X eid %(x)s, S name "system"', {'x': eid}) + self.commit() + source.reset_caches() + rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) + self.assertEqual(len(rset), 1) + e = rset.get_entity(0, 0) + self.assertEqual(e.eid, eid) + self.assertEqual(e.cw_metainformation(), {'source': {'type': u'native', + 'uri': u'system', + 'use-cwuri-as-url': False}, + 'type': 'CWUser', + 'extid': None}) + self.assertEqual(e.cw_source[0].name, 'system') + self.assertTrue(e.creation_date) + self.assertTrue(e.modification_date) + source.pull_data(self.session) + rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) + self.assertEqual(len(rset), 1) + self.assertTrue(self.repo.system_source.authenticate( + self.session, 'syt', password='syt')) + + +class LDAPUserSourceTC(LDAPFeedSourceTC): + test_db_id = 'ldap-user' + tags = CubicWebTC.tags | Tags(('ldap')) + + @classmethod + def pre_setup_database(cls, session, config): + session.create_entity('CWSource', name=u'ldapuser', type=u'ldapuser', + url=URL, config=CONFIG) + session.commit() + # XXX keep it there + session.execute('CWUser U') + + def assertMetadata(self, entity): + self.assertEqual(entity.creation_date, None) + self.assertEqual(entity.modification_date, None) + + def test_synchronize(self): + source = self.repo.sources_by_uri['ldapuser'] + source.synchronize() + + def test_base(self): + # check a known one + rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) + e = rset.get_entity(0, 0) + self.assertEqual(e.login, 'syt') + e.complete() + self.assertMetadata(e) + self.assertEqual(e.firstname, None) + self.assertEqual(e.surname, None) + self.assertEqual(e.in_group[0].name, 'users') + self.assertEqual(e.owned_by[0].login, 'syt') + self.assertEqual(e.created_by, ()) + self.assertEqual(e.primary_email[0].address, 'Sylvain Thenault') + # email content should be indexed on the user + rset = self.sexecute('CWUser X WHERE X has_text "thenault"') + self.assertEqual(rset.rows, [[e.eid]]) + + def test_not(self): + eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + rset = self.sexecute('CWUser X WHERE NOT X eid %s' % eid) + self.assert_(rset) + self.assert_(not eid in (r[0] for r in rset)) + + def test_multiple(self): + seid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + aeid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'adim'})[0][0] + rset = self.sexecute('CWUser X, Y WHERE X login %(syt)s, Y login %(adim)s', + {'syt': 'syt', 'adim': 'adim'}) + self.assertEqual(rset.rows, [[seid, aeid]]) + rset = self.sexecute('Any X,Y,L WHERE X login L, X login %(syt)s, Y login %(adim)s', + {'syt': 'syt', 'adim': 'adim'}) + self.assertEqual(rset.rows, [[seid, aeid, 'syt']]) + + def test_in(self): + seid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + aeid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'adim'})[0][0] + rset = self.sexecute('Any X,L ORDERBY L WHERE X login IN("%s", "%s"), X login L' % ('syt', 'adim')) + self.assertEqual(rset.rows, [[aeid, 'adim'], [seid, 'syt']]) + + def test_relations(self): + eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + rset = self.sexecute('Any X,E WHERE X is CWUser, X login L, X primary_email E') + self.assert_(eid in (r[0] for r in rset)) + rset = self.sexecute('Any X,L,E WHERE X is CWUser, X login L, X primary_email E') + self.assert_('syt' in (r[1] for r in rset)) + + def test_count(self): + nbusers = self.sexecute('Any COUNT(X) WHERE X is CWUser')[0][0] + # just check this is a possible number + self.assert_(nbusers > 1, nbusers) + self.assert_(nbusers < 30, nbusers) + + def test_upper(self): + eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + rset = self.sexecute('Any UPPER(L) WHERE X eid %s, X login L' % eid) + self.assertEqual(rset[0][0], 'syt'.upper()) + + def test_unknown_attr(self): + eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] + rset = self.sexecute('Any L,C,M WHERE X eid %s, X login L, ' + 'X creation_date C, X modification_date M' % eid) + self.assertEqual(rset[0][0], 'syt') + self.assertEqual(rset[0][1], None) + self.assertEqual(rset[0][2], None) + + def test_sort(self): + logins = [l for l, in self.sexecute('Any L ORDERBY L WHERE X login L')] + self.assertEqual(logins, sorted(logins)) + + def test_lower_sort(self): + logins = [l for l, in self.sexecute('Any L ORDERBY lower(L) WHERE X login L')] + self.assertEqual(logins, sorted(logins)) + + def test_or(self): + rset = self.sexecute('DISTINCT Any X WHERE X login %(login)s OR (X in_group G, G name "managers")', + {'login': 'syt'}) + self.assertEqual(len(rset), 2, rset.rows) # syt + admin + + def test_nonregr_set_owned_by(self): + # test that when a user coming from ldap is triggering a transition + # the related TrInfo has correct owner information + self.sexecute('SET X in_group G WHERE X login %(syt)s, G name "managers"', {'syt': 'syt'}) + self.commit() + syt = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}).get_entity(0, 0) + self.assertEqual([g.name for g in syt.in_group], ['managers', 'users']) + cnx = self.login('syt', password='syt') + cu = cnx.cursor() + adim = cu.execute('CWUser X WHERE X login %(login)s', {'login': 'adim'}).get_entity(0, 0) + iworkflowable = adim.cw_adapt_to('IWorkflowable') + iworkflowable.fire_transition('deactivate') + try: + cnx.commit() + adim.cw_clear_all_caches() + self.assertEqual(adim.in_state[0].name, 'deactivated') + trinfo = iworkflowable.latest_trinfo() + self.assertEqual(trinfo.owned_by[0].login, 'syt') + # select from_state to skip the user's creation TrInfo + rset = self.sexecute('Any U ORDERBY D DESC WHERE WF wf_info_for X,' + 'WF creation_date D, WF from_state FS,' + 'WF owned_by U?, X eid %(x)s', + {'x': adim.eid}) + self.assertEqual(rset.rows, [[syt.eid]]) + finally: + # restore db state + self.restore_connection() + adim = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'adim'}).get_entity(0, 0) + adim.cw_adapt_to('IWorkflowable').fire_transition('activate') + self.sexecute('DELETE X in_group G WHERE X login %(syt)s, G name "managers"', {'syt': 'syt'}) + + def test_same_column_names(self): + self.sexecute('Any X, Y WHERE X copain Y, X login "comme", Y login "cochon"') + + def test_multiple_entities_from_different_sources(self): + req = self.request() + self.create_user(req, 'cochon') + self.assertTrue(self.sexecute('Any X,Y WHERE X login %(syt)s, Y login "cochon"', {'syt': 'syt'})) + + def test_exists1(self): + self.session.set_cnxset() + self.session.create_entity('CWGroup', name=u'bougloup1') + self.session.create_entity('CWGroup', name=u'bougloup2') + self.sexecute('SET U in_group G WHERE G name ~= "bougloup%", U login "admin"') + self.sexecute('SET U in_group G WHERE G name = "bougloup1", U login %(syt)s', {'syt': 'syt'}) + rset = self.sexecute('Any L,SN ORDERBY L WHERE X in_state S, ' + 'S name SN, X login L, EXISTS(X in_group G, G name ~= "bougloup%")') + self.assertEqual(rset.rows, [['admin', 'activated'], ['syt', 'activated']]) + + def test_exists2(self): + req = self.request() + self.create_user(req, 'comme') + self.create_user(req, 'cochon') + self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') + rset = self.sexecute('Any GN ORDERBY GN WHERE X in_group G, G name GN, ' + '(G name "managers" OR EXISTS(X copain T, T login in ("comme", "cochon")))') + self.assertEqual(rset.rows, [['managers'], ['users']]) + + def test_exists3(self): + req = self.request() + self.create_user(req, 'comme') + self.create_user(req, 'cochon') + self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') + self.assertTrue(self.sexecute('Any X, Y WHERE X copain Y, X login "comme", Y login "cochon"')) + self.sexecute('SET X copain Y WHERE X login %(syt)s, Y login "cochon"', {'syt': 'syt'}) + self.assertTrue(self.sexecute('Any X, Y WHERE X copain Y, X login %(syt)s, Y login "cochon"', {'syt': 'syt'})) + rset = self.sexecute('Any GN,L WHERE X in_group G, X login L, G name GN, G name "managers" ' + 'OR EXISTS(X copain T, T login in ("comme", "cochon"))') + self.assertEqual(sorted(rset.rows), [['managers', 'admin'], ['users', 'comme'], ['users', 'syt']]) + + def test_exists4(self): + req = self.request() + self.create_user(req, 'comme') + self.create_user(req, 'cochon', groups=('users', 'guests')) + self.create_user(req, 'billy') + self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') + self.sexecute('SET X copain Y WHERE X login "cochon", Y login "cochon"') + self.sexecute('SET X copain Y WHERE X login "comme", Y login "billy"') + self.sexecute('SET X copain Y WHERE X login %(syt)s, Y login "billy"', {'syt': 'syt'}) + # search for group name, login where + # CWUser copain with "comme" or "cochon" AND same login as the copain + # OR + # CWUser in_state activated AND not copain with billy + # + # SO we expect everybody but "comme" and "syt" + rset= self.sexecute('Any GN,L WHERE X in_group G, X login L, G name GN, ' + 'EXISTS(X copain T, T login L, T login in ("comme", "cochon")) OR ' + 'EXISTS(X in_state S, S name "activated", NOT X copain T2, T2 login "billy")') + all = self.sexecute('Any GN, L WHERE X in_group G, X login L, G name GN') + all.rows.remove(['users', 'comme']) + all.rows.remove(['users', 'syt']) + self.assertEqual(sorted(rset.rows), sorted(all.rows)) + + def test_exists5(self): + req = self.request() + self.create_user(req, 'comme') + self.create_user(req, 'cochon', groups=('users', 'guests')) + self.create_user(req, 'billy') + self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') + self.sexecute('SET X copain Y WHERE X login "cochon", Y login "cochon"') + self.sexecute('SET X copain Y WHERE X login "comme", Y login "billy"') + self.sexecute('SET X copain Y WHERE X login %(syt)s, Y login "cochon"', {'syt': 'syt'}) + rset= self.sexecute('Any L WHERE X login L, ' + 'EXISTS(X copain T, T login in ("comme", "cochon")) AND ' + 'NOT EXISTS(X copain T2, T2 login "billy")') + self.assertEqual(sorted(rset.rows), [['cochon'], ['syt']]) + rset= self.sexecute('Any GN,L WHERE X in_group G, X login L, G name GN, ' + 'EXISTS(X copain T, T login in ("comme", "cochon")) AND ' + 'NOT EXISTS(X copain T2, T2 login "billy")') + self.assertEqual(sorted(rset.rows), [['guests', 'cochon'], + ['users', 'cochon'], + ['users', 'syt']]) + + def test_cd_restriction(self): + rset = self.sexecute('CWUser X WHERE X creation_date > "2009-02-01"') + # admin/anon but no ldap user since it doesn't support creation_date + self.assertEqual(sorted(e.login for e in rset.entities()), + ['admin', 'anon']) + + def test_union(self): + afeids = self.sexecute('State X') + ueids = self.sexecute('CWUser X') + rset = self.sexecute('(Any X WHERE X is State) UNION (Any X WHERE X is CWUser)') + self.assertEqual(sorted(r[0] for r in rset.rows), + sorted(r[0] for r in afeids + ueids)) + + def _init_security_test(self): + req = self.request() + self.create_user(req, 'iaminguestsgrouponly', groups=('guests',)) + cnx = self.login('iaminguestsgrouponly') + return cnx.cursor() + + def test_security1(self): + cu = self._init_security_test() + rset = cu.execute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) + self.assertEqual(rset.rows, []) + rset = cu.execute('Any X WHERE X login "iaminguestsgrouponly"') + self.assertEqual(len(rset.rows), 1) + + def test_security2(self): + cu = self._init_security_test() + rset = cu.execute('Any X WHERE X has_text %(syt)s', {'syt': 'syt'}) + self.assertEqual(rset.rows, []) + rset = cu.execute('Any X WHERE X has_text "iaminguestsgrouponly"') + self.assertEqual(len(rset.rows), 1) + + def test_security3(self): + cu = self._init_security_test() + rset = cu.execute('Any F WHERE X has_text %(syt)s, X firstname F', {'syt': 'syt'}) + self.assertEqual(rset.rows, []) + rset = cu.execute('Any F WHERE X has_text "iaminguestsgrouponly", X firstname F') + self.assertEqual(rset.rows, [[None]]) + + def test_nonregr1(self): + self.sexecute('Any X,AA ORDERBY AA DESC WHERE E eid %(x)s, E owned_by X, ' + 'X modification_date AA', + {'x': self.session.user.eid}) + + def test_nonregr2(self): + self.sexecute('Any X,L,AA WHERE E eid %(x)s, E owned_by X, ' + 'X login L, X modification_date AA', + {'x': self.session.user.eid}) + + def test_nonregr3(self): + self.sexecute('Any X,AA ORDERBY AA DESC WHERE E eid %(x)s, ' + 'X modification_date AA', + {'x': self.session.user.eid}) + + def test_nonregr4(self): + emaileid = self.sexecute('INSERT EmailAddress X: X address "toto@logilab.org"')[0][0] + self.sexecute('Any X,AA WHERE X use_email Y, Y eid %(x)s, X modification_date AA', + {'x': emaileid}) + + def test_nonregr5(self): + # original jpl query: + # Any X, NOW - CD, P WHERE P is Project, U interested_in P, U is CWUser, + # U login "sthenault", X concerns P, X creation_date CD ORDERBY CD DESC LIMIT 5 + rql = ('Any X, NOW - CD, P ORDERBY CD DESC LIMIT 5 WHERE P bookmarked_by U, ' + 'U login "%s", P is X, X creation_date CD') % self.session.user.login + self.sexecute(rql, )#{'x': }) + + def test_nonregr6(self): + self.sexecute('Any B,U,UL GROUPBY B,U,UL WHERE B created_by U?, B is File ' + 'WITH U,UL BEING (Any U,UL WHERE ME eid %(x)s, (EXISTS(U identity ME) ' + 'OR (EXISTS(U in_group G, G name IN("managers", "staff")))) ' + 'OR (EXISTS(U in_group H, ME in_group H, NOT H name "users")), U login UL, U is CWUser)', + {'x': self.session.user.eid}) + +class GlobTrFuncTC(TestCase): + + def test_count(self): + trfunc = GlobTrFunc('count', 0) + res = trfunc.apply([[1], [2], [3], [4]]) + self.assertEqual(res, [[4]]) + trfunc = GlobTrFunc('count', 1) + res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) + self.assertEqual(res, [[1, 2], [2, 1], [3, 1]]) + + def test_sum(self): + trfunc = GlobTrFunc('sum', 0) + res = trfunc.apply([[1], [2], [3], [4]]) + self.assertEqual(res, [[10]]) + trfunc = GlobTrFunc('sum', 1) + res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) + self.assertEqual(res, [[1, 7], [2, 4], [3, 6]]) + + def test_min(self): + trfunc = GlobTrFunc('min', 0) + res = trfunc.apply([[1], [2], [3], [4]]) + self.assertEqual(res, [[1]]) + trfunc = GlobTrFunc('min', 1) + res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) + self.assertEqual(res, [[1, 2], [2, 4], [3, 6]]) + + def test_max(self): + trfunc = GlobTrFunc('max', 0) + res = trfunc.apply([[1], [2], [3], [4]]) + self.assertEqual(res, [[4]]) + trfunc = GlobTrFunc('max', 1) + res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) + self.assertEqual(res, [[1, 5], [2, 4], [3, 6]]) + + +class RQL2LDAPFilterTC(RQLGeneratorTC): + + tags = RQLGeneratorTC.tags | Tags(('ldap')) + + @property + def schema(self): + """return the application schema""" + return self._schema + + def setUp(self): + self.handler = get_test_db_handler(LDAPUserSourceTC.config) + self.handler.build_db_cache('ldap-user', LDAPUserSourceTC.pre_setup_database) + self.handler.restore_database('ldap-user') + self._repo = repo = self.handler.get_repo() + self._schema = repo.schema + super(RQL2LDAPFilterTC, self).setUp() + ldapsource = repo.sources[-1] + self.cnxset = repo._get_cnxset() + session = mock_object(cnxset=self.cnxset) + self.o = RQL2LDAPFilter(ldapsource, session) + self.ldapclasses = ''.join(ldapsource.base_filters) + + def tearDown(self): + self._repo.turn_repo_off() + super(RQL2LDAPFilterTC, self).tearDown() + + def test_base(self): + rqlst = self._prepare('CWUser X WHERE X login "toto"').children[0] + self.assertEqual(self.o.generate(rqlst, 'X')[1], + '(&%s(uid=toto))' % self.ldapclasses) + + def test_kwargs(self): + rqlst = self._prepare('CWUser X WHERE X login %(x)s').children[0] + self.o._args = {'x': "toto"} + self.assertEqual(self.o.generate(rqlst, 'X')[1], + '(&%s(uid=toto))' % self.ldapclasses) + + def test_get_attr(self): + rqlst = self._prepare('Any X WHERE E firstname X, E eid 12').children[0] + self.assertRaises(UnknownEid, self.o.generate, rqlst, 'E') + + +if __name__ == '__main__': + unittest_main() diff -r 7920f439f383 -r be91151107f6 server/test/unittest_ldapuser.py --- a/server/test/unittest_ldapuser.py Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,632 +0,0 @@ -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr -# -# This file is part of CubicWeb. -# -# CubicWeb is free software: you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 2.1 of the License, or (at your option) -# any later version. -# -# CubicWeb is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with CubicWeb. If not, see . -"""cubicweb.server.sources.ldapusers unit and functional tests""" - -import os -import shutil -import time -from os.path import join, exists -import subprocess -import tempfile - -from logilab.common.testlib import TestCase, unittest_main, mock_object, Tags - -from cubicweb import AuthenticationError -from cubicweb.devtools.testlib import CubicWebTC -from cubicweb.devtools.repotest import RQLGeneratorTC -from cubicweb.devtools.httptest import get_available_port -from cubicweb.devtools import get_test_db_handler - -from cubicweb.server.sources.ldapuser import GlobTrFunc, UnknownEid, RQL2LDAPFilter - -CONFIG = u'user-base-dn=ou=People,dc=cubicweb,dc=test' -URL = None - -def create_slapd_configuration(cls): - global URL - slapddir = tempfile.mkdtemp('cw-unittest-ldap') - config = cls.config - slapdconf = join(config.apphome, "slapd.conf") - confin = file(join(config.apphome, "slapd.conf.in")).read() - confstream = file(slapdconf, 'w') - confstream.write(confin % {'apphome': config.apphome, 'testdir': slapddir}) - confstream.close() - # fill ldap server with some data - ldiffile = join(config.apphome, "ldap_test.ldif") - config.info('Initing ldap database') - cmdline = "/usr/sbin/slapadd -f %s -l %s -c" % (slapdconf, ldiffile) - subprocess.call(cmdline, shell=True) - - #ldapuri = 'ldapi://' + join(basedir, "ldapi").replace('/', '%2f') - port = get_available_port(xrange(9000, 9100)) - host = 'localhost:%s' % port - ldapuri = 'ldap://%s' % host - cmdline = ["/usr/sbin/slapd", "-f", slapdconf, "-h", ldapuri, "-d", "0"] - config.info('Starting slapd:', ' '.join(cmdline)) - cls.slapd_process = subprocess.Popen(cmdline) - time.sleep(0.2) - if cls.slapd_process.poll() is None: - config.info('slapd started with pid %s' % cls.slapd_process.pid) - else: - raise EnvironmentError('Cannot start slapd with cmdline="%s" (from directory "%s")' % - (" ".join(cmdline), os.getcwd())) - URL = u'ldap://%s' % host - return slapddir - -def terminate_slapd(cls): - config = cls.config - if cls.slapd_process and cls.slapd_process.returncode is None: - config.info('terminating slapd') - if hasattr(cls.slapd_process, 'terminate'): - cls.slapd_process.terminate() - else: - import os, signal - os.kill(cls.slapd_process.pid, signal.SIGTERM) - cls.slapd_process.wait() - config.info('DONE') - -class LDAPTestBase(CubicWebTC): - loglevel = 'ERROR' - - @classmethod - def setUpClass(cls): - from cubicweb.cwctl import init_cmdline_log_threshold - init_cmdline_log_threshold(cls.config, cls.loglevel) - cls._tmpdir = create_slapd_configuration(cls) - - @classmethod - def tearDownClass(cls): - terminate_slapd(cls) - try: - shutil.rmtree(cls._tmpdir) - except: - pass - -class CheckWrongGroup(LDAPTestBase): - - def test_wrong_group(self): - self.session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', - url=URL, config=CONFIG) - self.commit() - with self.session.repo.internal_session(safe=True) as session: - source = self.session.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0,0) - config = source.repo_source.check_config(source) - # inject a bogus group here, along with at least a valid one - config['user-default-group'] = ('thisgroupdoesnotexists','users') - source.repo_source.update_config(source, config) - session.commit(free_cnxset=False) - # here we emitted an error log entry - stats = source.repo_source.pull_data(session, force=True, raise_on_error=True) - session.commit() - -class DeleteStuffFromLDAPFeedSourceTC(LDAPTestBase): - test_db_id = 'ldap-feed' - - @classmethod - def pre_setup_database(cls, session, config): - session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', - url=URL, config=CONFIG) - session.commit() - with session.repo.internal_session(safe=True) as isession: - lfsource = isession.repo.sources_by_uri['ldapuser'] - stats = lfsource.pull_data(isession, force=True, raise_on_error=True) - - def _pull(self): - with self.session.repo.internal_session() as isession: - lfsource = isession.repo.sources_by_uri['ldapuser'] - stats = lfsource.pull_data(isession, force=True, raise_on_error=True) - isession.commit() - - def test_a_filter_inactivate(self): - """ filtered out people should be deactivated, unable to authenticate """ - source = self.session.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0,0) - config = source.repo_source.check_config(source) - # filter with adim's phone number - config['user-filter'] = u'(%s=%s)' % ('telephoneNumber', '109') - source.repo_source.update_config(source, config) - self.commit() - self._pull() - self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') - self.assertEqual(self.execute('Any N WHERE U login "syt", ' - 'U in_state S, S name N').rows[0][0], - 'deactivated') - self.assertEqual(self.execute('Any N WHERE U login "adim", ' - 'U in_state S, S name N').rows[0][0], - 'activated') - # unfilter, syt should be activated again - config['user-filter'] = u'' - source.repo_source.update_config(source, config) - self.commit() - self._pull() - self.assertEqual(self.execute('Any N WHERE U login "syt", ' - 'U in_state S, S name N').rows[0][0], - 'activated') - self.assertEqual(self.execute('Any N WHERE U login "adim", ' - 'U in_state S, S name N').rows[0][0], - 'activated') - - def test_delete(self): - """ delete syt, pull, check deactivation, repull, - readd syt, pull, check activation - """ - uri = self.repo.sources_by_uri['ldapuser'].urls[0] - deletecmd = ("ldapdelete -H %s 'uid=syt,ou=People,dc=cubicweb,dc=test' " - "-v -x -D cn=admin,dc=cubicweb,dc=test -w'cw'" % uri) - os.system(deletecmd) - self._pull() - self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') - self.assertEqual(self.execute('Any N WHERE U login "syt", ' - 'U in_state S, S name N').rows[0][0], - 'deactivated') - # check that it doesn't choke - self._pull() - # reset the fscking ldap thing - self.tearDownClass() - self.setUpClass() - self._pull() - self.assertEqual(self.execute('Any N WHERE U login "syt", ' - 'U in_state S, S name N').rows[0][0], - 'activated') - # test reactivating the user isn't enough to authenticate, as the native source - # refuse to authenticate user from other sources - os.system(deletecmd) - self._pull() - user = self.execute('CWUser U WHERE U login "syt"').get_entity(0, 0) - user.cw_adapt_to('IWorkflowable').fire_transition('activate') - self.commit() - self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') - -class LDAPFeedSourceTC(LDAPTestBase): - test_db_id = 'ldap-feed' - - @classmethod - def pre_setup_database(cls, session, config): - session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', - url=URL, config=CONFIG) - session.commit() - isession = session.repo.internal_session(safe=True) - lfsource = isession.repo.sources_by_uri['ldapuser'] - stats = lfsource.pull_data(isession, force=True, raise_on_error=True) - - def setUp(self): - super(LDAPFeedSourceTC, self).setUp() - # ldap source url in the database may use a different port as the one - # just attributed - lfsource = self.repo.sources_by_uri['ldapuser'] - lfsource.urls = [URL] - - def assertMetadata(self, entity): - self.assertTrue(entity.creation_date) - self.assertTrue(entity.modification_date) - - def test_authenticate(self): - source = self.repo.sources_by_uri['ldapuser'] - self.session.set_cnxset() - # ensure we won't be logged against - self.assertRaises(AuthenticationError, - source.authenticate, self.session, 'toto', 'toto') - self.assertTrue(source.authenticate(self.session, 'syt', 'syt')) - self.assertTrue(self.repo.connect('syt', password='syt')) - - def test_base(self): - # check a known one - rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) - e = rset.get_entity(0, 0) - self.assertEqual(e.login, 'syt') - e.complete() - self.assertMetadata(e) - self.assertEqual(e.firstname, None) - self.assertEqual(e.surname, None) - self.assertEqual(e.in_group[0].name, 'users') - self.assertEqual(e.owned_by[0].login, 'syt') - self.assertEqual(e.created_by, ()) - self.assertEqual(e.primary_email[0].address, 'Sylvain Thenault') - # email content should be indexed on the user - rset = self.sexecute('CWUser X WHERE X has_text "thenault"') - self.assertEqual(rset.rows, [[e.eid]]) - - def test_copy_to_system_source(self): - source = self.repo.sources_by_uri['ldapuser'] - eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - self.sexecute('SET X cw_source S WHERE X eid %(x)s, S name "system"', {'x': eid}) - self.commit() - source.reset_caches() - rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) - self.assertEqual(len(rset), 1) - e = rset.get_entity(0, 0) - self.assertEqual(e.eid, eid) - self.assertEqual(e.cw_metainformation(), {'source': {'type': u'native', - 'uri': u'system', - 'use-cwuri-as-url': False}, - 'type': 'CWUser', - 'extid': None}) - self.assertEqual(e.cw_source[0].name, 'system') - self.assertTrue(e.creation_date) - self.assertTrue(e.modification_date) - source.pull_data(self.session) - rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) - self.assertEqual(len(rset), 1) - self.assertTrue(self.repo.system_source.authenticate( - self.session, 'syt', password='syt')) - - -class LDAPUserSourceTC(LDAPFeedSourceTC): - test_db_id = 'ldap-user' - tags = CubicWebTC.tags | Tags(('ldap')) - - @classmethod - def pre_setup_database(cls, session, config): - session.create_entity('CWSource', name=u'ldapuser', type=u'ldapuser', - url=URL, config=CONFIG) - session.commit() - # XXX keep it there - session.execute('CWUser U') - - def assertMetadata(self, entity): - self.assertEqual(entity.creation_date, None) - self.assertEqual(entity.modification_date, None) - - def test_synchronize(self): - source = self.repo.sources_by_uri['ldapuser'] - source.synchronize() - - def test_base(self): - # check a known one - rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) - e = rset.get_entity(0, 0) - self.assertEqual(e.login, 'syt') - e.complete() - self.assertMetadata(e) - self.assertEqual(e.firstname, None) - self.assertEqual(e.surname, None) - self.assertEqual(e.in_group[0].name, 'users') - self.assertEqual(e.owned_by[0].login, 'syt') - self.assertEqual(e.created_by, ()) - self.assertEqual(e.primary_email[0].address, 'Sylvain Thenault') - # email content should be indexed on the user - rset = self.sexecute('CWUser X WHERE X has_text "thenault"') - self.assertEqual(rset.rows, [[e.eid]]) - - def test_not(self): - eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - rset = self.sexecute('CWUser X WHERE NOT X eid %s' % eid) - self.assert_(rset) - self.assert_(not eid in (r[0] for r in rset)) - - def test_multiple(self): - seid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - aeid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'adim'})[0][0] - rset = self.sexecute('CWUser X, Y WHERE X login %(syt)s, Y login %(adim)s', - {'syt': 'syt', 'adim': 'adim'}) - self.assertEqual(rset.rows, [[seid, aeid]]) - rset = self.sexecute('Any X,Y,L WHERE X login L, X login %(syt)s, Y login %(adim)s', - {'syt': 'syt', 'adim': 'adim'}) - self.assertEqual(rset.rows, [[seid, aeid, 'syt']]) - - def test_in(self): - seid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - aeid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'adim'})[0][0] - rset = self.sexecute('Any X,L ORDERBY L WHERE X login IN("%s", "%s"), X login L' % ('syt', 'adim')) - self.assertEqual(rset.rows, [[aeid, 'adim'], [seid, 'syt']]) - - def test_relations(self): - eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - rset = self.sexecute('Any X,E WHERE X is CWUser, X login L, X primary_email E') - self.assert_(eid in (r[0] for r in rset)) - rset = self.sexecute('Any X,L,E WHERE X is CWUser, X login L, X primary_email E') - self.assert_('syt' in (r[1] for r in rset)) - - def test_count(self): - nbusers = self.sexecute('Any COUNT(X) WHERE X is CWUser')[0][0] - # just check this is a possible number - self.assert_(nbusers > 1, nbusers) - self.assert_(nbusers < 30, nbusers) - - def test_upper(self): - eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - rset = self.sexecute('Any UPPER(L) WHERE X eid %s, X login L' % eid) - self.assertEqual(rset[0][0], 'syt'.upper()) - - def test_unknown_attr(self): - eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] - rset = self.sexecute('Any L,C,M WHERE X eid %s, X login L, ' - 'X creation_date C, X modification_date M' % eid) - self.assertEqual(rset[0][0], 'syt') - self.assertEqual(rset[0][1], None) - self.assertEqual(rset[0][2], None) - - def test_sort(self): - logins = [l for l, in self.sexecute('Any L ORDERBY L WHERE X login L')] - self.assertEqual(logins, sorted(logins)) - - def test_lower_sort(self): - logins = [l for l, in self.sexecute('Any L ORDERBY lower(L) WHERE X login L')] - self.assertEqual(logins, sorted(logins)) - - def test_or(self): - rset = self.sexecute('DISTINCT Any X WHERE X login %(login)s OR (X in_group G, G name "managers")', - {'login': 'syt'}) - self.assertEqual(len(rset), 2, rset.rows) # syt + admin - - def test_nonregr_set_owned_by(self): - # test that when a user coming from ldap is triggering a transition - # the related TrInfo has correct owner information - self.sexecute('SET X in_group G WHERE X login %(syt)s, G name "managers"', {'syt': 'syt'}) - self.commit() - syt = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'syt'}).get_entity(0, 0) - self.assertEqual([g.name for g in syt.in_group], ['managers', 'users']) - cnx = self.login('syt', password='syt') - cu = cnx.cursor() - adim = cu.execute('CWUser X WHERE X login %(login)s', {'login': 'adim'}).get_entity(0, 0) - iworkflowable = adim.cw_adapt_to('IWorkflowable') - iworkflowable.fire_transition('deactivate') - try: - cnx.commit() - adim.cw_clear_all_caches() - self.assertEqual(adim.in_state[0].name, 'deactivated') - trinfo = iworkflowable.latest_trinfo() - self.assertEqual(trinfo.owned_by[0].login, 'syt') - # select from_state to skip the user's creation TrInfo - rset = self.sexecute('Any U ORDERBY D DESC WHERE WF wf_info_for X,' - 'WF creation_date D, WF from_state FS,' - 'WF owned_by U?, X eid %(x)s', - {'x': adim.eid}) - self.assertEqual(rset.rows, [[syt.eid]]) - finally: - # restore db state - self.restore_connection() - adim = self.sexecute('CWUser X WHERE X login %(login)s', {'login': 'adim'}).get_entity(0, 0) - adim.cw_adapt_to('IWorkflowable').fire_transition('activate') - self.sexecute('DELETE X in_group G WHERE X login %(syt)s, G name "managers"', {'syt': 'syt'}) - - def test_same_column_names(self): - self.sexecute('Any X, Y WHERE X copain Y, X login "comme", Y login "cochon"') - - def test_multiple_entities_from_different_sources(self): - req = self.request() - self.create_user(req, 'cochon') - self.assertTrue(self.sexecute('Any X,Y WHERE X login %(syt)s, Y login "cochon"', {'syt': 'syt'})) - - def test_exists1(self): - self.session.set_cnxset() - self.session.create_entity('CWGroup', name=u'bougloup1') - self.session.create_entity('CWGroup', name=u'bougloup2') - self.sexecute('SET U in_group G WHERE G name ~= "bougloup%", U login "admin"') - self.sexecute('SET U in_group G WHERE G name = "bougloup1", U login %(syt)s', {'syt': 'syt'}) - rset = self.sexecute('Any L,SN ORDERBY L WHERE X in_state S, ' - 'S name SN, X login L, EXISTS(X in_group G, G name ~= "bougloup%")') - self.assertEqual(rset.rows, [['admin', 'activated'], ['syt', 'activated']]) - - def test_exists2(self): - req = self.request() - self.create_user(req, 'comme') - self.create_user(req, 'cochon') - self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') - rset = self.sexecute('Any GN ORDERBY GN WHERE X in_group G, G name GN, ' - '(G name "managers" OR EXISTS(X copain T, T login in ("comme", "cochon")))') - self.assertEqual(rset.rows, [['managers'], ['users']]) - - def test_exists3(self): - req = self.request() - self.create_user(req, 'comme') - self.create_user(req, 'cochon') - self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') - self.assertTrue(self.sexecute('Any X, Y WHERE X copain Y, X login "comme", Y login "cochon"')) - self.sexecute('SET X copain Y WHERE X login %(syt)s, Y login "cochon"', {'syt': 'syt'}) - self.assertTrue(self.sexecute('Any X, Y WHERE X copain Y, X login %(syt)s, Y login "cochon"', {'syt': 'syt'})) - rset = self.sexecute('Any GN,L WHERE X in_group G, X login L, G name GN, G name "managers" ' - 'OR EXISTS(X copain T, T login in ("comme", "cochon"))') - self.assertEqual(sorted(rset.rows), [['managers', 'admin'], ['users', 'comme'], ['users', 'syt']]) - - def test_exists4(self): - req = self.request() - self.create_user(req, 'comme') - self.create_user(req, 'cochon', groups=('users', 'guests')) - self.create_user(req, 'billy') - self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') - self.sexecute('SET X copain Y WHERE X login "cochon", Y login "cochon"') - self.sexecute('SET X copain Y WHERE X login "comme", Y login "billy"') - self.sexecute('SET X copain Y WHERE X login %(syt)s, Y login "billy"', {'syt': 'syt'}) - # search for group name, login where - # CWUser copain with "comme" or "cochon" AND same login as the copain - # OR - # CWUser in_state activated AND not copain with billy - # - # SO we expect everybody but "comme" and "syt" - rset= self.sexecute('Any GN,L WHERE X in_group G, X login L, G name GN, ' - 'EXISTS(X copain T, T login L, T login in ("comme", "cochon")) OR ' - 'EXISTS(X in_state S, S name "activated", NOT X copain T2, T2 login "billy")') - all = self.sexecute('Any GN, L WHERE X in_group G, X login L, G name GN') - all.rows.remove(['users', 'comme']) - all.rows.remove(['users', 'syt']) - self.assertEqual(sorted(rset.rows), sorted(all.rows)) - - def test_exists5(self): - req = self.request() - self.create_user(req, 'comme') - self.create_user(req, 'cochon', groups=('users', 'guests')) - self.create_user(req, 'billy') - self.sexecute('SET X copain Y WHERE X login "comme", Y login "cochon"') - self.sexecute('SET X copain Y WHERE X login "cochon", Y login "cochon"') - self.sexecute('SET X copain Y WHERE X login "comme", Y login "billy"') - self.sexecute('SET X copain Y WHERE X login %(syt)s, Y login "cochon"', {'syt': 'syt'}) - rset= self.sexecute('Any L WHERE X login L, ' - 'EXISTS(X copain T, T login in ("comme", "cochon")) AND ' - 'NOT EXISTS(X copain T2, T2 login "billy")') - self.assertEqual(sorted(rset.rows), [['cochon'], ['syt']]) - rset= self.sexecute('Any GN,L WHERE X in_group G, X login L, G name GN, ' - 'EXISTS(X copain T, T login in ("comme", "cochon")) AND ' - 'NOT EXISTS(X copain T2, T2 login "billy")') - self.assertEqual(sorted(rset.rows), [['guests', 'cochon'], - ['users', 'cochon'], - ['users', 'syt']]) - - def test_cd_restriction(self): - rset = self.sexecute('CWUser X WHERE X creation_date > "2009-02-01"') - # admin/anon but no ldap user since it doesn't support creation_date - self.assertEqual(sorted(e.login for e in rset.entities()), - ['admin', 'anon']) - - def test_union(self): - afeids = self.sexecute('State X') - ueids = self.sexecute('CWUser X') - rset = self.sexecute('(Any X WHERE X is State) UNION (Any X WHERE X is CWUser)') - self.assertEqual(sorted(r[0] for r in rset.rows), - sorted(r[0] for r in afeids + ueids)) - - def _init_security_test(self): - req = self.request() - self.create_user(req, 'iaminguestsgrouponly', groups=('guests',)) - cnx = self.login('iaminguestsgrouponly') - return cnx.cursor() - - def test_security1(self): - cu = self._init_security_test() - rset = cu.execute('CWUser X WHERE X login %(login)s', {'login': 'syt'}) - self.assertEqual(rset.rows, []) - rset = cu.execute('Any X WHERE X login "iaminguestsgrouponly"') - self.assertEqual(len(rset.rows), 1) - - def test_security2(self): - cu = self._init_security_test() - rset = cu.execute('Any X WHERE X has_text %(syt)s', {'syt': 'syt'}) - self.assertEqual(rset.rows, []) - rset = cu.execute('Any X WHERE X has_text "iaminguestsgrouponly"') - self.assertEqual(len(rset.rows), 1) - - def test_security3(self): - cu = self._init_security_test() - rset = cu.execute('Any F WHERE X has_text %(syt)s, X firstname F', {'syt': 'syt'}) - self.assertEqual(rset.rows, []) - rset = cu.execute('Any F WHERE X has_text "iaminguestsgrouponly", X firstname F') - self.assertEqual(rset.rows, [[None]]) - - def test_nonregr1(self): - self.sexecute('Any X,AA ORDERBY AA DESC WHERE E eid %(x)s, E owned_by X, ' - 'X modification_date AA', - {'x': self.session.user.eid}) - - def test_nonregr2(self): - self.sexecute('Any X,L,AA WHERE E eid %(x)s, E owned_by X, ' - 'X login L, X modification_date AA', - {'x': self.session.user.eid}) - - def test_nonregr3(self): - self.sexecute('Any X,AA ORDERBY AA DESC WHERE E eid %(x)s, ' - 'X modification_date AA', - {'x': self.session.user.eid}) - - def test_nonregr4(self): - emaileid = self.sexecute('INSERT EmailAddress X: X address "toto@logilab.org"')[0][0] - self.sexecute('Any X,AA WHERE X use_email Y, Y eid %(x)s, X modification_date AA', - {'x': emaileid}) - - def test_nonregr5(self): - # original jpl query: - # Any X, NOW - CD, P WHERE P is Project, U interested_in P, U is CWUser, - # U login "sthenault", X concerns P, X creation_date CD ORDERBY CD DESC LIMIT 5 - rql = ('Any X, NOW - CD, P ORDERBY CD DESC LIMIT 5 WHERE P bookmarked_by U, ' - 'U login "%s", P is X, X creation_date CD') % self.session.user.login - self.sexecute(rql, )#{'x': }) - - def test_nonregr6(self): - self.sexecute('Any B,U,UL GROUPBY B,U,UL WHERE B created_by U?, B is File ' - 'WITH U,UL BEING (Any U,UL WHERE ME eid %(x)s, (EXISTS(U identity ME) ' - 'OR (EXISTS(U in_group G, G name IN("managers", "staff")))) ' - 'OR (EXISTS(U in_group H, ME in_group H, NOT H name "users")), U login UL, U is CWUser)', - {'x': self.session.user.eid}) - -class GlobTrFuncTC(TestCase): - - def test_count(self): - trfunc = GlobTrFunc('count', 0) - res = trfunc.apply([[1], [2], [3], [4]]) - self.assertEqual(res, [[4]]) - trfunc = GlobTrFunc('count', 1) - res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) - self.assertEqual(res, [[1, 2], [2, 1], [3, 1]]) - - def test_sum(self): - trfunc = GlobTrFunc('sum', 0) - res = trfunc.apply([[1], [2], [3], [4]]) - self.assertEqual(res, [[10]]) - trfunc = GlobTrFunc('sum', 1) - res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) - self.assertEqual(res, [[1, 7], [2, 4], [3, 6]]) - - def test_min(self): - trfunc = GlobTrFunc('min', 0) - res = trfunc.apply([[1], [2], [3], [4]]) - self.assertEqual(res, [[1]]) - trfunc = GlobTrFunc('min', 1) - res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) - self.assertEqual(res, [[1, 2], [2, 4], [3, 6]]) - - def test_max(self): - trfunc = GlobTrFunc('max', 0) - res = trfunc.apply([[1], [2], [3], [4]]) - self.assertEqual(res, [[4]]) - trfunc = GlobTrFunc('max', 1) - res = trfunc.apply([[1, 2], [2, 4], [3, 6], [1, 5]]) - self.assertEqual(res, [[1, 5], [2, 4], [3, 6]]) - - -class RQL2LDAPFilterTC(RQLGeneratorTC): - - tags = RQLGeneratorTC.tags | Tags(('ldap')) - - @property - def schema(self): - """return the application schema""" - return self._schema - - def setUp(self): - self.handler = get_test_db_handler(LDAPUserSourceTC.config) - self.handler.build_db_cache('ldap-user', LDAPUserSourceTC.pre_setup_database) - self.handler.restore_database('ldap-user') - self._repo = repo = self.handler.get_repo() - self._schema = repo.schema - super(RQL2LDAPFilterTC, self).setUp() - ldapsource = repo.sources[-1] - self.cnxset = repo._get_cnxset() - session = mock_object(cnxset=self.cnxset) - self.o = RQL2LDAPFilter(ldapsource, session) - self.ldapclasses = ''.join(ldapsource.base_filters) - - def tearDown(self): - self._repo.turn_repo_off() - super(RQL2LDAPFilterTC, self).tearDown() - - def test_base(self): - rqlst = self._prepare('CWUser X WHERE X login "toto"').children[0] - self.assertEqual(self.o.generate(rqlst, 'X')[1], - '(&%s(uid=toto))' % self.ldapclasses) - - def test_kwargs(self): - rqlst = self._prepare('CWUser X WHERE X login %(x)s').children[0] - self.o._args = {'x': "toto"} - self.assertEqual(self.o.generate(rqlst, 'X')[1], - '(&%s(uid=toto))' % self.ldapclasses) - - def test_get_attr(self): - rqlst = self._prepare('Any X WHERE E firstname X, E eid 12').children[0] - self.assertRaises(UnknownEid, self.o.generate, rqlst, 'E') - - -if __name__ == '__main__': - unittest_main() diff -r 7920f439f383 -r be91151107f6 server/test/unittest_session.py --- a/server/test/unittest_session.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/test/unittest_session.py Mon Apr 15 13:18:01 2013 +0200 @@ -17,6 +17,7 @@ # with CubicWeb. If not, see . from cubicweb.devtools.testlib import CubicWebTC +from cubicweb.server.session import HOOKS_ALLOW_ALL, HOOKS_DENY_ALL class InternalSessionTC(CubicWebTC): def test_dbapi_query(self): @@ -29,36 +30,36 @@ def test_hooks_control(self): session = self.session - self.assertEqual(session.hooks_mode, session.HOOKS_ALLOW_ALL) - self.assertEqual(session.disabled_hook_categories, set()) - self.assertEqual(session.enabled_hook_categories, set()) - self.assertEqual(len(session._tx_data), 1) + self.assertEqual(HOOKS_ALLOW_ALL, session.hooks_mode) + self.assertEqual(set(), session.disabled_hook_categories) + self.assertEqual(set(), session.enabled_hook_categories) + self.assertEqual(1, len(session._txs)) with session.deny_all_hooks_but('metadata'): - self.assertEqual(session.hooks_mode, session.HOOKS_DENY_ALL) - self.assertEqual(session.disabled_hook_categories, set()) - self.assertEqual(session.enabled_hook_categories, set(('metadata',))) + self.assertEqual(HOOKS_DENY_ALL, session.hooks_mode) + self.assertEqual(set(), session.disabled_hook_categories) + self.assertEqual(set(('metadata',)), session.enabled_hook_categories) session.commit() - self.assertEqual(session.hooks_mode, session.HOOKS_DENY_ALL) - self.assertEqual(session.disabled_hook_categories, set()) - self.assertEqual(session.enabled_hook_categories, set(('metadata',))) + self.assertEqual(HOOKS_DENY_ALL, session.hooks_mode) + self.assertEqual(set(), session.disabled_hook_categories) + self.assertEqual(set(('metadata',)), session.enabled_hook_categories) session.rollback() - self.assertEqual(session.hooks_mode, session.HOOKS_DENY_ALL) - self.assertEqual(session.disabled_hook_categories, set()) - self.assertEqual(session.enabled_hook_categories, set(('metadata',))) + self.assertEqual(HOOKS_DENY_ALL, session.hooks_mode) + self.assertEqual(set(), session.disabled_hook_categories) + self.assertEqual(set(('metadata',)), session.enabled_hook_categories) with session.allow_all_hooks_but('integrity'): - self.assertEqual(session.hooks_mode, session.HOOKS_ALLOW_ALL) - self.assertEqual(session.disabled_hook_categories, set(('integrity',))) - self.assertEqual(session.enabled_hook_categories, set(('metadata',))) # not changed in such case - self.assertEqual(session.hooks_mode, session.HOOKS_DENY_ALL) - self.assertEqual(session.disabled_hook_categories, set()) - self.assertEqual(session.enabled_hook_categories, set(('metadata',))) + self.assertEqual(HOOKS_ALLOW_ALL, session.hooks_mode) + self.assertEqual(set(('integrity',)), session.disabled_hook_categories) + self.assertEqual(set(('metadata',)), session.enabled_hook_categories) # not changed in such case + self.assertEqual(HOOKS_DENY_ALL, session.hooks_mode) + self.assertEqual(set(), session.disabled_hook_categories) + self.assertEqual(set(('metadata',)), session.enabled_hook_categories) # leaving context manager with no transaction running should reset the # transaction local storage (and associated cnxset) - self.assertEqual(session._tx_data, {}) - self.assertEqual(session.cnxset, None) - self.assertEqual(session.hooks_mode, session.HOOKS_ALLOW_ALL) - self.assertEqual(session.disabled_hook_categories, set()) - self.assertEqual(session.enabled_hook_categories, set()) + self.assertEqual({}, session._txs) + self.assertEqual(None, session.cnxset) + self.assertEqual(HOOKS_ALLOW_ALL, session.hooks_mode, session.HOOKS_ALLOW_ALL) + self.assertEqual(set(), session.disabled_hook_categories) + self.assertEqual(set(), session.enabled_hook_categories) if __name__ == '__main__': diff -r 7920f439f383 -r be91151107f6 server/test/unittest_undo.py --- a/server/test/unittest_undo.py Fri Apr 12 11:39:10 2013 +0200 +++ b/server/test/unittest_undo.py Mon Apr 15 13:18:01 2013 +0200 @@ -19,6 +19,8 @@ from cubicweb import ValidationError from cubicweb.devtools.testlib import CubicWebTC +import cubicweb.server.session +from cubicweb.server.session import Transaction as OldTransaction from cubicweb.transaction import * from cubicweb.server.sources.native import UndoTransactionException, _UndoException @@ -28,12 +30,19 @@ def setup_database(self): req = self.request() - self.session.undo_actions = True self.toto = self.create_user(req, 'toto', password='toto', groups=('users',), commit=False) self.txuuid = self.commit() + def setUp(self): + class Transaction(OldTransaction): + """Force undo feature to be turned on in all case""" + undo_actions = property(lambda tx: True, lambda x, y:None) + cubicweb.server.session.Transaction = Transaction + super(UndoableTransactionTC, self).setUp() + def tearDown(self): + cubicweb.server.session.Transaction = OldTransaction self.restore_connection() self.session.undo_support = set() super(UndoableTransactionTC, self).tearDown() diff -r 7920f439f383 -r be91151107f6 skeleton/debian/control.tmpl --- a/skeleton/debian/control.tmpl Fri Apr 12 11:39:10 2013 +0200 +++ b/skeleton/debian/control.tmpl Mon Apr 15 13:18:01 2013 +0200 @@ -2,9 +2,9 @@ Section: web Priority: optional Maintainer: %(author)s <%(author-email)s> -Build-Depends: debhelper (>= 7), python (>=2.5), python-support +Build-Depends: debhelper (>= 7), python (>= 2.6), python-support Standards-Version: 3.9.3 -XS-Python-Version: >= 2.5 +XS-Python-Version: >= 2.6 Package: %(distname)s Architecture: all diff -r 7920f439f383 -r be91151107f6 sobjects/cwxmlparser.py --- a/sobjects/cwxmlparser.py Fri Apr 12 11:39:10 2013 +0200 +++ b/sobjects/cwxmlparser.py Mon Apr 15 13:18:01 2013 +0200 @@ -42,7 +42,7 @@ from yams.constraints import BASE_CONVERTERS from yams.schema import role_name as rn -from cubicweb import ValidationError, RegistryException, typed_eid +from cubicweb import ValidationError, RegistryException from cubicweb.view import Component from cubicweb.server.sources import datafeed from cubicweb.server.hook import match_rtype @@ -326,10 +326,10 @@ item['cwtype'] = unicode(node.tag) item.setdefault('cwsource', None) try: - item['eid'] = typed_eid(item['eid']) + item['eid'] = int(item['eid']) except KeyError: # cw < 3.11 compat mode XXX - item['eid'] = typed_eid(node.find('eid').text) + item['eid'] = int(node.find('eid').text) item['cwuri'] = node.find('cwuri').text rels = {} for child in node: diff -r 7920f439f383 -r be91151107f6 sobjects/ldapparser.py --- a/sobjects/ldapparser.py Fri Apr 12 11:39:10 2013 +0200 +++ b/sobjects/ldapparser.py Mon Apr 15 13:18:01 2013 +0200 @@ -176,7 +176,7 @@ return self._cw.execute('Any X WHERE X is CWGroup, X name %(name)s', {'name': name}).get_entity(0, 0) except IndexError: - self.error('group %r referenced by source configuration %r does not exist' - % (name, self.source.uri)) + self.error('group %r referenced by source configuration %r does not exist', + name, self.source.uri) return None diff -r 7920f439f383 -r be91151107f6 sobjects/textparsers.py --- a/sobjects/textparsers.py Fri Apr 12 11:39:10 2013 +0200 +++ b/sobjects/textparsers.py Mon Apr 15 13:18:01 2013 +0200 @@ -26,7 +26,7 @@ import re -from cubicweb import UnknownEid, typed_eid +from cubicweb import UnknownEid from cubicweb.view import Component @@ -66,7 +66,7 @@ def parse(self, caller, text): for trname, eid in self.instr_rgx.findall(text): try: - entity = self._cw.entity_from_eid(typed_eid(eid)) + entity = self._cw.entity_from_eid(int(eid)) except UnknownEid: self.error("can't get entity with eid %s", eid) continue diff -r 7920f439f383 -r be91151107f6 test/unittest_vregistry.py --- a/test/unittest_vregistry.py Fri Apr 12 11:39:10 2013 +0200 +++ b/test/unittest_vregistry.py Mon Apr 15 13:18:01 2013 +0200 @@ -54,23 +54,23 @@ def test_load_subinterface_based_appobjects(self): - self.vreg.register_objects([join(BASE, 'web', 'views', 'iprogress.py')]) - # check progressbar was kicked - self.assertFalse(self.vreg['views'].get('progressbar')) + self.vreg.register_objects([join(BASE, 'web', 'views', 'idownloadable.py')]) + # check downloadlink was kicked + self.assertFalse(self.vreg['views'].get('downloadlink')) # we've to emulate register_objects to add custom MyCard objects path = [join(BASE, 'entities', '__init__.py'), join(BASE, 'entities', 'adapters.py'), - join(BASE, 'web', 'views', 'iprogress.py')] + join(BASE, 'web', 'views', 'idownloadable.py')] filemods = self.vreg.init_registration(path, None) for filepath, modname in filemods: self.vreg.load_file(filepath, modname) - class CardIProgressAdapter(EntityAdapter): - __regid__ = 'IProgress' + class CardIDownloadableAdapter(EntityAdapter): + __regid__ = 'IDownloadable' self.vreg._loadedmods[__name__] = {} - self.vreg.register(CardIProgressAdapter) + self.vreg.register(CardIDownloadableAdapter) self.vreg.initialization_completed() # check progressbar isn't kicked - self.assertEqual(len(self.vreg['views']['progressbar']), 1) + self.assertEqual(len(self.vreg['views']['downloadlink']), 1) def test_properties(self): self.vreg.reset() diff -r 7920f439f383 -r be91151107f6 utils.py --- a/utils.py Fri Apr 12 11:39:10 2013 +0200 +++ b/utils.py Mon Apr 15 13:18:01 2013 +0200 @@ -263,10 +263,7 @@ def add_post_inline_script(self, content): self.post_inlined_scripts.append(content) - def add_onload(self, jscode, jsoncall=_MARKER): - if jsoncall is not _MARKER: - warn('[3.7] specifying jsoncall is not needed anymore', - DeprecationWarning, stacklevel=2) + def add_onload(self, jscode): self.add_post_inline_script(u"""$(cw).one('server-response', function(event) { %s});""" % jscode) @@ -567,14 +564,6 @@ return 'javascript: ' + PERCENT_IN_URLQUOTE_RE.sub(r'%25', javascript_code) -@deprecated('[3.7] merge_dicts is deprecated') -def merge_dicts(dict1, dict2): - """update a copy of `dict1` with `dict2`""" - dict1 = dict(dict1) - dict1.update(dict2) - return dict1 - - def parse_repo_uri(uri): """ transform a command line uri into a (protocol, hostport, appid), e.g: -> 'inmemory', None, '' diff -r 7920f439f383 -r be91151107f6 web/data/cubicweb.gmap.js --- a/web/data/cubicweb.gmap.js Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,87 +0,0 @@ -/** - * :organization: Logilab - * :copyright: 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. - * :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr - */ - -Widgets.GMapWidget = defclass('GMapWidget', null, { - __init__: function(wdgnode) { - // Assume we have imported google maps JS - if (GBrowserIsCompatible()) { - var uselabelstr = wdgnode.getAttribute('cubicweb:uselabel'); - var uselabel = true; - if (uselabelstr) { - if (uselabelstr == 'True') { - uselabel = true; - } - else { - uselabel = false; - } - } - var map = new GMap2(wdgnode); - map.addControl(new GSmallMapControl()); - var jsonurl = wdgnode.getAttribute('cubicweb:loadurl'); - var self = this; // bind this to a local variable - jQuery.getJSON(jsonurl, function(geodata) { - var zoomLevel; - var center; - var latlngbounds = new GLatLngBounds( ); - for (var i = 0; i < geodata.markers.length; i++) { - var marker = geodata.markers[i]; - var latlng = new GLatLng(marker.latitude, marker.longitude); - latlngbounds.extend( latlng ); - } - if (geodata.zoomlevel) { - zoomLevel = geodata.zoomlevel; - } else { - zoomLevel = map.getBoundsZoomLevel( latlngbounds ) - 1; - } - if (geodata.center) { - center = new GLatng(geodata.center.latitude, geodata.center.longitude); - } else { - center = latlngbounds.getCenter(); - } - map.setCenter(center, zoomLevel); - for (var i = 0; i < geodata.markers.length; i++) { - var marker = geodata.markers[i]; - self.createMarker(map, marker, i + 1, uselabel); - } - }); - jQuery(wdgnode).after(this.legendBox); - } else { // incompatible browser - jQuery.unload(GUnload); - } - }, - - createMarker: function(map, marker, i, uselabel) { - var point = new GLatLng(marker.latitude, marker.longitude); - var icon = new GIcon(); - icon.image = marker.icon[0]; - icon.iconSize = new GSize(marker.icon[1][0], marker.icon[1][1]); - icon.iconAnchor = new GPoint(marker.icon[2][0], marker.icon[2][1]); - if (marker.icon[3]) { - icon.shadow4 = marker.icon[3]; - } - if (typeof LabeledMarker == "undefined") { - var gmarker = new GMarker(point, { - icon: icon, - title: marker.title - }); - } else { - var gmarker = new LabeledMarker(point, { - icon: icon, - title: marker.title, - labelText: uselabel ? '' + i + '': '', - labelOffset: new GSize(2, - 32) - }); - } - map.addOverlay(gmarker); - GEvent.addListener(gmarker, 'click', function() { - jQuery.post(marker.bubbleUrl, function(data) { - map.openInfoWindowHtml(point, data); - }); - }); - } - -}); - diff -r 7920f439f383 -r be91151107f6 web/data/cubicweb.iprogress.css --- a/web/data/cubicweb.iprogress.css Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,78 +0,0 @@ -/* - * :organization: Logilab - * :copyright: 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. - * :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr - */ - -/******************************************************************************/ -/* progressbar */ -/******************************************************************************/ - -.done { background:red } - -.inprogress { background:green } - -.overpassed { background: yellow} - - -canvas.progressbar { - border:1px solid black; -} - -.progressbarback { - border: 1px solid #000000; - background: transparent; - height: 10px; - width: 100px; -} - -/******************************************************************************/ -/* progress table */ -/******************************************************************************/ - -table.progress { - /* The default table view */ - margin: 10px 0px 1em; - width: 100%; - font-size: 0.9167em; -} - -table.progress th { - white-space: nowrap; - font-weight: bold; - background: %(listingHeaderBgColor)s; - padding: 2px 4px; - font-size:8pt; -} - -table.progress th, -table.progress td { - border: 1px solid %(listingBorderColor)s; -} - -table.progress td { - text-align: right; - padding: 2px 3px; -} - -table.progress th.tdleft, -table.progress td.tdleft { - text-align: left; - padding: 2px 3px 2px 5px; -} - -table.progress tr.highlighted { - background-color: %(listingHighlightedBgColor)s; -} - -table.progress tr.highlighted .progressbarback { - border: 1px solid %(listingHighlightedBgColor)s; -} - -table.progress .progressbarback { - border: 1px solid #777; -} - -.progress_data { - padding-right: 3px; -} \ No newline at end of file diff -r 7920f439f383 -r be91151107f6 web/data/cubicweb.iprogress.js --- a/web/data/cubicweb.iprogress.js Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,63 +0,0 @@ -function ProgressBar() { - this.budget = 100; - this.todo = 100; - this.done = 100; - this.color_done = "green"; - this.color_budget = "blue"; - this.color_todo = "#cccccc"; // grey - this.height = 16; - this.middle = this.height / 2; - this.radius = 4; -} - -ProgressBar.prototype.draw_one_rect = function(ctx, pos, color, fill) { - ctx.beginPath(); - ctx.lineWidth = 1; - ctx.strokeStyle = color; - if (fill) { - ctx.fillStyle = color; - ctx.fillRect(0, 0, pos, this.middle * 2); - } else { - ctx.lineWidth = 2; - ctx.strokeStyle = "black"; - ctx.moveTo(pos, 0); - ctx.lineTo(pos, this.middle * 2); - ctx.stroke(); - } -}; - -ProgressBar.prototype.draw_one_circ = function(ctx, pos, color) { - ctx.beginPath(); - ctx.lineWidth = 2; - ctx.strokeStyle = color; - ctx.moveTo(0, this.middle); - ctx.lineTo(pos, this.middle); - ctx.arc(pos, this.middle, this.radius, 0, Math.PI * 2, true); - ctx.stroke(); -}; - -ProgressBar.prototype.draw_circ = function(ctx) { - this.draw_one_circ(ctx, this.budget, this.color_budget); - this.draw_one_circ(ctx, this.todo, this.color_todo); - this.draw_one_circ(ctx, this.done, this.color_done); -}; - -ProgressBar.prototype.draw_rect = function(ctx) { - this.draw_one_rect(ctx, this.todo, this.color_todo, true); - this.draw_one_rect(ctx, this.done, this.color_done, true); - this.draw_one_rect(ctx, this.budget, this.color_budget, false); -}; - -function draw_progressbar(cid, done, todo, budget, color) { - var canvas = document.getElementById(cid); - if (canvas.getContext) { - var ctx = canvas.getContext("2d"); - var bar = new ProgressBar(); - bar.budget = budget; - bar.todo = todo; - bar.done = done; - bar.color_done = color; - bar.draw_rect(ctx); - } -} - diff -r 7920f439f383 -r be91151107f6 web/data/cubicweb.mailform.css --- a/web/data/cubicweb.mailform.css Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,66 +0,0 @@ -/* styles for the email form (views/massmailing.py) - * - * :organization: Logilab - * :copyright: 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. - * :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr - */ - -form#sendmail { - border: 1px solid #DBDCE3; - background-color: #E9F5F7; - font-family:Verdana,Tahoma,Arial,sans-serif; - padding: 1em 1ex; -} - -table.headersform { - width: 100%; -} - -form#sendmail td#buttonbar { - padding: 0.5ex 0ex; -} - -table.headersform td.hlabel { - padding-top: 0.5ex; - color: #444444; - text-align: right; -} - -table.headersform td.hvalue { - padding-top: 0.5ex; - padding-left: 0.5em; - width: 100%; -} - -table.headersform td.hvalue input#mailsubj { - width: 47em; -} - -form#sendmail div#toolbar { - margin: 0.5em 0em; - height: 29px; -} - -form#sendmail div#toolbar ul { - list-style-image: none; - list-style-position: outside; - list-style-type:none; - margin:0px; - padding:0px; - /* border: 1px solid #DBDCE3; */ -} - -form#sendmail div#toolbar li { - background: none; - padding-left: 1em; - float: left; -} - -form#sendmail div#toolbar li a { - font-family: Verdana,Tahoma,Arial,sans-serif; - color: #444444; -} - -div#substitutions { - padding-left: 1em; -} diff -r 7920f439f383 -r be91151107f6 web/data/gmap.utility.labeledmarker.js --- a/web/data/gmap.utility.labeledmarker.js Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2 +0,0 @@ -/* http://code.google.com/p/gmaps-utility-library/source/browse/trunk/labeledmarker/ */ -function LabeledMarker(latlng,opt_opts){this.latlng_=latlng;this.opts_=opt_opts;this.labelText_=opt_opts.labelText||"";this.labelClass_=opt_opts.labelClass||"LabeledMarker_markerLabel";this.labelOffset_=opt_opts.labelOffset||new GSize(0,0);this.clickable_=opt_opts.clickable||true;this.title_=opt_opts.title||"";this.labelVisibility_=true;if(opt_opts.draggable){opt_opts.draggable=false}GMarker.apply(this,arguments)};LabeledMarker.prototype=new GMarker(new GLatLng(0,0));LabeledMarker.prototype.initialize=function(map){GMarker.prototype.initialize.apply(this,arguments);this.map_=map;this.div_=document.createElement("div");this.div_.className=this.labelClass_;this.div_.innerHTML=this.labelText_;this.div_.style.position="absolute";this.div_.style.cursor="pointer";this.div_.title=this.title_;map.getPane(G_MAP_MARKER_PANE).appendChild(this.div_);if(this.clickable_){function newEventPassthru(obj,event){return function(){GEvent.trigger(obj,event)}}var eventPassthrus=['click','dblclick','mousedown','mouseup','mouseover','mouseout'];for(var i=0;i. -""" - -""" - -from logilab.common.testlib import TestCase, unittest_main - -from cubicweb.web.views.embedding import prefix_links - -class UILIBTC(TestCase): - - - def test_prefix_links(self): - """suppose we are embedding http://embedded.com/page1.html""" - orig = ['perdu ?', - 'perdu ?', - 'perdu ?', - 'perdu ?', - '', - '', - '', - ] - expected = ['perdu ?', - 'perdu ?', - 'perdu ?', - 'perdu ?', - '', - '', - '', - ] - for orig_a, expected_a in zip(orig, expected): - got = prefix_links(orig_a, 'PREFIX', 'http://embedded.com/page1.html') - self.assertEqual(got, expected_a) - -if __name__ == '__main__': - unittest_main() - diff -r 7920f439f383 -r be91151107f6 web/views/autoform.py --- a/web/views/autoform.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/autoform.py Mon Apr 15 13:18:01 2013 +0200 @@ -127,7 +127,7 @@ from logilab.common.decorators import iclassmethod, cached from logilab.common.deprecation import deprecated -from cubicweb import typed_eid, neg_role, uilib +from cubicweb import neg_role, uilib from cubicweb.schema import display_name from cubicweb.view import EntityView from cubicweb.predicates import ( @@ -415,7 +415,7 @@ subjs, rtype, objs = rstr.split(':') for subj in subjs.split('_'): for obj in objs.split('_'): - yield typed_eid(subj), rtype, typed_eid(obj) + yield int(subj), rtype, int(obj) def delete_relations(req, rdefs): """delete relations from the repository""" @@ -460,12 +460,12 @@ def _add_pending(req, eidfrom, rel, eidto, kind): key = 'pending_%s' % kind pendings = req.session.data.setdefault(key, set()) - pendings.add( (typed_eid(eidfrom), rel, typed_eid(eidto)) ) + pendings.add( (int(eidfrom), rel, int(eidto)) ) def _remove_pending(req, eidfrom, rel, eidto, kind): key = 'pending_%s' % kind pendings = req.session.data[key] - pendings.remove( (typed_eid(eidfrom), rel, typed_eid(eidto)) ) + pendings.remove( (int(eidfrom), rel, int(eidto)) ) @ajaxfunc(output_type='json') def remove_pending_insert(self, (eidfrom, rel, eidto)): @@ -606,7 +606,7 @@ for pendingid in pending_inserts: eidfrom, rtype, eidto = pendingid.split(':') pendingid = 'id' + pendingid - if typed_eid(eidfrom) == entity.eid: # subject + if int(eidfrom) == entity.eid: # subject label = display_name(form._cw, rtype, 'subject', entity.__regid__) reid = eidto diff -r 7920f439f383 -r be91151107f6 web/views/basecontrollers.py --- a/web/views/basecontrollers.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/basecontrollers.py Mon Apr 15 13:18:01 2013 +0200 @@ -27,7 +27,7 @@ from logilab.common.deprecation import deprecated from cubicweb import (NoSelectableObject, ObjectNotFound, ValidationError, - AuthenticationError, typed_eid, UndoTransactionException, + AuthenticationError, UndoTransactionException, Forbidden) from cubicweb.utils import json_dumps from cubicweb.predicates import (authenticated_user, anonymous_user, @@ -176,7 +176,7 @@ if not '__linkto' in req.form: return if eid is None: - eid = typed_eid(req.form['eid']) + eid = int(req.form['eid']) for linkto in req.list_form_param('__linkto', pop=True): rtype, eids, target = linkto.split(':') assert target in ('subject', 'object') @@ -186,7 +186,7 @@ else: rql = 'SET Y %s X WHERE X eid %%(x)s, Y eid %%(y)s' % rtype for teid in eids: - req.execute(rql, {'x': eid, 'y': typed_eid(teid)}) + req.execute(rql, {'x': eid, 'y': int(teid)}) def _validation_error(req, ex): @@ -271,7 +271,6 @@ return ajax_controller.publish(rset) -# XXX move to massmailing class MailBugReportController(Controller): __regid__ = 'reportbug' __select__ = match_form_params('description') diff -r 7920f439f383 -r be91151107f6 web/views/bookmark.py --- a/web/views/bookmark.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/bookmark.py Mon Apr 15 13:18:01 2013 +0200 @@ -22,7 +22,7 @@ from logilab.mtconverter import xml_escape -from cubicweb import Unauthorized, typed_eid +from cubicweb import Unauthorized from cubicweb.predicates import is_instance, one_line_rset from cubicweb.web import action, component, htmlwidgets, formwidgets as fw from cubicweb.web.views import uicfg, primary @@ -137,4 +137,4 @@ @ajaxfunc def delete_bookmark(self, beid): rql = 'DELETE B bookmarked_by U WHERE B eid %(b)s, U eid %(u)s' - self._cw.execute(rql, {'b': typed_eid(beid), 'u' : self._cw.user.eid}) + self._cw.execute(rql, {'b': int(beid), 'u' : self._cw.user.eid}) diff -r 7920f439f383 -r be91151107f6 web/views/editcontroller.py --- a/web/views/editcontroller.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/editcontroller.py Mon Apr 15 13:18:01 2013 +0200 @@ -25,7 +25,7 @@ from rql.utils import rqlvar_maker -from cubicweb import Binary, ValidationError, typed_eid +from cubicweb import Binary, ValidationError from cubicweb.view import EntityAdapter, implements_adapter_compat from cubicweb.predicates import is_instance from cubicweb.web import (INTERNAL_FIELD_VALUE, RequestError, NothingToEdit, @@ -67,7 +67,7 @@ def valerror_eid(eid): try: - return typed_eid(eid) + return int(eid) except (ValueError, TypeError): return eid @@ -221,7 +221,7 @@ todelete = self._cw.list_form_param('__delete', formparams, pop=True) autoform.delete_relations(self._cw, todelete) if '__cloned_eid' in formparams: - entity.copy_relations(typed_eid(formparams['__cloned_eid'])) + entity.copy_relations(int(formparams['__cloned_eid'])) if is_main_entity: # only execute linkto for the main entity self.execute_linkto(entity.eid) return eid diff -r 7920f439f383 -r be91151107f6 web/views/editviews.py --- a/web/views/editviews.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/editviews.py Mon Apr 15 13:18:01 2013 +0200 @@ -23,7 +23,6 @@ from logilab.common.decorators import cached from logilab.mtconverter import xml_escape -from cubicweb import typed_eid from cubicweb.view import EntityView, StartupView from cubicweb.predicates import (one_line_rset, non_final_entity, match_search_state) @@ -53,7 +52,7 @@ def filter_box_context_info(self): entity = self.cw_rset.get_entity(0, 0) role, eid, rtype, etype = self._cw.search_state[1] - assert entity.eid == typed_eid(eid) + assert entity.eid == int(eid) # the default behaviour is to fetch all unrelated entities and display # them. Use fetch_order and not fetch_unrelated_order as sort method # since the latter is mainly there to select relevant items in the combo diff -r 7920f439f383 -r be91151107f6 web/views/embedding.py --- a/web/views/embedding.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/embedding.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -19,172 +19,20 @@ functionality. """ -__docformat__ = "restructuredtext en" -_ = unicode - -import re -from urlparse import urljoin -from urllib2 import urlopen, Request, HTTPError -from urllib import quote as urlquote # XXX should use view.url_quote method - -from logilab.mtconverter import guess_encoding - -from cubicweb.predicates import (one_line_rset, score_entity, implements, - adaptable, match_search_state) -from cubicweb.interfaces import IEmbedable -from cubicweb.view import NOINDEX, NOFOLLOW, EntityAdapter, implements_adapter_compat -from cubicweb.uilib import soup2xhtml -from cubicweb.web.controller import Controller -from cubicweb.web.action import Action -from cubicweb.web.views import basetemplates - - -class IEmbedableAdapter(EntityAdapter): - """interface for embedable entities""" - __needs_bw_compat__ = True - __regid__ = 'IEmbedable' - __select__ = implements(IEmbedable, warn=False) # XXX for bw compat, should be abstract - - @implements_adapter_compat('IEmbedable') - def embeded_url(self): - """embed action interface""" - raise NotImplementedError - - -class ExternalTemplate(basetemplates.TheMainTemplate): - """template embeding an external web pages into CubicWeb web interface - """ - __regid__ = 'external' +from logilab.common.deprecation import class_moved, moved - def call(self, body): - # XXX fallback to HTML 4 mode when embeding ? - self.set_request_content_type() - self._cw.search_state = ('normal',) - self.template_header(self.content_type, None, self._cw._('external page'), - [NOINDEX, NOFOLLOW]) - self.content_header() - self.w(body) - self.content_footer() - self.template_footer() - - -class EmbedController(Controller): - __regid__ = 'embed' - template = 'external' - - def publish(self, rset=None): - req = self._cw - if 'custom_css' in req.form: - req.add_css(req.form['custom_css']) - embedded_url = req.form['url'] - allowed = self._cw.vreg.config['embed-allowed'] - _ = req._ - if allowed is None or not allowed.match(embedded_url): - body = '

%s

%s

' % ( - _('error while embedding page'), - _('embedding this url is forbidden')) - else: - prefix = req.build_url(self.__regid__, url='') - authorization = req.get_header('Authorization') - if authorization: - headers = {'Authorization' : authorization} - else: - headers = {} - try: - body = embed_external_page(embedded_url, prefix, - headers, req.form.get('custom_css')) - body = soup2xhtml(body, self._cw.encoding) - except HTTPError as err: - body = '

%s

%s

' % ( - _('error while embedding page'), err) - rset = self.process_rql() - return self._cw.vreg['views'].main_template(req, self.template, - rset=rset, body=body) - +try: + from cubes.embed.views import * -def entity_has_embedable_url(entity): - """return 1 if the entity provides an allowed embedable url""" - url = entity.cw_adapt_to('IEmbedable').embeded_url() - if not url or not url.strip(): - return 0 - allowed = entity._cw.vreg.config['embed-allowed'] - if allowed is None or not allowed.match(url): - return 0 - return 1 - - -class EmbedAction(Action): - """display an 'embed' link on entity implementing `embeded_url` method - if the returned url match embeding configuration - """ - __regid__ = 'embed' - __select__ = (one_line_rset() & match_search_state('normal') - & adaptable('IEmbedable') - & score_entity(entity_has_embedable_url)) - - title = _('embed') - - def url(self, row=0): - entity = self.cw_rset.get_entity(row, 0) - url = urljoin(self._cw.base_url(), entity.cw_adapt_to('IEmbedable').embeded_url()) - if 'rql' in self._cw.form: - return self._cw.build_url('embed', url=url, rql=self._cw.form['rql']) - return self._cw.build_url('embed', url=url) - - - -# functions doing necessary substitutions to embed an external html page ###### - - -BODY_RGX = re.compile('(.*?)', re.I | re.S | re.U) -HREF_RGX = re.compile('. """Specific views for entities implementing IGeocodable""" -__docformat__ = "restructuredtext en" - -from cubicweb.interfaces import IGeocodable -from cubicweb.view import EntityView, EntityAdapter, implements_adapter_compat -from cubicweb.predicates import implements, adaptable -from cubicweb.utils import json_dumps - -class IGeocodableAdapter(EntityAdapter): - """interface required by geocoding views such as gmap-view""" - __needs_bw_compat__ = True - __regid__ = 'IGeocodable' - __select__ = implements(IGeocodable, warn=False) # XXX for bw compat, should be abstract - - @property - @implements_adapter_compat('IGeocodable') - def latitude(self): - """returns the latitude of the entity in degree (-90 < float < +90)""" - raise NotImplementedError +try: + from cubes.geocoding.views import (IGeocodableAdapter, + GeocodingJsonView, + GoogleMapBubbleView, + GoogleMapsView, + GoogeMapsLegend) - @property - @implements_adapter_compat('IGeocodable') - def longitude(self): - """returns the longitude of the entity in degree (-180 < float < +180)""" - raise NotImplementedError - - @implements_adapter_compat('IGeocodable') - def marker_icon(self): - """returns the icon that should be used as the marker. - - an icon is defined by a 4-uple: - - (icon._url, icon.size, icon.iconAnchor, icon.shadow) - """ - return (self._cw.uiprops['GMARKER_ICON'], (20, 34), (4, 34), None) - - -class GeocodingJsonView(EntityView): - __regid__ = 'geocoding-json' - __select__ = adaptable('IGeocodable') - - binary = True - templatable = False - content_type = 'application/json' + from logilab.common.deprecation import class_moved - def call(self): - zoomlevel = self._cw.form.pop('zoomlevel', None) - extraparams = self._cw.form.copy() - extraparams.pop('vid', None) - extraparams.pop('rql', None) - markers = [] - for entity in self.cw_rset.entities(): - igeocodable = entity.cw_adapt_to('IGeocodable') - # remove entities that don't define latitude and longitude - if not (igeocodable.latitude and igeocodable.longitude): - continue - markers.append(self.build_marker_data(entity, igeocodable, - extraparams)) - if not markers: - return - geodata = { - 'markers': markers, - } - if zoomlevel: - geodata['zoomlevel'] = int(zoomlevel) - self.w(json_dumps(geodata)) - - def build_marker_data(self, entity, igeocodable, extraparams): - return {'latitude': igeocodable.latitude, - 'longitude': igeocodable.longitude, - 'icon': igeocodable.marker_icon(), - 'title': entity.dc_long_title(), - 'bubbleUrl': entity.absolute_url( - vid='gmap-bubble', __notemplate=1, **extraparams), - } - - -class GoogleMapBubbleView(EntityView): - __regid__ = 'gmap-bubble' - __select__ = adaptable('IGeocodable') - - def cell_call(self, row, col): - entity = self.cw_rset.get_entity(row, col) - self.w(u'
%s
' % entity.view('oneline')) - # FIXME: we should call something like address-view if available - - -class GoogleMapsView(EntityView): - __regid__ = 'gmap-view' - __select__ = adaptable('IGeocodable') - - paginable = False - - def call(self, gmap_key, width=400, height=400, uselabel=True, urlparams=None): - self._cw.demote_to_html() - self._cw.add_js('http://maps.google.com/maps?sensor=false&file=api&v=2&key=%s' - % gmap_key, localfile=False) - self._cw.add_js( ('cubicweb.widgets.js', 'cubicweb.gmap.js', 'gmap.utility.labeledmarker.js') ) - rql = self.cw_rset.printable_rql() - if urlparams is None: - loadurl = self._cw.build_url(rql=rql, vid='geocoding-json') - else: - loadurl = self._cw.build_url(rql=rql, vid='geocoding-json', **urlparams) - self.w(u'
' - % (width, height, loadurl, uselabel)) - - -class GoogeMapsLegend(EntityView): - __regid__ = 'gmap-legend' - - def call(self): - self.w(u'
    ') - for rowidx in xrange(len(self.cw_rset)): - self.w(u'
  1. ') - self.wview('listitem', self.cw_rset, row=rowidx, col=0) - self.w(u'
    2. ') - self.w(u'
') + msg = '[3.17] cubicweb.web.views.igeocodable moved to cubes.geocoding.views' + IGeocodableAdapter = class_moved(IGeocodableAdapter, message=msg) + GeocodingJsonView = class_moved(GeocodingJsonView, message=msg) + GoogleMapBubbleView = class_moved(GoogleMapBubbleView, message=msg) + GoogleMapsView = class_moved(GoogleMapsView, message=msg) + GoogeMapsLegend = class_moved(GoogeMapsLegend, message=msg) +except ImportError: + from cubicweb.web import LOGGER + LOGGER.warning('[3.17] igeocoding extracted to cube geocoding that was not found. try installing it.') diff -r 7920f439f383 -r be91151107f6 web/views/iprogress.py --- a/web/views/iprogress.py Fri Apr 12 11:39:10 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,260 +0,0 @@ -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr -# -# This file is part of CubicWeb. -# -# CubicWeb is free software: you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 2.1 of the License, or (at your option) -# any later version. -# -# CubicWeb is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with CubicWeb. If not, see . -"""Specific views for entities implementing IProgress/IMileStone""" - -__docformat__ = "restructuredtext en" -_ = unicode - -from math import floor - -from logilab.common.deprecation import class_deprecated -from logilab.mtconverter import xml_escape - -from cubicweb.utils import make_uid -from cubicweb.predicates import adaptable -from cubicweb.schema import display_name -from cubicweb.view import EntityView -from cubicweb.web.views.tableview import EntityAttributesTableView - - -class ProgressTableView(EntityAttributesTableView): - """The progress table view is able to display progress information - of any object implement IMileStone. - - The default layout is composoed of 7 columns : parent task, - milestone, state, estimated date, cost, progressbar, and todo_by - - The view accepts an optional ``columns`` paramater that lets you - remove or reorder some of those columns. - - To add new columns, you should extend this class, define a new - ``columns`` class attribute and implement corresponding - build_COLNAME_cell methods - - header_for_COLNAME methods allow to customize header's label - """ - __metaclass__ = class_deprecated - __deprecation_warning__ = '[3.14] %(cls)s is deprecated' - - __regid__ = 'progress_table_view' - __select__ = adaptable('IMileStone') - title = _('task progression') - table_css = "progress" - css_files = ('cubicweb.iprogress.css',) - - # default columns of the table - columns = (_('project'), _('milestone'), _('state'), _('eta_date'), - _('cost'), _('progress'), _('todo_by')) - - def cell_call(self, row, col): - _ = self._cw._ - entity = self.cw_rset.get_entity(row, col) - infos = {} - for col in self.columns: - meth = getattr(self, 'build_%s_cell' % col, None) - # find the build method or try to find matching attribute - if meth: - content = meth(entity) - else: - content = entity.printable_value(col) - infos[col] = content - cssclass = entity.cw_adapt_to('IMileStone').progress_class() - self.w(u"""""" % cssclass) - line = u''.join(u'%%(%s)s' % col for col in self.columns) - self.w(line % infos) - self.w(u'\n') - - ## header management ###################################################### - - def header_for_project(self, sample): - """use entity's parent type as label""" - return display_name(self._cw, sample.cw_adapt_to('IMileStone').parent_type) - - def header_for_milestone(self, sample): - """use entity's type as label""" - return display_name(self._cw, sample.__regid__) - - ## cell management ######################################################## - def build_project_cell(self, entity): - """``project`` column cell renderer""" - project = entity.cw_adapt_to('IMileStone').get_main_task() - if project: - return project.view('incontext') - return self._cw._('no related project') - - def build_milestone_cell(self, entity): - """``milestone`` column cell renderer""" - return entity.view('incontext') - - def build_state_cell(self, entity): - """``state`` column cell renderer""" - return xml_escape(entity.cw_adapt_to('IWorkflowable').printable_state) - - def build_eta_date_cell(self, entity): - """``eta_date`` column cell renderer""" - imilestone = entity.cw_adapt_to('IMileStone') - if imilestone.finished(): - return self._cw.format_date(imilestone.completion_date()) - formated_date = self._cw.format_date(imilestone.initial_prevision_date()) - if imilestone.in_progress(): - eta_date = self._cw.format_date(imilestone.eta_date()) - _ = self._cw._ - if formated_date: - formated_date += u' (%s %s)' % (_('expected:'), eta_date) - else: - formated_date = u'%s %s' % (_('expected:'), eta_date) - return formated_date - - def build_todo_by_cell(self, entity): - """``todo_by`` column cell renderer""" - imilestone = entity.cw_adapt_to('IMileStone') - return u', '.join(p.view('outofcontext') for p in imilestone.contractors()) - - def build_cost_cell(self, entity): - """``cost`` column cell renderer""" - _ = self._cw._ - imilestone = entity.cw_adapt_to('IMileStone') - pinfo = imilestone.progress_info() - totalcost = pinfo.get('estimatedcorrected', pinfo['estimated']) - missing = pinfo.get('notestimatedcorrected', pinfo.get('notestimated', 0)) - costdescr = [] - if missing: - # XXX: link to unestimated entities - costdescr.append(_('%s not estimated') % missing) - estimated = pinfo['estimated'] - if estimated and estimated != totalcost: - costdescr.append(_('initial estimation %s') % estimated) - if costdescr: - return u'%s (%s)' % (totalcost, ', '.join(costdescr)) - return unicode(totalcost) - - def build_progress_cell(self, entity): - """``progress`` column cell renderer""" - return entity.view('progressbar') - - -class InContextProgressTableView(ProgressTableView): - """this views redirects to ``progress_table_view`` but removes - the ``project`` column - """ - __metaclass__ = class_deprecated - __deprecation_warning__ = '[3.14] %(cls)s is deprecated' - __regid__ = 'ic_progress_table_view' - - def call(self, columns=None): - view = self._cw.vreg['views'].select('progress_table_view', self._cw, - rset=self.cw_rset) - columns = list(columns or view.columns) - try: - columns.remove('project') - except ValueError: - self.info('[ic_progress_table_view] could not remove project from columns') - view.render(w=self.w, columns=columns) - - -class ProgressBarView(EntityView): - """displays a progress bar""" - __metaclass__ = class_deprecated - __deprecation_warning__ = '[3.14] %(cls)s is deprecated' - __regid__ = 'progressbar' - __select__ = adaptable('IProgress') - - title = _('progress bar') - - precision = 0.1 - red_threshold = 1.1 - orange_threshold = 1.05 - yellow_threshold = 1 - - @classmethod - def overrun(cls, iprogress): - done = iprogress.done or 0 - todo = iprogress.todo or 0 - budget = iprogress.revised_cost or 0 - if done + todo > budget: - overrun = done + todo - budget - else: - overrun = 0 - if overrun < cls.precision: - overrun = 0 - return overrun - - @classmethod - def overrun_percentage(cls, iprogress): - budget = iprogress.revised_cost or 0 - if budget == 0: - return 0 - return cls.overrun(iprogress) * 100. / budget - - def cell_call(self, row, col): - self._cw.add_css('cubicweb.iprogress.css') - self._cw.add_js('cubicweb.iprogress.js') - entity = self.cw_rset.get_entity(row, col) - iprogress = entity.cw_adapt_to('IProgress') - done = iprogress.done or 0 - todo = iprogress.todo or 0 - budget = iprogress.revised_cost or 0 - if budget == 0: - pourcent = 100 - else: - pourcent = done*100./budget - if pourcent > 100.1: - color = 'red' - elif todo+done > self.red_threshold*budget: - color = 'red' - elif todo+done > self.orange_threshold*budget: - color = 'orange' - elif todo+done > self.yellow_threshold*budget: - color = 'yellow' - else: - color = 'green' - if pourcent < 0: - pourcent = 0 - - if floor(done) == done or done>100: - done_str = '%i' % done - else: - done_str = '%.1f' % done - if floor(budget) == budget or budget>100: - budget_str = '%i' % budget - else: - budget_str = '%.1f' % budget - - title = u'%s/%s = %i%%' % (done_str, budget_str, pourcent) - short_title = title - overrunpercent = self.overrun_percentage(iprogress) - if overrunpercent: - overrun = self.overrun(iprogress) - title += u' overrun +%sj (+%i%%)' % (overrun, overrunpercent) - if floor(overrun) == overrun or overrun > 100: - short_title += u' +%i' % overrun - else: - short_title += u' +%.1f' % overrun - # write bars - maxi = max(done+todo, budget) - if maxi == 0: - maxi = 1 - cid = make_uid('progress_bar') - self._cw.html_headers.add_onload( - 'draw_progressbar("canvas%s", %i, %i, %i, "%s");' % - (cid, int(100.*done/maxi), int(100.*(done+todo)/maxi), - int(100.*budget/maxi), color)) - self.w(u'%s
' - u'' - % (xml_escape(short_title), cid)) diff -r 7920f439f383 -r be91151107f6 web/views/isioc.py --- a/web/views/isioc.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/isioc.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -20,144 +20,16 @@ http://sioc-project.org """ -__docformat__ = "restructuredtext en" -_ = unicode - -from logilab.mtconverter import xml_escape - -from cubicweb.view import EntityView, EntityAdapter, implements_adapter_compat -from cubicweb.predicates import implements, adaptable -from cubicweb.interfaces import ISiocItem, ISiocContainer - - -class ISIOCItemAdapter(EntityAdapter): - """interface for entities which may be represented as an ISIOC items""" - __needs_bw_compat__ = True - __regid__ = 'ISIOCItem' - __select__ = implements(ISiocItem, warn=False) # XXX for bw compat, should be abstract - - @implements_adapter_compat('ISIOCItem') - def isioc_content(self): - """return item's content""" - raise NotImplementedError - - @implements_adapter_compat('ISIOCItem') - def isioc_container(self): - """return container entity""" - raise NotImplementedError - - @implements_adapter_compat('ISIOCItem') - def isioc_type(self): - """return container type (post, BlogPost, MailMessage)""" - raise NotImplementedError +from logilab.common.deprecation import class_moved - @implements_adapter_compat('ISIOCItem') - def isioc_replies(self): - """return replies items""" - raise NotImplementedError - - @implements_adapter_compat('ISIOCItem') - def isioc_topics(self): - """return topics items""" - raise NotImplementedError - - -class ISIOCContainerAdapter(EntityAdapter): - """interface for entities which may be represented as an ISIOC container""" - __needs_bw_compat__ = True - __regid__ = 'ISIOCContainer' - __select__ = implements(ISiocContainer, warn=False) # XXX for bw compat, should be abstract - - @implements_adapter_compat('ISIOCContainer') - def isioc_type(self): - """return container type (forum, Weblog, MailingList)""" - raise NotImplementedError - - @implements_adapter_compat('ISIOCContainer') - def isioc_items(self): - """return contained items""" - raise NotImplementedError - - -class SIOCView(EntityView): - __regid__ = 'sioc' - __select__ = adaptable('ISIOCItem', 'ISIOCContainer') - title = _('sioc') - templatable = False - content_type = 'text/xml' +try: + from cubes.sioc.views import * - def call(self): - self.w(u'\n' % self._cw.encoding) - self.w(u'''\n''') - for i in xrange(self.cw_rset.rowcount): - self.cell_call(i, 0) - self.w(u'\n') - - def cell_call(self, row, col): - self.wview('sioc_element', self.cw_rset, row=row, col=col) - -class SIOCContainerView(EntityView): - __regid__ = 'sioc_element' - __select__ = adaptable('ISIOCContainer') - templatable = False - content_type = 'text/xml' - - def cell_call(self, row, col): - entity = self.cw_rset.complete_entity(row, col) - isioc = entity.cw_adapt_to('ISIOCContainer') - isioct = isioc.isioc_type() - self.w(u'\n' - % (isioct, xml_escape(entity.absolute_url()))) - self.w(u'%s' - % xml_escape(entity.dc_title())) - self.w(u'%s' - % entity.creation_date) # XXX format - self.w(u'%s' - % entity.modification_date) # XXX format - self.w(u'')#entity.isioc_items() - self.w(u'\n' % isioct) - - -class SIOCItemView(EntityView): - __regid__ = 'sioc_element' - __select__ = adaptable('ISIOCItem') - templatable = False - content_type = 'text/xml' - - def cell_call(self, row, col): - entity = self.cw_rset.complete_entity(row, col) - isioc = entity.cw_adapt_to('ISIOCItem') - isioct = isioc.isioc_type() - self.w(u'\n' - % (isioct, xml_escape(entity.absolute_url()))) - self.w(u'%s' - % xml_escape(entity.dc_title())) - self.w(u'%s' - % entity.creation_date) # XXX format - self.w(u'%s' - % entity.modification_date) # XXX format - content = isioc.isioc_content() - if content: - self.w(u'%s' % xml_escape(content)) - container = isioc.isioc_container() - if container: - self.w(u'\n' - % xml_escape(container.absolute_url())) - if entity.creator: - self.w(u'\n') - self.w(u'\n' - % xml_escape(entity.creator.absolute_url())) - self.w(entity.creator.view('foaf')) - self.w(u'\n') - self.w(u'\n') - self.w(u'')#entity.isioc_topics() - self.w(u'')#entity.isioc_replies() - self.w(u' \n' % isioct) - + ISIOCItemAdapter = class_moved(ISIOCItemAdapter, message='[3.17] ISIOCItemAdapter moved to cubes.isioc.views') + ISIOCContainerAdapter = class_moved(ISIOCContainerAdapter, message='[3.17] ISIOCContainerAdapter moved to cubes.isioc.views') + SIOCView = class_moved(SIOCView, message='[3.17] SIOCView moved to cubes.is.view') + SIOCContainerView = class_moved(SIOCContainerView, message='[3.17] SIOCContainerView moved to cubes.is.view') + SIOCItemView = class_moved(SIOCItemView, message='[3.17] SIOCItemView moved to cubes.is.view') +except ImportError: + from cubicweb.web import LOGGER + LOGGER.warning('[3.17] isioc extracted to cube sioc that was not found. try installing it.') diff -r 7920f439f383 -r be91151107f6 web/views/magicsearch.py --- a/web/views/magicsearch.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/magicsearch.py Mon Apr 15 13:18:01 2013 +0200 @@ -29,7 +29,7 @@ from rql.utils import rqlvar_maker from rql.nodes import Relation -from cubicweb import Unauthorized, typed_eid +from cubicweb import Unauthorized from cubicweb.view import Component from cubicweb.web.views.ajaxcontroller import ajaxfunc @@ -254,7 +254,7 @@ """ # if this is an integer, then directly go to eid try: - eid = typed_eid(word) + eid = int(word) return 'Any X WHERE X eid %(x)s', {'x': eid}, 'x' except ValueError: etype = self._get_entity_type(word) diff -r 7920f439f383 -r be91151107f6 web/views/massmailing.py --- a/web/views/massmailing.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/massmailing.py Mon Apr 15 13:18:01 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -17,161 +17,24 @@ # with CubicWeb. If not, see . """Mass mailing handling: send mail to entities adaptable to IEmailable""" -__docformat__ = "restructuredtext en" -_ = unicode - -import operator -from functools import reduce - -from cubicweb.predicates import (is_instance, authenticated_user, - adaptable, match_form_params) -from cubicweb.view import EntityView -from cubicweb.web import (Redirect, stdmsgs, controller, action, - form, formfields as ff) -from cubicweb.web.formwidgets import CheckBox, TextInput, AjaxWidget, ImgButton -from cubicweb.web.views import forms, formrenderers - - -class SendEmailAction(action.Action): - __regid__ = 'sendemail' - # XXX should check email is set as well - __select__ = (action.Action.__select__ - & authenticated_user() - & adaptable('IEmailable')) - - title = _('send email') - category = 'mainactions' - - def url(self): - params = {'vid': 'massmailing', '__force_display': 1} - if 'rql' in self._cw.form: - params['rql'] = self._cw.form['rql'] - return self._cw.build_url(self._cw.relative_path(includeparams=False), - **params) - - -def recipient_vocabulary(form, field): - vocab = [(entity.cw_adapt_to('IEmailable').get_email(), unicode(entity.eid)) - for entity in form.cw_rset.entities()] - return [(label, value) for label, value in vocab if label] +try: + from cubes.massmailing.views import (SendEmailAction, + recipient_vocabulary, + MassMailingForm, + MassMailingFormRenderer, + MassMailingFormView, + SendMailController) -class MassMailingForm(forms.FieldsForm): - __regid__ = 'massmailing' - - needs_js = ('cubicweb.edition.js', 'cubicweb.widgets.js',) - needs_css = ('cubicweb.mailform.css') - domid = 'sendmail' - action = 'sendmail' - - sender = ff.StringField(widget=TextInput({'disabled': 'disabled'}), - label=_('From:'), - value=lambda form, field: '%s <%s>' % ( - form._cw.user.dc_title(), - form._cw.user.cw_adapt_to('IEmailable').get_email())) - recipient = ff.StringField(widget=CheckBox(), label=_('Recipients:'), - choices=recipient_vocabulary, - value= lambda form, field: [entity.eid for entity in form.cw_rset.entities() - if entity.cw_adapt_to('IEmailable').get_email()]) - subject = ff.StringField(label=_('Subject:'), max_length=256) - mailbody = ff.StringField(widget=AjaxWidget(wdgtype='TemplateTextField', - inputid='mailbody')) - - form_buttons = [ImgButton('sendbutton', "javascript: $('#sendmail').submit()", - _('send email'), 'SEND_EMAIL_ICON'), - ImgButton('cancelbutton', "javascript: history.back()", - _(stdmsgs.BUTTON_CANCEL[0]), stdmsgs.BUTTON_CANCEL[1])] - form_renderer_id = __regid__ - - def __init__(self, *args, **kwargs): - super(MassMailingForm, self).__init__(*args, **kwargs) - field = self.field_by_name('mailbody') - field.widget.attrs['cubicweb:variables'] = ','.join(self.get_allowed_substitutions()) - - def get_allowed_substitutions(self): - attrs = [] - for coltype in self.cw_rset.column_types(0): - entity = self._cw.vreg['etypes'].etype_class(coltype)(self._cw) - attrs.append(entity.cw_adapt_to('IEmailable').allowed_massmail_keys()) - return sorted(reduce(operator.and_, attrs)) - - def build_substitutions_help(self): - insertLink = u'%%(%s)s' - substs = (u'
%s
' % (insertLink % (subst, subst)) - for subst in self.get_allowed_substitutions()) - helpmsg = self._cw._('You can use any of the following substitutions in your text') - return u'
%s%s
' % ( - helpmsg, u'\n'.join(substs)) - - -class MassMailingFormRenderer(formrenderers.FormRenderer): - __regid__ = 'massmailing' + from logilab.common.deprecation import class_moved, moved - def _render_fields(self, fields, w, form): - w(u'') - for field in fields: - if field.name == 'mailbody': - w(u'
') - self._render_toolbar(w, form) - w(u'') - w(u'') - w(u'' % self.render_label(form, field)) - w(u'') - w(u'' % form.build_substitutions_help()) - w(u'') - else: - w(u'') - w(u'
') - else: - w(u'
%s') - w(field.render(form, self)) - if field.name == 'mailbody': - w(u'%s
') - - def _render_toolbar(self, w, form): - w(u'
') - w(u'
    ') - for button in form.form_buttons: - w(u'
  • %s
    • ' % button.render(form)) - w(u'
') - w(u'
') - - def render_buttons(self, w, form): - pass - - -class MassMailingFormView(form.FormViewMixIn, EntityView): - __regid__ = 'massmailing' - __select__ = authenticated_user() & adaptable('IEmailable') - - def call(self): - form = self._cw.vreg['forms'].select('massmailing', self._cw, - rset=self.cw_rset) - form.render(w=self.w) - - -class SendMailController(controller.Controller): - __regid__ = 'sendmail' - __select__ = authenticated_user() & match_form_params('recipient', 'mailbody', 'subject') - - def recipients(self): - """returns an iterator on email's recipients as entities""" - eids = self._cw.form['recipient'] - # eids may be a string if only one recipient was specified - if isinstance(eids, basestring): - rset = self._cw.execute('Any X WHERE X eid %(x)s', {'x': eids}) - else: - rset = self._cw.execute('Any X WHERE X eid in (%s)' % (','.join(eids))) - return rset.entities() - - def publish(self, rset=None): - # XXX this allows users with access to an cubicweb instance to use it as - # a mail relay - body = self._cw.form['mailbody'] - subject = self._cw.form['subject'] - for recipient in self.recipients(): - iemailable = recipient.cw_adapt_to('IEmailable') - text = body % iemailable.as_email_context() - self.sendmail(iemailable.get_email(), subject, text) - url = self._cw.build_url(__message=self._cw._('emails successfully sent')) - raise Redirect(url) + msg = '[3.17] cubicweb.web.views.massmailing moved to cubes.massmailing.views' + SendEmailAction = class_moved(SendEmailAction, message=msg) + recipient_vocabulary = moved('cubes.massmailing.views', 'recipient_vocabulary') + MassMailingForm = class_moved(MassMailingForm, message=msg) + MassMailingFormRenderer = class_moved(MassMailingFormRenderer, message=msg) + MassMailingFormView = class_moved(MassMailingFormView, message=msg) + SendMailController = class_moved(SendMailController, message=msg) +except ImportError: + from cubicweb.web import LOGGER + LOGGER.warning('[3.17] massmailing extracted to cube massmailing that was not found. try installing it.') diff -r 7920f439f383 -r be91151107f6 web/views/reledit.py --- a/web/views/reledit.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/reledit.py Mon Apr 15 13:18:01 2013 +0200 @@ -29,7 +29,7 @@ from logilab.common.deprecation import deprecated, class_renamed from logilab.common.decorators import cached -from cubicweb import neg_role, typed_eid +from cubicweb import neg_role from cubicweb.schema import display_name from cubicweb.utils import json, json_dumps from cubicweb.predicates import non_final_entity, match_kwargs @@ -402,7 +402,7 @@ req = self._cw args = dict((x, req.form[x]) for x in ('formid', 'rtype', 'role', 'reload', 'action')) - rset = req.eid_rset(typed_eid(self._cw.form['eid'])) + rset = req.eid_rset(int(self._cw.form['eid'])) try: args['reload'] = json.loads(args['reload']) except ValueError: # not true/false, an absolute url diff -r 7920f439f383 -r be91151107f6 web/views/urlpublishing.py --- a/web/views/urlpublishing.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/urlpublishing.py Mon Apr 15 13:18:01 2013 +0200 @@ -59,7 +59,7 @@ from rql import TypeResolverException -from cubicweb import RegistryException, typed_eid +from cubicweb import RegistryException from cubicweb.web import NotFound, Redirect, component @@ -165,7 +165,7 @@ if len(parts) != 1: raise PathDontMatch() try: - rset = req.execute('Any X WHERE X eid %(x)s', {'x': typed_eid(parts[0])}) + rset = req.execute('Any X WHERE X eid %(x)s', {'x': int(parts[0])}) except ValueError: raise PathDontMatch() if rset.rowcount == 0: @@ -222,7 +222,7 @@ 'x', 'Substitute') if attrname == 'eid': try: - rset = req.execute(st.as_string(), {'x': typed_eid(value)}) + rset = req.execute(st.as_string(), {'x': int(value)}) except (ValueError, TypeResolverException): # conflicting eid/type raise PathDontMatch() diff -r 7920f439f383 -r be91151107f6 web/views/urlrewrite.py --- a/web/views/urlrewrite.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/views/urlrewrite.py Mon Apr 15 13:18:01 2013 +0200 @@ -19,7 +19,6 @@ import re -from cubicweb import typed_eid from cubicweb.uilib import domid from cubicweb.appobject import AppObject @@ -186,7 +185,7 @@ except KeyError: kwargs[key] = value if cachekey is not None and key in cachekey: - kwargs[key] = typed_eid(value) + kwargs[key] = int(value) if setuser: kwargs['u'] = req.user.eid for param in rqlformparams: diff -r 7920f439f383 -r be91151107f6 web/webconfig.py --- a/web/webconfig.py Fri Apr 12 11:39:10 2013 +0200 +++ b/web/webconfig.py Mon Apr 15 13:18:01 2013 +0200 @@ -110,6 +110,14 @@ 'group': 'web', 'level': 3, }), # web configuration + ('ui-cube', + {'type' : 'string', + 'default': None, + 'help': 'the name of the UI cube that will be loaded before all other '\ + 'cubes. Setting this value to None will instruct cubicweb not to load '\ + 'any extra cube.', + 'group': 'web', 'level': 3, + }), ('https-url', {'type' : 'string', 'default': None, diff -r 7920f439f383 -r be91151107f6 wsgi/request.py --- a/wsgi/request.py Fri Apr 12 11:39:10 2013 +0200 +++ b/wsgi/request.py Mon Apr 15 13:18:01 2013 +0200 @@ -38,13 +38,14 @@ class CubicWebWsgiRequest(CubicWebRequestBase): - """most of this code COMES FROM DJANO + """most of this code COMES FROM DJANGO """ def __init__(self, environ, vreg): self.environ = environ self.path = environ['PATH_INFO'] self.method = environ['REQUEST_METHOD'].upper() + self.content = environ['wsgi.input'] headers_in = dict((normalize_header(k[5:]), v) for k, v in self.environ.items() if k.startswith('HTTP_')) diff -r 7920f439f383 -r be91151107f6 xy.py --- a/xy.py Fri Apr 12 11:39:10 2013 +0200 +++ b/xy.py Mon Apr 15 13:18:01 2013 +0200 @@ -23,7 +23,6 @@ xy.register_prefix('dc', 'http://purl.org/dc/elements/1.1/') xy.register_prefix('foaf', 'http://xmlns.com/foaf/0.1/') xy.register_prefix('doap', 'http://usefulinc.com/ns/doap#') -xy.register_prefix('sioc', 'http://rdfs.org/sioc/ns#') xy.register_prefix('owl', 'http://www.w3.org/2002/07/owl#') xy.register_prefix('dcterms', 'http://purl.org/dc/terms/')