# HG changeset patch # User Sylvain Thénault # Date 1269422611 -3600 # Node ID 02b52bf9f5f8e0f2aedeaeee3593cc107288f48f # Parent c25da7573ebd19bafaed771c0ea597a5a35eaddb# Parent 4247066fd3dea5327acaaef7365eb8a6322da947 oldstable is now 3.6 diff -r c25da7573ebd -r 02b52bf9f5f8 .hgtags --- a/.hgtags Fri Feb 12 15:18:00 2010 +0100 +++ b/.hgtags Wed Mar 24 10:23:31 2010 +0100 @@ -98,3 +98,14 @@ 4920121d41f28c8075a4f00461911677396fc566 cubicweb-debian-version-3.5.11-1 98af3d02b83e7635207781289cc3445fb0829951 cubicweb-version-3.5.12 4281e1e2d76b9a37f38c0eeb1cbdcaa2fac6533c cubicweb-debian-version-3.5.12-1 +5f957e351b0a60d5c5fff60c560b04e666c3a8c6 cubicweb-version-3.6.0 +17e88f2485d1ea1fb8a3926a274637ce19e95d69 cubicweb-debian-version-3.6.0-1 +450804da3ab2476b7ede0c1f956235b4c239734f cubicweb-version-3.6.0 +d2ba93fcb8da95ceab08f48f8149a480215f149c cubicweb-debian-version-3.6.0-1 +4ae30c9ca11b1edad67d25b76fce672171d02023 cubicweb-version-3.6.1 +b9cdfe3341d1228687515d9af8686971ad5e6f5c cubicweb-debian-version-3.6.1-1 +0a16f07112b90fb61d2e905855fece77e5a7e39c cubicweb-debian-version-3.6.1-2 +bfebe3d14d5390492925fc294dfdafad890a7104 cubicweb-version-3.6.2 +f3b4bb9121a0e7ee5961310ff79e61c890948a77 cubicweb-debian-version-3.6.2-1 +9c342fa4f1b73e06917d7dc675949baff442108b cubicweb-version-3.6.3 +f9fce56d6a0c2bc6c4b497b66039a8bbbbdc8074 cubicweb-debian-version-3.6.3-1 diff -r c25da7573ebd -r 02b52bf9f5f8 MANIFEST.in --- a/MANIFEST.in Fri Feb 12 15:18:00 2010 +0100 +++ b/MANIFEST.in Wed Mar 24 10:23:31 2010 +0100 @@ -17,7 +17,6 @@ recursive-include i18n *.pot *.po recursive-include schemas *.py *.sql.* -recursive-include common/test/data * recursive-include entities/test/data * recursive-include sobjects/test/data * recursive-include server/test/data * diff -r c25da7573ebd -r 02b52bf9f5f8 __init__.py --- a/__init__.py Fri Feb 12 15:18:00 2010 +0100 +++ b/__init__.py Wed Mar 24 10:23:31 2010 +0100 @@ -7,7 +7,6 @@ :license: Library General Public License version 2 - http://www.gnu.org/licenses """ __docformat__ = "restructuredtext en" -from cubicweb.__pkginfo__ import version as __version__ import __builtin__ # '_' is available in builtins to mark internationalized string but should @@ -19,9 +18,7 @@ import sys, os, logging from StringIO import StringIO -from urllib import quote as urlquote, unquote as urlunquote -from logilab.common.decorators import cached from logilab.common.logging_ext import set_log_methods @@ -30,6 +27,8 @@ else: logging.basicConfig() +from cubicweb.__pkginfo__ import version as __version__ + set_log_methods(sys.modules[__name__], logging.getLogger('cubicweb')) @@ -57,293 +56,12 @@ "Binary objects must use raw strings, not %s" % data.__class__ StringIO.write(self, data) - -class RequestSessionMixIn(object): - """mixin class containing stuff shared by server session and web request - """ - def __init__(self, vreg): - self.vreg = vreg - try: - encoding = vreg.property_value('ui.encoding') - except: # no vreg or property not registered - encoding = 'utf-8' - self.encoding = encoding - # cache result of execution for (rql expr / eids), - # should be emptied on commit/rollback of the server session / web - # connection - self.local_perm_cache = {} - - def property_value(self, key): - if self.user: - return self.user.property_value(key) - return self.vreg.property_value(key) - - def etype_rset(self, etype, size=1): - """return a fake result set for a particular entity type""" - from cubicweb.rset import ResultSet - rset = ResultSet([('A',)]*size, '%s X' % etype, - description=[(etype,)]*size) - def get_entity(row, col=0, etype=etype, req=self, rset=rset): - return req.vreg.etype_class(etype)(req, rset, row, col) - rset.get_entity = get_entity - return self.decorate_rset(rset) - - def eid_rset(self, eid, etype=None): - """return a result set for the given eid without doing actual query - (we have the eid, we can suppose it exists and user has access to the - entity) - """ - from cubicweb.rset import ResultSet - eid = typed_eid(eid) - if etype is None: - etype = self.describe(eid)[0] - rset = ResultSet([(eid,)], 'Any X WHERE X eid %(x)s', {'x': eid}, - [(etype,)]) - return self.decorate_rset(rset) - - def empty_rset(self): - """return an empty result set. This is used e.g. to substitute - to a real result set if the user doesn't have permission to - access the results of a query. - """ - from cubicweb.rset import ResultSet - return self.decorate_rset(ResultSet([], 'Any X WHERE X eid -1')) - - def entity_from_eid(self, eid, etype=None): - try: - return self.entity_cache(eid) - except KeyError: - rset = self.eid_rset(eid, etype) - entity = rset.get_entity(0, 0) - self.set_entity_cache(entity) - return entity - - def entity_cache(self, eid): - raise KeyError - def set_entity_cache(self, entity): - pass - - def create_entity(self, etype, _cw_unsafe=False, **kwargs): - """add a new entity of the given type - - Example (in a shell session): - - c = create_entity('Company', name=u'Logilab') - create_entity('Person', works_for=c, firstname=u'John', lastname=u'Doe') - - """ - if _cw_unsafe: - execute = self.unsafe_execute - else: - execute = self.execute - rql = 'INSERT %s X' % etype - relations = [] - restrictions = set() - cachekey = [] - pending_relations = [] - for attr, value in kwargs.items(): - if isinstance(value, (tuple, list, set, frozenset)): - if len(value) == 1: - value = iter(value).next() - else: - del kwargs[attr] - pending_relations.append( (attr, value) ) - continue - if hasattr(value, 'eid'): # non final relation - rvar = attr.upper() - # XXX safer detection of object relation - if attr.startswith('reverse_'): - relations.append('%s %s X' % (rvar, attr[len('reverse_'):])) - else: - relations.append('X %s %s' % (attr, rvar)) - restriction = '%s eid %%(%s)s' % (rvar, attr) - if not restriction in restrictions: - restrictions.add(restriction) - cachekey.append(attr) - kwargs[attr] = value.eid - else: # attribute - relations.append('X %s %%(%s)s' % (attr, attr)) - if relations: - rql = '%s: %s' % (rql, ', '.join(relations)) - if restrictions: - rql = '%s WHERE %s' % (rql, ', '.join(restrictions)) - created = execute(rql, kwargs, cachekey).get_entity(0, 0) - for attr, values in pending_relations: - if attr.startswith('reverse_'): - restr = 'Y %s X' % attr[len('reverse_'):] - else: - restr = 'X %s Y' % attr - execute('SET %s WHERE X eid %%(x)s, Y eid IN (%s)' % ( - restr, ','.join(str(r.eid) for r in values)), - {'x': created.eid}, 'x') - return created - - # url generation methods ################################################## - - def build_url(self, *args, **kwargs): - """return an absolute URL using params dictionary key/values as URL - parameters. Values are automatically URL quoted, and the - publishing method to use may be specified or will be guessed. - """ - # use *args since we don't want first argument to be "anonymous" to - # avoid potential clash with kwargs - assert len(args) == 1, 'only 0 or 1 non-named-argument expected' - method = args[0] - base_url = kwargs.pop('base_url', None) - if base_url is None: - base_url = self.base_url() - if '_restpath' in kwargs: - assert method == 'view', method - path = kwargs.pop('_restpath') - else: - path = method - if not kwargs: - return u'%s%s' % (base_url, path) - return u'%s%s?%s' % (base_url, path, self.build_url_params(**kwargs)) +# use this dictionary to rename entity types while keeping bw compat +ETYPE_NAME_MAP = {} - - def build_url_params(self, **kwargs): - """return encoded params to incorporate them in an URL""" - args = [] - for param, values in kwargs.items(): - if not isinstance(values, (list, tuple)): - values = (values,) - for value in values: - args.append(u'%s=%s' % (param, self.url_quote(value))) - return '&'.join(args) - - def url_quote(self, value, safe=''): - """urllib.quote is not unicode safe, use this method to do the - necessary encoding / decoding. Also it's designed to quote each - part of a url path and so the '/' character will be encoded as well. - """ - if isinstance(value, unicode): - quoted = urlquote(value.encode(self.encoding), safe=safe) - return unicode(quoted, self.encoding) - return urlquote(str(value), safe=safe) - - def url_unquote(self, quoted): - """returns a unicode unquoted string - - decoding is based on `self.encoding` which is the encoding - used in `url_quote` - """ - if isinstance(quoted, unicode): - quoted = quoted.encode(self.encoding) - try: - return unicode(urlunquote(quoted), self.encoding) - except UnicodeDecodeError: # might occurs on manually typed URLs - return unicode(urlunquote(quoted), 'iso-8859-1') - - - # session's user related methods ##################################### - - @cached - def user_data(self): - """returns a dictionnary with this user's information""" - userinfo = {} - if self.is_internal_session: - userinfo['login'] = "cubicweb" - userinfo['name'] = "cubicweb" - userinfo['email'] = "" - return userinfo - user = self.actual_session().user - userinfo['login'] = user.login - userinfo['name'] = user.name() - userinfo['email'] = user.get_email() - return userinfo - - def is_internal_session(self): - """overrided on the server-side""" - return False - - # abstract methods to override according to the web front-end ############# - - def base_url(self): - """return the root url of the instance""" - raise NotImplementedError - - def decorate_rset(self, rset): - """add vreg/req (at least) attributes to the given result set """ - raise NotImplementedError - - def describe(self, eid): - """return a tuple (type, sourceuri, extid) for the entity with id """ - raise NotImplementedError - - -# XXX 2.45 is allowing nicer entity type names, use this map for bw compat -ETYPE_NAME_MAP = {# 3.2 migration - 'ECache': 'CWCache', - 'EUser': 'CWUser', - 'EGroup': 'CWGroup', - 'EProperty': 'CWProperty', - 'EFRDef': 'CWAttribute', - 'ENFRDef': 'CWRelation', - 'ERType': 'CWRType', - 'EEType': 'CWEType', - 'EConstraintType': 'CWConstraintType', - 'EConstraint': 'CWConstraint', - 'EPermission': 'CWPermission', - # 2.45 migration - 'Eetype': 'CWEType', - 'Ertype': 'CWRType', - 'Efrdef': 'CWAttribute', - 'Enfrdef': 'CWRelation', - 'Econstraint': 'CWConstraint', - 'Econstrainttype': 'CWConstraintType', - 'Epermission': 'CWPermission', - 'Egroup': 'CWGroup', - 'Euser': 'CWUser', - 'Eproperty': 'CWProperty', - 'Emailaddress': 'EmailAddress', - 'Rqlexpression': 'RQLExpression', - 'Trinfo': 'TrInfo', - } - - - -# XXX cubic web cube migration map -CW_MIGRATION_MAP = {'erudi': 'cubicweb', - - 'eaddressbook': 'addressbook', - 'ebasket': 'basket', - 'eblog': 'blog', - 'ebook': 'book', - 'ecomment': 'comment', - 'ecompany': 'company', - 'econference': 'conference', - 'eemail': 'email', - 'eevent': 'event', - 'eexpense': 'expense', - 'efile': 'file', - 'einvoice': 'invoice', - 'elink': 'link', - 'emailinglist': 'mailinglist', - 'eperson': 'person', - 'eshopcart': 'shopcart', - 'eskillmat': 'skillmat', - 'etask': 'task', - 'eworkcase': 'workcase', - 'eworkorder': 'workorder', - 'ezone': 'zone', - 'i18ncontent': 'i18ncontent', - 'svnfile': 'vcsfile', - - 'eclassschemes': 'keyword', - 'eclassfolders': 'folder', - 'eclasstags': 'tag', - - 'jpl': 'jpl', - 'jplintra': 'jplintra', - 'jplextra': 'jplextra', - 'jplorg': 'jplorg', - 'jplrecia': 'jplrecia', - 'crm': 'crm', - 'agueol': 'agueol', - 'docaster': 'docaster', - 'asteretud': 'asteretud', - } +# XXX cubic web cube migration map. See if it's worth keeping this mecanism +# to help in cube renaming +CW_MIGRATION_MAP = {} def neg_role(role): if role == 'subject': @@ -362,9 +80,6 @@ except AttributeError: return neg_role(obj.role) -def underline_title(title, car='-'): - return title+'\n'+(car*len(title)) - class CubicWebEventManager(object): """simple event / callback manager. diff -r c25da7573ebd -r 02b52bf9f5f8 __pkginfo__.py --- a/__pkginfo__.py Fri Feb 12 15:18:00 2010 +0100 +++ b/__pkginfo__.py Wed Mar 24 10:23:31 2010 +0100 @@ -7,7 +7,7 @@ distname = "cubicweb" modname = "cubicweb" -numversion = (3, 5, 12) +numversion = (3, 6, 3) version = '.'.join(str(num) for num in numversion) license = 'LGPL' @@ -48,7 +48,6 @@ scripts = [s for s in glob.glob(join('bin', 'cubicweb-*')) if not s.endswith('.bat')] include_dirs = [join('test', 'data'), - join('common', 'test', 'data'), join('server', 'test', 'data'), join('web', 'test', 'data'), join('devtools', 'test', 'data'), diff -r c25da7573ebd -r 02b52bf9f5f8 _exceptions.py --- a/_exceptions.py Fri Feb 12 15:18:00 2010 +0100 +++ b/_exceptions.py Wed Mar 24 10:23:31 2010 +0100 @@ -20,7 +20,7 @@ if self.args: return self.msg % tuple(self.args) return self.msg - return ' '.join(str(arg) for arg in self.args) + return ' '.join(unicode(arg) for arg in self.args) class ConfigurationError(CubicWebException): @@ -49,7 +49,11 @@ class AuthenticationError(ConnectionError): """raised when a bad connection id is given or when an attempt to establish - a connection failed""" + a connection failed + """ + def __init__(self, *args, **kwargs): + super(AuthenticationError, self).__init__(*args) + self.__dict__.update(kwargs) class BadConnectionId(ConnectionError): """raised when a bad connection id is given or when an attempt to establish diff -r c25da7573ebd -r 02b52bf9f5f8 _gcdebug.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/_gcdebug.py Wed Mar 24 10:23:31 2010 +0100 @@ -0,0 +1,87 @@ + +import gc, types, weakref + +from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema + +listiterator = type(iter([])) + +IGNORE_CLASSES = ( + type, tuple, dict, list, set, frozenset, type(len), + weakref.ref, weakref.WeakKeyDictionary, + listiterator, + property, classmethod, + types.ModuleType, types.FunctionType, types.MethodType, + types.MemberDescriptorType, types.GetSetDescriptorType, + ) + +def _get_counted_class(obj, classes): + for cls in classes: + if isinstance(obj, cls): + return cls + raise AssertionError() + +def gc_info(countclasses, + ignoreclasses=IGNORE_CLASSES, + viewreferrersclasses=(), showobjs=False, maxlevel=1): + gc.collect() + gc.collect() + counters = {} + ocounters = {} + for obj in gc.get_objects(): + if isinstance(obj, countclasses): + cls = _get_counted_class(obj, countclasses) + try: + counters[cls.__name__] += 1 + except KeyError: + counters[cls.__name__] = 1 + elif not isinstance(obj, ignoreclasses): + try: + key = '%s.%s' % (obj.__class__.__module__, + obj.__class__.__name__) + except AttributeError: + key = str(obj) + try: + ocounters[key] += 1 + except KeyError: + ocounters[key] = 1 + if isinstance(obj, viewreferrersclasses): + print ' ', obj, referrers(obj, showobjs, maxlevel) + return counters, ocounters, gc.garbage + + +def referrers(obj, showobj=False, maxlevel=1): + objreferrers = _referrers(obj, maxlevel) + try: + return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x))) + for x in objreferrers)) + except TypeError: + s = set() + unhashable = [] + for x in objreferrers: + try: + s.add(x) + except TypeError: + unhashable.append(x) + return sorted(s) + unhashable + +def _referrers(obj, maxlevel, _seen=None, _level=0): + interesting = [] + if _seen is None: + _seen = set() + for x in gc.get_referrers(obj): + if id(x) in _seen: + continue + _seen.add(id(x)) + if isinstance(x, types.FrameType): + continue + if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)): + continue + if isinstance(x, (list, tuple, set, dict, listiterator)): + if _level >= maxlevel: + pass + #interesting.append(x) + else: + interesting += _referrers(x, maxlevel, _seen, _level+1) + else: + interesting.append(x) + return interesting diff -r c25da7573ebd -r 02b52bf9f5f8 appobject.py --- a/appobject.py Fri Feb 12 15:18:00 2010 +0100 +++ b/appobject.py Wed Mar 24 10:23:31 2010 +0100 @@ -11,29 +11,11 @@ import types from logging import getLogger -from datetime import datetime, timedelta, time +from warnings import warn -from logilab.common.decorators import classproperty from logilab.common.deprecation import deprecated from logilab.common.logging_ext import set_log_methods -from rql.nodes import VariableRef, SubQuery -from rql.stmts import Union, Select - -from cubicweb import Unauthorized, NoSelectableObject -from cubicweb.utils import UStringIO, ustrftime, strptime, todate, todatetime - -ONESECOND = timedelta(0, 1, 0) -CACHE_REGISTRY = {} - - -class Cache(dict): - def __init__(self): - super(Cache, self).__init__() - _now = datetime.now() - self.cache_creation_date = _now - self.latest_cache_lookup = _now - # selector base classes and operations ######################################## @@ -42,12 +24,13 @@ would be overkill:: @objectify_selector - def yes(cls, *args, **kwargs): + def one(cls, *args, **kwargs): return 1 """ return type(selector_func.__name__, (Selector,), - {'__call__': lambda self, *args, **kwargs: selector_func(*args, **kwargs)}) + {'__doc__': selector_func.__doc__, + '__call__': lambda self, *a, **kw: selector_func(*a, **kw)}) def _instantiate_selector(selector): @@ -100,11 +83,15 @@ return AndSelector(self, other) def __rand__(self, other): return AndSelector(other, self) + def __iand__(self, other): + raise NotImplementedError('cant use inplace & (binary and)') def __or__(self, other): return OrSelector(self, other) def __ror__(self, other): return OrSelector(other, self) + def __ior__(self, other): + raise NotImplementedError('cant use inplace | (binary or)') def __invert__(self): return NotSelector(self) @@ -195,9 +182,13 @@ class yes(Selector): - """return arbitrary score + """Return the score given as parameter, with a default score of 0.5 so any + other selector take precedence. - default score of 0.5 so any other selector take precedence + Usually used for appobjects which can be selected whatever the context, or + also sometimes to add arbitrary points to a score. + + Take care, `yes(0)` could be named 'no'... """ def __init__(self, score=0.5): self.score = score @@ -220,7 +211,7 @@ :__registry__: name of the registry for this object (string like 'views', 'templates'...) - :id: + :__regid__: object's identifier in the registry (string like 'main', 'primary', 'folder_box') :__select__: @@ -229,341 +220,201 @@ Moreover, the `__abstract__` attribute may be set to True to indicate that a appobject is abstract and should not be registered. - At registration time, the following attributes are set on the class: - :vreg: - the instance's registry - :schema: - the instance's schema - :config: - the instance's configuration + At selection time, the following attributes are set on the instance: + + :_cw: + current request + :cw_extra_kwargs: + other received arguments - At selection time, the following attributes are set on the instance: - :req: - current request - :rset: + only if rset is found in arguments (in which case rset/row/col will be + removed from cwextra_kwargs): + + :cw_rset: context result set or None - :row: + :cw_row: if a result set is set and the context is about a particular cell in the result set, and not the result set as a whole, specify the row number we are interested in, else None - :col: + :cw_col: if a result set is set and the context is about a particular cell in the result set, and not the result set as a whole, specify the col number we are interested in, else None """ __registry__ = None - id = None + __regid__ = None __select__ = yes() @classmethod - def classid(cls): - """returns a unique identifier for the appobject""" - return '%s.%s' % (cls.__module__, cls.__name__) - - # XXX bw compat code - @classmethod - def build___select__(cls): - for klass in cls.mro(): - if klass.__name__ == 'AppObject': - continue # the bw compat __selector__ is there - klassdict = klass.__dict__ - if ('__select__' in klassdict and '__selectors__' in klassdict - and '__selgenerated__' not in klassdict): - raise TypeError("__select__ and __selectors__ can't be used together on class %s" % cls) - if '__selectors__' in klassdict and '__selgenerated__' not in klassdict: - cls.__selgenerated__ = True - # case where __selectors__ is defined locally (but __select__ - # is in a parent class) - selectors = klassdict['__selectors__'] - if len(selectors) == 1: - # micro optimization: don't bother with AndSelector if there's - # only one selector - select = _instantiate_selector(selectors[0]) - else: - select = AndSelector(*selectors) - cls.__select__ = select - - @classmethod - def registered(cls, registry): + def __registered__(cls, registry): """called by the registry when the appobject has been registered. It must return the object that will be actually registered (this may be the right hook to create an instance for example). By default the appobject is returned without any transformation. """ - cls.build___select__() - cls.vreg = registry.vreg - cls.schema = registry.schema - cls.config = registry.config + try: # XXX < 3.6 bw compat + pdefs = cls.property_defs + except AttributeError: + pdefs = getattr(cls, 'cw_property_defs', {}) + else: + warn('property_defs is deprecated, use cw_property_defs in %s' + % cls, DeprecationWarning) + for propid, pdef in pdefs.items(): + pdef = pdef.copy() # may be shared + pdef['default'] = getattr(cls, propid, pdef['default']) + pdef['sitewide'] = getattr(cls, 'site_wide', pdef.get('sitewide')) + registry.vreg.register_property(cls._cwpropkey(propid), **pdef) + assert callable(cls.__select__), cls return cls - @classmethod - def vreg_initialization_completed(cls): - pass + def __init__(self, req, **extra): + super(AppObject, self).__init__() + self._cw = req + try: + self.cw_rset = extra.pop('rset') + self.cw_row = extra.pop('row', None) + self.cw_col = extra.pop('col', None) + except KeyError: + pass + self.cw_extra_kwargs = extra - # Eproperties definition: - # key: id of the property (the actual CWProperty key is build using - # .. - # value: tuple (property type, vocabfunc, default value, property description) - # possible types are those used by `logilab.common.configuration` + # persistent class properties ############################################## + # + # optional `cw_property_defs` dict on a class defines available persistent + # properties for this class: + # + # * key: id of the property (the actual CWProperty key is build using + # .. + # * value: tuple (property type, vocabfunc, default value, property description) + # possible types are those used by `logilab.common.configuration` # # notice that when it exists multiple objects with the same id (adaptation, # overriding) only the first encountered definition is considered, so those # objects can't try to have different default values for instance. - - property_defs = {} - - @classmethod - def register_properties(cls): - for propid, pdef in cls.property_defs.items(): - pdef = pdef.copy() # may be shared - pdef['default'] = getattr(cls, propid, pdef['default']) - pdef['sitewide'] = getattr(cls, 'site_wide', pdef.get('sitewide')) - cls.vreg.register_property(cls.propkey(propid), **pdef) + # + # you can then access to a property value using self.cw_propval, where self + # is an instance of class @classmethod - def propkey(cls, propid): - return '%s.%s.%s' % (cls.__registry__, cls.id, propid) - - @classproperty - @deprecated('use __select__ and & or | operators') - def __selectors__(cls): - selector = cls.__select__ - if isinstance(selector, AndSelector): - return tuple(selector.selectors) - if not isinstance(selector, tuple): - selector = (selector,) - return selector - - def __init__(self, req=None, rset=None, row=None, col=None, **extra): - super(AppObject, self).__init__() - self.req = req - self.rset = rset - self.row = row - self.col = col - self.extra_kwargs = extra - - def get_cache(self, cachename): - """ - NOTE: cachename should be dotted names as in : - - cubicweb.mycache - - cubes.blog.mycache - - etc. - """ - if cachename in CACHE_REGISTRY: - cache = CACHE_REGISTRY[cachename] - else: - cache = CACHE_REGISTRY[cachename] = Cache() - _now = datetime.now() - if _now > cache.latest_cache_lookup + ONESECOND: - ecache = self.req.execute('Any C,T WHERE C is CWCache, C name %(name)s, C timestamp T', - {'name':cachename}).get_entity(0,0) - cache.latest_cache_lookup = _now - if not ecache.valid(cache.cache_creation_date): - cache.clear() - cache.cache_creation_date = _now - return cache - - def propval(self, propid): - assert self.req - return self.req.property_value(self.propkey(propid)) - - def limited_rql(self): - """return a printable rql for the result set associated to the object, - with limit/offset correctly set according to maximum page size and - currently displayed page when necessary + def _cwpropkey(cls, propid): + """return cw property key for the property of the given id for this + class """ - # try to get page boundaries from the navigation component - # XXX we should probably not have a ref to this component here (eg in - # cubicweb.common) - nav = self.vreg['components'].select_object('navigation', self.req, - rset=self.rset) - if nav: - start, stop = nav.page_boundaries() - rql = self._limit_offset_rql(stop - start, start) - # result set may have be limited manually in which case navigation won't - # apply - elif self.rset.limited: - rql = self._limit_offset_rql(*self.rset.limited) - # navigation component doesn't apply and rset has not been limited, no - # need to limit query - else: - rql = self.rset.printable_rql() - return rql + return '%s.%s.%s' % (cls.__registry__, cls.__regid__, propid) - def _limit_offset_rql(self, limit, offset): - rqlst = self.rset.syntax_tree() - if len(rqlst.children) == 1: - select = rqlst.children[0] - olimit, ooffset = select.limit, select.offset - select.limit, select.offset = limit, offset - rql = rqlst.as_string(kwargs=self.rset.args) - # restore original limit/offset - select.limit, select.offset = olimit, ooffset - else: - newselect = Select() - newselect.limit = limit - newselect.offset = offset - aliases = [VariableRef(newselect.get_variable(vref.name, i)) - for i, vref in enumerate(rqlst.selection)] - newselect.set_with([SubQuery(aliases, rqlst)], check=False) - newunion = Union() - newunion.append(newselect) - rql = rqlst.as_string(kwargs=self.rset.args) - rqlst.parent = None - return rql + def cw_propval(self, propid): + """return cw property value associated to key - def view(self, __vid, rset=None, __fallback_oid=None, __registry='views', - **kwargs): - """shortcut to self.vreg.view method avoiding to pass self.req""" - return self.vreg[__registry].render(__vid, self.req, __fallback_oid, - rset=rset, **kwargs) - - def initialize_varmaker(self): - varmaker = self.req.get_page_data('rql_varmaker') - if varmaker is None: - varmaker = self.req.varmaker - self.req.set_page_data('rql_varmaker', varmaker) - self.varmaker = varmaker - - # url generation methods ################################################## - - controller = 'view' - - def build_url(self, *args, **kwargs): - """return an absolute URL using params dictionary key/values as URL - parameters. Values are automatically URL quoted, and the - publishing method to use may be specified or will be guessed. + .. """ - # use *args since we don't want first argument to be "anonymous" to - # avoid potential clash with kwargs - if args: - assert len(args) == 1, 'only 0 or 1 non-named-argument expected' - method = args[0] - else: - method = None - # XXX I (adim) think that if method is passed explicitly, we should - # not try to process it and directly call req.build_url() - if method is None: - method = self.controller - if method == 'view' and self.req.from_controller() == 'view' and \ - not '_restpath' in kwargs: - method = self.req.relative_path(includeparams=False) or 'view' - return self.req.build_url(method, **kwargs) + return self._cw.property_value(self._cwpropkey(propid)) + + # deprecated ############################################################### + + @property + @deprecated('[3.6] use self.__regid__') + def id(self): + return self.__regid__ - # various resources accessors ############################################# + @property + @deprecated('[3.6] use self._cw.vreg') + def vreg(self): + return self._cw.vreg - def entity(self, row, col=0): - """short cut to get an entity instance for a particular row/column - (col default to 0) - """ - return self.rset.get_entity(row, col) + @property + @deprecated('[3.6] use self._cw.vreg.schema') + def schema(self): + return self._cw.vreg.schema + + @property + @deprecated('[3.6] use self._cw.vreg.config') + def config(self): + return self._cw.vreg.config - def complete_entity(self, row, col=0, skip_bytes=True): - """short cut to get an completed entity instance for a particular - row (all instance's attributes have been fetched) - """ - entity = self.entity(row, col) - entity.complete(skip_bytes=skip_bytes) - return entity + @property + @deprecated('[3.6] use self._cw') + def req(self): + return self._cw + + @deprecated('[3.6] use self.cw_rset') + def get_rset(self): + return self.cw_rset + @deprecated('[3.6] use self.cw_rset') + def set_rset(self, rset): + self.cw_rset = rset + rset = property(get_rset, set_rset) - def user_rql_callback(self, args, msg=None): - """register a user callback to execute some rql query and return an url - to call it ready to be inserted in html - """ - def rqlexec(req, rql, args=None, key=None): - req.execute(rql, args, key) - return self.user_callback(rqlexec, args, msg) + @property + @deprecated('[3.6] use self.cw_row') + def row(self): + return self.cw_row - def user_callback(self, cb, args, msg=None, nonify=False): - """register the given user callback and return an url to call it ready to be - inserted in html - """ - from simplejson import dumps - self.req.add_js('cubicweb.ajax.js') - cbname = self.req.register_onetime_callback(cb, *args) - msg = dumps(msg or '') - return "javascript:userCallbackThenReloadPage('%s', %s)" % ( - cbname, msg) + @property + @deprecated('[3.6] use self.cw_col') + def col(self): + return self.cw_col - # formating methods ####################################################### + @property + @deprecated('[3.6] use self.cw_extra_kwargs') + def extra_kwargs(self): + return self.cw_extra_kwargs - def tal_render(self, template, variables): - """render a precompiled page template with variables in the given - dictionary as context - """ - from cubicweb.ext.tal import CubicWebContext - context = CubicWebContext() - context.update({'self': self, 'rset': self.rset, '_' : self.req._, - 'req': self.req, 'user': self.req.user}) - context.update(variables) - output = UStringIO() - template.expand(context, output) - return output.getvalue() + @deprecated('[3.6] use self._cw.view') + def view(self, *args, **kwargs): + return self._cw.view(*args, **kwargs) + + @property + @deprecated('[3.6] use self._cw.varmaker') + def varmaker(self): + return self._cw.varmaker + + @deprecated('[3.6] use self._cw.get_cache') + def get_cache(self, cachename): + return self._cw.get_cache(cachename) + + @deprecated('[3.6] use self._cw.build_url') + def build_url(self, *args, **kwargs): + return self._cw.build_url(*args, **kwargs) + @deprecated('[3.6] use self.cw_rset.limited_rql') + def limited_rql(self): + return self.cw_rset.limited_rql() + + @deprecated('[3.6] use self.cw_rset.complete_entity(row,col) instead') + def complete_entity(self, row, col=0, skip_bytes=True): + return self.cw_rset.complete_entity(row, col, skip_bytes) + + @deprecated('[3.6] use self.cw_rset.get_entity(row,col) instead') + def entity(self, row, col=0): + return self.cw_rset.get_entity(row, col) + + @deprecated('[3.6] use self._cw.user_rql_callback') + def user_rql_callback(self, args, msg=None): + return self._cw.user_rql_callback(args, msg) + + @deprecated('[3.6] use self._cw.user_callback') + def user_callback(self, cb, args, msg=None, nonify=False): + return self._cw.user_callback(cb, args, msg, nonify) + + @deprecated('[3.6] use self._cw.format_date') def format_date(self, date, date_format=None, time=False): - """return a string for a date time according to instance's - configuration - """ - if date: - if date_format is None: - if time: - date_format = self.req.property_value('ui.datetime-format') - else: - date_format = self.req.property_value('ui.date-format') - return ustrftime(date, date_format) - return u'' + return self._cw.format_date(date, date_format, time) + @deprecated('[3.6] use self._cw.format_time') def format_time(self, time): - """return a string for a time according to instance's - configuration - """ - if time: - return ustrftime(time, self.req.property_value('ui.time-format')) - return u'' + return self._cw.format_time(time) + @deprecated('[3.6] use self._cw.format_float') def format_float(self, num): - """return a string for floating point number according to instance's - configuration """ - if num is not None: - return self.req.property_value('ui.float-format') % num - return u'' + return self._cw.format_float(num) + @deprecated('[3.6] use self._cw.parse_datetime') def parse_datetime(self, value, etype='Datetime'): - """get a datetime or time from a string (according to etype) - Datetime formatted as Date are accepted - """ - assert etype in ('Datetime', 'Date', 'Time'), etype - # XXX raise proper validation error - if etype == 'Datetime': - format = self.req.property_value('ui.datetime-format') - try: - return todatetime(strptime(value, format)) - except ValueError: - pass - elif etype == 'Time': - format = self.req.property_value('ui.time-format') - try: - # (adim) I can't find a way to parse a Time with a custom format - date = strptime(value, format) # this returns a DateTime - return time(date.hour, date.minute, date.second) - except ValueError: - raise ValueError('can\'t parse %r (expected %s)' % (value, format)) - try: - format = self.req.property_value('ui.date-format') - dt = strptime(value, format) - if etype == 'Datetime': - return todatetime(dt) - return todate(dt) - except ValueError: - raise ValueError('can\'t parse %r (expected %s)' % (value, format)) + return self._cw.parse_datetime(value, etype) - # security related methods ################################################ - - def ensure_ro_rql(self, rql): - """raise an exception if the given rql is not a select query""" - first = rql.split(' ', 1)[0].lower() - if first in ('insert', 'set', 'delete'): - raise Unauthorized(self.req._('only select queries are authorized')) + @deprecated('[3.6] use self.cw_propval') + def propval(self, propid): + return self._cw.property_value(self._cwpropkey(propid)) set_log_methods(AppObject, getLogger('cubicweb.appobject')) diff -r c25da7573ebd -r 02b52bf9f5f8 common/__init__.py --- a/common/__init__.py Fri Feb 12 15:18:00 2010 +0100 +++ b/common/__init__.py Wed Mar 24 10:23:31 2010 +0100 @@ -7,47 +7,3 @@ :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses """ -from logilab.common.adbh import FunctionDescr - -from cubicweb._exceptions import * # bw compat - -from rql.utils import register_function, iter_funcnode_variables - -class COMMA_JOIN(FunctionDescr): - supported_backends = ('postgres', 'sqlite',) - rtype = 'String' - - @classmethod - def st_description(cls, funcnode, mainindex, tr): - return ', '.join(sorted(term.get_description(mainindex, tr) - for term in iter_funcnode_variables(funcnode))) - -register_function(COMMA_JOIN) # XXX do not expose? - - -class CONCAT_STRINGS(COMMA_JOIN): - aggregat = True - -register_function(CONCAT_STRINGS) # XXX bw compat - -class GROUP_CONCAT(CONCAT_STRINGS): - supported_backends = ('mysql', 'postgres', 'sqlite',) - -register_function(GROUP_CONCAT) - - -class LIMIT_SIZE(FunctionDescr): - supported_backends = ('postgres', 'sqlite',) - rtype = 'String' - - @classmethod - def st_description(cls, funcnode, mainindex, tr): - return funcnode.children[0].get_description(mainindex, tr) - -register_function(LIMIT_SIZE) - - -class TEXT_LIMIT_SIZE(LIMIT_SIZE): - supported_backends = ('mysql', 'postgres', 'sqlite',) - -register_function(TEXT_LIMIT_SIZE) diff -r c25da7573ebd -r 02b52bf9f5f8 common/appobject.py --- a/common/appobject.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -"""pre 3.2 bw compat - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -# pylint: disable-msg=W0614,W0401 -from warnings import warn -warn('moved to cubicweb.appobject', DeprecationWarning, stacklevel=2) -from cubicweb.appobject import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/entity.py --- a/common/entity.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,12 +0,0 @@ -"""pre 3.2 bw compat - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -# pylint: disable-msg=W0614,W0401 -from warnings import warn -warn('moved to cubicweb.entity', DeprecationWarning, stacklevel=2) -from cubicweb.entity import * -from cubicweb.entity import _marker diff -r c25da7573ebd -r 02b52bf9f5f8 common/i18n.py --- a/common/i18n.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,99 +0,0 @@ -"""Some i18n/gettext utilities. - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -import re -import os -import sys -from os.path import join, basename, splitext, exists -from glob import glob - -from cubicweb.toolsutils import create_dir - -def extract_from_tal(files, output_file): - """extract i18n strings from tal and write them into the given output file - using standard python gettext marker (_) - """ - output = open(output_file, 'w') - for filepath in files: - for match in re.finditer('i18n:(content|replace)="([^"]+)"', open(filepath).read()): - print >> output, '_("%s")' % match.group(2) - output.close() - - -def add_msg(w, msgid, msgctx=None): - """write an empty pot msgid definition""" - if isinstance(msgid, unicode): - msgid = msgid.encode('utf-8') - if msgctx: - if isinstance(msgctx, unicode): - msgctx = msgctx.encode('utf-8') - w('msgctxt "%s"\n' % msgctx) - msgid = msgid.replace('"', r'\"').splitlines() - if len(msgid) > 1: - w('msgid ""\n') - for line in msgid: - w('"%s"' % line.replace('"', r'\"')) - else: - w('msgid "%s"\n' % msgid[0]) - w('msgstr ""\n\n') - - -def execute(cmd): - """display the command, execute it and raise an Exception if returned - status != 0 - """ - print cmd.replace(os.getcwd() + os.sep, '') - from subprocess import call - status = call(cmd, shell=True) - if status != 0: - raise Exception('status = %s' % status) - - -def available_catalogs(i18ndir=None): - if i18ndir is None: - wildcard = '*.po' - else: - wildcard = join(i18ndir, '*.po') - for popath in glob(wildcard): - lang = splitext(basename(popath))[0] - yield lang, popath - - -def compile_i18n_catalogs(sourcedirs, destdir, langs): - """generate .mo files for a set of languages into the `destdir` i18n directory - """ - from logilab.common.fileutils import ensure_fs_mode - print '-> compiling %s catalogs...' % destdir - errors = [] - for lang in langs: - langdir = join(destdir, lang, 'LC_MESSAGES') - if not exists(langdir): - create_dir(langdir) - pofiles = [join(path, '%s.po' % lang) for path in sourcedirs] - pofiles = [pof for pof in pofiles if exists(pof)] - mergedpo = join(destdir, '%s_merged.po' % lang) - try: - # merge instance/cubes messages catalogs with the stdlib's one - execute('msgcat --use-first --sort-output --strict -o "%s" %s' - % (mergedpo, ' '.join('"%s"' % f for f in pofiles))) - # make sure the .mo file is writeable and compiles with *msgfmt* - applmo = join(destdir, lang, 'LC_MESSAGES', 'cubicweb.mo') - try: - ensure_fs_mode(applmo) - except OSError: - pass # suppose not exists - execute('msgfmt "%s" -o "%s"' % (mergedpo, applmo)) - except Exception, ex: - errors.append('while handling language %s: %s' % (lang, ex)) - try: - # clean everything - os.unlink(mergedpo) - except Exception: - continue - return errors diff -r c25da7573ebd -r 02b52bf9f5f8 common/mail.py --- a/common/mail.py Fri Feb 12 15:18:00 2010 +0100 +++ b/common/mail.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,272 +1,5 @@ -"""Common utilies to format / semd emails. - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -from base64 import b64encode, b64decode -from itertools import repeat -from time import time -from email.MIMEMultipart import MIMEMultipart -from email.MIMEText import MIMEText -from email.MIMEImage import MIMEImage -from email.Header import Header -try: - from socket import gethostname -except ImportError: - def gethostname(): # gae - return 'XXX' - -from cubicweb.view import EntityView -from cubicweb.entity import Entity - -def header(ustring): - return Header(ustring.encode('UTF-8'), 'UTF-8') - -def addrheader(uaddr, uname=None): - # even if an email address should be ascii, encode it using utf8 since - # automatic tests may generate non ascii email address - addr = uaddr.encode('UTF-8') - if uname: - return '%s <%s>' % (header(uname).encode(), addr) - return addr - - -def construct_message_id(appid, eid, withtimestamp=True): - if withtimestamp: - addrpart = 'eid=%s×tamp=%.10f' % (eid, time()) - else: - addrpart = 'eid=%s' % eid - # we don't want any equal sign nor trailing newlines - leftpart = b64encode(addrpart, '.-').rstrip().rstrip('=') - return '<%s@%s.%s>' % (leftpart, appid, gethostname()) - - -def parse_message_id(msgid, appid): - if msgid[0] == '<': - msgid = msgid[1:] - if msgid[-1] == '>': - msgid = msgid[:-1] - try: - values, qualif = msgid.split('@') - padding = len(values) % 4 - values = b64decode(str(values + '='*padding), '.-') - values = dict(v.split('=') for v in values.split('&')) - fromappid, host = qualif.split('.', 1) - except: - return None - if appid != fromappid or host != gethostname(): - return None - return values - - -def format_mail(uinfo, to_addrs, content, subject="", - cc_addrs=(), msgid=None, references=(), config=None): - """Sends an Email to 'e_addr' with content 'content', and subject 'subject' - - to_addrs and cc_addrs are expected to be a list of email address without - name - """ - assert type(content) is unicode, repr(content) - msg = MIMEText(content.encode('UTF-8'), 'plain', 'UTF-8') - # safety: keep only the first newline - subject = subject.splitlines()[0] - msg['Subject'] = header(subject) - if uinfo.get('email'): - email = uinfo['email'] - elif config and config['sender-addr']: - email = unicode(config['sender-addr']) - else: - email = u'' - if uinfo.get('name'): - name = uinfo['name'] - elif config and config['sender-addr']: - name = unicode(config['sender-name']) - else: - name = u'' - msg['From'] = addrheader(email, name) - if config and config['sender-addr'] and config['sender-addr'] != email: - appaddr = addrheader(config['sender-addr'], config['sender-name']) - msg['Reply-to'] = '%s, %s' % (msg['From'], appaddr) - elif email: - msg['Reply-to'] = msg['From'] - if config is not None: - msg['X-CW'] = config.appid - unique_addrs = lambda addrs: sorted(set(addr for addr in addrs if addr is not None)) - msg['To'] = ', '.join(addrheader(addr) for addr in unique_addrs(to_addrs)) - if cc_addrs: - msg['Cc'] = ', '.join(addrheader(addr) for addr in unique_addrs(cc_addrs)) - if msgid: - msg['Message-id'] = msgid - if references: - msg['References'] = ', '.join(references) - return msg - - -class HtmlEmail(MIMEMultipart): - - def __init__(self, subject, textcontent, htmlcontent, - sendermail=None, sendername=None, recipients=None, ccrecipients=None): - MIMEMultipart.__init__(self, 'related') - self['Subject'] = header(subject) - self.preamble = 'This is a multi-part message in MIME format.' - # Attach alternative text message - alternative = MIMEMultipart('alternative') - self.attach(alternative) - msgtext = MIMEText(textcontent.encode('UTF-8'), 'plain', 'UTF-8') - alternative.attach(msgtext) - # Attach html message - msghtml = MIMEText(htmlcontent.encode('UTF-8'), 'html', 'UTF-8') - alternative.attach(msghtml) - if sendermail or sendername: - self['From'] = addrheader(sendermail, sendername) - if recipients: - self['To'] = ', '.join(addrheader(addr) for addr in recipients if addr is not None) - if ccrecipients: - self['Cc'] = ', '.join(addrheader(addr) for addr in ccrecipients if addr is not None) - - def attach_image(self, data, htmlId): - image = MIMEImage(data) - image.add_header('Content-ID', '<%s>' % htmlId) - self.attach(image) - - -class NotificationView(EntityView): - """abstract view implementing the "email" API (eg to simplify sending - notification) - """ - # XXX refactor this class to work with len(rset) > 1 - - msgid_timestamp = True - - # this is usually the method to call - def render_and_send(self, **kwargs): - """generate and send an email message for this view""" - delayed = kwargs.pop('delay_to_commit', None) - for recipients, msg in self.render_emails(**kwargs): - if delayed is None: - self.send(recipients, msg) - elif delayed: - self.send_on_commit(recipients, msg) - else: - self.send_now(recipients, msg) - - def cell_call(self, row, col=0, **kwargs): - self.w(self.req._(self.content) % self.context(**kwargs)) - - def render_emails(self, **kwargs): - """generate and send emails for this view (one per recipient)""" - self._kwargs = kwargs - recipients = self.recipients() - if not recipients: - self.info('skipping %s notification, no recipients', self.id) - return - if self.rset is not None: - entity = self.entity(self.row or 0, self.col or 0) - # if the view is using timestamp in message ids, no way to reference - # previous email - if not self.msgid_timestamp: - refs = [self.construct_message_id(eid) - for eid in entity.notification_references(self)] - else: - refs = () - msgid = self.construct_message_id(entity.eid) - else: - refs = () - msgid = None - req = self.req - self.user_data = req.user_data() - origlang = req.lang - for something in recipients: - if isinstance(something, Entity): - # hi-jack self.req to get a session for the returned user - self.req = self.req.hijack_user(something) - emailaddr = something.get_email() - else: - emailaddr, lang = something - self.req.set_language(lang) - # since the same view (eg self) may be called multiple time and we - # need a fresh stream at each iteration, reset it explicitly - self.w = None - # XXX call render before subject to set .row/.col attributes on the - # view - try: - content = self.render(row=0, col=0, **kwargs) - subject = self.subject() - except SkipEmail: - continue - except Exception, ex: - # shouldn't make the whole transaction fail because of rendering - # error (unauthorized or such) - self.exception(str(ex)) - continue - msg = format_mail(self.user_data, [emailaddr], content, subject, - config=self.config, msgid=msgid, references=refs) - yield [emailaddr], msg - # restore language - req.set_language(origlang) - - # recipients / email sending ############################################### - - def recipients(self): - """return a list of either 2-uple (email, language) or user entity to - who this email should be sent - """ - # use super_session when available, we don't want to consider security - # when selecting recipients_finder - try: - req = self.req.super_session - except AttributeError: - req = self.req - finder = self.vreg['components'].select('recipients_finder', req, - rset=self.rset, - row=self.row or 0, - col=self.col or 0) - return finder.recipients() - - def send_now(self, recipients, msg): - self.config.sendmails([(msg, recipients)]) - - def send_on_commit(self, recipients, msg): - raise NotImplementedError - - send = send_now - - # email generation helpers ################################################# - - def construct_message_id(self, eid): - return construct_message_id(self.config.appid, eid, self.msgid_timestamp) - - def format_field(self, attr, value): - return ':%(attr)s: %(value)s' % {'attr': attr, 'value': value} - - def format_section(self, attr, value): - return '%(attr)s\n%(ul)s\n%(value)s\n' % { - 'attr': attr, 'ul': '-'*len(attr), 'value': value} - - def subject(self): - entity = self.entity(self.row or 0, self.col or 0) - subject = self.req._(self.message) - etype = entity.dc_type() - eid = entity.eid - login = self.user_data['login'] - return self.req._('%(subject)s %(etype)s #%(eid)s (%(login)s)') % locals() - - def context(self, **kwargs): - entity = self.entity(self.row or 0, self.col or 0) - for key, val in kwargs.iteritems(): - if val and isinstance(val, unicode) and val.strip(): - kwargs[key] = self.req._(val) - kwargs.update({'user': self.user_data['login'], - 'eid': entity.eid, - 'etype': entity.dc_type(), - 'url': entity.absolute_url(), - 'title': entity.dc_long_title(),}) - return kwargs - - -class SkipEmail(Exception): - """raise this if you decide to skip an email during its generation""" +"""pre 3.6 bw compat""" +# pylint: disable-msg=W0614,W0401 +from warnings import warn +warn('moved to cubicweb.mail', DeprecationWarning, stacklevel=2) +from cubicweb.mail import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/migration.py --- a/common/migration.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,373 +0,0 @@ -"""utilities for instances migration - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -import sys -import os -import logging -import tempfile -from os.path import exists, join, basename, splitext - -from logilab.common.decorators import cached -from logilab.common.configuration import REQUIRED, read_old_config -from logilab.common.shellutils import ASK - -from cubicweb import ConfigurationError - - -def filter_scripts(config, directory, fromversion, toversion, quiet=True): - """return a list of paths of migration files to consider to upgrade - from a version to a greater one - """ - from logilab.common.changelog import Version # doesn't work with appengine - assert fromversion - assert toversion - assert isinstance(fromversion, tuple), fromversion.__class__ - assert isinstance(toversion, tuple), toversion.__class__ - assert fromversion <= toversion, (fromversion, toversion) - if not exists(directory): - if not quiet: - print directory, "doesn't exists, no migration path" - return [] - if fromversion == toversion: - return [] - result = [] - for fname in os.listdir(directory): - if fname.endswith('.pyc') or fname.endswith('.pyo') \ - or fname.endswith('~'): - continue - fpath = join(directory, fname) - try: - tver, mode = fname.split('_', 1) - except ValueError: - continue - mode = mode.split('.', 1)[0] - if not config.accept_mode(mode): - continue - try: - tver = Version(tver) - except ValueError: - continue - if tver <= fromversion: - continue - if tver > toversion: - continue - result.append((tver, fpath)) - # be sure scripts are executed in order - return sorted(result) - - -IGNORED_EXTENSIONS = ('.swp', '~') - - -def execscript_confirm(scriptpath): - """asks for confirmation before executing a script and provides the - ability to show the script's content - """ - while True: - answer = ASK.ask('Execute %r ?' % scriptpath, ('Y','n','show'), 'Y') - if answer == 'n': - return False - elif answer == 'show': - stream = open(scriptpath) - scriptcontent = stream.read() - stream.close() - print - print scriptcontent - print - else: - return True - -def yes(*args, **kwargs): - return True - - -class MigrationHelper(object): - """class holding CubicWeb Migration Actions used by migration scripts""" - - def __init__(self, config, interactive=True, verbosity=1): - self.config = config - if config: - # no config on shell to a remote instance - self.config.init_log(logthreshold=logging.ERROR, debug=True) - # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything - self.verbosity = verbosity - self.need_wrap = True - if not interactive or not verbosity: - self.confirm = yes - self.execscript_confirm = yes - else: - self.execscript_confirm = execscript_confirm - self._option_changes = [] - self.__context = {'confirm': self.confirm, - 'config': self.config, - 'interactive_mode': interactive, - } - - def __getattribute__(self, name): - try: - return object.__getattribute__(self, name) - except AttributeError: - cmd = 'cmd_%s' % name - if hasattr(self, cmd): - meth = getattr(self, cmd) - return lambda *args, **kwargs: self.interact(args, kwargs, - meth=meth) - raise - raise AttributeError(name) - - def repo_connect(self): - return self.config.repository() - - def migrate(self, vcconf, toupgrade, options): - """upgrade the given set of cubes - - `cubes` is an ordered list of 3-uple: - (cube, fromversion, toversion) - """ - if options.fs_only: - # monkey path configuration.accept_mode so database mode (e.g. Any) - # won't be accepted - orig_accept_mode = self.config.accept_mode - def accept_mode(mode): - if mode == 'Any': - return False - return orig_accept_mode(mode) - self.config.accept_mode = accept_mode - # may be an iterator - toupgrade = tuple(toupgrade) - vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade) - ctx = self.__context - ctx['versions_map'] = vmap - if self.config.accept_mode('Any') and 'cubicweb' in vmap: - migrdir = self.config.migration_scripts_dir() - self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py')) - for cube, fromversion, toversion in toupgrade: - if cube == 'cubicweb': - migrdir = self.config.migration_scripts_dir() - else: - migrdir = self.config.cube_migration_scripts_dir(cube) - scripts = filter_scripts(self.config, migrdir, fromversion, toversion) - if scripts: - prevversion = None - for version, script in scripts: - # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call - # cube_upgraded once all script of X.Y.Z have been executed - if prevversion is not None and version != prevversion: - self.cube_upgraded(cube, prevversion) - prevversion = version - self.cmd_process_script(script) - self.cube_upgraded(cube, toversion) - else: - self.cube_upgraded(cube, toversion) - - def cube_upgraded(self, cube, version): - pass - - def shutdown(self): - pass - - def interact(self, args, kwargs, meth): - """execute the given method according to user's confirmation""" - msg = 'Execute command: %s(%s) ?' % ( - meth.__name__[4:], - ', '.join([repr(arg) for arg in args] + - ['%s=%r' % (n,v) for n,v in kwargs.items()])) - if 'ask_confirm' in kwargs: - ask_confirm = kwargs.pop('ask_confirm') - else: - ask_confirm = True - if not ask_confirm or self.confirm(msg): - return meth(*args, **kwargs) - - def confirm(self, question, shell=True, abort=True, retry=False, default='y'): - """ask for confirmation and return true on positive answer - - if `retry` is true the r[etry] answer may return 2 - """ - possibleanswers = ['y','n'] - if abort: - possibleanswers.append('abort') - if shell: - possibleanswers.append('shell') - if retry: - possibleanswers.append('retry') - try: - answer = ASK.ask(question, possibleanswers, default) - except (EOFError, KeyboardInterrupt): - answer = 'abort' - if answer == 'n': - return False - if answer == 'retry': - return 2 - if answer == 'abort': - raise SystemExit(1) - if shell and answer == 'shell': - self.interactive_shell() - return self.confirm(question) - return True - - def interactive_shell(self): - self.confirm = yes - self.need_wrap = False - # avoid '_' to be added to builtins by sys.display_hook - def do_not_add___to_builtins(obj): - if obj is not None: - print repr(obj) - sys.displayhook = do_not_add___to_builtins - local_ctx = self._create_context() - try: - import readline - from rlcompleter import Completer - except ImportError: - # readline not available - pass - else: - readline.set_completer(Completer(local_ctx).complete) - readline.parse_and_bind('tab: complete') - home_key = 'HOME' - if sys.platform == 'win32': - home_key = 'USERPROFILE' - histfile = os.path.join(os.environ[home_key], ".eshellhist") - try: - readline.read_history_file(histfile) - except IOError: - pass - from code import interact - banner = """entering the migration python shell -just type migration commands or arbitrary python code and type ENTER to execute it -type "exit" or Ctrl-D to quit the shell and resume operation""" - # give custom readfunc to avoid http://bugs.python.org/issue1288615 - def unicode_raw_input(prompt): - return unicode(raw_input(prompt), sys.stdin.encoding) - interact(banner, readfunc=unicode_raw_input, local=local_ctx) - readline.write_history_file(histfile) - # delete instance's confirm attribute to avoid questions - del self.confirm - self.need_wrap = True - - @cached - def _create_context(self): - """return a dictionary to use as migration script execution context""" - context = self.__context - for attr in dir(self): - if attr.startswith('cmd_'): - if self.need_wrap: - context[attr[4:]] = getattr(self, attr[4:]) - else: - context[attr[4:]] = getattr(self, attr) - return context - - def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs): - """execute a migration script - in interactive mode, display the migration script path, ask for - confirmation and execute it if confirmed - """ - migrscript = os.path.normpath(migrscript) - if migrscript.endswith('.py'): - script_mode = 'python' - elif migrscript.endswith('.txt') or migrscript.endswith('.rst'): - script_mode = 'doctest' - else: - raise Exception('This is not a valid cubicweb shell input') - if not self.execscript_confirm(migrscript): - return - scriptlocals = self._create_context().copy() - if script_mode == 'python': - if funcname is None: - pyname = '__main__' - else: - pyname = splitext(basename(migrscript))[0] - scriptlocals.update({'__file__': migrscript, '__name__': pyname}) - execfile(migrscript, scriptlocals) - if funcname is not None: - try: - func = scriptlocals[funcname] - self.info('found %s in locals', funcname) - assert callable(func), '%s (%s) is not callable' % (func, funcname) - except KeyError: - self.critical('no %s in script %s', funcname, migrscript) - return None - return func(*args, **kwargs) - else: # script_mode == 'doctest' - import doctest - doctest.testfile(migrscript, module_relative=False, - optionflags=doctest.ELLIPSIS, globs=scriptlocals) - - def cmd_option_renamed(self, oldname, newname): - """a configuration option has been renamed""" - self._option_changes.append(('renamed', oldname, newname)) - - def cmd_option_group_change(self, option, oldgroup, newgroup): - """a configuration option has been moved in another group""" - self._option_changes.append(('moved', option, oldgroup, newgroup)) - - def cmd_option_added(self, optname): - """a configuration option has been added""" - self._option_changes.append(('added', optname)) - - def cmd_option_removed(self, optname): - """a configuration option has been removed""" - # can safely be ignored - #self._option_changes.append(('removed', optname)) - - def cmd_option_type_changed(self, optname, oldtype, newvalue): - """a configuration option's type has changed""" - self._option_changes.append(('typechanged', optname, oldtype, newvalue)) - - def cmd_add_cubes(self, cubes): - """modify the list of used cubes in the in-memory config - returns newly inserted cubes, including dependencies - """ - if isinstance(cubes, basestring): - cubes = (cubes,) - origcubes = self.config.cubes() - newcubes = [p for p in self.config.expand_cubes(cubes) - if not p in origcubes] - if newcubes: - for cube in cubes: - assert cube in newcubes - self.config.add_cubes(newcubes) - return newcubes - - def cmd_remove_cube(self, cube, removedeps=False): - if removedeps: - toremove = self.config.expand_cubes([cube]) - else: - toremove = (cube,) - origcubes = self.config._cubes - basecubes = [c for c in origcubes if not c in toremove] - self.config._cubes = tuple(self.config.expand_cubes(basecubes)) - removed = [p for p in origcubes if not p in self.config._cubes] - if not cube in removed: - raise ConfigurationError("can't remove cube %s, " - "used as a dependency" % cube) - return removed - - def rewrite_configuration(self): - # import locally, show_diffs unavailable in gae environment - from cubicweb.toolsutils import show_diffs - configfile = self.config.main_config_file() - if self._option_changes: - read_old_config(self.config, self._option_changes, configfile) - fd, newconfig = tempfile.mkstemp() - for optdescr in self._option_changes: - if optdescr[0] == 'added': - optdict = self.config.get_option_def(optdescr[1]) - if optdict.get('default') is REQUIRED: - self.config.input_option(optdescr[1], optdict) - self.config.generate_config(open(newconfig, 'w')) - show_diffs(configfile, newconfig) - os.close(fd) - if exists(newconfig): - os.unlink(newconfig) - - -from logging import getLogger -from cubicweb import set_log_methods -set_log_methods(MigrationHelper, getLogger('cubicweb.migration')) diff -r c25da7573ebd -r 02b52bf9f5f8 common/mixins.py --- a/common/mixins.py Fri Feb 12 15:18:00 2010 +0100 +++ b/common/mixins.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,315 +1,5 @@ -"""mixins of entity/views organized somewhat in a graph or tree structure - - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -from logilab.common.deprecation import deprecated -from logilab.common.decorators import cached - -from cubicweb import typed_eid -from cubicweb.selectors import implements -from cubicweb.interfaces import IEmailable, ITree - - -class TreeMixIn(object): - """base tree-mixin providing the tree interface - - This mixin has to be inherited explicitly and configured using the - tree_attribute, parent_target and children_target class attribute to - benefit from this default implementation - """ - tree_attribute = None - # XXX misnamed - parent_target = 'subject' - children_target = 'object' - - def different_type_children(self, entities=True): - """return children entities of different type as this entity. - - according to the `entities` parameter, return entity objects or the - equivalent result set - """ - res = self.related(self.tree_attribute, self.children_target, - entities=entities) - if entities: - return [e for e in res if e.e_schema != self.e_schema] - return res.filtered_rset(lambda x: x.e_schema != self.e_schema, self.col) - - def same_type_children(self, entities=True): - """return children entities of the same type as this entity. - - according to the `entities` parameter, return entity objects or the - equivalent result set - """ - res = self.related(self.tree_attribute, self.children_target, - entities=entities) - if entities: - return [e for e in res if e.e_schema == self.e_schema] - return res.filtered_rset(lambda x: x.e_schema == self.e_schema, self.col) - - def iterchildren(self, _done=None): - if _done is None: - _done = set() - for child in self.children(): - if child.eid in _done: - self.error('loop in %s tree', self.id.lower()) - continue - yield child - _done.add(child.eid) - - def prefixiter(self, _done=None): - if _done is None: - _done = set() - if self.eid in _done: - return - yield self - _done.add(self.eid) - for child in self.iterchildren(_done): - try: - for entity in child.prefixiter(_done): - yield entity - except AttributeError: - pass - - @cached - def path(self): - """returns the list of eids from the root object to this object""" - path = [] - parent = self - while parent: - if parent.eid in path: - self.error('loop in %s tree', self.id.lower()) - break - path.append(parent.eid) - try: - # check we are not leaving the tree - if (parent.tree_attribute != self.tree_attribute or - parent.parent_target != self.parent_target): - break - parent = parent.parent() - except AttributeError: - break - - path.reverse() - return path - - def iterparents(self): - def _uptoroot(self): - curr = self - while True: - curr = curr.parent() - if curr is None: - break - yield curr - return _uptoroot(self) - - def notification_references(self, view): - """used to control References field of email send on notification - for this entity. `view` is the notification view. - - Should return a list of eids which can be used to generate message ids - of previously sent email - """ - return self.path()[:-1] - - - ## ITree interface ######################################################## - def parent(self): - """return the parent entity if any, else None (e.g. if we are on the - root - """ - try: - return self.related(self.tree_attribute, self.parent_target, - entities=True)[0] - except (KeyError, IndexError): - return None - - def children(self, entities=True, sametype=False): - """return children entities - - according to the `entities` parameter, return entity objects or the - equivalent result set - """ - if sametype: - return self.same_type_children(entities) - else: - return self.related(self.tree_attribute, self.children_target, - entities=entities) - - def children_rql(self): - return self.related_rql(self.tree_attribute, self.children_target) - - def is_leaf(self): - return len(self.children()) == 0 - - def is_root(self): - return self.parent() is None - - def root(self): - """return the root object""" - return self.req.entity_from_eid(self.path()[0]) - - -class EmailableMixIn(object): - """base mixin providing the default get_email() method used by - the massmailing view - - NOTE: The default implementation is based on the - primary_email / use_email scheme - """ - __implements__ = (IEmailable,) - - def get_email(self): - if getattr(self, 'primary_email', None): - return self.primary_email[0].address - if getattr(self, 'use_email', None): - return self.use_email[0].address - return None - - @classmethod - def allowed_massmail_keys(cls): - """returns a set of allowed email substitution keys - - The default is to return the entity's attribute list but an - entity class might override this method to allow extra keys. - For instance, the Person class might want to return a `companyname` - key. - """ - return set(rschema.type - for rschema, attrtype in cls.e_schema.attribute_definitions() - if attrtype.type not in ('Password', 'Bytes')) - - def as_email_context(self): - """returns the dictionary as used by the sendmail controller to - build email bodies. - - NOTE: the dictionary keys should match the list returned by the - `allowed_massmail_keys` method. - """ - return dict( (attr, getattr(self, attr)) for attr in self.allowed_massmail_keys() ) - - -"""pluggable mixins system: plug classes registered in MI_REL_TRIGGERS on entity -classes which have the relation described by the dict's key. - -NOTE: pluggable mixins can't override any method of the 'explicit' user classes tree -(eg without plugged classes). This includes bases Entity and AnyEntity classes. -""" -MI_REL_TRIGGERS = { - ('primary_email', 'subject'): EmailableMixIn, - ('use_email', 'subject'): EmailableMixIn, - } - - - -def _done_init(done, view, row, col): - """handle an infinite recursion safety belt""" - if done is None: - done = set() - entity = view.entity(row, col) - if entity.eid in done: - msg = entity.req._('loop in %(rel)s relation (%(eid)s)') % { - 'rel': entity.tree_attribute, - 'eid': entity.eid - } - return None, msg - done.add(entity.eid) - return done, entity - - -class TreeViewMixIn(object): - """a recursive tree view""" - id = 'tree' - item_vid = 'treeitem' - __select__ = implements(ITree) - - def call(self, done=None, **kwargs): - if done is None: - done = set() - super(TreeViewMixIn, self).call(done=done, **kwargs) - - def cell_call(self, row, col=0, vid=None, done=None, **kwargs): - done, entity = _done_init(done, self, row, col) - if done is None: - # entity is actually an error message - self.w(u'
  • %s
    • ' % entity) - return - self.open_item(entity) - entity.view(vid or self.item_vid, w=self.w, **kwargs) - relatedrset = entity.children(entities=False) - self.wview(self.id, relatedrset, 'null', done=done, **kwargs) - self.close_item(entity) - - def open_item(self, entity): - self.w(u'
  • \n' % entity.id.lower()) - def close_item(self, entity): - self.w(u'
    • \n') - - -class TreePathMixIn(object): - """a recursive path view""" - id = 'path' - item_vid = 'oneline' - separator = u' > ' - - def call(self, **kwargs): - self.w(u'
    ') - super(TreePathMixIn, self).call(**kwargs) - self.w(u'
    ') - - def cell_call(self, row, col=0, vid=None, done=None, **kwargs): - done, entity = _done_init(done, self, row, col) - if done is None: - # entity is actually an error message - self.w(u'%s' % entity) - return - parent = entity.parent() - if parent: - parent.view(self.id, w=self.w, done=done) - self.w(self.separator) - entity.view(vid or self.item_vid, w=self.w) - - -class ProgressMixIn(object): - """provide default implementations for IProgress interface methods""" - # This is an adapter isn't it ? - - @property - def cost(self): - return self.progress_info()['estimated'] - - @property - def revised_cost(self): - return self.progress_info().get('estimatedcorrected', self.cost) - - @property - def done(self): - return self.progress_info()['done'] - - @property - def todo(self): - return self.progress_info()['todo'] - - @cached - def progress_info(self): - raise NotImplementedError() - - def finished(self): - return not self.in_progress() - - def in_progress(self): - raise NotImplementedError() - - def progress(self): - try: - return 100. * self.done / self.revised_cost - except ZeroDivisionError: - # total cost is 0 : if everything was estimated, task is completed - if self.progress_info().get('notestimated'): - return 0. - return 100 +"""pre 3.6 bw compat""" +# pylint: disable-msg=W0614,W0401 +from warnings import warn +warn('moved to cubicweb.mixins', DeprecationWarning, stacklevel=2) +from cubicweb.mixins import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/mttransforms.py --- a/common/mttransforms.py Fri Feb 12 15:18:00 2010 +0100 +++ b/common/mttransforms.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,93 +1,5 @@ -"""mime type transformation engine for cubicweb, based on mtconverter - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -from logilab import mtconverter - -from logilab.mtconverter.engine import TransformEngine -from logilab.mtconverter.transform import Transform -from logilab.mtconverter import (register_base_transforms, - register_pil_transforms, - register_pygments_transforms) - -from cubicweb.common.uilib import rest_publish, html_publish - -HTML_MIMETYPES = ('text/html', 'text/xhtml', 'application/xhtml+xml') - -# CubicWeb specific transformations - -class rest_to_html(Transform): - inputs = ('text/rest', 'text/x-rst') - output = 'text/html' - def _convert(self, trdata): - return rest_publish(trdata.appobject, trdata.decode()) - -class html_to_html(Transform): - inputs = HTML_MIMETYPES - output = 'text/html' - def _convert(self, trdata): - return html_publish(trdata.appobject, trdata.data) - - -# Instantiate and configure the transformation engine - -mtconverter.UNICODE_POLICY = 'replace' - -ENGINE = TransformEngine() -ENGINE.add_transform(rest_to_html()) -ENGINE.add_transform(html_to_html()) - -try: - from cubicweb.ext.tal import compile_template -except ImportError: - HAS_TAL = False - from cubicweb import schema - schema.NEED_PERM_FORMATS.remove('text/cubicweb-page-template') - -else: - HAS_TAL = True - - class ept_to_html(Transform): - inputs = ('text/cubicweb-page-template',) - output = 'text/html' - output_encoding = 'utf-8' - def _convert(self, trdata): - value = trdata.encode(self.output_encoding) - return trdata.appobject.tal_render(compile_template(value), {}) - - ENGINE.add_transform(ept_to_html()) - -if register_pil_transforms(ENGINE, verb=False): - HAS_PIL_TRANSFORMS = True -else: - HAS_PIL_TRANSFORMS = False - -try: - from logilab.mtconverter.transforms import pygmentstransforms - for mt in ('text/plain',) + HTML_MIMETYPES: - try: - pygmentstransforms.mimetypes.remove(mt) - except ValueError: - continue - register_pygments_transforms(ENGINE, verb=False) - - def patch_convert(cls): - def _convert(self, trdata, origconvert=cls._convert): - try: - trdata.appobject.req.add_css('pygments.css') - except AttributeError: # session has no add_css, only http request - pass - return origconvert(self, trdata) - cls._convert = _convert - patch_convert(pygmentstransforms.PygmentsHTMLTransform) - - HAS_PYGMENTS_TRANSFORMS = True -except ImportError: - HAS_PYGMENTS_TRANSFORMS = False - -register_base_transforms(ENGINE, verb=False) +"""pre 3.6 bw compat""" +# pylint: disable-msg=W0614,W0401 +from warnings import warn +warn('moved to cubicweb.mttransforms', DeprecationWarning, stacklevel=2) +from cubicweb.mttransforms import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/schema.py --- a/common/schema.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -"""pre 3.0 bw compat - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -# pylint: disable-msg=W0614,W0401 -from warnings import warn -warn('moved to cubicweb.schema', DeprecationWarning, stacklevel=2) -from cubicweb.schema import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/selectors.py --- a/common/selectors.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,12 +0,0 @@ -"""pre 3.2 bw compat - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -# pylint: disable-msg=W0614,W0401 -from warnings import warn -warn('moved to cubicweb.selectors', DeprecationWarning, stacklevel=2) -from cubicweb.selectors import * -from cubicweb.selectors import _rql_condition diff -r c25da7573ebd -r 02b52bf9f5f8 common/tags.py --- a/common/tags.py Fri Feb 12 15:18:00 2010 +0100 +++ b/common/tags.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,49 +1,5 @@ -"""helper classes to generate simple (X)HTML tags - -:organization: Logilab -:copyright: 2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -from cubicweb.common.uilib import simple_sgml_tag, sgml_attributes - -class tag(object): - def __init__(self, name, escapecontent=True): - self.name = name - self.escapecontent = escapecontent - - def __call__(self, __content=None, **attrs): - attrs.setdefault('escapecontent', self.escapecontent) - return simple_sgml_tag(self.name, __content, **attrs) - -button = tag('button') -input = tag('input') -textarea = tag('textarea') -a = tag('a') -span = tag('span') -div = tag('div', False) -img = tag('img') -label = tag('label') -option = tag('option') -h1 = tag('h1') -h2 = tag('h2') -h3 = tag('h3') -h4 = tag('h4') -h5 = tag('h5') -tr = tag('tr') -th = tag('th') -td = tag('td') - -def select(name, id=None, multiple=False, options=[], **attrs): - if multiple: - attrs['multiple'] = 'multiple' - if id: - attrs['id'] = id - attrs['name'] = name - html = [u'') - return u'\n'.join(html) - +"""pre 3.6 bw compat""" +# pylint: disable-msg=W0614,W0401 +from warnings import warn +warn('moved to cubicweb.tags', DeprecationWarning, stacklevel=2) +from cubicweb.tags import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/bootstrap_cubes --- a/common/test/data/bootstrap_cubes Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ - diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.0.3_Any.py --- a/common/test/data/migration/0.0.3_Any.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -""" - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -coucou diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.0.4_Any.py --- a/common/test/data/migration/0.0.4_Any.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -""" - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -coucou diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.1.0_Any.py --- a/common/test/data/migration/0.1.0_Any.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -""" - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -coucou diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.1.0_common.py --- a/common/test/data/migration/0.1.0_common.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -"""common to all configuration - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.1.0_repository.py --- a/common/test/data/migration/0.1.0_repository.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -"""repository specific - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.1.0_web.py --- a/common/test/data/migration/0.1.0_web.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -"""web only - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/0.1.2_Any.py --- a/common/test/data/migration/0.1.2_Any.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -""" - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -coucou diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/migration/depends.map --- a/common/test/data/migration/depends.map Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,5 +0,0 @@ -0.0.2: 2.3.0 -0.0.3: 2.4.0 -# missing 0.0.4 entry, that's alright -0.1.0: 2.6.0 -0.1.2: 2.10.0 diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/server_migration/2.10.2_Any.sql diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/server_migration/2.5.0_Any.sql diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/server_migration/2.6.0_Any.sql diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/data/server_migration/bootstrapmigration_repository.py --- a/common/test/data/server_migration/bootstrapmigration_repository.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -"""allways executed before all others in server migration - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/unittest_mail.py --- a/common/test/unittest_mail.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -"""unit tests for module cubicweb.common.mail - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" - -import os -import sys - -from logilab.common.testlib import unittest_main -from logilab.common.umessage import message_from_string - -from cubicweb.devtools.apptest import EnvBasedTC -from cubicweb.common.mail import format_mail - - -def getlogin(): - """avoid usinng os.getlogin() because of strange tty / stdin problems - (man 3 getlogin) - Another solution would be to use $LOGNAME, $USER or $USERNAME - """ - if sys.platform != 'win32': - import pwd - return pwd.getpwuid(os.getuid())[0] - else: - return os.environ.get('USERNAME') - - -class EmailTC(EnvBasedTC): - - def test_format_mail(self): - self.set_option('sender-addr', 'bim@boum.fr') - self.set_option('sender-name', 'BimBam') - - mail = format_mail({'name': 'oim', 'email': 'oim@logilab.fr'}, - ['test@logilab.fr'], u'un petit cöucou', u'bïjour', - config=self.config) - self.assertLinesEquals(mail.as_string(), """\ -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Content-Transfer-Encoding: base64 -Subject: =?utf-8?q?b=C3=AFjour?= -From: =?utf-8?q?oim?= -Reply-to: =?utf-8?q?oim?= , =?utf-8?q?BimBam?= -X-CW: data -To: test@logilab.fr - -dW4gcGV0aXQgY8O2dWNvdQ== -""") - msg = message_from_string(mail.as_string()) - self.assertEquals(msg.get('subject'), u'bïjour') - self.assertEquals(msg.get('from'), u'oim ') - self.assertEquals(msg.get('to'), u'test@logilab.fr') - self.assertEquals(msg.get('reply-to'), u'oim , BimBam ') - self.assertEquals(msg.get_payload(decode=True), u'un petit cöucou') - - - def test_format_mail_euro(self): - mail = format_mail({'name': u'oîm', 'email': u'oim@logilab.fr'}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €') - self.assertLinesEquals(mail.as_string(), """\ -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Content-Transfer-Encoding: base64 -Subject: =?utf-8?b?YsOvam91ciDigqw=?= -From: =?utf-8?q?o=C3=AEm?= -Reply-to: =?utf-8?q?o=C3=AEm?= -To: test@logilab.fr - -dW4gcGV0aXQgY8O2dWNvdSDigqw= -""") - msg = message_from_string(mail.as_string()) - self.assertEquals(msg.get('subject'), u'bïjour €') - self.assertEquals(msg.get('from'), u'oîm ') - self.assertEquals(msg.get('to'), u'test@logilab.fr') - self.assertEquals(msg.get('reply-to'), u'oîm ') - self.assertEquals(msg.get_payload(decode=True), u'un petit cöucou €') - - - def test_format_mail_from_reply_to(self): - # no sender-name, sender-addr in the configuration - self.set_option('sender-name', '') - self.set_option('sender-addr', '') - msg = format_mail({'name': u'', 'email': u''}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €', - config=self.config) - self.assertEquals(msg.get('from'), u'') - self.assertEquals(msg.get('reply-to'), None) - msg = format_mail({'name': u'tutu', 'email': u'tutu@logilab.fr'}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €', - config=self.config) - msg = message_from_string(msg.as_string()) - self.assertEquals(msg.get('from'), u'tutu ') - self.assertEquals(msg.get('reply-to'), u'tutu ') - msg = format_mail({'name': u'tutu', 'email': u'tutu@logilab.fr'}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €') - msg = message_from_string(msg.as_string()) - self.assertEquals(msg.get('from'), u'tutu ') - self.assertEquals(msg.get('reply-to'), u'tutu ') - # set sender name and address as expected - self.set_option('sender-name', 'cubicweb-test') - self.set_option('sender-addr', 'cubicweb-test@logilab.fr') - # anonymous notification: no name and no email specified - msg = format_mail({'name': u'', 'email': u''}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €', - config=self.config) - msg = message_from_string(msg.as_string()) - self.assertEquals(msg.get('from'), u'cubicweb-test ') - self.assertEquals(msg.get('reply-to'), u'cubicweb-test ') - # anonymous notification: only email specified - msg = format_mail({'email': u'tutu@logilab.fr'}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €', - config=self.config) - msg = message_from_string(msg.as_string()) - self.assertEquals(msg.get('from'), u'cubicweb-test ') - self.assertEquals(msg.get('reply-to'), u'cubicweb-test , cubicweb-test ') - # anonymous notification: only name specified - msg = format_mail({'name': u'tutu'}, - ['test@logilab.fr'], u'un petit cöucou €', u'bïjour €', - config=self.config) - msg = message_from_string(msg.as_string()) - self.assertEquals(msg.get('from'), u'tutu ') - self.assertEquals(msg.get('reply-to'), u'tutu ') - - - -if __name__ == '__main__': - unittest_main() - diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/unittest_migration.py --- a/common/test/unittest_migration.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,106 +0,0 @@ -"""cubicweb.common.migration unit tests - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" - -from os.path import abspath -from logilab.common.testlib import TestCase, unittest_main - -from cubicweb.devtools import TestServerConfiguration -from cubicweb.devtools.apptest import TestEnvironment - -from cubicweb.cwconfig import CubicWebConfiguration -from cubicweb.common.migration import MigrationHelper, filter_scripts -from cubicweb.server.migractions import ServerMigrationHelper - - -class Schema(dict): - def has_entity(self, e_type): - return self.has_key(e_type) - -SMIGRDIR = abspath('data/server_migration') + '/' -TMIGRDIR = abspath('data/migration') + '/' - -class MigrTestConfig(TestServerConfiguration): - verbosity = 0 - def migration_scripts_dir(cls): - return SMIGRDIR - - def cube_migration_scripts_dir(cls, cube): - return TMIGRDIR - -class MigrationToolsTC(TestCase): - def setUp(self): - self.config = MigrTestConfig('data') - from yams.schema import Schema - self.config.load_schema = lambda expand_cubes=False: Schema('test') - self.config.__class__.cubicweb_appobject_path = frozenset() - self.config.__class__.cube_appobject_path = frozenset() - - def test_filter_scripts_base(self): - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,3,0), (2,4,0)), - []) - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,4,0), (2,5,0)), - [((2, 5, 0), SMIGRDIR+'2.5.0_Any.sql')]) - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,5,0), (2,6,0)), - [((2, 6, 0), SMIGRDIR+'2.6.0_Any.sql')]) - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,4,0), (2,6,0)), - [((2, 5, 0), SMIGRDIR+'2.5.0_Any.sql'), - ((2, 6, 0), SMIGRDIR+'2.6.0_Any.sql')]) - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,5,0), (2,5,1)), - []) - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,5,0), (2,10,2)), - [((2, 6, 0), SMIGRDIR+'2.6.0_Any.sql'), - ((2, 10, 2), SMIGRDIR+'2.10.2_Any.sql')]) - self.assertListEquals(filter_scripts(self.config, SMIGRDIR, (2,5,1), (2,6,0)), - [((2, 6, 0), SMIGRDIR+'2.6.0_Any.sql')]) - - self.assertListEquals(filter_scripts(self.config, TMIGRDIR, (0,0,2), (0,0,3)), - [((0, 0, 3), TMIGRDIR+'0.0.3_Any.py')]) - self.assertListEquals(filter_scripts(self.config, TMIGRDIR, (0,0,2), (0,0,4)), - [((0, 0, 3), TMIGRDIR+'0.0.3_Any.py'), - ((0, 0, 4), TMIGRDIR+'0.0.4_Any.py')]) - - def test_filter_scripts_for_mode(self): - self.assertIsInstance(self.config.migration_handler(), ServerMigrationHelper) - config = CubicWebConfiguration('data') - config.verbosity = 0 - self.assert_(not isinstance(config.migration_handler(), ServerMigrationHelper)) - self.assertIsInstance(config.migration_handler(), MigrationHelper) - config = self.config - config.__class__.name = 'twisted' - self.assertListEquals(filter_scripts(config, TMIGRDIR, (0,0,4), (0,1,0)), - [((0, 1 ,0), TMIGRDIR+'0.1.0_common.py'), - ((0, 1 ,0), TMIGRDIR+'0.1.0_web.py')]) - config.__class__.name = 'repository' - self.assertListEquals(filter_scripts(config, TMIGRDIR, (0,0,4), (0,1,0)), - [((0, 1 ,0), TMIGRDIR+'0.1.0_Any.py'), - ((0, 1 ,0), TMIGRDIR+'0.1.0_common.py'), - ((0, 1 ,0), TMIGRDIR+'0.1.0_repository.py')]) - config.__class__.name = 'all-in-one' - self.assertListEquals(filter_scripts(config, TMIGRDIR, (0,0,4), (0,1,0)), - [((0, 1 ,0), TMIGRDIR+'0.1.0_Any.py'), - ((0, 1 ,0), TMIGRDIR+'0.1.0_common.py'), - ((0, 1 ,0), TMIGRDIR+'0.1.0_repository.py'), - ((0, 1 ,0), TMIGRDIR+'0.1.0_web.py')]) - config.__class__.name = 'repository' - - -from cubicweb.devtools import ApptestConfiguration, init_test_database, cleanup_sqlite - -class BaseCreationTC(TestCase): - - def test_db_creation(self): - """make sure database can be created""" - config = ApptestConfiguration('data') - source = config.sources()['system'] - self.assertEquals(source['db-driver'], 'sqlite') - cleanup_sqlite(source['db-name'], removecube=True) - init_test_database(driver=source['db-driver'], config=config) - - -if __name__ == '__main__': - unittest_main() diff -r c25da7573ebd -r 02b52bf9f5f8 common/test/unittest_uilib.py --- a/common/test/unittest_uilib.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -"""unittests for cubicweb.common.uilib - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" - -__docformat__ = "restructuredtext en" - -from logilab.common.testlib import TestCase, unittest_main -from logilab.common.tree import Node - -from cubicweb.common import uilib - -class UILIBTC(TestCase): - - def test_remove_tags(self): - """make sure remove_tags remove all tags""" - data = [ - ('

    Hello

    ', 'Hello'), - ('

    Hello spam

    ', 'Hello spam'), - ('
    Hello', 'Hello'), - ('

    ', ''), - ] - for text, expected in data: - got = uilib.remove_html_tags(text) - self.assertEquals(got, expected) - - def test_fallback_safe_cut(self): - self.assertEquals(uilib.fallback_safe_cut(u'ab cd', 4), u'ab c...') - self.assertEquals(uilib.fallback_safe_cut(u'ab cd', 5), u'ab cd') - self.assertEquals(uilib.fallback_safe_cut(u'ab &d', 4), u'ab &...') - self.assertEquals(uilib.fallback_safe_cut(u'ab &d ef', 5), u'ab &d...') - self.assertEquals(uilib.fallback_safe_cut(u'ab ìd', 4), u'ab ì...') - self.assertEquals(uilib.fallback_safe_cut(u'& &d ef', 4), u'& &d...') - - def test_lxml_safe_cut(self): - self.assertEquals(uilib.safe_cut(u'aaa
    aaad
    ef', 4), u'

    aaa

    a...
    ') - self.assertEquals(uilib.safe_cut(u'aaa
    aaad
    ef', 7), u'

    aaa

    aaad
    ...') - self.assertEquals(uilib.safe_cut(u'aaa
    aaad
    ', 7), u'

    aaa

    aaad
    ') - # Missing ellipsis due to space management but we don't care - self.assertEquals(uilib.safe_cut(u'ab &d', 4), u'

    ab &...

    ') - - def test_cut(self): - """tests uilib.cut() behaviour""" - data = [ - ('hello', 'hello'), - ('hello world', 'hello wo...'), - ("hellO' world", "hellO..."), - ] - for text, expected in data: - got = uilib.cut(text, 8) - self.assertEquals(got, expected) - - def test_text_cut(self): - """tests uilib.text_cut() behaviour with no text""" - data = [('',''), - ("""Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse -cillum dolore eu fugiat nulla pariatur.""", - "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod \ -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \ -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo \ -consequat."), - ("""Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam, -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat Duis aute irure dolor in reprehenderit in voluptate velit esse -cillum dolore eu fugiat nulla pariatur Excepteur sint occaecat cupidatat non -proident, sunt in culpa qui officia deserunt mollit anim id est laborum -""", - "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod \ -tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam, \ -quis nostrud exercitation ullamco laboris nisi"), - ] - for text, expected in data: - got = uilib.text_cut(text, 30) - self.assertEquals(got, expected) - -if __name__ == '__main__': - unittest_main() - diff -r c25da7573ebd -r 02b52bf9f5f8 common/uilib.py --- a/common/uilib.py Fri Feb 12 15:18:00 2010 +0100 +++ b/common/uilib.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,383 +1,5 @@ -# -*- coding: utf-8 -*- -"""user interface libraries - -contains some functions designed to help implementation of cubicweb user interface - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -import csv -import re -from StringIO import StringIO - -from logilab.mtconverter import xml_escape, html_unescape - -from cubicweb.utils import ustrftime - -def rql_for_eid(eid): - """return the rql query necessary to fetch entity with the given eid. This - function should only be used to generate link with rql inside, not to give - to cursor.execute (in which case you won't benefit from rql cache). - - :Parameters: - - `eid`: the eid of the entity we should search - :rtype: str - :return: the rql query - """ - return 'Any X WHERE X eid %s' % eid - - -def printable_value(req, attrtype, value, props=None, displaytime=True): - """return a displayable value (i.e. unicode string)""" - if value is None or attrtype == 'Bytes': - return u'' - if attrtype == 'String': - # don't translate empty value if you don't want strange results - if props is not None and value and props.get('internationalizable'): - return req._(value) - return value - if attrtype == 'Date': - return ustrftime(value, req.property_value('ui.date-format')) - if attrtype == 'Time': - return ustrftime(value, req.property_value('ui.time-format')) - if attrtype == 'Datetime': - if displaytime: - return ustrftime(value, req.property_value('ui.datetime-format')) - return ustrftime(value, req.property_value('ui.date-format')) - if attrtype == 'Boolean': - if value: - return req._('yes') - return req._('no') - if attrtype == 'Float': - value = req.property_value('ui.float-format') % value - return unicode(value) - - -# text publishing ############################################################# - -try: - from cubicweb.ext.rest import rest_publish # pylint: disable-msg=W0611 -except ImportError: - def rest_publish(entity, data): - """default behaviour if docutils was not found""" - return xml_escape(data) - -TAG_PROG = re.compile(r'', re.U) -def remove_html_tags(text): - """Removes HTML tags from text - - >>> remove_html_tags('hi world') - 'hi world' - >>> - """ - return TAG_PROG.sub('', text) - - -REF_PROG = re.compile(r"([^<]*)", re.U) -def _subst_rql(view, obj): - delim, rql, descr = obj.groups() - return u'%s' % (view.build_url(rql=rql), descr) - -def html_publish(view, text): - """replace links by """ - if not text: - return u'' - return REF_PROG.sub(lambda obj, view=view:_subst_rql(view, obj), text) - -# fallback implementation, nicer one defined below if lxml is available -def soup2xhtml(data, encoding): - # normalize line break - # see http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 - return u'\n'.join(data.splitlines()) - -# fallback implementation, nicer one defined below if lxml> 2.0 is available -def safe_cut(text, length): - """returns a string of length based on , removing any html - tags from given text if cut is necessary.""" - if text is None: - return u'' - noenttext = html_unescape(text) - text_nohtml = remove_html_tags(noenttext) - # try to keep html tags if text is short enough - if len(text_nohtml) <= length: - return text - # else if un-tagged text is too long, cut it - return xml_escape(text_nohtml[:length] + u'...') - -fallback_safe_cut = safe_cut - - -try: - from lxml import etree -except (ImportError, AttributeError): - # gae environment: lxml not available - pass -else: - - def soup2xhtml(data, encoding): - """tidy (at least try) html soup and return the result - Note: the function considers a string with no surrounding tag as valid - if
    `data`
    can be parsed by an XML parser - """ - # normalize line break - # see http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 - data = u'\n'.join(data.splitlines()) - # XXX lxml 1.1 support still needed ? - xmltree = etree.HTML('
    %s
    ' % data) - # NOTE: lxml 1.1 (etch platforms) doesn't recognize - # the encoding=unicode parameter (lxml 2.0 does), this is - # why we specify an encoding and re-decode to unicode later - body = etree.tostring(xmltree[0], encoding=encoding) - # remove and and decode to unicode - return body[11:-13].decode(encoding) - - if hasattr(etree.HTML('
    test
    '), 'iter'): - - def safe_cut(text, length): - """returns an html document of length based on , - and cut is necessary. - """ - if text is None: - return u'' - dom = etree.HTML(text) - curlength = 0 - add_ellipsis = False - for element in dom.iter(): - if curlength >= length: - parent = element.getparent() - parent.remove(element) - if curlength == length and (element.text or element.tail): - add_ellipsis = True - else: - if element.text is not None: - element.text = cut(element.text, length - curlength) - curlength += len(element.text) - if element.tail is not None: - if curlength < length: - element.tail = cut(element.tail, length - curlength) - curlength += len(element.tail) - elif curlength == length: - element.tail = '...' - else: - element.tail = '' - text = etree.tounicode(dom[0])[6:-7] # remove wrapping - if add_ellipsis: - return text + u'...' - return text - -def text_cut(text, nbwords=30, gotoperiod=True): - """from the given plain text, return a text with at least words, - trying to go to the end of the current sentence. - - :param nbwords: the minimum number of words required - :param gotoperiod: specifies if the function should try to go to - the first period after the cut (i.e. finish - the sentence if possible) - - Note that spaces are normalized. - """ - if text is None: - return u'' - words = text.split() - text = u' '.join(words) # normalize spaces - textlength = minlength = len(' '.join(words[:nbwords])) - if gotoperiod: - textlength = text.find('.', minlength) + 1 - if textlength == 0: # no period found - textlength = minlength - return text[:textlength] - -def cut(text, length): - """returns a string of a maximum length based on - (approximatively, since if text has been cut, '...' is added to the end of the string, - resulting in a string of len + 3) - """ - if text is None: - return u'' - if len(text) <= length: - return text - # else if un-tagged text is too long, cut it - return text[:length] + u'...' - - - -# HTML generation helper functions ############################################ - -HTML4_EMPTY_TAGS = frozenset(('base', 'meta', 'link', 'hr', 'br', 'param', - 'img', 'area', 'input', 'col')) - -def sgml_attributes(attrs): - return u' '.join(u'%s="%s"' % (attr, xml_escape(unicode(value))) - for attr, value in sorted(attrs.items()) - if value is not None) - -def simple_sgml_tag(tag, content=None, escapecontent=True, **attrs): - """generation of a simple sgml tag (eg without children tags) easier - - content and attri butes will be escaped - """ - value = u'<%s' % tag - if attrs: - try: - attrs['class'] = attrs.pop('klass') - except KeyError: - pass - value += u' ' + sgml_attributes(attrs) - if content: - if escapecontent: - content = xml_escape(unicode(content)) - value += u'>%s' % (content, tag) - else: - if tag in HTML4_EMPTY_TAGS: - value += u' />' - else: - value += u'>' % tag - return value - -def tooltipize(text, tooltip, url=None): - """make an HTML tooltip""" - url = url or '#' - return u'    %s' % (url, tooltip, text) - -def toggle_action(nodeid): - """builds a HTML link that uses the js toggleVisibility function""" - return u"javascript: toggleVisibility('%s')" % nodeid - -def toggle_link(nodeid, label): - """builds a HTML link that uses the js toggleVisibility function""" - return u'%s' % (toggle_action(nodeid), label) - - -def ureport_as_html(layout): - from logilab.common.ureports import HTMLWriter - formater = HTMLWriter(True) - stream = StringIO() #UStringIO() don't want unicode assertion - formater.format(layout, stream) - res = stream.getvalue() - if isinstance(res, str): - res = unicode(res, 'UTF8') - return res - -# traceback formatting ######################################################## - -import traceback - -def rest_traceback(info, exception): - """return a ReST formated traceback""" - res = [u'Traceback\n---------\n::\n'] - for stackentry in traceback.extract_tb(info[2]): - res.append(u'\tFile %s, line %s, function %s' % tuple(stackentry[:3])) - if stackentry[3]: - res.append(u'\t %s' % stackentry[3].decode('utf-8', 'replace')) - res.append(u'\n') - try: - res.append(u'\t Error: %s\n' % exception) - except: - pass - return u'\n'.join(res) - - -def html_traceback(info, exception, title='', - encoding='ISO-8859-1', body=''): - """ return an html formatted traceback from python exception infos. - """ - tcbk = info[2] - stacktb = traceback.extract_tb(tcbk) - strings = [] - if body: - strings.append(u'
    ') - # FIXME - strings.append(body) - strings.append(u'
    ') - if title: - strings.append(u'

    %s

    '% xml_escape(title)) - try: - strings.append(u'

    %s

    ' % xml_escape(str(exception)).replace("\n","
    ")) - except UnicodeError: - pass - strings.append(u'
    ') - for index, stackentry in enumerate(stacktb): - strings.append(u'File %s, line ' - u'%s, function ' - u'%s:
    '%( - xml_escape(stackentry[0]), stackentry[1], xml_escape(stackentry[2]))) - if stackentry[3]: - string = xml_escape(stackentry[3]).decode('utf-8', 'replace') - strings.append(u'  %s
    \n' % (string)) - # add locals info for each entry - try: - local_context = tcbk.tb_frame.f_locals - html_info = [] - chars = 0 - for name, value in local_context.iteritems(): - value = xml_escape(repr(value)) - info = u'%s=%s, ' % (name, value) - line_length = len(name) + len(value) - chars += line_length - # 150 is the result of *years* of research ;-) (CSS might be helpful here) - if chars > 150: - info = u'
    ' + info - chars = line_length - html_info.append(info) - boxid = 'ctxlevel%d' % index - strings.append(u'[%s]' % toggle_link(boxid, '+')) - strings.append(u'' % - (boxid, ''.join(html_info))) - tcbk = tcbk.tb_next - except Exception: - pass # doesn't really matter if we have no context info - strings.append(u'
    ') - return '\n'.join(strings) - -# csv files / unicode support ################################################# - -class UnicodeCSVWriter: - """proxies calls to csv.writer.writerow to be able to deal with unicode""" - - def __init__(self, wfunc, encoding, **kwargs): - self.writer = csv.writer(self, **kwargs) - self.wfunc = wfunc - self.encoding = encoding - - def write(self, data): - self.wfunc(data) - - def writerow(self, row): - csvrow = [] - for elt in row: - if isinstance(elt, unicode): - csvrow.append(elt.encode(self.encoding)) - else: - csvrow.append(str(elt)) - self.writer.writerow(csvrow) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -# some decorators ############################################################# - -class limitsize(object): - def __init__(self, maxsize): - self.maxsize = maxsize - - def __call__(self, function): - def newfunc(*args, **kwargs): - ret = function(*args, **kwargs) - if isinstance(ret, basestring): - return ret[:self.maxsize] - return ret - return newfunc - - -def htmlescape(function): - def newfunc(*args, **kwargs): - ret = function(*args, **kwargs) - assert isinstance(ret, basestring) - return xml_escape(ret) - return newfunc +"""pre 3.6 bw compat""" +# pylint: disable-msg=W0614,W0401 +from warnings import warn +warn('moved to cubicweb.uilib', DeprecationWarning, stacklevel=2) +from cubicweb.uilib import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/utils.py --- a/common/utils.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -"""pre 3.2 bw compat - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -# pylint: disable-msg=W0614,W0401 -from warnings import warn -warn('moved to cubicweb.utils', DeprecationWarning, stacklevel=2) -from cubicweb.utils import * diff -r c25da7573ebd -r 02b52bf9f5f8 common/view.py --- a/common/view.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -"""pre 3.2 bw compat - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -# pylint: disable-msg=W0614,W0401 -from warnings import warn -warn('moved to cubicweb.view', DeprecationWarning, stacklevel=2) -from cubicweb.view import * diff -r c25da7573ebd -r 02b52bf9f5f8 crypto.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/crypto.py Wed Mar 24 10:23:31 2010 +0100 @@ -0,0 +1,35 @@ +"""Simple cryptographic routines, based on python-crypto. + +:organization: Logilab +:copyright: 2009-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" +__docformat__ = "restructuredtext en" + +from pickle import dumps, loads +from base64 import b64encode, b64decode + +from Crypto.Cipher import Blowfish + + +_CYPHERERS = {} +def _cypherer(seed): + try: + return _CYPHERERS[seed] + except KeyError: + _CYPHERERS[seed] = Blowfish.new(seed, Blowfish.MODE_ECB) + return _CYPHERERS[seed] + + +def encrypt(data, seed): + string = dumps(data) + string = string + '*' * (8 - len(string) % 8) + string = b64encode(_cypherer(seed).encrypt(string)) + return unicode(string) + + +def decrypt(string, seed): + # pickle ignores trailing characters so we do not need to strip them off + string = _cypherer(seed).decrypt(b64decode(string)) + return loads(string) diff -r c25da7573ebd -r 02b52bf9f5f8 cwconfig.py --- a/cwconfig.py Fri Feb 12 15:18:00 2010 +0100 +++ b/cwconfig.py Wed Mar 24 10:23:31 2010 +0100 @@ -78,12 +78,13 @@ import sys import os import logging +import tempfile from smtplib import SMTP from threading import Lock from os.path import exists, join, expanduser, abspath, normpath, basename, isdir -import tempfile +from warnings import warn -from logilab.common.decorators import cached +from logilab.common.decorators import cached, classproperty from logilab.common.deprecation import deprecated from logilab.common.logging_ext import set_log_methods, init_log from logilab.common.configuration import (Configuration, Method, @@ -114,7 +115,7 @@ def possible_configurations(directory): """return a list of installed configurations in a directory - according to *-ctl files + according to \*-ctl files """ return [name for name in ('repository', 'twisted', 'all-in-one') if exists(join(directory, '%s.conf' % name))] @@ -216,7 +217,9 @@ if os.environ.get('APYCOT_ROOT'): mode = 'test' - if CWDEV: + # allow to test cubes within apycot using cubicweb not installed by + # apycot + if __file__.startswith(os.environ['APYCOT_ROOT']): CUBES_DIR = '%(APYCOT_ROOT)s/local/share/cubicweb/cubes/' % os.environ # create __init__ file file(join(CUBES_DIR, '__init__.py'), 'w').close() @@ -235,7 +238,7 @@ options = ( ('log-threshold', {'type' : 'string', # XXX use a dedicated type? - 'default': 'ERROR', + 'default': 'WARNING', 'help': 'server\'s log level', 'group': 'main', 'inputlevel': 1, }), @@ -290,11 +293,6 @@ this option is set to yes", 'group': 'email', 'inputlevel': 2, }), - ('disable-appobjects', - {'type' : 'csv', 'default': (), - 'help': 'comma separated list of identifiers of application objects (.) to disable', - 'group': 'appobjects', 'inputlevel': 2, - }), ) # static and class methods used to get instance independant resources ## @@ -332,7 +330,7 @@ cubes = set() for directory in cls.cubes_search_path(): if not os.path.exists(directory): - self.error('unexistant directory in cubes search path: %s' + cls.error('unexistant directory in cubes search path: %s' % directory) continue for cube in os.listdir(directory): @@ -351,10 +349,18 @@ path.append(directory) except KeyError: pass - if not cls.CUBES_DIR in path: + if not cls.CUBES_DIR in path and exists(cls.CUBES_DIR): path.append(cls.CUBES_DIR) return path + @classproperty + def extrapath(cls): + extrapath = {} + for cubesdir in cls.cubes_search_path(): + if cubesdir != cls.CUBES_DIR: + extrapath[cubesdir] = 'cubes' + return extrapath + @classmethod def cube_dir(cls, cube): """return the cube directory for the given cube id, @@ -470,7 +476,7 @@ from logilab.common.modutils import load_module_from_file cls.cls_adjust_sys_path() for ctlfile in ('web/webctl.py', 'etwist/twctl.py', - 'server/serverctl.py', 'hercule.py', + 'server/serverctl.py', 'devtools/devctl.py', 'goa/goactl.py'): if exists(join(CW_SOFTWARE_ROOT, ctlfile)): try: @@ -480,14 +486,23 @@ (ctlfile, err)) cls.info('loaded cubicweb-ctl plugin %s', ctlfile) for cube in cls.available_cubes(): - pluginfile = join(cls.cube_dir(cube), 'ecplugin.py') + oldpluginfile = join(cls.cube_dir(cube), 'ecplugin.py') + pluginfile = join(cls.cube_dir(cube), 'ccplugin.py') initfile = join(cls.cube_dir(cube), '__init__.py') if exists(pluginfile): try: + __import__('cubes.%s.ccplugin' % cube) + cls.info('loaded cubicweb-ctl plugin from %s', cube) + except: + cls.exception('while loading plugin %s', pluginfile) + elif exists(oldpluginfile): + warn('[3.6] %s: ecplugin module should be renamed to ccplugin' % cube, + DeprecationWarning) + try: __import__('cubes.%s.ecplugin' % cube) cls.info('loaded cubicweb-ctl plugin from %s', cube) except: - cls.exception('while loading plugin %s', pluginfile) + cls.exception('while loading plugin %s', oldpluginfile) elif exists(initfile): try: __import__('cubes.%s' % cube) @@ -556,6 +571,7 @@ return vregpath def __init__(self): + register_stored_procedures() ConfigurationMixIn.__init__(self) self.adjust_sys_path() self.load_defaults() @@ -624,16 +640,18 @@ """base class for cubicweb server and web configurations""" INSTANCES_DATA_DIR = None - if CubicWebNoAppConfiguration.mode == 'test': + if os.environ.get('APYCOT_ROOT'): root = os.environ['APYCOT_ROOT'] REGISTRY_DIR = '%s/etc/cubicweb.d/' % root + if not exists(REGISTRY_DIR): + os.makedirs(REGISTRY_DIR) RUNTIME_DIR = tempfile.gettempdir() - if CWDEV: + # allow to test cubes within apycot using cubicweb not installed by + # apycot + if __file__.startswith(os.environ['APYCOT_ROOT']): MIGRATION_DIR = '%s/local/share/cubicweb/migration/' % root else: MIGRATION_DIR = '/usr/share/cubicweb/migration/' - if not exists(REGISTRY_DIR): - os.makedirs(REGISTRY_DIR) else: if CubicWebNoAppConfiguration.mode == 'user': REGISTRY_DIR = expanduser('~/etc/cubicweb.d/') @@ -862,11 +880,15 @@ if exists(sitefile) and not sitefile in self._site_loaded: self._load_site_cubicweb(sitefile) self._site_loaded.add(sitefile) - self.warning('site_erudi.py is deprecated, should be renamed to site_cubicweb.py') + self.warning('[3.5] site_erudi.py is deprecated, should be renamed to site_cubicweb.py') def _load_site_cubicweb(self, sitefile): - from logilab.common.modutils import load_module_from_file - module = load_module_from_file(sitefile) + # XXX extrapath argument to load_module_from_file only in lgc > 0.46 + from logilab.common.modutils import load_module_from_modpath, modpath_from_file + def load_module_from_file(filepath, path=None, use_sys=1, extrapath=None): + return load_module_from_modpath(modpath_from_file(filepath, extrapath), + path, use_sys) + module = load_module_from_file(sitefile, extrapath=self.extrapath) self.info('%s loaded', sitefile) # cube specific options if getattr(module, 'options', None): @@ -896,7 +918,7 @@ """return available translation for an instance, by looking for compiled catalog - take *args to be usable as a vocabulary method + take \*args to be usable as a vocabulary method """ from glob import glob yield 'en' # ensure 'en' is yielded even if no .mo found @@ -935,11 +957,11 @@ def migration_handler(self): """return a migration handler instance""" - from cubicweb.common.migration import MigrationHelper + from cubicweb.migration import MigrationHelper return MigrationHelper(self, verbosity=self.verbosity) def i18ncompile(self, langs=None): - from cubicweb.common import i18n + from cubicweb import i18n if langs is None: langs = self.available_languages() i18ndir = join(self.apphome, 'i18n') @@ -976,3 +998,61 @@ # alias to get a configuration instance from an instance id instance_configuration = CubicWebConfiguration.config_for application_configuration = deprecated('use instance_configuration')(instance_configuration) + + +_EXT_REGISTERED = False +def register_stored_procedures(): + from logilab.common.adbh import FunctionDescr + from rql.utils import register_function, iter_funcnode_variables + + global _EXT_REGISTERED + if _EXT_REGISTERED: + return + _EXT_REGISTERED = True + + class COMMA_JOIN(FunctionDescr): + supported_backends = ('postgres', 'sqlite',) + rtype = 'String' + + @classmethod + def st_description(cls, funcnode, mainindex, tr): + return ', '.join(sorted(term.get_description(mainindex, tr) + for term in iter_funcnode_variables(funcnode))) + + register_function(COMMA_JOIN) # XXX do not expose? + + + class CONCAT_STRINGS(COMMA_JOIN): + aggregat = True + + register_function(CONCAT_STRINGS) # XXX bw compat + + class GROUP_CONCAT(CONCAT_STRINGS): + supported_backends = ('mysql', 'postgres', 'sqlite',) + + register_function(GROUP_CONCAT) + + + class LIMIT_SIZE(FunctionDescr): + supported_backends = ('postgres', 'sqlite',) + rtype = 'String' + + @classmethod + def st_description(cls, funcnode, mainindex, tr): + return funcnode.children[0].get_description(mainindex, tr) + + register_function(LIMIT_SIZE) + + + class TEXT_LIMIT_SIZE(LIMIT_SIZE): + supported_backends = ('mysql', 'postgres', 'sqlite',) + + register_function(TEXT_LIMIT_SIZE) + + + + class FSPATH(FunctionDescr): + supported_backends = ('postgres', 'sqlite',) + rtype = 'Bytes' + + register_function(FSPATH) diff -r c25da7573ebd -r 02b52bf9f5f8 cwctl.py --- a/cwctl.py Fri Feb 12 15:18:00 2010 +0100 +++ b/cwctl.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,25 +1,35 @@ -"""%%prog %s [options] %s +"""the cubicweb-ctl tool, based on logilab.common.clcommands to +provide a pluggable commands system. + -CubicWeb main instances controller. +:organization: Logilab +:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -%s""" +""" +__docformat__ = "restructuredtext en" +# *ctl module should limit the number of import to be imported as quickly as +# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash +# completion). So import locally in command helpers. import sys from os import remove, listdir, system, pathsep try: from os import kill, getpgid except ImportError: - def kill(*args): pass - def getpgid(): pass + def kill(*args): + """win32 kill implementation""" + def getpgid(): + """win32 getpgid implementation""" from os.path import exists, join, isfile, isdir, dirname, abspath from logilab.common.clcommands import register_commands, pop_arg from logilab.common.shellutils import ASK -from cubicweb import ConfigurationError, ExecutionError, BadCommandUsage, underline_title +from cubicweb import ConfigurationError, ExecutionError, BadCommandUsage from cubicweb.cwconfig import CubicWebConfiguration as cwcfg, CWDEV, CONFIGURATIONS -from cubicweb.toolsutils import Command, main_run, rm, create_dir +from cubicweb.toolsutils import Command, main_run, rm, create_dir, underline_title def wait_process_end(pid, maxtry=10, waittime=1): """wait for a process to actually die""" @@ -158,6 +168,84 @@ # base commands ############################################################### +def version_strictly_lower(a, b): + from logilab.common.changelog import Version + if a: + a = Version(a) + if b: + b = Version(b) + return a < b + +def max_version(a, b): + from logilab.common.changelog import Version + return str(max(Version(a), Version(b))) + +class ConfigurationProblem(object): + """Each cube has its own list of dependencies on other cubes/versions. + + The ConfigurationProblem is used to record the loaded cubes, then to detect + inconsistencies in their dependencies. + + See configuration management on wikipedia for litterature. + """ + + def __init__(self): + self.cubes = {} + + def add_cube(self, name, info): + self.cubes[name] = info + + def solve(self): + self.warnings = [] + self.errors = [] + self.read_constraints() + for cube, versions in sorted(self.constraints.items()): + oper, version = None, None + # simplify constraints + if versions: + for constraint in versions: + op, ver = constraint + if oper is None: + oper = op + version = ver + elif op == '>=' and oper == '>=': + version = max_version(ver, version) + else: + print 'unable to handle this case', oper, version, op, ver + # "solve" constraint satisfaction problem + if cube not in self.cubes: + self.errors.append( ('add', cube, version) ) + elif versions: + lower_strict = version_strictly_lower(self.cubes[cube].version, version) + if oper in ('>=','='): + if lower_strict: + self.errors.append( ('update', cube, version) ) + else: + print 'unknown operator', oper + + def read_constraints(self): + self.constraints = {} + self.reverse_constraints = {} + for cube, info in self.cubes.items(): + if hasattr(info,'__depends_cubes__'): + use = info.__depends_cubes__ + if not isinstance(use, dict): + use = dict((key, None) for key in use) + self.warnings.append('cube %s should define __depends_cubes__ as a dict not a list') + else: + self.warnings.append('cube %s should define __depends_cubes__' % cube) + use = dict((key, None) for key in info.__use__) + for name, constraint in use.items(): + self.constraints.setdefault(name,set()) + if constraint: + try: + oper, version = constraint.split() + self.constraints[name].add( (oper, version) ) + except: + self.warnings.append('cube %s depends on %s but constraint badly formatted: %s' + % (cube, name, constraint)) + self.reverse_constraints.setdefault(name, set()).add(cube) + class ListCommand(Command): """List configurations, cubes and instances. @@ -185,6 +273,7 @@ continue print ' ', line print + cfgpb = ConfigurationProblem() try: cubesdir = pathsep.join(cwcfg.cubes_search_path()) namesize = max(len(x) for x in cwcfg.available_cubes()) @@ -200,6 +289,7 @@ try: tinfo = cwcfg.cube_pkginfo(cube) tversion = tinfo.version + cfgpb.add_cube(cube, tinfo) except ConfigurationError: tinfo = None tversion = '[missing cube information]' @@ -235,7 +325,21 @@ else: print 'No instance available in %s' % regdir print - + # configuration management problem solving + cfgpb.solve() + if cfgpb.warnings: + print 'Warnings:\n', '\n'.join('* '+txt for txt in cfgpb.warnings) + if cfgpb.errors: + print 'Errors:' + for op, cube, version in cfgpb.errors: + if op == 'add': + print '* cube', cube, + if version: + print ' version', version, + print 'is not installed, but required by %s' % ' '.join(cfgpb.reverse_constraints[cube]) + else: + print '* cube %s version %s is installed, but version %s is required by (%s)' % ( + cube, cfgpb.cubes[cube].version, version, ', '.join(cfgpb.reverse_constraints[cube])) class CreateInstanceCommand(Command): """Create an instance from a cube. This is an unified @@ -297,14 +401,18 @@ # create the registry directory for this instance print '\n'+underline_title('Creating the instance %s' % appid) create_dir(config.apphome) - # load site_cubicweb from the cubes dir (if any) - config.load_site_cubicweb() # cubicweb-ctl configuration print '\n'+underline_title('Configuring the instance (%s.conf)' % configname) config.input_config('main', self.config.config_level) # configuration'specific stuff print helper.bootstrap(cubes, self.config.config_level) + # input for cubes specific options + for section in set(sect.lower() for sect, opt, optdict in config.all_options() + if optdict.get('inputlevel') <= self.config.config_level): + if section not in ('main', 'email', 'pyro'): + print '\n' + underline_title('%s options' % section) + config.input_config(section, self.config.config_level) # write down configuration config.save() self._handle_win32(config, appid) @@ -312,7 +420,7 @@ # handle i18n files structure # in the first cube given print '-> preparing i18n catalogs' - from cubicweb.common import i18n + from cubicweb import i18n langs = [lang for lang, _ in i18n.available_catalogs(join(templdirs[0], 'i18n'))] errors = config.i18ncompile(langs) if errors: @@ -690,7 +798,7 @@ # * install new languages # * recompile catalogs # in the first componant given - from cubicweb.common import i18n + from cubicweb import i18n templdir = cwcfg.cube_dir(config.cubes()[0]) langs = [lang for lang, _ in i18n.available_catalogs(join(templdir, 'i18n'))] errors = config.i18ncompile(langs) @@ -883,7 +991,12 @@ def run(args): """command line tool""" cwcfg.load_cwctl_plugins() - main_run(args, __doc__) + main_run(args, """%%prog %s [options] %s + +The CubicWeb swiss-knife. + +%s""" +) if __name__ == '__main__': run(sys.argv[1:]) diff -r c25da7573ebd -r 02b52bf9f5f8 cwvreg.py --- a/cwvreg.py Fri Feb 12 15:18:00 2010 +0100 +++ b/cwvreg.py Wed Mar 24 10:23:31 2010 +0100 @@ -8,7 +8,7 @@ __docformat__ = "restructuredtext en" _ = unicode -from logilab.common.decorators import cached, clear_cache, monkeypatch +from logilab.common.decorators import cached, clear_cache from logilab.common.deprecation import deprecated from logilab.common.modutils import cleanup_sys_modules @@ -18,7 +18,7 @@ ObjectNotFound, NoSelectableObject, RegistryNotFound, RegistryOutOfDate, CW_EVENT_MANAGER, onevent) from cubicweb.utils import dump_class -from cubicweb.vregistry import VRegistry, Registry +from cubicweb.vregistry import VRegistry, Registry, class_regid from cubicweb.rtags import RTAGS @@ -58,14 +58,7 @@ def schema(self): return self.vreg.schema - def initialization_completed(self): - # call vreg_initialization_completed on appobjects and print - # registry content - for appobjects in self.itervalues(): - for appobject in appobjects: - # XXX kill vreg_initialization_completed - appobject.vreg_initialization_completed() - + @deprecated('[3.6] select object, then use obj.render()') def render(self, __oid, req, __fallback_oid=None, rset=None, initargs=None, **kwargs): """Select object with the given id (`__oid`) then render it. If the @@ -90,20 +83,22 @@ obj = self.select(__fallback_oid, req, rset=rset, **initargs) return obj.render(**kwargs) + @deprecated('[3.6] use select_or_none and test for obj.cw_propval("visible")') def select_vobject(self, oid, *args, **kwargs): - selected = self.select_object(oid, *args, **kwargs) - if selected and selected.propval('visible'): + selected = self.select_or_none(oid, *args, **kwargs) + if selected and selected.cw_propval('visible'): return selected return None - def possible_vobjects(self, *args, **kwargs): + def poss_visible_objects(self, *args, **kwargs): """return an ordered list of possible app objects in a given registry, supposing they support the 'visible' and 'order' properties (as most visualizable objects) """ return sorted([x for x in self.possible_objects(*args, **kwargs) - if x.propval('visible')], - key=lambda x: x.propval('order')) + if x.cw_propval('visible')], + key=lambda x: x.cw_propval('order')) + possible_vobjects = deprecated('[3.6] use poss_visible_objects()')(poss_visible_objects) VRegistry.REGISTRY_FACTORY[None] = CWRegistry @@ -117,17 +112,27 @@ super(ETypeRegistry, self).initialization_completed() # clear etype cache if you don't want to run into deep weirdness clear_cache(self, 'etype_class') + clear_cache(self, 'parent_classes') def register(self, obj, **kwargs): - oid = kwargs.get('oid') or obj.id + oid = kwargs.get('oid') or class_regid(obj) if oid != 'Any' and not oid in self.schema: self.error('don\'t register %s, %s type not defined in the ' - 'schema', obj, obj.id) + 'schema', obj, oid) return kwargs['clear'] = True super(ETypeRegistry, self).register(obj, **kwargs) @cached + def parent_classes(self, etype): + if etype == 'Any': + return [self.etype_class('Any')] + eschema = self.schema.eschema(etype) + parents = [self.etype_class(e.type) for e in eschema.ancestors()] + parents.append(self.etype_class('Any')) + return parents + + @cached def etype_class(self, etype): """return an entity class for the given entity type. @@ -138,42 +143,42 @@ """ etype = str(etype) if etype == 'Any': - return self.select('Any', 'Any') + objects = self['Any'] + assert len(objects) == 1, objects + return objects[0] eschema = self.schema.eschema(etype) baseschemas = [eschema] + eschema.ancestors() # browse ancestors from most specific to most generic and try to find an # associated custom entity class - cls = None for baseschema in baseschemas: try: btype = ETYPE_NAME_MAP[baseschema] except KeyError: btype = str(baseschema) - if cls is None: - try: - objects = self[btype] - assert len(objects) == 1, objects - if btype == etype: - cls = objects[0] - else: - cls = self.etype_class(btype) - except ObjectNotFound: - continue - else: - # ensure parent classes are built first - self.etype_class(btype) - if cls is None: + try: + objects = self[btype] + assert len(objects) == 1, objects + if btype == etype: + cls = objects[0] + else: + # recurse to ensure issubclass(etype_class('Child'), + # etype_class('Parent')) + cls = self.etype_class(btype) + break + except ObjectNotFound: + pass + else: # no entity class for any of the ancestors, fallback to the default # one objects = self['Any'] assert len(objects) == 1, objects cls = objects[0] - # make a copy event if cls.id == etype, else we may have pb for client - # application using multiple connections to different repositories (eg - # shingouz) + # make a copy event if cls.__regid__ == etype, else we may have pb for + # client application using multiple connections to different + # repositories (eg shingouz) cls = dump_class(cls, etype) - cls.id = etype - cls.__initialize__() + cls.__regid__ = etype + cls.__initialize__(self.schema) return cls VRegistry.REGISTRY_FACTORY['etypes'] = ETypeRegistry @@ -181,12 +186,13 @@ class ViewsRegistry(CWRegistry): - def main_template(self, req, oid='main-template', **kwargs): + def main_template(self, req, oid='main-template', rset=None, **kwargs): """display query by calling the given template (default to main), and returning the output as a string instead of requiring the [w]rite method as argument """ - res = self.render(oid, req, **kwargs) + obj = self.select(oid, req, rset=rset, **kwargs) + res = obj.render(**kwargs) if isinstance(res, unicode): return res.encode(req.encoding) assert isinstance(res, str) @@ -201,7 +207,7 @@ if vid[0] == '_': continue try: - view = self.select_best(views, req, rset=rset, **kwargs) + view = self._select_best(views, req, rset=rset, **kwargs) if view.linkable(): yield view except NoSelectableObject: @@ -217,7 +223,7 @@ def possible_actions(self, req, rset=None, **kwargs): if rset is None: - actions = self.possible_vobjects(req, rset=rset, **kwargs) + actions = self.poss_visible_objects(req, rset=rset, **kwargs) else: actions = rset.possible_actions(**kwargs) # cached implementation result = {} @@ -334,7 +340,7 @@ def register_if_interface_found(self, obj, ifaces, **kwargs): """register an object but remove it if no entity class implements one of - the given interfaces + the given interfaces at the end of the registration process """ self.register(obj, **kwargs) if not isinstance(ifaces, (tuple, list)): @@ -354,35 +360,23 @@ if force_reload is None: force_reload = self.config.debugmode try: - self._register_objects(path, force_reload) + super(CubicWebVRegistry, self).register_objects( + path, force_reload, self.config.extrapath) except RegistryOutOfDate: CW_EVENT_MANAGER.emit('before-registry-reload') # modification detected, reset and reload self.reset(path, force_reload) - self._register_objects(path, force_reload) + super(CubicWebVRegistry, self).register_objects( + path, force_reload, self.config.extrapath) CW_EVENT_MANAGER.emit('after-registry-reload') - def _register_objects(self, path, force_reload=None): - """overriden to remove objects requiring a missing interface""" - extrapath = {} - for cubesdir in self.config.cubes_search_path(): - if cubesdir != self.config.CUBES_DIR: - extrapath[cubesdir] = 'cubes' - if super(CubicWebVRegistry, self).register_objects(path, force_reload, - extrapath): - self.initialization_completed() - # don't check rtags if we don't want to cleanup_interface_sobjects - for rtag in RTAGS: - rtag.init(self.schema, - check=self.config.cleanup_interface_sobjects) + def initialization_completed(self): + """cw specific code once vreg initialization is completed: - def initialization_completed(self): - for regname, reg in self.iteritems(): - self.debug('available in registry %s: %s', regname, sorted(reg)) - reg.initialization_completed() - for appobjects in reg.itervalues(): - for appobjectcls in appobjects: - appobjectcls.register_properties() + * remove objects requiring a missing interface, unless + config.cleanup_interface_sobjects is false + * init rtags + """ # we may want to keep interface dependent objects (e.g.for i18n # catalog generation) if self.config.cleanup_interface_sobjects: @@ -409,6 +403,11 @@ # clear needs_iface so we don't try to remove some not-anymore-in # objects on automatic reloading self._needs_iface.clear() + super(CubicWebVRegistry, self).initialization_completed() + for rtag in RTAGS: + # don't check rtags if we don't want to cleanup_interface_sobjects + rtag.init(self.schema, check=self.config.cleanup_interface_sobjects) + # rql parsing utilities #################################################### @@ -469,7 +468,7 @@ try: return self['propertyvalues'][key] except KeyError: - return self['propertydefs'][key]['default'] + return self.property_info(key)['default'] def typed_value(self, key, value): """value is an unicode string, return it correctly typed. Let potential diff -r c25da7573ebd -r 02b52bf9f5f8 dbapi.py --- a/dbapi.py Fri Feb 12 15:18:00 2010 +0100 +++ b/dbapi.py Wed Mar 24 10:23:31 2010 +0100 @@ -19,22 +19,18 @@ from logilab.common.decorators import monkeypatch from logilab.common.deprecation import deprecated -from cubicweb import ETYPE_NAME_MAP, ConnectionError, RequestSessionMixIn -from cubicweb import cwvreg, cwconfig +from cubicweb import ETYPE_NAME_MAP, ConnectionError, cwvreg, cwconfig +from cubicweb.req import RequestSessionBase + _MARKER = object() def _fake_property_value(self, name): try: - return super(dbapi.DBAPIRequest, self).property_value(name) + return super(DBAPIRequest, self).property_value(name) except KeyError: return '' -def _fix_cls_attrs(reg, appobject): - appobject.vreg = reg.vreg - appobject.schema = reg.schema - appobject.config = reg.config - def multiple_connections_fix(): """some monkey patching necessary when an application has to deal with several connections to different repositories. It tries to hide buggy class @@ -42,21 +38,6 @@ registries. """ defaultcls = cwvreg.VRegistry.REGISTRY_FACTORY[None] - orig_select_best = defaultcls.orig_select_best = defaultcls.select_best - @monkeypatch(defaultcls) - def select_best(self, appobjects, *args, **kwargs): - """return an instance of the most specific object according - to parameters - - raise NoSelectableObject if no object apply - """ - for appobjectcls in appobjects: - _fix_cls_attrs(self, appobjectcls) - selected = orig_select_best(self, appobjects, *args, **kwargs) - # redo the same thing on the instance so it won't use equivalent class - # attributes (which may change) - _fix_cls_attrs(self, selected) - return selected etypescls = cwvreg.VRegistry.REGISTRY_FACTORY['etypes'] orig_etype_class = etypescls.orig_etype_class = etypescls.etype_class @@ -73,8 +54,6 @@ return usercls def multiple_connections_unfix(): - defaultcls = cwvreg.VRegistry.REGISTRY_FACTORY[None] - defaultcls.select_best = defaultcls.orig_select_best etypescls = cwvreg.VRegistry.REGISTRY_FACTORY['etypes'] etypescls.etype_class = etypescls.orig_etype_class @@ -110,20 +89,21 @@ except Exception, ex: raise ConnectionError(str(ex)) -def repo_connect(repo, login, password, cnxprops=None): +def repo_connect(repo, login, **kwargs): """Constructor to create a new connection to the CubicWeb repository. Returns a Connection instance. """ - cnxprops = cnxprops or ConnectionProperties('inmemory') - cnxid = repo.connect(unicode(login), password, cnxprops=cnxprops) - cnx = Connection(repo, cnxid, cnxprops) - if cnxprops.cnxtype == 'inmemory': + if not 'cnxprops' in kwargs: + kwargs['cnxprops'] = ConnectionProperties('inmemory') + cnxid = repo.connect(unicode(login), **kwargs) + cnx = Connection(repo, cnxid, kwargs['cnxprops']) + if kwargs['cnxprops'].cnxtype == 'inmemory': cnx.vreg = repo.vreg return cnx -def connect(database=None, login=None, password=None, host=None, group=None, - cnxprops=None, setvreg=True, mulcnx=True, initlog=True): +def connect(database=None, login=None, host=None, group=None, + cnxprops=None, setvreg=True, mulcnx=True, initlog=True, **kwargs): """Constructor for creating a connection to the CubicWeb repository. Returns a Connection object. @@ -153,11 +133,11 @@ vreg.set_schema(schema) else: vreg = None - cnx = repo_connect(repo, login, password, cnxprops) + cnx = repo_connect(repo, login, cnxprops=cnxprops, **kwargs) cnx.vreg = vreg return cnx -def in_memory_cnx(config, login, password): +def in_memory_cnx(config, login, **kwargs): """usefull method for testing and scripting to get a dbapi.Connection object connected to an in-memory repository instance """ @@ -170,11 +150,11 @@ repo = get_repository('inmemory', config=config, vreg=vreg) # connection to the CubicWeb repository cnxprops = ConnectionProperties('inmemory') - cnx = repo_connect(repo, login, password, cnxprops=cnxprops) + cnx = repo_connect(repo, login, cnxprops=cnxprops, **kwargs) return repo, cnx -class DBAPIRequest(RequestSessionMixIn): +class DBAPIRequest(RequestSessionBase): def __init__(self, vreg, cnx=None): super(DBAPIRequest, self).__init__(vreg) @@ -220,7 +200,7 @@ except KeyError: # this occurs usually during test execution self._ = self.__ = unicode - self.pgettext = lambda x,y: y + self.pgettext = lambda x, y: y self.debug('request default language: %s', self.lang) def decorate_rset(self, rset): @@ -261,6 +241,8 @@ def get_session_data(self, key, default=None, pop=False): """return value associated to `key` in session data""" + if self.cnx is None: + return None # before the connection has been established return self.cnx.get_session_data(key, default, pop) def set_session_data(self, key, value): diff -r c25da7573ebd -r 02b52bf9f5f8 debian.hardy/rules --- a/debian.hardy/rules Fri Feb 12 15:18:00 2010 +0100 +++ b/debian.hardy/rules Wed Mar 24 10:23:31 2010 +0100 @@ -33,7 +33,6 @@ dh_clean dh_installdirs - #python setup.py install_lib --no-compile --install-dir=debian/cubicweb-common/usr/lib/python2.4/site-packages/ python setup.py -q install --no-compile --prefix=debian/tmp/usr # Put all the python library and data in cubicweb-common @@ -41,17 +40,15 @@ dh_install -vi # cwctl in the cubicweb-ctl package rm -f debian/cubicweb-common/usr/share/pyshared/cubicweb/cwctl.py - # hercule in the cubicweb-client package - rm -f debian/cubicweb-common/usr/share/pyshared/cubicweb/hercule.py # Remove unittests directory (should be available in cubicweb-dev only) rm -rf debian/cubicweb-server/usr/lib/${PY_VERSION}/site-packages/cubicweb/server/test + rm -rf debian/cubicweb-server/usr/lib/${PY_VERSION}/site-packages/cubicweb/hooks/test rm -rf debian/cubicweb-server/usr/lib/${PY_VERSION}/site-packages/cubicweb/sobjects/test rm -rf debian/cubicweb-web/usr/lib/${PY_VERSION}/site-packages/cubicweb/web/test rm -rf debian/cubicweb-twisted/usr/lib/${PY_VERSION}/site-packages/cubicweb/etwist/test rm -rf debian/cubicweb-common/usr/lib/${PY_VERSION}/site-packages/cubicweb/ext/test - rm -rf debian/cubicweb-common/usr/lib/${PY_VERSION}/site-packages/cubicweb/common/test rm -rf debian/cubicweb-common/usr/lib/${PY_VERSION}/site-packages/cubicweb/entities/test # cubes directory must be managed as a valid python module diff -r c25da7573ebd -r 02b52bf9f5f8 debian/changelog --- a/debian/changelog Fri Feb 12 15:18:00 2010 +0100 +++ b/debian/changelog Wed Mar 24 10:23:31 2010 +0100 @@ -1,3 +1,37 @@ +cubicweb (3.6.3-1) unstable; urgency=low + + * remove postgresql-contrib from cubicweb dependency (using tsearch + which is included with postgres >= 8.3) + * add postgresql-client | mysql-client to cubicweb-server dependencies using two + new cubicweb-[postgresql|mysql]-support virtual packages (necessary for + dump/restore of database) + + -- Sylvain Thénault Wed, 24 Mar 2010 07:50:47 +0100 + +cubicweb (3.6.2-1) unstable; urgency=low + + * new upstream release + + -- Sylvain Thénault Thu, 11 Mar 2010 19:49:04 +0100 + +cubicweb (3.6.1-2) unstable; urgency=low + + * remove depends to python-elementtree (included in python>=2.5) + + -- Pierre-Yves David Fri, 04 Mar 2010 14:43:01 +0100 + +cubicweb (3.6.1-1) unstable; urgency=low + + * new upstream release + + -- Sylvain Thénault Fri, 26 Feb 2010 14:14:01 +0100 + +cubicweb (3.6.0-1) unstable; urgency=low + + * new upstream release + + -- Sylvain Thénault Wed, 10 Feb 2010 09:44:58 +0100 + cubicweb (3.5.12-1) unstable; urgency=low * new upstream release diff -r c25da7573ebd -r 02b52bf9f5f8 debian/control --- a/debian/control Fri Feb 12 15:18:00 2010 +0100 +++ b/debian/control Wed Mar 24 10:23:31 2010 +0100 @@ -15,8 +15,8 @@ Package: cubicweb Architecture: all XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, cubicweb-server (= ${source:Version}), cubicweb-twisted (= ${source:Version}), cubicweb-client (= ${source:Version}) -XB-Recommends: (postgresql, postgresql-plpython, postgresql-contrib) | mysql | sqlite3 +Depends: ${python:Depends}, cubicweb-server (= ${source:Version}), cubicweb-twisted (= ${source:Version}) +XB-Recommends: (postgresql, postgresql-plpython) | mysql | sqlite3 Recommends: postgresql | mysql | sqlite3 Description: the complete CubicWeb framework CubicWeb is a semantic web application framework. @@ -33,7 +33,8 @@ Conflicts: cubicweb-multisources Replaces: cubicweb-multisources Provides: cubicweb-multisources -Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-indexer (>= 0.6.1), python-psycopg2 | python-mysqldb | python-pysqlite2 +# postgresql/mysql -client packages for backup/restore of non local database +Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-indexer (>= 0.6.1), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2 Recommends: pyro, cubicweb-documentation (= ${source:Version}) Description: server part of the CubicWeb framework CubicWeb is a semantic web application framework. @@ -43,6 +44,26 @@ This package provides the repository server part of the library and necessary shared data files such as the schema library. +Package: cubicweb-postgresql-support +Architecture: all +# postgresql-client packages for backup/restore of non local database +Depends: python-psycopg2, postgresql-client +Description: postgres support for the CubicWeb framework + CubicWeb is a semantic web application framework. + . + This virtual package provides dependancies to use postgres for the + cubicweb repository. + +Package: cubicweb-mysql-support +Architecture: all +# mysql-client packages for backup/restore of non local database +Depends: python-mysqldb, mysql-client +Description: mysql support for the CubicWeb framework + CubicWeb is a semantic web application framework. + . + This virtual package provides dependancies to use mysql for the + cubicweb repository. + Package: cubicweb-twisted Architecture: all @@ -62,8 +83,8 @@ Package: cubicweb-web Architecture: all XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), python-simplejson (>= 1.3), python-elementtree -Recommends: python-docutils, python-vobject, fckeditor, python-fyzz, python-pysixt, fop +Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), python-simplejson (>= 1.3) +Recommends: python-docutils, python-vobject, fckeditor, python-fyzz, python-pysixt, fop, python-imaging Description: web interface library for the CubicWeb framework CubicWeb is a semantic web application framework. . @@ -77,8 +98,8 @@ Package: cubicweb-common Architecture: all XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, graphviz, gettext, python-logilab-mtconverter (>= 0.6.0), python-logilab-common (>= 0.46.0), python-yams (>= 0.25.0), python-rql (>= 0.22.3), python-lxml -Recommends: python-simpletal (>= 4.0) +Depends: ${python:Depends}, graphviz, gettext, python-logilab-mtconverter (>= 0.6.0), python-logilab-common (>= 0.48.1), python-yams (>= 0.28.0), python-rql (>= 0.24.0), python-lxml +Recommends: python-simpletal (>= 4.0), python-crypto Conflicts: cubicweb-core Replaces: cubicweb-core Description: common library for the CubicWeb framework @@ -100,17 +121,6 @@ to automatically start and stop CubicWeb applications on boot or shutdown. -Package: cubicweb-client -Architecture: all -XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, cubicweb-ctl (= ${source:Version}), pyro -Description: RQL command line client for the CubicWeb framework - CubicWeb is a semantic web application framework. - . - This package provides a RQL (Relation Query Language) command line client using - pyro to connect to a repository server. - - Package: cubicweb-dev Architecture: all XB-Python-Version: ${python:Versions} diff -r c25da7573ebd -r 02b52bf9f5f8 debian/cubicweb-client.install.in --- a/debian/cubicweb-client.install.in Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/hercule.py usr/lib/PY_VERSION/site-packages/cubicweb diff -r c25da7573ebd -r 02b52bf9f5f8 debian/cubicweb-dev.install.in --- a/debian/cubicweb-dev.install.in Fri Feb 12 15:18:00 2010 +0100 +++ b/debian/cubicweb-dev.install.in Wed Mar 24 10:23:31 2010 +0100 @@ -1,11 +1,11 @@ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/devtools/ usr/lib/PY_VERSION/site-packages/cubicweb/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/skeleton/ usr/lib/PY_VERSION/site-packages/cubicweb/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/test usr/lib/PY_VERSION/site-packages/cubicweb/ -debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/common/test usr/lib/PY_VERSION/site-packages/cubicweb/common/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/entities/test usr/lib/PY_VERSION/site-packages/cubicweb/entities/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/ext/test usr/lib/PY_VERSION/site-packages/cubicweb/ext/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/server/test usr/lib/PY_VERSION/site-packages/cubicweb/server/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/sobjects/test usr/lib/PY_VERSION/site-packages/cubicweb/sobjects/ +debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/hooks/test usr/lib/PY_VERSION/site-packages/cubicweb/sobjects/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/web/test usr/lib/PY_VERSION/site-packages/cubicweb/web/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/etwist/test usr/lib/PY_VERSION/site-packages/cubicweb/etwist/ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/goa/test usr/lib/PY_VERSION/site-packages/cubicweb/goa/ diff -r c25da7573ebd -r 02b52bf9f5f8 debian/cubicweb-server.install.in --- a/debian/cubicweb-server.install.in Fri Feb 12 15:18:00 2010 +0100 +++ b/debian/cubicweb-server.install.in Wed Mar 24 10:23:31 2010 +0100 @@ -1,4 +1,5 @@ debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/server/ usr/lib/PY_VERSION/site-packages/cubicweb +debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/hooks/ usr/lib/PY_VERSION/site-packages/cubicweb debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/sobjects/ usr/lib/PY_VERSION/site-packages/cubicweb debian/tmp/usr/lib/PY_VERSION/site-packages/cubicweb/schemas/ usr/lib/PY_VERSION/site-packages/cubicweb debian/tmp/usr/share/cubicweb/migration/ usr/share/cubicweb/ diff -r c25da7573ebd -r 02b52bf9f5f8 debian/rules --- a/debian/rules Fri Feb 12 15:18:00 2010 +0100 +++ b/debian/rules Wed Mar 24 10:23:31 2010 +0100 @@ -41,18 +41,16 @@ dh_install -vi # cwctl in the cubicweb-ctl package rm -f debian/cubicweb-common/usr/share/pyshared/cubicweb/cwctl.py - # hercule in the cubicweb-client package - rm -f debian/cubicweb-common/usr/share/pyshared/cubicweb/hercule.py dh_lintian # Remove unittests directory (should be available in cubicweb-dev only) rm -rf debian/cubicweb-server/usr/lib/${PY_VERSION}/site-packages/cubicweb/server/test + rm -rf debian/cubicweb-server/usr/lib/${PY_VERSION}/site-packages/cubicweb/hooks/test rm -rf debian/cubicweb-server/usr/lib/${PY_VERSION}/site-packages/cubicweb/sobjects/test rm -rf debian/cubicweb-web/usr/lib/${PY_VERSION}/site-packages/cubicweb/web/test rm -rf debian/cubicweb-twisted/usr/lib/${PY_VERSION}/site-packages/cubicweb/etwist/test rm -rf debian/cubicweb-common/usr/lib/${PY_VERSION}/site-packages/cubicweb/ext/test - rm -rf debian/cubicweb-common/usr/lib/${PY_VERSION}/site-packages/cubicweb/common/test rm -rf debian/cubicweb-common/usr/lib/${PY_VERSION}/site-packages/cubicweb/entities/test # cubes directory must be managed as a valid python module diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/__init__.py --- a/devtools/__init__.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/__init__.py Wed Mar 24 10:23:31 2010 +0100 @@ -13,20 +13,56 @@ from os.path import (abspath, join, exists, basename, dirname, normpath, split, isfile, isabs) -from cubicweb import CW_SOFTWARE_ROOT, ConfigurationError -from cubicweb.utils import strptime -from cubicweb.toolsutils import read_config -from cubicweb.cwconfig import CubicWebConfiguration, merge_options +from logilab.common.date import strptime +from cubicweb import CW_SOFTWARE_ROOT, ConfigurationError, schema, cwconfig from cubicweb.server.serverconfig import ServerConfiguration from cubicweb.etwist.twconfig import TwistedConfiguration +cwconfig.CubicWebConfiguration.cls_adjust_sys_path() + +# db auto-population configuration ############################################# + +SYSTEM_ENTITIES = schema.SCHEMA_TYPES | set(( + 'CWGroup', 'CWUser', 'CWProperty', + 'Workflow', 'State', 'BaseTransition', 'Transition', 'WorkflowTransition', + 'TrInfo', 'SubWorkflowExitPoint', + )) + +SYSTEM_RELATIONS = schema.META_RTYPES | set(( + # workflow related + 'workflow_of', 'state_of', 'transition_of', 'initial_state', 'default_workflow', + 'allowed_transition', 'destination_state', 'from_state', 'to_state', + 'condition', 'subworkflow', 'subworkflow_state', 'subworkflow_exit', + 'custom_workflow', 'in_state', 'wf_info_for', + # cwproperty + 'for_user', + # schema definition + 'specializes', + 'relation_type', 'from_entity', 'to_entity', + 'constrained_by', 'cstrtype', 'widget', + 'read_permission', 'update_permission', 'delete_permission', 'add_permission', + # permission + 'in_group', 'require_group', 'require_permission', + # deducted from other relations + 'primary_email', + )) + +# content validation configuration ############################################# + # validators are used to validate (XML, DTD, whatever) view's content # validators availables are : # 'dtd' : validates XML + declared DTD # 'xml' : guarantees XML is well formed # None : do not try to validate anything + +# {'vid': validator} VIEW_VALIDATORS = {} + + +# cubicweb test configuration ################################################## + BASE_URL = 'http://testing.fr/cubicweb/' + DEFAULT_SOURCES = {'system': {'adapter' : 'native', 'db-encoding' : 'UTF-8', #'ISO-8859-1', 'db-user' : u'admin', @@ -40,13 +76,14 @@ }, } + class TestServerConfiguration(ServerConfiguration): mode = 'test' set_language = False read_instance_schema = False bootstrap_schema = False init_repository = True - options = merge_options(ServerConfiguration.options + ( + options = cwconfig.merge_options(ServerConfiguration.options + ( ('anonymous-user', {'type' : 'string', 'default': None, @@ -66,10 +103,11 @@ def __init__(self, appid, log_threshold=logging.CRITICAL+10): ServerConfiguration.__init__(self, appid) - self.global_set_option('log-file', None) self.init_log(log_threshold, force=True) # need this, usually triggered by cubicweb-ctl self.load_cwctl_plugins() + self.global_set_option('anonymous-user', 'anon') + self.global_set_option('anonymous-password', 'anon') anonymous_user = TwistedConfiguration.anonymous_user.im_func @@ -81,6 +119,11 @@ return abspath('..') appdatahome = apphome + def load_configuration(self): + super(TestServerConfiguration, self).load_configuration() + self.global_set_option('anonymous-user', 'anon') + self.global_set_option('anonymous-password', 'anon') + def main_config_file(self): """return instance's control configuration file""" return join(self.apphome, '%s.conf' % self.name) @@ -118,30 +161,11 @@ sources = DEFAULT_SOURCES return sources - def load_defaults(self): - super(TestServerConfiguration, self).load_defaults() - # note: don't call global set option here, OptionManager may not yet be initialized - # add anonymous user - self.set_option('anonymous-user', 'anon') - self.set_option('anonymous-password', 'anon') - # uncomment the line below if you want rql queries to be logged - #self.set_option('query-log-file', '/tmp/test_rql_log.' + `os.getpid()`) - self.set_option('sender-name', 'cubicweb-test') - self.set_option('sender-addr', 'cubicweb-test@logilab.fr') - try: - send_to = '%s@logilab.fr' % os.getlogin() - except (OSError, AttributeError): - send_to = '%s@logilab.fr' % (os.environ.get('USER') - or os.environ.get('USERNAME') - or os.environ.get('LOGNAME')) - self.set_option('sender-addr', send_to) - self.set_option('default-dest-addrs', send_to) - self.set_option('base-url', BASE_URL) - class BaseApptestConfiguration(TestServerConfiguration, TwistedConfiguration): repo_method = 'inmemory' - options = merge_options(TestServerConfiguration.options + TwistedConfiguration.options) + options = cwconfig.merge_options(TestServerConfiguration.options + + TwistedConfiguration.options) cubicweb_appobject_path = TestServerConfiguration.cubicweb_appobject_path | TwistedConfiguration.cubicweb_appobject_path cube_appobject_path = TestServerConfiguration.cube_appobject_path | TwistedConfiguration.cube_appobject_path @@ -163,98 +187,84 @@ BaseApptestConfiguration.__init__(self, appid, log_threshold=log_threshold) self.init_repository = sourcefile is None self.sourcefile = sourcefile - import re - self.global_set_option('embed-allowed', re.compile('.*')) -class RealDatabaseConfiguration(ApptestConfiguration): - init_repository = False - sourcesdef = {'system': {'adapter' : 'native', - 'db-encoding' : 'UTF-8', #'ISO-8859-1', - 'db-user' : u'admin', - 'db-password' : 'gingkow', - 'db-name' : 'seotest', - 'db-driver' : 'postgres', - 'db-host' : None, - }, - 'admin' : {'login': u'admin', - 'password': u'gingkow', - }, - } +# test database handling ####################################################### - def __init__(self, appid, log_threshold=logging.CRITICAL, sourcefile=None): - ApptestConfiguration.__init__(self, appid) - self.init_repository = False +def init_test_database(config=None, configdir='data'): + """init a test database for a specific driver""" + from cubicweb.dbapi import in_memory_cnx + config = config or TestServerConfiguration(configdir) + sources = config.sources() + driver = sources['system']['db-driver'] + if driver == 'sqlite': + init_test_database_sqlite(config) + elif driver == 'postgres': + init_test_database_postgres(config) + else: + raise ValueError('no initialization function for driver %r' % driver) + config._cubes = None # avoid assertion error + repo, cnx = in_memory_cnx(config, unicode(sources['admin']['login']), + password=sources['admin']['password'] or 'xxx') + if driver == 'sqlite': + install_sqlite_patch(repo.querier) + return repo, cnx - def sources(self): - """ - By default, we run tests with the sqlite DB backend. - One may use its own configuration by just creating a - 'sources' file in the test directory from wich tests are - launched. - """ - self._sources = self.sourcesdef - return self._sources +def reset_test_database(config): + """init a test database for a specific driver""" + driver = config.sources()['system']['db-driver'] + if driver == 'sqlite': + reset_test_database_sqlite(config) + else: + raise ValueError('no reset function for driver %r' % driver) -def buildconfig(dbuser, dbpassword, dbname, adminuser, adminpassword, dbhost=None): - """convenience function that builds a real-db configuration class""" - sourcesdef = {'system': {'adapter' : 'native', - 'db-encoding' : 'UTF-8', #'ISO-8859-1', - 'db-user' : dbuser, - 'db-password' : dbpassword, - 'db-name' : dbname, - 'db-driver' : 'postgres', - 'db-host' : dbhost, - }, - 'admin' : {'login': adminuser, - 'password': adminpassword, - }, - } - return type('MyRealDBConfig', (RealDatabaseConfiguration,), - {'sourcesdef': sourcesdef}) +### postgres test database handling ############################################ -def loadconfig(filename): - """convenience function that builds a real-db configuration class - from a file - """ - return type('MyRealDBConfig', (RealDatabaseConfiguration,), - {'sourcesdef': read_config(filename)}) +def init_test_database_postgres(config): + """initialize a fresh sqlite databse used for testing purpose""" + if config.init_repository: + from cubicweb.server import init_repository + init_repository(config, interactive=False, drop=True) -class LivetestConfiguration(BaseApptestConfiguration): - init_repository = False +### sqlite test database handling ############################################## + +def cleanup_sqlite(dbfile, removetemplate=False): + try: + os.remove(dbfile) + os.remove('%s-journal' % dbfile) + except OSError: + pass + if removetemplate: + try: + os.remove('%s-template' % dbfile) + except OSError: + pass - def __init__(self, cube=None, sourcefile=None, pyro_name=None, - log_threshold=logging.CRITICAL): - TestServerConfiguration.__init__(self, cube, log_threshold=log_threshold) - self.appid = pyro_name or cube - # don't change this, else some symlink problems may arise in some - # environment (e.g. mine (syt) ;o) - # XXX I'm afraid this test will prevent to run test from a production - # environment - self._sources = None - # instance cube test - if cube is not None: - self.apphome = self.cube_dir(cube) - elif 'web' in os.getcwd().split(os.sep): - # web test - self.apphome = join(normpath(join(dirname(__file__), '..')), 'web') - else: - # cube test - self.apphome = abspath('..') - self.sourcefile = sourcefile - self.global_set_option('realm', '') - self.use_pyro = pyro_name is not None +def reset_test_database_sqlite(config): + import shutil + dbfile = config.sources()['system']['db-name'] + cleanup_sqlite(dbfile) + template = '%s-template' % dbfile + if exists(template): + shutil.copy(template, dbfile) + return True + return False - def pyro_enabled(self): - if self.use_pyro: - return True - else: - return False +def init_test_database_sqlite(config): + """initialize a fresh sqlite databse used for testing purpose""" + # remove database file if it exists + dbfile = config.sources()['system']['db-name'] + if not reset_test_database_sqlite(config): + # initialize the database + import shutil + from cubicweb.server import init_repository + init_repository(config, interactive=False) + dbfile = config.sources()['system']['db-name'] + shutil.copy(dbfile, '%s-template' % dbfile) -CubicWebConfiguration.cls_adjust_sys_path() def install_sqlite_patch(querier): """This patch hotfixes the following sqlite bug : @@ -293,58 +303,3 @@ return new_execute querier.__class__.execute = wrap_execute(querier.__class__.execute) querier.__class__._devtools_sqlite_patched = True - -def init_test_database(driver='sqlite', configdir='data', config=None, - vreg=None): - """init a test database for a specific driver""" - from cubicweb.dbapi import in_memory_cnx - if vreg and not config: - config = vreg.config - config = config or TestServerConfiguration(configdir) - source = config.sources() - if driver == 'sqlite': - init_test_database_sqlite(config, source) - elif driver == 'postgres': - init_test_database_postgres(config, source) - else: - raise ValueError('no initialization function for driver %r' % driver) - config._cubes = None # avoid assertion error - repo, cnx = in_memory_cnx(vreg or config, unicode(source['admin']['login']), - source['admin']['password'] or 'xxx') - if driver == 'sqlite': - install_sqlite_patch(repo.querier) - return repo, cnx - -def init_test_database_postgres(config, source, vreg=None): - """initialize a fresh sqlite databse used for testing purpose""" - if config.init_repository: - from cubicweb.server import init_repository - init_repository(config, interactive=False, drop=True, vreg=vreg) - -def cleanup_sqlite(dbfile, removetemplate=False): - try: - os.remove(dbfile) - os.remove('%s-journal' % dbfile) - except OSError: - pass - if removetemplate: - try: - os.remove('%s-template' % dbfile) - except OSError: - pass - -def init_test_database_sqlite(config, source, vreg=None): - """initialize a fresh sqlite databse used for testing purpose""" - import shutil - # remove database file if it exists (actually I know driver == 'sqlite' :) - dbfile = source['system']['db-name'] - source['system']['db-name'] = os.path.abspath(dbfile) - cleanup_sqlite(dbfile) - template = '%s-template' % dbfile - if exists(template): - shutil.copy(template, dbfile) - else: - # initialize the database - from cubicweb.server import init_repository - init_repository(config, interactive=False, vreg=vreg) - shutil.copy(dbfile, template) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/_apptest.py --- a/devtools/_apptest.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,207 +0,0 @@ -"""Hidden internals for the devtools.apptest module - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -import sys, traceback - -from logilab.common.pytest import pause_tracing, resume_tracing - -import yams.schema - -from cubicweb.dbapi import repo_connect, ConnectionProperties, ProgrammingError -from cubicweb.cwvreg import CubicWebVRegistry - -from cubicweb.web.application import CubicWebPublisher -from cubicweb.web import Redirect - -from cubicweb.devtools import ApptestConfiguration, init_test_database -from cubicweb.devtools.fake import FakeRequest - -SYSTEM_ENTITIES = ('CWGroup', 'CWUser', - 'CWAttribute', 'CWRelation', - 'CWConstraint', 'CWConstraintType', 'CWProperty', - 'CWEType', 'CWRType', - 'Workflow', 'State', 'BaseTransition', 'Transition', 'WorkflowTransition', 'TrInfo', 'SubWorkflowExitPoint', - 'RQLExpression', - ) -SYSTEM_RELATIONS = ( - # virtual relation - 'identity', - # metadata - 'is', 'is_instance_of', 'owned_by', 'created_by', 'specializes', - # workflow related - 'workflow_of', 'state_of', 'transition_of', 'initial_state', 'allowed_transition', - 'destination_state', 'in_state', 'wf_info_for', 'from_state', 'to_state', - 'condition', 'subworkflow', 'subworkflow_state', 'subworkflow_exit', - # permission - 'in_group', 'require_group', 'require_permission', - 'read_permission', 'update_permission', 'delete_permission', 'add_permission', - # eproperty - 'for_user', - # schema definition - 'relation_type', 'from_entity', 'to_entity', - 'constrained_by', 'cstrtype', 'widget', - # deducted from other relations - 'primary_email', - ) - -def unprotected_entities(app_schema, strict=False): - """returned a Set of each non final entity type, excluding CWGroup, and CWUser... - """ - if strict: - protected_entities = yams.schema.BASE_TYPES - else: - protected_entities = yams.schema.BASE_TYPES.union(set(SYSTEM_ENTITIES)) - entities = set(app_schema.entities()) - return entities - protected_entities - - -def ignore_relations(*relations): - global SYSTEM_RELATIONS - SYSTEM_RELATIONS += relations - - -class TestEnvironment(object): - """TestEnvironment defines a context (e.g. a config + a given connection) in - which the tests are executed - """ - - def __init__(self, appid, reporter=None, verbose=False, - configcls=ApptestConfiguration, requestcls=FakeRequest): - config = configcls(appid) - self.requestcls = requestcls - self.cnx = None - config.db_perms = False - source = config.sources()['system'] - if verbose: - print "init test database ..." - self.vreg = vreg = CubicWebVRegistry(config) - self.admlogin = source['db-user'] - # restore database <=> init database - self.restore_database() - if verbose: - print "init done" - config.repository = lambda x=None: self.repo - self.app = CubicWebPublisher(config, vreg=vreg) - self.verbose = verbose - schema = self.vreg.schema - # else we may run into problems since email address are ususally share in app tests - # XXX should not be necessary anymore - schema.rschema('primary_email').set_rproperty('CWUser', 'EmailAddress', 'composite', False) - self.deletable_entities = unprotected_entities(schema) - - def restore_database(self): - """called by unittests' tearDown to restore the original database - """ - try: - pause_tracing() - if self.cnx: - self.cnx.close() - source = self.vreg.config.sources()['system'] - self.repo, self.cnx = init_test_database(driver=source['db-driver'], - vreg=self.vreg) - self._orig_cnx = self.cnx - resume_tracing() - except: - resume_tracing() - traceback.print_exc() - sys.exit(1) - # XXX cnx decoration is usually done by the repository authentication manager, - # necessary in authentication tests - self.cnx.vreg = self.vreg - self.cnx.login = source['db-user'] - self.cnx.password = source['db-password'] - - - def create_user(self, login, groups=('users',), req=None): - req = req or self.create_request() - cursor = self._orig_cnx.cursor(req) - rset = cursor.execute('INSERT CWUser X: X login %(login)s, X upassword %(passwd)s', - {'login': unicode(login), 'passwd': login.encode('utf8')}) - user = rset.get_entity(0, 0) - cursor.execute('SET X in_group G WHERE X eid %%(x)s, G name IN(%s)' - % ','.join(repr(g) for g in groups), - {'x': user.eid}, 'x') - user.clear_related_cache('in_group', 'subject') - self._orig_cnx.commit() - return user - - def login(self, login, password=None): - if login == self.admlogin: - self.restore_connection() - else: - self.cnx = repo_connect(self.repo, unicode(login), - password or str(login), - ConnectionProperties('inmemory')) - if login == self.vreg.config.anonymous_user()[0]: - self.cnx.anonymous_connection = True - return self.cnx - - def restore_connection(self): - if not self.cnx is self._orig_cnx: - try: - self.cnx.close() - except ProgrammingError: - pass # already closed - self.cnx = self._orig_cnx - - ############################################################################ - - def execute(self, rql, args=None, eidkey=None, req=None): - """executes , builds a resultset, and returns a couple (rset, req) - where req is a FakeRequest - """ - req = req or self.create_request(rql=rql) - return self.cnx.cursor(req).execute(unicode(rql), args, eidkey) - - def create_request(self, rql=None, **kwargs): - """executes , builds a resultset, and returns a - couple (rset, req) where req is a FakeRequest - """ - if rql: - kwargs['rql'] = rql - req = self.requestcls(self.vreg, form=kwargs) - req.set_connection(self.cnx) - return req - - def get_rset_and_req(self, rql, optional_args=None, args=None, eidkey=None): - """executes , builds a resultset, and returns a - couple (rset, req) where req is a FakeRequest - """ - return (self.execute(rql, args, eidkey), - self.create_request(rql=rql, **optional_args or {})) - - -class ExistingTestEnvironment(TestEnvironment): - - def __init__(self, appid, sourcefile, verbose=False): - config = ApptestConfiguration(appid, sourcefile=sourcefile) - if verbose: - print "init test database ..." - source = config.sources()['system'] - self.vreg = CubicWebVRegistry(config) - self.cnx = init_test_database(driver=source['db-driver'], - vreg=self.vreg)[1] - if verbose: - print "init done" - self.app = CubicWebPublisher(config, vreg=self.vreg) - self.verbose = verbose - # this is done when the publisher is opening a connection - self.cnx.vreg = self.vreg - - def setup(self, config=None): - """config is passed by TestSuite but is ignored in this environment""" - cursor = self.cnx.cursor() - self.last_eid = cursor.execute('Any X WHERE X creation_date D ORDERBY D DESC LIMIT 1').rows[0][0] - - def cleanup(self): - """cancel inserted elements during tests""" - cursor = self.cnx.cursor() - cursor.execute('DELETE Any X WHERE X eid > %(x)s', {'x' : self.last_eid}, eid_key='x') - print "cleaning done" - self.cnx.commit() diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/apptest.py --- a/devtools/apptest.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,522 +0,0 @@ -"""This module provides misc utilities to test instances - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -from copy import deepcopy - -import simplejson - -from logilab.common.testlib import TestCase -from logilab.common.pytest import nocoverage -from logilab.common.umessage import message_from_string - -from logilab.common.deprecation import deprecated - -from cubicweb.devtools import init_test_database, TestServerConfiguration, ApptestConfiguration -from cubicweb.devtools._apptest import TestEnvironment -from cubicweb.devtools.fake import FakeRequest - -from cubicweb.dbapi import repo_connect, ConnectionProperties, ProgrammingError - - -MAILBOX = [] -class Email: - def __init__(self, recipients, msg): - self.recipients = recipients - self.msg = msg - - @property - def message(self): - return message_from_string(self.msg) - - @property - def subject(self): - return self.message.get('Subject') - - @property - def content(self): - return self.message.get_payload(decode=True) - - def __repr__(self): - return '' % (','.join(self.recipients), - self.message.get('Subject')) - -class MockSMTP: - def __init__(self, server, port): - pass - def close(self): - pass - def sendmail(self, helo_addr, recipients, msg): - MAILBOX.append(Email(recipients, msg)) - -from cubicweb import cwconfig -cwconfig.SMTP = MockSMTP - - -def get_versions(self, checkversions=False): - """return the a dictionary containing cubes used by this instance - as key with their version as value, including cubicweb version. This is a - public method, not requiring a session id. - - replace Repository.get_versions by this method if you don't want versions - checking - """ - vcconf = {'cubicweb': self.config.cubicweb_version()} - self.config.bootstrap_cubes() - for pk in self.config.cubes(): - version = self.config.cube_version(pk) - vcconf[pk] = version - self.config._cubes = None - return vcconf - - -@property -def late_binding_env(self): - """builds TestEnvironment as late as possible""" - if not hasattr(self, '_env'): - self.__class__._env = TestEnvironment('data', configcls=self.configcls, - requestcls=self.requestcls) - return self._env - - -class autoenv(type): - """automatically set environment on EnvBasedTC subclasses if necessary - """ - def __new__(mcs, name, bases, classdict): - env = classdict.get('env') - # try to find env in one of the base classes - if env is None: - for base in bases: - env = getattr(base, 'env', None) - if env is not None: - classdict['env'] = env - break - if not classdict.get('__abstract__') and not classdict.get('env'): - classdict['env'] = late_binding_env - return super(autoenv, mcs).__new__(mcs, name, bases, classdict) - - -class EnvBasedTC(TestCase): - """abstract class for test using an apptest environment - """ - __metaclass__ = autoenv - __abstract__ = True - env = None - configcls = ApptestConfiguration - requestcls = FakeRequest - - # user / session management ############################################### - - def user(self, req=None): - if req is None: - req = self.env.create_request() - return self.env.cnx.user(req) - else: - return req.user - - def create_user(self, *args, **kwargs): - return self.env.create_user(*args, **kwargs) - - def login(self, login, password=None): - return self.env.login(login, password) - - def restore_connection(self): - self.env.restore_connection() - - # db api ################################################################## - - @nocoverage - def cursor(self, req=None): - return self.env.cnx.cursor(req or self.request()) - - @nocoverage - def execute(self, *args, **kwargs): - return self.env.execute(*args, **kwargs) - - @nocoverage - def commit(self): - self.env.cnx.commit() - - @nocoverage - def rollback(self): - try: - self.env.cnx.rollback() - except ProgrammingError: - pass - - # other utilities ######################################################### - def set_debug(self, debugmode): - from cubicweb.server import set_debug - set_debug(debugmode) - - @property - def config(self): - return self.vreg.config - - def session(self): - """return current server side session (using default manager account)""" - return self.env.repo._sessions[self.env.cnx.sessionid] - - def request(self, *args, **kwargs): - """return a web interface request""" - return self.env.create_request(*args, **kwargs) - - @nocoverage - def rset_and_req(self, *args, **kwargs): - return self.env.get_rset_and_req(*args, **kwargs) - - def entity(self, rql, args=None, eidkey=None, req=None): - return self.execute(rql, args, eidkey, req=req).get_entity(0, 0) - - def etype_instance(self, etype, req=None): - req = req or self.request() - e = self.env.vreg['etypes'].etype_class(etype)(req) - e.eid = None - return e - - def add_entity(self, etype, **kwargs): - rql = ['INSERT %s X' % etype] - - # dict for replacement in RQL Request - rql_args = {} - - if kwargs: # - rql.append(':') - # dict to define new entities variables - entities = {} - - # assignement part of the request - sub_rql = [] - for key, value in kwargs.iteritems(): - # entities - if hasattr(value, 'eid'): - new_value = "%s__" % key.upper() - - entities[new_value] = value.eid - rql_args[new_value] = value.eid - - sub_rql.append("X %s %s" % (key, new_value)) - # final attributes - else: - sub_rql.append('X %s %%(%s)s' % (key, key)) - rql_args[key] = value - rql.append(', '.join(sub_rql)) - - - if entities: - rql.append('WHERE') - # WHERE part of the request (to link entity to they eid) - sub_rql = [] - for key, value in entities.iteritems(): - sub_rql.append("%s eid %%(%s)s" % (key, key)) - rql.append(', '.join(sub_rql)) - - rql = ' '.join(rql) - rset = self.execute(rql, rql_args) - return rset.get_entity(0, 0) - - def set_option(self, optname, value): - self.vreg.config.global_set_option(optname, value) - - def pviews(self, req, rset): - return sorted((a.id, a.__class__) for a in self.vreg['views'].possible_views(req, rset=rset)) - - def pactions(self, req, rset, skipcategories=('addrelated', 'siteactions', 'useractions', 'footer')): - return [(a.id, a.__class__) for a in self.vreg['actions'].possible_vobjects(req, rset=rset) - if a.category not in skipcategories] - - def action_submenu(self, req, rset, id): - return self._test_action(self.vreg['actions'].select(id, req, rset=rset)) - - def _test_action(self, action): - class fake_menu(list): - @property - def items(self): - return self - class fake_box(object): - def mk_action(self, label, url, **kwargs): - return (label, url) - def box_action(self, action, **kwargs): - return (action.title, action.url()) - submenu = fake_menu() - action.fill_menu(fake_box(), submenu) - return submenu - - def pactions_by_cats(self, req, rset, categories=('addrelated',)): - return [(a.id, a.__class__) for a in self.vreg['actions'].possible_vobjects(req, rset=rset) - if a.category in categories] - - paddrelactions = deprecated()(pactions_by_cats) - - def pactionsdict(self, req, rset, skipcategories=('addrelated', 'siteactions', 'useractions')): - res = {} - for a in self.vreg['actions'].possible_vobjects(req, rset=rset): - if a.category not in skipcategories: - res.setdefault(a.category, []).append(a.__class__) - return res - - - def remote_call(self, fname, *args): - """remote call simulation""" - dump = simplejson.dumps - args = [dump(arg) for arg in args] - req = self.request(fname=fname, pageid='123', arg=args) - ctrl = self.vreg['controllers'].select('json', req) - return ctrl.publish(), req - - # default test setup and teardown ######################################### - - def setup_database(self): - pass - - def setUp(self): - self.restore_connection() - session = self.session() - #self.maxeid = self.execute('Any MAX(X)') - session.set_pool() - self.maxeid = session.system_sql('SELECT MAX(eid) FROM entities').fetchone()[0] - self.app = self.env.app - self.vreg = self.env.app.vreg - self.schema = self.vreg.schema - self.vreg.config.mode = 'test' - # set default-dest-addrs to a dumb email address to avoid mailbox or - # mail queue pollution - self.set_option('default-dest-addrs', ['whatever']) - self.setup_database() - self.commit() - MAILBOX[:] = [] # reset mailbox - - @nocoverage - def tearDown(self): - self.rollback() - # self.env.restore_database() - self.env.restore_connection() - self.session().unsafe_execute('DELETE Any X WHERE X eid > %s' % self.maxeid) - self.commit() - - # global resources accessors ############################################### - -# XXX -try: - from cubicweb.web import Redirect - from urllib import unquote -except ImportError: - pass # cubicweb-web not installed -else: - class ControllerTC(EnvBasedTC): - def setUp(self): - super(ControllerTC, self).setUp() - self.req = self.request() - self.ctrl = self.vreg['controllers'].select('edit', self.req) - - def publish(self, req): - assert req is self.ctrl.req - try: - result = self.ctrl.publish() - req.cnx.commit() - except Redirect: - req.cnx.commit() - raise - return result - - def expect_redirect_publish(self, req=None): - if req is not None: - self.ctrl = self.vreg['controllers'].select('edit', req) - else: - req = self.req - try: - res = self.publish(req) - except Redirect, ex: - try: - path, params = ex.location.split('?', 1) - except: - path, params = ex.location, "" - req._url = path - cleanup = lambda p: (p[0], unquote(p[1])) - params = dict(cleanup(p.split('=', 1)) for p in params.split('&') if p) - return req.relative_path(False), params # path.rsplit('/', 1)[-1], params - else: - self.fail('expected a Redirect exception') - - -def make_late_binding_repo_property(attrname): - @property - def late_binding(self): - """builds cnx as late as possible""" - if not hasattr(self, attrname): - # sets explicit test mode here to avoid autoreload - from cubicweb.cwconfig import CubicWebConfiguration - CubicWebConfiguration.mode = 'test' - cls = self.__class__ - config = self.repo_config or TestServerConfiguration('data') - cls._repo, cls._cnx = init_test_database('sqlite', config=config) - return getattr(self, attrname) - return late_binding - - -class autorepo(type): - """automatically set repository on RepositoryBasedTC subclasses if necessary - """ - def __new__(mcs, name, bases, classdict): - repo = classdict.get('repo') - # try to find repo in one of the base classes - if repo is None: - for base in bases: - repo = getattr(base, 'repo', None) - if repo is not None: - classdict['repo'] = repo - break - if name != 'RepositoryBasedTC' and not classdict.get('repo'): - classdict['repo'] = make_late_binding_repo_property('_repo') - classdict['cnx'] = make_late_binding_repo_property('_cnx') - return super(autorepo, mcs).__new__(mcs, name, bases, classdict) - - -class RepositoryBasedTC(TestCase): - """abstract class for test using direct repository connections - """ - __metaclass__ = autorepo - repo_config = None # set a particular config instance if necessary - - # user / session management ############################################### - - def create_user(self, user, groups=('users',), password=None, commit=True): - if password is None: - password = user - eid = self.execute('INSERT CWUser X: X login %(x)s, X upassword %(p)s', - {'x': unicode(user), 'p': password})[0][0] - groups = ','.join(repr(group) for group in groups) - self.execute('SET X in_group Y WHERE X eid %%(x)s, Y name IN (%s)' % groups, - {'x': eid}) - if commit: - self.commit() - self.session.reset_pool() - return eid - - def login(self, login, password=None): - cnx = repo_connect(self.repo, unicode(login), password or login, - ConnectionProperties('inmemory')) - self.cnxs.append(cnx) - return cnx - - def current_session(self): - return self.repo._sessions[self.cnxs[-1].sessionid] - - def restore_connection(self): - assert len(self.cnxs) == 1, self.cnxs - cnx = self.cnxs.pop() - try: - cnx.close() - except Exception, ex: - print "exception occured while closing connection", ex - - # db api ################################################################## - - def execute(self, rql, args=None, eid_key=None): - assert self.session.id == self.cnxid - rset = self.__execute(self.cnxid, rql, args, eid_key) - rset.vreg = self.vreg - rset.req = self.session - # call to set_pool is necessary to avoid pb when using - # instance entities for convenience - self.session.set_pool() - return rset - - def commit(self): - self.__commit(self.cnxid) - self.session.set_pool() - - def rollback(self): - self.__rollback(self.cnxid) - self.session.set_pool() - - def close(self): - self.__close(self.cnxid) - - # other utilities ######################################################### - - def set_debug(self, debugmode): - from cubicweb.server import set_debug - set_debug(debugmode) - - def set_option(self, optname, value): - self.vreg.config.global_set_option(optname, value) - - def add_entity(self, etype, **kwargs): - restrictions = ', '.join('X %s %%(%s)s' % (key, key) for key in kwargs) - rql = 'INSERT %s X' % etype - if kwargs: - rql += ': %s' % ', '.join('X %s %%(%s)s' % (key, key) for key in kwargs) - rset = self.execute(rql, kwargs) - return rset.get_entity(0, 0) - - def default_user_password(self): - config = self.repo.config #TestConfiguration('data') - user = unicode(config.sources()['system']['db-user']) - passwd = config.sources()['system']['db-password'] - return user, passwd - - def close_connections(self): - for cnx in self.cnxs: - try: - cnx.rollback() - cnx.close() - except: - continue - self.cnxs = [] - - pactions = EnvBasedTC.pactions.im_func - pactionsdict = EnvBasedTC.pactionsdict.im_func - - # default test setup and teardown ######################################### - - def _prepare(self): - MAILBOX[:] = [] # reset mailbox - if hasattr(self, 'cnxid'): - return - repo = self.repo - self.__execute = repo.execute - self.__commit = repo.commit - self.__rollback = repo.rollback - self.__close = repo.close - self.cnxid = self.cnx.sessionid - self.session = repo._sessions[self.cnxid] - self.cnxs = [] - # reset caches, they may introduce bugs among tests - repo._type_source_cache = {} - repo._extid_cache = {} - repo.querier._rql_cache = {} - for source in repo.sources: - source.reset_caches() - for s in repo.sources: - if hasattr(s, '_cache'): - s._cache = {} - - @property - def config(self): - return self.repo.config - - @property - def vreg(self): - return self.repo.vreg - - @property - def schema(self): - return self.repo.schema - - def setUp(self): - self._prepare() - self.session.set_pool() - self.maxeid = self.session.system_sql('SELECT MAX(eid) FROM entities').fetchone()[0] - - def tearDown(self): - self.close_connections() - self.rollback() - self.session.unsafe_execute('DELETE Any X WHERE X eid > %(x)s', {'x': self.maxeid}) - self.commit() - diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/dataimport.py --- a/devtools/dataimport.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/dataimport.py Wed Mar 24 10:23:31 2010 +0100 @@ -39,7 +39,7 @@ # create controller ctl = CWImportController(RQLObjectStore()) - ctl.askerror = True + ctl.askerror = 1 ctl.generators = GENERATORS ctl.store._checkpoint = checkpoint ctl.store._rql = rql @@ -48,19 +48,60 @@ ctl.run() sys.exit(0) + +.. BUG fichier à une colonne pose un problème de parsing +.. TODO rollback() """ __docformat__ = "restructuredtext en" -import sys, csv, traceback +import sys +import csv +import traceback +import os.path as osp +from StringIO import StringIO +from copy import copy from logilab.common import shellutils +from logilab.common.date import strptime +from logilab.common.decorators import cached +from logilab.common.deprecation import deprecated -def utf8csvreader(file, encoding='utf-8', separator=',', quote='"'): - """A csv reader that accepts files with any encoding and outputs - unicode strings.""" - for row in csv.reader(file, delimiter=separator, quotechar=quote): + +def ucsvreader_pb(filepath, encoding='utf-8', separator=',', quote='"', + skipfirst=False, withpb=True): + """same as ucsvreader but a progress bar is displayed as we iter on rows""" + if not osp.exists(filepath): + raise Exception("file doesn't exists: %s" % filepath) + rowcount = int(shellutils.Execute('wc -l "%s"' % filepath).out.strip().split()[0]) + if skipfirst: + rowcount -= 1 + if withpb: + pb = shellutils.ProgressBar(rowcount, 50) + for urow in ucsvreader(file(filepath), encoding, separator, quote, skipfirst): + yield urow + if withpb: + pb.update() + print ' %s rows imported' % rowcount + +def ucsvreader(stream, encoding='utf-8', separator=',', quote='"', + skipfirst=False): + """A csv reader that accepts files with any encoding and outputs unicode + strings + """ + it = iter(csv.reader(stream, delimiter=separator, quotechar=quote)) + if skipfirst: + it.next() + for row in it: yield [item.decode(encoding) for item in row] +def commit_every(nbit, store, it): + for i, x in enumerate(it): + yield x + if nbit is not None and i % nbit: + store.checkpoint() + if nbit is not None: + store.checkpoint() + def lazytable(reader): """The first row is taken to be the header of the table and used to output a dict for each row of data. @@ -71,63 +112,133 @@ for row in reader: yield dict(zip(header, row)) -def tell(msg): - print msg - -# base sanitizing functions ##### - -def capitalize_if_unicase(txt): - if txt.isupper() or txt.islower(): - return txt.capitalize() - return txt - -def no_space(txt): - return txt.replace(' ','') - -def no_uspace(txt): - return txt.replace(u'\xa0','') - -def no_dash(txt): - return txt.replace('-','') - -def alldigits(txt): - if txt.isdigit(): - return txt - else: - return u'' - -def strip(txt): - return txt.strip() - -# base checks ##### - -def check_doubles(buckets): - """Extract the keys that have more than one item in their bucket.""" - return [(key, len(value)) for key,value in buckets.items() if len(value) > 1] - -# make entity helper ##### - def mk_entity(row, map): """Return a dict made from sanitized mapped values. + ValidationError can be raised on unexpected values found in checkers + >>> row = {'myname': u'dupont'} >>> map = [('myname', u'name', (capitalize_if_unicase,))] >>> mk_entity(row, map) {'name': u'Dupont'} + >>> row = {'myname': u'dupont', 'optname': u''} + >>> map = [('myname', u'name', (capitalize_if_unicase,)), + ... ('optname', u'MARKER', (optional,))] + >>> mk_entity(row, map) + {'name': u'Dupont'} """ res = {} + assert isinstance(row, dict) + assert isinstance(map, list) for src, dest, funcs in map: + assert not (required in funcs and optional in funcs), \ + "optional and required checks are exclusive" res[dest] = row[src] - for func in funcs: - res[dest] = func(res[dest]) + try: + for func in funcs: + res[dest] = func(res[dest]) + if res[dest] is None: + break + except ValueError, err: + raise ValueError('error with %r field: %s' % (src, err)) return res -# object stores + +# user interactions ############################################################ + +def tell(msg): + print msg + +def confirm(question): + """A confirm function that asks for yes/no/abort and exits on abort.""" + answer = shellutils.ASK.ask(question, ('Y', 'n', 'abort'), 'Y') + if answer == 'abort': + sys.exit(1) + return answer == 'Y' + + +class catch_error(object): + """Helper for @contextmanager decorator.""" + + def __init__(self, ctl, key='unexpected error', msg=None): + self.ctl = ctl + self.key = key + self.msg = msg + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + if type is not None: + if issubclass(type, (KeyboardInterrupt, SystemExit)): + return # re-raise + if self.ctl.catcherrors: + self.ctl.record_error(self.key, None, type, value, traceback) + return True # silent + + +# base sanitizing/coercing functions ########################################### + +def optional(value): + """validation error will not been raised if you add this checker in chain""" + if value: + return value + return None + +def required(value): + """raise ValueError is value is empty + + This check should be often found in last position in the chain. + """ + if value: + return value + raise ValueError("required") + +def todatetime(format='%d/%m/%Y'): + """return a transformation function to turn string input value into a + `datetime.datetime` instance, using given format. + + Follow it by `todate` or `totime` functions from `logilab.common.date` if + you want a `date`/`time` instance instead of `datetime`. + """ + def coerce(value): + return strptime(value, format) + return coerce + +def call_transform_method(methodname, *args, **kwargs): + """return value returned by calling the given method on input""" + def coerce(value): + return getattr(value, methodname)(*args, **kwargs) + return coerce + +def call_check_method(methodname, *args, **kwargs): + """check value returned by calling the given method on input is true, + else raise ValueError + """ + def check(value): + if getattr(value, methodname)(*args, **kwargs): + return value + raise ValueError('%s not verified on %r' % (methodname, value)) + return check + +# base integrity checking functions ############################################ + +def check_doubles(buckets): + """Extract the keys that have more than one item in their bucket.""" + return [(k, len(v)) for k, v in buckets.items() if len(v) > 1] + +def check_doubles_not_none(buckets): + """Extract the keys that have more than one item in their bucket.""" + return [(k, len(v)) for k, v in buckets.items() + if k is not None and len(v) > 1] + + +# object stores ################################################################# class ObjectStore(object): - """Store objects in memory for faster testing. Will not - enforce the constraints of the schema and hence will miss - some problems. + """Store objects in memory for *faster* validation (development mode) + + But it will not enforce the constraints of the schema and hence will miss some problems >>> store = ObjectStore() >>> user = {'login': 'johndoe'} @@ -136,7 +247,6 @@ >>> store.add('CWUser', group) >>> store.relate(user['eid'], 'in_group', group['eid']) """ - def __init__(self): self.items = [] self.eids = {} @@ -156,24 +266,66 @@ self.eids[eid] = item self.types.setdefault(type, []).append(eid) - def relate(self, eid_from, rtype, eid_to): - eids_valid = (eid_from < len(self.items) and eid_to <= len(self.items)) - assert eids_valid, 'eid error %s %s' % (eid_from, eid_to) - self.relations.add( (eid_from, rtype, eid_to) ) + def relate(self, eid_from, rtype, eid_to, inlined=False): + """Add new relation (reverse type support is available) - def build_index(self, name, type, func): + >>> 1,2 = eid_from, eid_to + >>> self.relate(eid_from, 'in_group', eid_to) + 1, 'in_group', 2 + >>> self.relate(eid_from, 'reverse_in_group', eid_to) + 2, 'in_group', 1 + """ + if rtype.startswith('reverse_'): + eid_from, eid_to = eid_to, eid_from + rtype = rtype[8:] + relation = eid_from, rtype, eid_to + self.relations.add(relation) + return relation + + def build_index(self, name, type, func=None): index = {} + if func is None or not callable(func): + func = lambda x: x['eid'] for eid in self.types[type]: index.setdefault(func(self.eids[eid]), []).append(eid) + assert index, "new index '%s' cannot be empty" % name self.indexes[name] = index - def get_many(self, name, key): - return self.indexes[name].get(key, []) + def build_rqlindex(self, name, type, key, rql, rql_params=False, func=None): + """build an index by rql query + + rql should return eid in first column + ctl.store.build_index('index_name', 'users', 'login', 'Any U WHERE U is CWUser') + """ + rset = self.rql(rql, rql_params or {}) + for entity in rset.entities(): + getattr(entity, key) # autopopulate entity with key attribute + self.eids[entity.eid] = dict(entity) + if entity.eid not in self.types.setdefault(type, []): + self.types[type].append(entity.eid) + assert self.types[type], "new index type '%s' cannot be empty (0 record found)" % type + + # Build index with specified key + func = lambda x: x[key] + self.build_index(name, type, func) - def get_one(self, name, key): + def fetch(self, name, key, unique=False, decorator=None): + """ + decorator is a callable method or an iterator of callable methods (usually a lambda function) + decorator=lambda x: x[:1] (first value is returned) + + We can use validation check function available in _entity + """ eids = self.indexes[name].get(key, []) - assert len(eids) == 1, 'expected a single one got %i' % len(eids) - return eids[0] + if decorator is not None: + if not hasattr(decorator, '__iter__'): + decorator = (decorator,) + for f in decorator: + eids = f(eids) + if unique: + assert len(eids) == 1, u'expected a single one value for key "%s" in index "%s". Got %i' % (key, name, len(eids)) + eids = eids[0] # FIXME maybe it's better to keep an iterator here ? + return eids def find(self, type, key, value): for idx in self.types[type]: @@ -181,27 +333,82 @@ if item[key] == value: yield item - def rql(self, query, args): - if self._rql: - return self._rql(query, args) + def rql(self, *args): + if self._rql is not None: + return self._rql(*args) def checkpoint(self): - if self._checkpoint: - self._checkpoint() + pass + + @property + def nb_inserted_entities(self): + return len(self.eids) + @property + def nb_inserted_types(self): + return len(self.types) + @property + def nb_inserted_relations(self): + return len(self.relations) + + @deprecated('[3.6] get_many() deprecated. Use fetch() instead') + def get_many(self, name, key): + return self.fetch(name, key, unique=False) + + @deprecated('[3.6] get_one() deprecated. Use fetch(..., unique=True) instead') + def get_one(self, name, key): + return self.fetch(name, key, unique=True) + class RQLObjectStore(ObjectStore): - """ObjectStore that works with an actual RQL repository.""" + """ObjectStore that works with an actual RQL repository (production mode)""" + _rql = None # bw compat + + def __init__(self, session=None, checkpoint=None): + ObjectStore.__init__(self) + if session is not None: + if not hasattr(session, 'set_pool'): + # connection + cnx = session + session = session.request() + session.set_pool = lambda : None + checkpoint = checkpoint or cnx.commit + else: + session.set_pool() + self.session = session + self._checkpoint = checkpoint or session.commit + elif checkpoint is not None: + self._checkpoint = checkpoint + # XXX .session + + def checkpoint(self): + self._checkpoint() + self.session.set_pool() + + def rql(self, *args): + if self._rql is not None: + return self._rql(*args) + return self.session.execute(*args) + + def create_entity(self, *args, **kwargs): + entity = self.session.create_entity(*args, **kwargs) + self.eids[entity.eid] = entity + self.types.setdefault(args[0], []).append(entity.eid) + return entity def _put(self, type, item): - query = ('INSERT %s X: ' % type) + ', '.join(['X %s %%(%s)s' % (key,key) for key in item]) + query = ('INSERT %s X: ' % type) + ', '.join('X %s %%(%s)s' % (k, k) + for k in item) return self.rql(query, item)[0][0] - def relate(self, eid_from, rtype, eid_to): - query = 'SET X %s Y WHERE X eid %%(from)s, Y eid %%(to)s' % rtype - self.rql(query, {'from': int(eid_from), 'to': int(eid_to)}) - self.relations.add( (eid_from, rtype, eid_to) ) + def relate(self, eid_from, rtype, eid_to, inlined=False): + # if reverse relation is found, eids are exchanged + eid_from, rtype, eid_to = super(RQLObjectStore, self).relate( + eid_from, rtype, eid_to) + self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, + {'x': int(eid_from), 'y': int(eid_to)}, ('x', 'y')) -# import controller ##### + +# the import controller ######################################################## class CWImportController(object): """Controller of the data import process. @@ -212,12 +419,17 @@ >>> ctl.run() """ - def __init__(self, store): + def __init__(self, store, askerror=0, catcherrors=None, tell=tell, + commitevery=50): self.store = store self.generators = None self.data = {} self.errors = None - self.askerror = False + self.askerror = askerror + if catcherrors is None: + catcherrors = askerror + self.catcherrors = catcherrors + self.commitevery = commitevery # set to None to do a single commit self._tell = tell def check(self, type, key, value): @@ -230,49 +442,311 @@ self.check(key, entity[key], None) entity[key] = default + def record_error(self, key, msg=None, type=None, value=None, tb=None): + tmp = StringIO() + if type is None: + traceback.print_exc(file=tmp) + else: + traceback.print_exception(type, value, tb, file=tmp) + print tmp.getvalue() + # use a list to avoid counting a errors instead of one + errorlog = self.errors.setdefault(key, []) + if msg is None: + errorlog.append(tmp.getvalue().splitlines()) + else: + errorlog.append( (msg, tmp.getvalue().splitlines()) ) + def run(self): self.errors = {} for func, checks in self.generators: self._checks = {} - func_name = func.__name__[4:] - question = 'Importation de %s' % func_name - self.tell(question) + func_name = func.__name__[4:] # XXX + self.tell("Import '%s'..." % func_name) try: func(self) except: - import StringIO - tmp = StringIO.StringIO() - traceback.print_exc(file=tmp) - print tmp.getvalue() - self.errors[func_name] = ('Erreur lors de la transformation', - tmp.getvalue().splitlines()) + if self.catcherrors: + self.record_error(func_name, 'While calling %s' % func.__name__) + else: + raise for key, func, title, help in checks: buckets = self._checks.get(key) if buckets: err = func(buckets) if err: self.errors[title] = (help, err) - self.store.checkpoint() - errors = sum(len(err[1]) for err in self.errors.values()) - self.tell('Importation terminée. (%i objets, %i types, %i relations et %i erreurs).' - % (len(self.store.eids), len(self.store.types), - len(self.store.relations), errors)) - if self.errors and self.askerror and confirm('Afficher les erreurs ?'): - import pprint - pprint.pprint(self.errors) + self.store.checkpoint() + nberrors = sum(len(err[1]) for err in self.errors.values()) + self.tell('\nImport completed: %i entities, %i types, %i relations and %i errors' + % (self.store.nb_inserted_entities, + self.store.nb_inserted_types, + self.store.nb_inserted_relations, + nberrors)) + if self.errors: + if self.askerror == 2 or (self.askerror and confirm('Display errors ?')): + from pprint import pformat + for errkey, error in self.errors.items(): + self.tell("\n%s (%s): %d\n" % (error[0], errkey, len(error[1]))) + self.tell(pformat(sorted(error[1]))) def get_data(self, key): return self.data.get(key) - def index(self, name, key, value): + def index(self, name, key, value, unique=False): + """create a new index + + If unique is set to True, only first occurence will be kept not the following ones + """ + if unique: + try: + if value in self.store.indexes[name][key]: + return + except KeyError: + # we're sure that one is the first occurence; so continue... + pass self.store.indexes.setdefault(name, {}).setdefault(key, []).append(value) def tell(self, msg): self._tell(msg) -def confirm(question): - """A confirm function that asks for yes/no/abort and exits on abort.""" - answer = shellutils.ASK.ask(question, ('Y','n','abort'), 'Y') - if answer == 'abort': - sys.exit(1) - return answer == 'Y' + def iter_and_commit(self, datakey): + """iter rows, triggering commit every self.commitevery iterations""" + return commit_every(self.commitevery, self.store, self.get_data(datakey)) + + + +from datetime import datetime +from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES + + +class NoHookRQLObjectStore(RQLObjectStore): + """ObjectStore that works with an actual RQL repository (production mode)""" + _rql = None # bw compat + + def __init__(self, session, metagen=None, baseurl=None): + super(NoHookRQLObjectStore, self).__init__(session) + self.source = session.repo.system_source + self.rschema = session.repo.schema.rschema + self.add_relation = self.source.add_relation + if metagen is None: + metagen = MetaGenerator(session, baseurl) + self.metagen = metagen + self._nb_inserted_entities = 0 + self._nb_inserted_types = 0 + self._nb_inserted_relations = 0 + self.rql = session.unsafe_execute + + def create_entity(self, etype, **kwargs): + for k, v in kwargs.iteritems(): + kwargs[k] = getattr(v, 'eid', v) + entity, rels = self.metagen.base_etype_dicts(etype) + entity = copy(entity) + entity._related_cache = {} + self.metagen.init_entity(entity) + entity.update(kwargs) + session = self.session + self.source.add_entity(session, entity) + self.source.add_info(session, entity, self.source, complete=False) + for rtype, targeteids in rels.iteritems(): + # targeteids may be a single eid or a list of eids + inlined = self.rschema(rtype).inlined + try: + for targeteid in targeteids: + self.add_relation(session, entity.eid, rtype, targeteid, + inlined) + except TypeError: + self.add_relation(session, entity.eid, rtype, targeteids, + inlined) + self._nb_inserted_entities += 1 + return entity + + def relate(self, eid_from, rtype, eid_to): + assert not rtype.startswith('reverse_') + self.add_relation(self.session, eid_from, rtype, eid_to, + self.rschema(rtype).inlined) + self._nb_inserted_relations += 1 + + @property + def nb_inserted_entities(self): + return self._nb_inserted_entities + @property + def nb_inserted_types(self): + return self._nb_inserted_types + @property + def nb_inserted_relations(self): + return self._nb_inserted_relations + + def _put(self, type, item): + raise RuntimeError('use create entity') + + +class MetaGenerator(object): + def __init__(self, session, baseurl=None): + self.session = session + self.source = session.repo.system_source + self.time = datetime.now() + if baseurl is None: + config = session.vreg.config + baseurl = config['base-url'] or config.default_base_url() + if not baseurl[-1] == '/': + baseurl += '/' + self.baseurl = baseurl + # attributes/relations shared by all entities of the same type + self.etype_attrs = [] + self.etype_rels = [] + # attributes/relations specific to each entity + self.entity_attrs = ['eid', 'cwuri'] + #self.entity_rels = [] XXX not handled (YAGNI?) + schema = session.vreg.schema + rschema = schema.rschema + for rtype in META_RTYPES: + if rtype in ('eid', 'cwuri') or rtype in VIRTUAL_RTYPES: + continue + if rschema(rtype).final: + self.etype_attrs.append(rtype) + else: + self.etype_rels.append(rtype) + if not schema._eid_index: + # test schema loaded from the fs + self.gen_is = self.test_gen_is + self.gen_is_instance_of = self.test_gen_is_instanceof + + @cached + def base_etype_dicts(self, etype): + entity = self.session.vreg['etypes'].etype_class(etype)(self.session) + # entity are "surface" copied, avoid shared dict between copies + del entity.cw_extra_kwargs + for attr in self.etype_attrs: + entity[attr] = self.generate(entity, attr) + rels = {} + for rel in self.etype_rels: + rels[rel] = self.generate(entity, rel) + return entity, rels + + def init_entity(self, entity): + for attr in self.entity_attrs: + entity[attr] = self.generate(entity, attr) + entity.eid = entity['eid'] + + def generate(self, entity, rtype): + return getattr(self, 'gen_%s' % rtype)(entity) + + def gen_eid(self, entity): + return self.source.create_eid(self.session) + + def gen_cwuri(self, entity): + return u'%seid/%s' % (self.baseurl, entity['eid']) + + def gen_creation_date(self, entity): + return self.time + def gen_modification_date(self, entity): + return self.time + + def gen_is(self, entity): + return entity.e_schema.eid + def gen_is_instance_of(self, entity): + eids = [] + for etype in entity.e_schema.ancestors() + [entity.e_schema]: + eids.append(entity.e_schema.eid) + return eids + + def gen_created_by(self, entity): + return self.session.user.eid + def gen_owned_by(self, entity): + return self.session.user.eid + + # implementations of gen_is / gen_is_instance_of to use during test where + # schema has been loaded from the fs (hence entity type schema eids are not + # known) + def test_gen_is(self, entity): + from cubicweb.hooks.metadata import eschema_eid + return eschema_eid(self.session, entity.e_schema) + def test_gen_is_instanceof(self, entity): + from cubicweb.hooks.metadata import eschema_eid + eids = [] + for eschema in entity.e_schema.ancestors() + [entity.e_schema]: + eids.append(eschema_eid(self.session, eschema)) + return eids + + +################################################################################ + +utf8csvreader = deprecated('[3.6] use ucsvreader instead')(ucsvreader) + +@deprecated('[3.6] use required') +def nonempty(value): + return required(value) + +@deprecated("[3.6] use call_check_method('isdigit')") +def alldigits(txt): + if txt.isdigit(): + return txt + else: + return u'' + +@deprecated("[3.7] too specific, will move away, copy me") +def capitalize_if_unicase(txt): + if txt.isupper() or txt.islower(): + return txt.capitalize() + return txt + +@deprecated("[3.7] too specific, will move away, copy me") +def yesno(value): + """simple heuristic that returns boolean value + + >>> yesno("Yes") + True + >>> yesno("oui") + True + >>> yesno("1") + True + >>> yesno("11") + True + >>> yesno("") + False + >>> yesno("Non") + False + >>> yesno("blablabla") + False + """ + if value: + return value.lower()[0] in 'yo1' + return False + +@deprecated("[3.7] use call_check_method('isalpha')") +def isalpha(value): + if value.isalpha(): + return value + raise ValueError("not all characters in the string alphabetic") + +@deprecated("[3.7] use call_transform_method('upper')") +def uppercase(txt): + return txt.upper() + +@deprecated("[3.7] use call_transform_method('lower')") +def lowercase(txt): + return txt.lower() + +@deprecated("[3.7] use call_transform_method('replace', ' ', '')") +def no_space(txt): + return txt.replace(' ','') + +@deprecated("[3.7] use call_transform_method('replace', u'\xa0', '')") +def no_uspace(txt): + return txt.replace(u'\xa0','') + +@deprecated("[3.7] use call_transform_method('replace', '-', '')") +def no_dash(txt): + return txt.replace('-','') + +@deprecated("[3.7] use call_transform_method('strip')") +def strip(txt): + return txt.strip() + +@deprecated("[3.7] use call_transform_method('replace', ',', '.'), float") +def decimal(value): + return comma_float(value) + +@deprecated('[3.7] use int builtin') +def integer(value): + return int(value) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/devctl.py --- a/devtools/devctl.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/devctl.py Wed Mar 24 10:23:31 2010 +0100 @@ -8,42 +8,44 @@ """ __docformat__ = "restructuredtext en" +# *ctl module should limit the number of import to be imported as quickly as +# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash +# completion). So import locally in command helpers. import sys from datetime import datetime -from os import mkdir, chdir, getcwd +from os import mkdir, chdir from os.path import join, exists, abspath, basename, normpath, split, isdir -from copy import deepcopy from warnings import warn from logilab.common import STD_BLACKLIST -from logilab.common.modutils import get_module_files -from logilab.common.textutils import splitstrip -from logilab.common.shellutils import ASK -from logilab.common.clcommands import register_commands +from logilab.common.clcommands import register_commands, pop_arg from cubicweb.__pkginfo__ import version as cubicwebversion -from cubicweb import (CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage, - underline_title) -from cubicweb.schema import CONSTRAINTS -from cubicweb.toolsutils import Command, copy_skeleton +from cubicweb import CW_SOFTWARE_ROOT as BASEDIR, BadCommandUsage +from cubicweb.toolsutils import Command, copy_skeleton, underline_title from cubicweb.web.webconfig import WebConfiguration from cubicweb.server.serverconfig import ServerConfiguration -class DevCubeConfiguration(ServerConfiguration, WebConfiguration): - """dummy config to get full library schema and entities""" + +class DevConfiguration(ServerConfiguration, WebConfiguration): + """dummy config to get full library schema and appobjects for + a cube or for cubicweb (without a home) + """ creating = True - cubicweb_appobject_path = ServerConfiguration.cubicweb_appobject_path | WebConfiguration.cubicweb_appobject_path - cube_appobject_path = ServerConfiguration.cube_appobject_path | WebConfiguration.cube_appobject_path + cleanup_interface_sobjects = False - def __init__(self, cube): - super(DevCubeConfiguration, self).__init__(cube) - if cube is None: - self._cubes = () + cubicweb_appobject_path = (ServerConfiguration.cubicweb_appobject_path + | WebConfiguration.cubicweb_appobject_path) + cube_appobject_path = (ServerConfiguration.cube_appobject_path + | WebConfiguration.cube_appobject_path) + + def __init__(self, *cubes): + super(DevConfiguration, self).__init__(cubes and cubes[0] or None) + if cubes: + self._cubes = self.reorder_cubes( + self.expand_cubes(cubes, with_recommends=True)) else: - self._cubes = self.reorder_cubes(self.expand_cubes(self.my_cubes(cube))) - - def my_cubes(self, cube): - return (cube,) + self.cube_dependencies(cube) + self.cube_recommends(cube) + self._cubes = () @property def apphome(self): @@ -54,16 +56,6 @@ pass def load_configuration(self): pass - - -class DevDepConfiguration(DevCubeConfiguration): - """configuration to use to generate cubicweb po files or to use as "library" configuration - to filter out message ids from cubicweb and dependencies of a cube - """ - - def my_cubes(self, cube): - return self.cube_dependencies(cube) + self.cube_recommends(cube) - def default_log_file(self): return None @@ -94,28 +86,29 @@ should be marked using '_' and extracted using xgettext """ from cubicweb.cwvreg import CubicWebVRegistry - cube = cubedir and split(cubedir)[-1] - libconfig = DevDepConfiguration(cube) - libconfig.cleanup_interface_sobjects = False - cleanup_sys_modules(libconfig) if cubedir: - config = DevCubeConfiguration(cube) - config.cleanup_interface_sobjects = False + cube = split(cubedir)[-1] + config = DevConfiguration(cube) + depcubes = list(config._cubes) + depcubes.remove(cube) + libconfig = DevConfiguration(*depcubes) else: - config = libconfig - libconfig = None + config = DevConfiguration() + cube = libconfig = None + cleanup_sys_modules(config) schema = config.load_schema(remove_unused_rtypes=False) vreg = CubicWebVRegistry(config) # set_schema triggers objects registrations vreg.set_schema(schema) w(DEFAULT_POT_HEAD) - _generate_schema_pot(w, vreg, schema, libconfig=libconfig, cube=cube) + _generate_schema_pot(w, vreg, schema, libconfig=libconfig) -def _generate_schema_pot(w, vreg, schema, libconfig=None, cube=None): - from cubicweb.common.i18n import add_msg +def _generate_schema_pot(w, vreg, schema, libconfig=None): + from copy import deepcopy + from cubicweb.i18n import add_msg from cubicweb.web import uicfg - from cubicweb.schema import META_RTYPES, SYSTEM_RTYPES + from cubicweb.schema import META_RTYPES, SYSTEM_RTYPES, CONSTRAINTS no_context_rtypes = META_RTYPES | SYSTEM_RTYPES w('# schema pot file, generated on %s\n' % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) w('# \n') @@ -125,19 +118,19 @@ if libconfig is not None: from cubicweb.cwvreg import CubicWebVRegistry, clear_rtag_objects libschema = libconfig.load_schema(remove_unused_rtypes=False) - rinlined = deepcopy(uicfg.autoform_is_inlined) + afs = deepcopy(uicfg.autoform_section) appearsin_addmenu = deepcopy(uicfg.actionbox_appearsin_addmenu) clear_rtag_objects() cleanup_sys_modules(libconfig) libvreg = CubicWebVRegistry(libconfig) libvreg.set_schema(libschema) # trigger objects registration - librinlined = uicfg.autoform_is_inlined + libafs = uicfg.autoform_section libappearsin_addmenu = uicfg.actionbox_appearsin_addmenu # prefill vregdone set list(_iter_vreg_objids(libvreg, vregdone)) else: libschema = {} - rinlined = uicfg.autoform_is_inlined + afs = uicfg.autoform_section appearsin_addmenu = uicfg.actionbox_appearsin_addmenu for cstrtype in CONSTRAINTS: add_msg(w, cstrtype) @@ -156,15 +149,17 @@ if eschema.final: continue for rschema, targetschemas, role in eschema.relation_definitions(True): + if rschema.final: + continue for tschema in targetschemas: - if rinlined.etype_get(eschema, rschema, role, tschema) and \ + fsections = afs.etype_get(eschema, rschema, role, tschema) + if 'main_inlined' in fsections and \ (libconfig is None or not - librinlined.etype_get(eschema, rschema, role, tschema)): + 'main_inlined' in libafs.etype_get( + eschema, rschema, role, tschema)): add_msg(w, 'add a %s' % tschema, 'inlined:%s.%s.%s' % (etype, rschema, role)) - add_msg(w, 'remove this %s' % tschema, - 'inlined:%s.%s.%s' % (etype, rschema, role)) - add_msg(w, 'This %s' % tschema, + add_msg(w, str(tschema), 'inlined:%s.%s.%s' % (etype, rschema, role)) if appearsin_addmenu.etype_get(eschema, rschema, role, tschema) and \ (libconfig is None or not @@ -184,7 +179,7 @@ # XXX also generate "creating ...' messages for actions in the # addrelated submenu w('# subject and object forms for each relation type\n') - w('# (no object form for final or symetric relation types)\n') + w('# (no object form for final or symmetric relation types)\n') w('\n') for rschema in sorted(schema.relations()): rtype = rschema.type @@ -204,7 +199,7 @@ for subjschema in rschema.subjects(): if not subjschema in libsubjects: add_msg(w, rtype, subjschema.type) - if not (schema.rschema(rtype).final or rschema.symetric): + if not (schema.rschema(rtype).final or rschema.symmetric): if rschema not in no_context_rtypes: libobjects = librschema and librschema.objects() or () for objschema in rschema.objects(): @@ -217,34 +212,24 @@ add_msg(w, '%s_description' % objid) add_msg(w, objid) -def _iter_vreg_objids(vreg, done, prefix=None): + +def _iter_vreg_objids(vreg, done): for reg, objdict in vreg.items(): for objects in objdict.values(): for obj in objects: - objid = '%s_%s' % (reg, obj.id) + objid = '%s_%s' % (reg, obj.__regid__) if objid in done: break - if obj.property_defs: + try: # XXX < 3.6 bw compat + pdefs = obj.property_defs + except AttributeError: + pdefs = getattr(obj, 'cw_property_defs', {}) + if pdefs: yield objid done.add(objid) break -def defined_in_library(etype, rtype, tetype, role): - """return true if the given relation definition exists in cubicweb's library - """ - if libschema is None: - return False - if role == 'subject': - subjtype, objtype = etype, tetype - else: - subjtype, objtype = tetype, etype - try: - return libschema.rschema(rtype).has_rdef(subjtype, objtype) - except KeyError: - return False - - LANGS = ('en', 'fr', 'es') I18NDIR = join(BASEDIR, 'i18n') DEFAULT_POT_HEAD = r'''msgid "" @@ -279,7 +264,8 @@ import yams from logilab.common.fileutils import ensure_fs_mode from logilab.common.shellutils import globfind, find, rm - from cubicweb.common.i18n import extract_from_tal, execute + from logilab.common.modutils import get_module_files + from cubicweb.i18n import extract_from_tal, execute tempdir = tempfile.mkdtemp() potfiles = [join(I18NDIR, 'static-messages.pot')] print '-> extract schema messages.' @@ -340,9 +326,10 @@ def run(self, args): """run the command with its specific arguments""" if args: - cubes = [DevCubeConfiguration.cube_dir(cube) for cube in args] + cubes = [DevConfiguration.cube_dir(cube) for cube in args] else: - cubes = [DevCubeConfiguration.cube_dir(cube) for cube in DevCubeConfiguration.available_cubes()] + cubes = [DevConfiguration.cube_dir(cube) + for cube in DevConfiguration.available_cubes()] cubes = [cubepath for cubepath in cubes if exists(join(cubepath, 'i18n'))] update_cubes_catalogs(cubes) @@ -372,7 +359,7 @@ import tempfile from logilab.common.fileutils import ensure_fs_mode from logilab.common.shellutils import find, rm - from cubicweb.common.i18n import extract_from_tal, execute + from cubicweb.i18n import extract_from_tal, execute toedit = [] cube = basename(normpath(cubedir)) tempdir = tempfile.mkdtemp() @@ -435,17 +422,18 @@ return toedit -class LiveServerCommand(Command): - """Run a server from within a cube directory. - """ - name = 'live-server' - arguments = '' - options = () +# XXX totally broken, fix it +# class LiveServerCommand(Command): +# """Run a server from within a cube directory. +# """ +# name = 'live-server' +# arguments = '' +# options = () - def run(self, args): - """run the command with its specific arguments""" - from cubicweb.devtools.livetest import runserver - runserver() +# def run(self, args): +# """run the command with its specific arguments""" +# from cubicweb.devtools.livetest import runserver +# runserver() class NewCubeCommand(Command): @@ -491,6 +479,7 @@ def run(self, args): + from logilab.common.shellutils import ASK if len(args) != 1: raise BadCommandUsage("exactly one argument (cube name) is expected") cubename, = args @@ -548,6 +537,8 @@ copy_skeleton(skeldir, cubedir, context) def _ask_for_dependancies(self): + from logilab.common.shellutils import ASK + from logilab.common.textutils import splitstrip includes = [] for stdtype in ServerConfiguration.available_cubes(): answer = ASK.ask("Depends on cube %s? " % stdtype, @@ -565,17 +556,16 @@ class ExamineLogCommand(Command): """Examine a rql log file. - usage: python exlog.py < rql.log - will print out the following table total execution time || number of occurences || rql query sorted by descending total execution time - chances are the lines at the top are the ones that will bring - the higher benefit after optimisation. Start there. + chances are the lines at the top are the ones that will bring the higher + benefit after optimisation. Start there. """ + arguments = '< rql.log' name = 'exlog' options = ( ) @@ -617,9 +607,72 @@ for clocktime, cputime, occ, rql in stat: print '%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime, cputime, occ, rql) + +class GenerateSchema(Command): + """Generate schema image for the given cube""" + name = "schema" + arguments = '' + options = [('output-file', {'type':'file', 'default': None, + 'metavar': '', 'short':'o', 'help':'output image file', + 'input':False}), + ('viewer', {'type': 'string', 'default':None, + 'short': "d", 'metavar':'', + 'help':'command use to view the generated file (empty for none)'} + ), + ('show-meta', {'action': 'store_true', 'default':False, + 'short': "m", 'metavar': "", + 'help':'include meta and internal entities in schema'} + ), + ('show-workflow', {'action': 'store_true', 'default':False, + 'short': "w", 'metavar': "", + 'help':'include workflow entities in schema'} + ), + ('show-cw-user', {'action': 'store_true', 'default':False, + 'metavar': "", + 'help':'include cubicweb user entities in schema'} + ), + ('exclude-type', {'type':'string', 'default':'', + 'short': "x", 'metavar': "", + 'help':'coma separated list of entity types to remove from view'} + ), + ('include-type', {'type':'string', 'default':'', + 'short': "i", 'metavar': "", + 'help':'coma separated list of entity types to include in view'} + ), + ] + + def run(self, args): + from subprocess import Popen + from tempfile import NamedTemporaryFile + from logilab.common.textutils import splitstrip + from yams import schema2dot, BASE_TYPES + from cubicweb.schema import (META_RTYPES, SCHEMA_TYPES, SYSTEM_RTYPES, + WORKFLOW_TYPES, INTERNAL_TYPES) + cubes = splitstrip(pop_arg(args, 1)) + dev_conf = DevConfiguration(*cubes) + schema = dev_conf.load_schema() + out, viewer = self['output-file'], self['viewer'] + if out is None: + tmp_file = NamedTemporaryFile(suffix=".svg") + out = tmp_file.name + skiptypes = BASE_TYPES | SCHEMA_TYPES + if not self['show-meta']: + skiptypes |= META_RTYPES | SYSTEM_RTYPES | INTERNAL_TYPES + if not self['show-workflow']: + skiptypes |= WORKFLOW_TYPES + if not self['show-cw-user']: + skiptypes |= set(('CWUser', 'CWGroup', 'EmailAddress')) + skiptypes |= set(self['exclude-type'].split(',')) + skiptypes -= set(self['include-type'].split(',')) + schema2dot.schema2dot(schema, out, skiptypes=skiptypes) + if viewer: + p = Popen((viewer, out)) + p.wait() + register_commands((UpdateCubicWebCatalogCommand, UpdateTemplateCatalogCommand, - LiveServerCommand, + #LiveServerCommand, NewCubeCommand, ExamineLogCommand, + GenerateSchema, )) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/fake.py --- a/devtools/fake.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/fake.py Wed Mar 24 10:23:31 2010 +0100 @@ -7,12 +7,11 @@ """ __docformat__ = "restructuredtext en" -from logilab.common.testlib import mock_object as Mock from logilab.common.adbh import get_adv_func_helper from indexer import get_indexer -from cubicweb import RequestSessionMixIn +from cubicweb.req import RequestSessionBase from cubicweb.cwvreg import CubicWebVRegistry from cubicweb.web.request import CubicWebRequestBase from cubicweb.devtools import BASE_URL, BaseApptestConfiguration @@ -81,15 +80,15 @@ def set_header(self, header, value, raw=True): """set an output HTTP header""" - pass + self._headers[header] = value def add_header(self, header, value): """set an output HTTP header""" - pass + self._headers[header] = value # XXX def remove_header(self, header): """remove an output HTTP header""" - pass + self._headers.pop(header, None) def get_header(self, header, default=None): """return the value associated with the given input header, @@ -97,16 +96,24 @@ """ return self._headers.get(header, default) - def set_cookie(self, cookie, key, maxage=300): + def set_cookie(self, cookie, key, maxage=300, expires=None): """set / update a cookie key by default, cookie will be available for the next 5 minutes """ - pass + morsel = cookie[key] + if maxage is not None: + morsel['Max-Age'] = maxage + if expires: + morsel['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S %z') + # make sure cookie is set on the correct path + morsel['path'] = self.base_url_path() + self.add_header('Set-Cookie', morsel.OutputString()) + self.add_header('Cookie', morsel.OutputString()) def remove_cookie(self, cookie, key): - """remove a cookie by expiring it""" - pass + self.remove_header('Set-Cookie') + self.remove_header('Cookie') def validate_cache(self): pass @@ -130,7 +137,7 @@ return True -class FakeSession(RequestSessionMixIn): +class FakeSession(RequestSessionBase): def __init__(self, repo=None, user=None): self.repo = repo self.vreg = getattr(self.repo, 'vreg', CubicWebVRegistry(FakeConfig(), initlog=False)) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/fill.py --- a/devtools/fill.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/fill.py Wed Mar 24 10:23:31 2010 +0100 @@ -10,47 +10,37 @@ from random import randint, choice from copy import deepcopy -from datetime import datetime, date, time#timedelta +from datetime import datetime, date, time, timedelta from decimal import Decimal +from logilab.common import attrdict from yams.constraints import (SizeConstraint, StaticVocabularyConstraint, - IntervalBoundConstraint) + IntervalBoundConstraint, BoundConstraint, + Attribute, actual_value) from rql.utils import decompose_b26 as base_decompose_b26 from cubicweb import Binary from cubicweb.schema import RQLConstraint +def custom_range(start, stop, step): + while start < stop: + yield start + start += step + def decompose_b26(index, ascii=False): """return a letter (base-26) decomposition of index""" if ascii: return base_decompose_b26(index) return base_decompose_b26(index, u'éabcdefghijklmnopqrstuvwxyz') -def get_choices(eschema, attrname): - """returns possible choices for 'attrname' - if attrname doesn't have ChoiceConstraint, return None - """ - for cst in eschema.constraints(attrname): - if isinstance(cst, StaticVocabularyConstraint): - return cst.vocabulary() - return None - - def get_max_length(eschema, attrname): """returns the maximum length allowed for 'attrname'""" - for cst in eschema.constraints(attrname): + for cst in eschema.rdef(attrname).constraints: if isinstance(cst, SizeConstraint) and cst.max: return cst.max return 300 #raise AttributeError('No Size constraint on attribute "%s"' % attrname) -def get_bounds(eschema, attrname): - for cst in eschema.constraints(attrname): - if isinstance(cst, IntervalBoundConstraint): - return cst.minvalue, cst.maxvalue - return None, None - - _GENERATED_VALUES = {} class _ValueGenerator(object): @@ -64,55 +54,52 @@ # some stuff ... return alist_of_acceptable_values # or None """ - self.e_schema = eschema self.choice_func = choice_func + self.eschema = eschema - def _generate_value(self, attrname, index, **kwargs): - if not self.e_schema.has_unique_values(attrname): - return self.__generate_value(attrname, index, **kwargs) - value = self.__generate_value(attrname, index, **kwargs) - while value in _GENERATED_VALUES.get((self.e_schema.type, attrname), ()): - index += 1 - value = self.__generate_value(attrname, index, **kwargs) - _GENERATED_VALUES.setdefault((self.e_schema.type, attrname), set()).add(value) + def generate_attribute_value(self, entity, attrname, index=1, **kwargs): + if attrname in entity: + return entity[attrname] + eschema = self.eschema + if not eschema.has_unique_values(attrname): + value = self.__generate_value(entity, attrname, index, **kwargs) + else: + value = self.__generate_value(entity, attrname, index, **kwargs) + while value in _GENERATED_VALUES.get((eschema, attrname), ()): + index += 1 + value = self.__generate_value(entity, attrname, index, **kwargs) + _GENERATED_VALUES.setdefault((eschema, attrname), set()).add(value) + entity[attrname] = value return value - def __generate_value(self, attrname, index, **kwargs): + def __generate_value(self, entity, attrname, index, **kwargs): """generates a consistent value for 'attrname'""" - attrtype = str(self.e_schema.destination(attrname)).lower() + eschema = self.eschema + attrtype = str(eschema.destination(attrname)).lower() # Before calling generate_%s functions, try to find values domain - etype = self.e_schema.type if self.choice_func is not None: - values_domain = self.choice_func(etype, attrname) + values_domain = self.choice_func(eschema, attrname) if values_domain is not None: return choice(values_domain) - gen_func = getattr(self, 'generate_%s_%s' % (self.e_schema.type, attrname), None) - if gen_func is None: - gen_func = getattr(self, 'generate_Any_%s' % attrname, None) + gen_func = getattr(self, 'generate_%s_%s' % (eschema, attrname), + getattr(self, 'generate_Any_%s' % attrname, None)) if gen_func is not None: - return gen_func(index, **kwargs) + return gen_func(entity, index, **kwargs) # If no specific values domain, then generate a dummy value gen_func = getattr(self, 'generate_%s' % (attrtype)) - return gen_func(attrname, index, **kwargs) + return gen_func(entity, attrname, index, **kwargs) - def generate_choice(self, attrname, index): - """generates a consistent value for 'attrname' if it's a choice""" - choices = get_choices(self.e_schema, attrname) - if choices is None: - return None - return unicode(choice(choices)) # FIXME - - def generate_string(self, attrname, index, format=None): + def generate_string(self, entity, attrname, index, format=None): """generates a consistent value for 'attrname' if it's a string""" # First try to get choices - choosed = self.generate_choice(attrname, index) + choosed = self.get_choice(entity, attrname) if choosed is not None: return choosed # All other case, generate a default string - attrlength = get_max_length(self.e_schema, attrname) + attrlength = get_max_length(self.eschema, attrname) num_len = numlen(index) if num_len >= attrlength: - ascii = self.e_schema.rproperty(attrname, 'internationalizable') + ascii = self.eschema.rdef(attrname).internationalizable return ('&'+decompose_b26(index, ascii))[:attrlength] # always use plain text when no format is specified attrprefix = attrname[:max(attrlength-num_len-1, 0)] @@ -131,69 +118,104 @@ value = u'é&%s%d' % (attrprefix, index) return value[:attrlength] - def generate_password(self, attrname, index): + def generate_password(self, entity, attrname, index): """generates a consistent value for 'attrname' if it's a password""" return u'toto' - def generate_integer(self, attrname, index): + def generate_integer(self, entity, attrname, index): """generates a consistent value for 'attrname' if it's an integer""" - choosed = self.generate_choice(attrname, index) - if choosed is not None: - return choosed - minvalue, maxvalue = get_bounds(self.e_schema, attrname) - if maxvalue is not None and maxvalue <= 0 and minvalue is None: - minvalue = maxvalue - index # i.e. randint(-index, 0) - else: - maxvalue = maxvalue or index - return randint(minvalue or 0, maxvalue) - + return self._constrained_generate(entity, attrname, 0, 1, index) generate_int = generate_integer - def generate_float(self, attrname, index): + def generate_float(self, entity, attrname, index): """generates a consistent value for 'attrname' if it's a float""" - return float(randint(-index, index)) + return self._constrained_generate(entity, attrname, 0.0, 1.0, index) - def generate_decimal(self, attrname, index): + def generate_decimal(self, entity, attrname, index): """generates a consistent value for 'attrname' if it's a float""" - return Decimal(str(self.generate_float(attrname, index))) + return Decimal(str(self.generate_float(entity, attrname, index))) - def generate_date(self, attrname, index): + def generate_datetime(self, entity, attrname, index): + """generates a random date (format is 'yyyy-mm-dd HH:MM')""" + base = datetime(randint(2000, 2004), randint(1, 12), randint(1, 28), 11, index%60) + return self._constrained_generate(entity, attrname, base, timedelta(hours=1), index) + + def generate_date(self, entity, attrname, index): """generates a random date (format is 'yyyy-mm-dd')""" - return date(randint(2000, 2004), randint(1, 12), randint(1, 28)) + base = date(randint(2000, 2010), 1, 1) + timedelta(randint(1, 365)) + return self._constrained_generate(entity, attrname, base, timedelta(days=1), index) - def generate_time(self, attrname, index): + def generate_time(self, entity, attrname, index): """generates a random time (format is ' HH:MM')""" return time(11, index%60) #'11:%02d' % (index % 60) - def generate_datetime(self, attrname, index): - """generates a random date (format is 'yyyy-mm-dd HH:MM')""" - return datetime(randint(2000, 2004), randint(1, 12), randint(1, 28), 11, index%60) - - - def generate_bytes(self, attrname, index, format=None): - # modpython way + def generate_bytes(self, entity, attrname, index, format=None): fakefile = Binary("%s%s" % (attrname, index)) fakefile.filename = u"file_%s" % attrname - fakefile.value = fakefile.getvalue() return fakefile - def generate_boolean(self, attrname, index): + def generate_boolean(self, entity, attrname, index): """generates a consistent value for 'attrname' if it's a boolean""" return index % 2 == 0 - def generate_Any_data_format(self, index, **kwargs): + def _constrained_generate(self, entity, attrname, base, step, index): + choosed = self.get_choice(entity, attrname) + if choosed is not None: + return choosed + # ensure index > 0 + index += 1 + minvalue, maxvalue = self.get_bounds(entity, attrname) + if maxvalue is None: + if minvalue is not None: + base = max(minvalue, base) + maxvalue = base + index * step + if minvalue is None: + minvalue = maxvalue - (index * step) # i.e. randint(-index, 0) + return choice(list(custom_range(minvalue, maxvalue, step))) + + def _actual_boundary(self, entity, boundary): + if isinstance(boundary, Attribute): + # ensure we've a value for this attribute + self.generate_attribute_value(entity, boundary.attr) + boundary = actual_value(boundary, entity) + return boundary + + def get_bounds(self, entity, attrname): + minvalue = maxvalue = None + for cst in self.eschema.rdef(attrname).constraints: + if isinstance(cst, IntervalBoundConstraint): + minvalue = self._actual_boundary(entity, cst.minvalue) + maxvalue = self._actual_boundary(entity, cst.maxvalue) + elif isinstance(cst, BoundConstraint): + if cst.operator[0] == '<': + maxvalue = self._actual_boundary(entity, cst.boundary) + else: + minvalue = self._actual_boundary(entity, cst.boundary) + return minvalue, maxvalue + + def get_choice(self, entity, attrname): + """generates a consistent value for 'attrname' if it has some static + vocabulary set, else return None. + """ + for cst in self.eschema.rdef(attrname).constraints: + if isinstance(cst, StaticVocabularyConstraint): + return unicode(choice(cst.vocabulary())) + return None + + # XXX nothing to do here + def generate_Any_data_format(self, entity, index, **kwargs): # data_format attribute of Image/File has no vocabulary constraint, we # need this method else stupid values will be set which make mtconverter # raise exception return u'application/octet-stream' - def generate_Any_content_format(self, index, **kwargs): + def generate_Any_content_format(self, entity, index, **kwargs): # content_format attribute of EmailPart has no vocabulary constraint, we # need this method else stupid values will be set which make mtconverter # raise exception return u'text/plain' - def generate_Image_data_format(self, index, **kwargs): + def generate_Image_data_format(self, entity, index, **kwargs): # data_format attribute of Image/File has no vocabulary constraint, we # need this method else stupid values will be set which make mtconverter # raise exception @@ -237,7 +259,7 @@ returns acceptable values for this attribute """ # XXX HACK, remove or fix asap - if etype in (('String', 'Int', 'Float', 'Boolean', 'Date', 'CWGroup', 'CWUser')): + if etype in set(('String', 'Int', 'Float', 'Boolean', 'Date', 'CWGroup', 'CWUser')): return [] queries = [] for index in xrange(entity_num): @@ -264,7 +286,7 @@ """ eschema = schema.eschema(etype) valgen = ValueGenerator(eschema, choice_func) - entity = {} + entity = attrdict() # preprocessing to deal with _format fields attributes = [] relatedfields = {} @@ -280,12 +302,11 @@ for attrname, attrschema in attributes: if attrname in relatedfields: # first generate a format and record it - format = valgen._generate_value(attrname + '_format', index) - entity[attrname + '_format'] = format + format = valgen.generate_attribute_value(entity, attrname + '_format', index) # then a value coherent with this format - value = valgen._generate_value(attrname, index, format=format) + value = valgen.generate_attribute_value(entity, attrname, index, format=format) else: - value = valgen._generate_value(attrname, index) + value = valgen.generate_attribute_value(entity, attrname, index) if form: # need to encode values if attrschema.type == 'Bytes': # twisted way @@ -303,7 +324,6 @@ value = fmt % value else: value = unicode(value) - entity[attrname] = value return entity @@ -340,10 +360,10 @@ def composite_relation(rschema): for obj in rschema.objects(): - if obj.objrproperty(rschema, 'composite') == 'subject': + if obj.rdef(rschema, 'object').composite == 'subject': return True for obj in rschema.subjects(): - if obj.subjrproperty(rschema, 'composite') == 'object': + if obj.rdef(rschema, 'subject').composite == 'object': return True return False @@ -372,11 +392,11 @@ oedict = deepcopy(edict) delayed = [] # for each couple (subjschema, objschema), insert relations - for subj, obj in rschema.iter_rdefs(): + for subj, obj in rschema.rdefs: sym.add( (subj, obj) ) - if rschema.symetric and (obj, subj) in sym: + if rschema.symmetric and (obj, subj) in sym: continue - subjcard, objcard = rschema.rproperty(subj, obj, 'cardinality') + subjcard, objcard = rschema.rdef(subj, obj).cardinality # process mandatory relations first if subjcard in '1+' or objcard in '1+' or composite_relation(rschema): for query, args in self.make_relation_queries(sedict, oedict, @@ -390,21 +410,22 @@ yield query, args def qargs(self, subjeids, objeids, subjcard, objcard, subjeid, objeid): - if subjcard in '?1': + if subjcard in '?1+': subjeids.remove(subjeid) - if objcard in '?1': + if objcard in '?1+': objeids.remove(objeid) return {'subjeid' : subjeid, 'objeid' : objeid} def make_relation_queries(self, sedict, oedict, rschema, subj, obj): - subjcard, objcard = rschema.rproperty(subj, obj, 'cardinality') + rdef = rschema.rdef(subj, obj) + subjcard, objcard = rdef.cardinality subjeids = sedict.get(subj, frozenset()) used = self.existingrels[rschema.type] preexisting_subjrels = set(subj for subj, obj in used) preexisting_objrels = set(obj for subj, obj in used) # if there are constraints, only select appropriate objeids q = self.rql_tmpl % rschema.type - constraints = [c for c in rschema.rproperty(subj, obj, 'constraints') + constraints = [c for c in rdef.constraints if isinstance(c, RQLConstraint)] if constraints: restrictions = ', '.join(c.restriction for c in constraints) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/htmlparser.py --- a/devtools/htmlparser.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/htmlparser.py Wed Mar 24 10:23:31 2010 +0100 @@ -174,3 +174,5 @@ except KeyError: continue return False + +VALMAP = {None: None, 'dtd': DTDValidator, 'xml': SaxOnlyValidator} diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/livetest.py --- a/devtools/livetest.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/livetest.py Wed Mar 24 10:23:31 2010 +0100 @@ -6,9 +6,10 @@ :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses """ +import os import socket import logging -from os.path import join, dirname, exists +from os.path import join, dirname, normpath, abspath from StringIO import StringIO #from twisted.application import service, strports @@ -21,10 +22,9 @@ from logilab.common.testlib import TestCase -import cubicweb.web from cubicweb.dbapi import in_memory_cnx from cubicweb.etwist.server import CubicWebRootResource -from cubicweb.devtools import LivetestConfiguration, init_test_database +from cubicweb.devtools import BaseApptestConfiguration, init_test_database @@ -50,25 +50,57 @@ +class LivetestConfiguration(BaseApptestConfiguration): + init_repository = False + + def __init__(self, cube=None, sourcefile=None, pyro_name=None, + log_threshold=logging.CRITICAL): + BaseApptestConfiguration.__init__(self, cube, log_threshold=log_threshold) + self.appid = pyro_name or cube + # don't change this, else some symlink problems may arise in some + # environment (e.g. mine (syt) ;o) + # XXX I'm afraid this test will prevent to run test from a production + # environment + self._sources = None + # instance cube test + if cube is not None: + self.apphome = self.cube_dir(cube) + elif 'web' in os.getcwd().split(os.sep): + # web test + self.apphome = join(normpath(join(dirname(__file__), '..')), 'web') + else: + # cube test + self.apphome = abspath('..') + self.sourcefile = sourcefile + self.global_set_option('realm', '') + self.use_pyro = pyro_name is not None + + def pyro_enabled(self): + if self.use_pyro: + return True + else: + return False + + + def make_site(cube, options=None): from cubicweb.etwist import twconfig # trigger configuration registration - sourcefile = options.sourcefile - config = LivetestConfiguration(cube, sourcefile, + config = LivetestConfiguration(cube, options.sourcefile, pyro_name=options.pyro_name, log_threshold=logging.DEBUG) - source = config.sources()['system'] - init_test_database(driver=source['db-driver'], config=config) + init_test_database(config=config) # if '-n' in sys.argv: # debug mode cubicweb = LivetestResource(config, debug=True) toplevel = cubicweb website = server.Site(toplevel) cube_dir = config.cube_dir(cube) + source = config.sources()['system'] for port in xrange(7777, 7798): try: reactor.listenTCP(port, channel.HTTPFactory(website)) saveconf(cube_dir, port, source['db-user'], source['db-password']) break - except CannotListenError, exc: + except CannotListenError: print "port %s already in use, I will try another one" % port else: raise @@ -119,7 +151,7 @@ # build a config, and get a connection self.config = LivetestConfiguration(self.cube, self.sourcefile) _, user, passwd, _ = loadconf() - self.repo, self.cnx = in_memory_cnx(self.config, user, passwd) + self.repo, self.cnx = in_memory_cnx(self.config, user, password=passwd) self.setup_db(self.cnx) def tearDown(self): diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/migrtest.py --- a/devtools/migrtest.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,153 +0,0 @@ -"""Migration test script - -* migration will be played into a chroot of the local machine -* the database server used can be configured -* test tested instance may be on another host - - -We are using postgres'.pgpass file. Here is a copy of postgres documentation -about that: - -The file .pgpass in a user's home directory or the file referenced by -PGPASSFILE can contain passwords to be used if the connection requires -a password (and no password has been specified otherwise). - - -This file should contain lines of the following format: - -hostname:port:database:username:password - -Each of the first four fields may be a literal value, or *, which -matches anything. The password field from the first line that matches -the current connection parameters will be used. (Therefore, put -more-specific entries first when you are using wildcards.) If an entry -needs to contain : or \, escape this character with \. A hostname of -localhost matches both host (TCP) and local (Unix domain socket) -connections coming from the local machine. - -The permissions on .pgpass must disallow any access to world or group; -achieve this by the command chmod 0600 ~/.pgpass. If the permissions -are less strict than this, the file will be ignored. - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -__docformat__ = "restructuredtext en" - -from os import system -from os.path import join, basename - -from logilab.common.shellutils import cp, rm - -from cubicweb.toolsutils import read_config -from cubicweb.server.utils import generate_sources_file - -# XXXX use db-copy instead - -# test environment configuration -chrootpath = '/sandbox/cubicwebtest' -tmpdbhost = 'crater' -tmpdbuser = 'syt' -tmpdbpasswd = 'syt' - -def play_migration(applhome, applhost='', sudo=False): - applid = dbname = basename(applhome) - testapplhome = join(chrootpath, applhome) - # copy instance into the chroot - if applhost: - system('scp -r %s:%s %s' % (applhost, applhome, testapplhome)) - else: - cp(applhome, testapplhome) -## # extract db parameters -## sources = read_config(join(testapplhome, 'sources')) -## dbname = sources['system']['db-name'] -## dbhost = sources['system'].get('db-host') or '' -## dbuser = sources['system'].get('db-user') or '' -## dbpasswd = sources['system'].get('db-password') or '' - # generate sources file - # XXX multisources - sources = {'system': {}} - sources['system']['db-encoding'] = 'UTF8' # XXX - sources['system']['db-name'] = dbname - sources['system']['db-host'] = None - sources['system']['db-user'] = tmpdbuser - sources['system']['db-password'] = None - generate_sources_file(applid, join(testapplhome, 'sources'), sources) -## # create postgres password file so we won't need anymore passwords -## # XXX may exist! -## pgpassfile = expanduser('~/.pgpass') -## pgpass = open(pgpassfile, 'w') -## if dbpasswd: -## pgpass.write('%s:*:%s:%s:%s\n' % (dbhost or applhost or 'localhost', -## dbname, dbuser, dbpasswd)) -## if tmpdbpasswd: -## pgpass.write('%s:*:%s:%s:%s\n' % (tmpdbhost or 'localhost', dbname, -## tmpdbuser, tmpdbpasswd)) -## pgpass.close() -## chmod(pgpassfile, 0600) - # dump db -## dumpcmd = 'pg_dump -Fc -U %s -f /tmp/%s.dump %s' % ( -## dbuser, dbname, dbname) -## if dbhost: -## dumpcmd += ' -h %s' % dbhost - dumpfile = '/tmp/%s.dump' % applid - dumpcmd = 'cubicweb-ctl db-dump --output=%s %s' % (dumpfile, applid) - if sudo: - dumpcmd = 'sudo %s' % dumpcmd - if applhost: - dumpcmd = 'ssh %s "%s"' % (applhost, dumpcmd) - if system(dumpcmd): - raise Exception('error while dumping the database') -## if not dbhost and applhost: - if applhost: - # retrieve the dump - if system('scp %s:%s %s' % (applhost, dumpfile, dumpfile)): - raise Exception('error while retreiving the dump') - # move the dump into the chroot - system('mv %s %s%s' % (dumpfile, chrootpath, dumpfile)) - # locate installed versions - vcconf = read_config(join(testapplhome, 'vc.conf')) - template = vcconf['TEMPLATE'] - cubicwebversion = vcconf['CW'] - templversion = vcconf['TEMPLATE_VERSION'] - # install the same versions cubicweb and template versions into the chroot - system('sudo chroot %s apt-get update' % chrootpath) - system('sudo chroot %s apt-get install cubicweb-server=%s cubicweb-client=%s' - % (chrootpath, cubicwebversion, cubicwebversion)) - system('sudo chroot %s apt-get install cubicweb-%s-appl-server=%s cubicweb-%s-appl-client=%s' - % (chrootpath, template, templversion, template, templversion)) - # update and upgrade to the latest version - system('sudo chroot %s apt-get install cubicweb-server cubicweb-client' % chrootpath) - system('sudo chroot %s apt-get install cubicweb-%s-appl-server cubicweb-%s-appl-client' - % (chrootpath, template, template)) - # create and fill the database - system('sudo chroot cubicweb-ctl db-restore %s %s' % (applid, dumpfile)) -## if not tmpdbhost: -## system('createdb -U %s -T template0 -E UTF8 %s' % (tmpdbuser, dbname)) -## system('pg_restore -U %s -O -Fc -d %s /tmp/%s.dump' -## % (tmpdbuser, dbname, dbname)) -## else: -## system('createdb -h %s -U %s -T template0 -E UTF8 %s' -## % (tmpdbhost, tmpdbuser, dbname)) -## system('pg_restore -h %s -U %s -O -Fc -d %s /tmp/%s.dump' -## % (tmpdbhost, tmpdbuser, dbname, dbname)) - # launch upgrade - system('sudo chroot %s cubicweb-ctl upgrade %s' % (chrootpath, applid)) - - # cleanup - rm(testapplhome) -## rm(pgpassfile) -## if tmpdbhost: -## system('dropdb -h %s -U %s %s' % (tmpdbuser, tmpdbhost, dbname)) -## else: -## system('dropdb -U %s %s' % (tmpdbuser, dbname)) -## if not dbhost and applhost: - if applhost: - system('ssh %s rm %s' % (applhost, dumpfile)) - rm('%s%s' % (chrootpath, dumpfile)) - - -if __name__ == '__main__': - play_migration('/etc/cubicweb.d/jpl', 'lepus') diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/realdbtest.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/devtools/realdbtest.py Wed Mar 24 10:23:31 2010 +0100 @@ -0,0 +1,42 @@ +from cubicweb import toolsutils +from cubicweb.devtools import DEFAULT_SOURCES, BaseApptestConfiguration + +class RealDatabaseConfiguration(BaseApptestConfiguration): + init_repository = False + sourcesdef = DEFAULT_SOURCES.copy() + + def sources(self): + """ + By default, we run tests with the sqlite DB backend. + One may use its own configuration by just creating a + 'sources' file in the test directory from wich tests are + launched. + """ + self._sources = self.sourcesdef + return self._sources + + +def buildconfig(dbuser, dbpassword, dbname, adminuser, adminpassword, dbhost=None): + """convenience function that builds a real-db configuration class""" + sourcesdef = {'system': {'adapter' : 'native', + 'db-encoding' : 'UTF-8', #'ISO-8859-1', + 'db-user' : dbuser, + 'db-password' : dbpassword, + 'db-name' : dbname, + 'db-driver' : 'postgres', + 'db-host' : dbhost, + }, + 'admin' : {'login': adminuser, + 'password': adminpassword, + }, + } + return type('MyRealDBConfig', (RealDatabaseConfiguration,), + {'sourcesdef': sourcesdef}) + + +def loadconfig(filename): + """convenience function that builds a real-db configuration class + from a file + """ + return type('MyRealDBConfig', (RealDatabaseConfiguration,), + {'sourcesdef': toolsutils.read_config(filename)}) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/repotest.py --- a/devtools/repotest.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/repotest.py Wed Mar 24 10:23:31 2010 +0100 @@ -174,31 +174,31 @@ rqlhelper._analyser.uid_func_mapping = {} return rqlhelper - def _prepare_plan(self, rql, kwargs=None): + def _prepare_plan(self, rql, kwargs=None, simplify=True): rqlhelper = self._rqlhelper() rqlst = rqlhelper.parse(rql) rqlhelper.compute_solutions(rqlst, kwargs=kwargs) - rqlhelper.simplify(rqlst) + if simplify: + rqlhelper.simplify(rqlst) for select in rqlst.children: select.solutions.sort() return self.o.plan_factory(rqlst, kwargs, self.session) def _prepare(self, rql, kwargs=None): - plan = self._prepare_plan(rql, kwargs) + plan = self._prepare_plan(rql, kwargs, simplify=False) plan.preprocess(plan.rqlst) rqlst = plan.rqlst.children[0] rqlst.solutions = remove_unused_solutions(rqlst, rqlst.solutions, {}, self.repo.schema)[0] return rqlst - def _user_session(self, groups=('guests',), ueid=None): + def user_groups_session(self, *groups): + """lightweight session using the current user with hi-jacked groups""" # use self.session.user.eid to get correct owned_by relation, unless explicit eid - if ueid is None: - ueid = self.session.user.eid - u = self.repo._build_user(self.session, ueid) + u = self.repo._build_user(self.session, self.session.user.eid) u._groups = set(groups) s = Session(u, self.repo) s._threaddata.pool = self.pool - return u, s + return s def execute(self, rql, args=None, eid_key=None, build_descr=True): return self.o.execute(self.session, rql, args, eid_key, build_descr) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/stresstester.py --- a/devtools/stresstester.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/stresstester.py Wed Mar 24 10:23:31 2010 +0100 @@ -26,8 +26,6 @@ :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses """ -__revision__ = "$Id: stresstester.py,v 1.3 2006-03-05 14:35:27 syt Exp $" - import os import sys import threading @@ -156,7 +154,7 @@ # get local access to the repository print "Creating repo", prof_file repo = Repository(config, prof_file) - cnxid = repo.connect(user, password) + cnxid = repo.connect(user, password=password) # connection to the CubicWeb repository repo_cnx = Connection(repo, cnxid) repo_cursor = repo_cnx.cursor() diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/test/data/schema.py --- a/devtools/test/data/schema.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/test/data/schema.py Wed Mar 24 10:23:31 2010 +0100 @@ -16,5 +16,5 @@ severity = String(vocabulary=('important', 'normal', 'minor'), default='normal') cost = Int() description = String(maxsize=4096, fulltextindexed=True) - identical_to = SubjectRelation('Bug', symetric=True) + identical_to = SubjectRelation('Bug', symmetric=True) diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/test/data/views.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/devtools/test/data/views.py Wed Mar 24 10:23:31 2010 +0100 @@ -0,0 +1,34 @@ +"""only for unit tests ! + +:organization: Logilab +:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" + +from cubicweb.view import EntityView +from cubicweb.selectors import implements + +HTML_PAGE = u""" + +

    Hello World !

    + + +""" + +class SimpleView(EntityView): + __regid__ = 'simple' + __select__ = implements('Bug',) + + def call(self, **kwargs): + self.cell_call(0, 0) + + def cell_call(self, row, col): + self.w(HTML_PAGE) + +class RaisingView(EntityView): + __regid__ = 'raising' + __select__ = implements('Bug',) + + def cell_call(self, row, col): + raise ValueError() diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/test/data/views/__init__.py diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/test/data/views/bug.py --- a/devtools/test/data/views/bug.py Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -"""only for unit tests ! - -:organization: Logilab -:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" - -from cubicweb.view import EntityView -from cubicweb.selectors import implements - -HTML_PAGE = u""" - -

    Hello World !

    - - -""" - -class SimpleView(EntityView): - id = 'simple' - __select__ = implements('Bug',) - - def call(self, **kwargs): - self.cell_call(0, 0) - - def cell_call(self, row, col): - self.w(HTML_PAGE) - -class RaisingView(EntityView): - id = 'raising' - __select__ = implements('Bug',) - - def cell_call(self, row, col): - raise ValueError() diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/test/unittest_dbfill.py --- a/devtools/test/unittest_dbfill.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/test/unittest_dbfill.py Wed Mar 24 10:23:31 2010 +0100 @@ -9,6 +9,7 @@ import os.path as osp import re +import datetime from logilab.common.testlib import TestCase, unittest_main @@ -22,10 +23,10 @@ class MyValueGenerator(ValueGenerator): - def generate_Bug_severity(self, index): + def generate_Bug_severity(self, entity, index): return u'dangerous' - def generate_Any_description(self, index, format=None): + def generate_Any_description(self, entity, index, format=None): return u'yo' @@ -41,7 +42,6 @@ def _available_Person_firstname(self, etype, attrname): return [f.strip() for f in file(osp.join(DATADIR, 'firstnames.txt'))] - def setUp(self): config = ApptestConfiguration('data') config.bootstrap_cubes() @@ -52,25 +52,14 @@ self.bug_valgen = MyValueGenerator(e_schema) self.config = config - def _check_date(self, date): - """checks that 'date' is well-formed""" - year = date.year - month = date.month - day = date.day - self.failUnless(day in range(1, 29), '%s not in [0;28]' % day) - self.failUnless(month in range(1, 13), '%s not in [1;12]' % month) - self.failUnless(year in range(2000, 2005), - '%s not in [2000;2004]' % year) - - def test_string(self): """test string generation""" - surname = self.person_valgen._generate_value('surname', 12) + surname = self.person_valgen.generate_attribute_value({}, 'surname', 12) self.assertEquals(surname, u'é&surname12') def test_domain_value(self): """test value generation from a given domain value""" - firstname = self.person_valgen._generate_value('firstname', 12) + firstname = self.person_valgen.generate_attribute_value({}, 'firstname', 12) possible_choices = self._choice_func('Person', 'firstname') self.failUnless(firstname in possible_choices, '%s not in %s' % (firstname, possible_choices)) @@ -79,38 +68,36 @@ """test choice generation""" # Test for random index for index in range(5): - sx_value = self.person_valgen._generate_value('civility', index) + sx_value = self.person_valgen.generate_attribute_value({}, 'civility', index) self.failUnless(sx_value in ('Mr', 'Mrs', 'Ms')) def test_integer(self): """test integer generation""" # Test for random index for index in range(5): - cost_value = self.bug_valgen._generate_value('cost', index) + cost_value = self.bug_valgen.generate_attribute_value({}, 'cost', index) self.failUnless(cost_value in range(index+1)) def test_date(self): """test date generation""" # Test for random index - for index in range(5): - date_value = self.person_valgen._generate_value('birthday', index) - self._check_date(date_value) + for index in range(10): + date_value = self.person_valgen.generate_attribute_value({}, 'birthday', index) + self.failUnless(isinstance(date_value, datetime.date)) def test_phone(self): """tests make_tel utility""" self.assertEquals(make_tel(22030405), '22 03 04 05') - def test_customized_generation(self): - self.assertEquals(self.bug_valgen._generate_value('severity', 12), + self.assertEquals(self.bug_valgen.generate_attribute_value({}, 'severity', 12), u'dangerous') - self.assertEquals(self.bug_valgen._generate_value('description', 12), + self.assertEquals(self.bug_valgen.generate_attribute_value({}, 'description', 12), u'yo') - self.assertEquals(self.person_valgen._generate_value('description', 12), + self.assertEquals(self.person_valgen.generate_attribute_value({}, 'description', 12), u'yo') - class ConstraintInsertionTC(TestCase): def test_writeme(self): diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/test/unittest_testlib.py --- a/devtools/test/unittest_testlib.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/test/unittest_testlib.py Wed Mar 24 10:23:31 2010 +0100 @@ -10,11 +10,11 @@ from unittest import TestSuite -from logilab.common.testlib import (TestCase, unittest_main, mock_object, +from logilab.common.testlib import (TestCase, unittest_main, SkipAwareTextTestRunner) + from cubicweb.devtools import htmlparser - -from cubicweb.devtools.testlib import WebTest, EnvBasedTC +from cubicweb.devtools.testlib import CubicWebTC class WebTestTC(TestCase): @@ -23,10 +23,10 @@ self.runner = SkipAwareTextTestRunner(stream=output) def test_error_raised(self): - class MyWebTest(WebTest): + class MyWebTest(CubicWebTC): def test_error_view(self): - self.add_entity('Bug', title=u"bt") + self.request().create_entity('Bug', title=u"bt") self.view('raising', self.execute('Bug B'), template=None) def test_correct_view(self): @@ -39,17 +39,6 @@ self.assertEquals(len(result.failures), 1) -class TestLibTC(EnvBasedTC): - def test_add_entity_with_relation(self): - bug = self.add_entity(u'Bug', title=u"toto") - self.add_entity(u'Bug', title=u"tata", identical_to=bug) - - rset = self.execute('Any BA WHERE BA is Bug, BA title "toto"') - self.assertEquals(len(rset), 1) - bug = tuple(rset.entities())[0] - self.assertEquals(bug.identical_to[0].title, "tata") - - HTML_PAGE = u""" diff -r c25da7573ebd -r 02b52bf9f5f8 devtools/testlib.py --- a/devtools/testlib.py Fri Feb 12 15:18:00 2010 +0100 +++ b/devtools/testlib.py Wed Mar 24 10:23:31 2010 +0100 @@ -1,4 +1,4 @@ -"""this module contains base classes for web tests +"""this module contains base classes and utilities for cubicweb tests :organization: Logilab :copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. @@ -7,85 +7,524 @@ """ __docformat__ = "restructuredtext en" +import os import sys +import re +from urllib import unquote from math import log -from logilab.common.debugger import Debugger -from logilab.common.testlib import InnerTest -from logilab.common.pytest import nocoverage +import simplejson + +import yams.schema -from cubicweb import ValidationError -from cubicweb.devtools import VIEW_VALIDATORS -from cubicweb.devtools.apptest import EnvBasedTC -from cubicweb.devtools._apptest import unprotected_entities, SYSTEM_RELATIONS -from cubicweb.devtools.htmlparser import DTDValidator, SaxOnlyValidator, HTMLValidator -from cubicweb.devtools.fill import insert_entity_queries, make_relations_queries +from logilab.common.testlib import TestCase, InnerTest +from logilab.common.pytest import nocoverage, pause_tracing, resume_tracing +from logilab.common.debugger import Debugger +from logilab.common.umessage import message_from_string +from logilab.common.decorators import cached, classproperty, clear_cache +from logilab.common.deprecation import deprecated -from cubicweb.sobjects.notification import NotificationView - -from cubicweb.vregistry import NoSelectableObject +from cubicweb import ValidationError, NoSelectableObject, AuthenticationError +from cubicweb import cwconfig, devtools, web, server +from cubicweb.dbapi import repo_connect, ConnectionProperties, ProgrammingError +from cubicweb.sobjects import notification +from cubicweb.web import Redirect, application +from cubicweb.devtools import SYSTEM_ENTITIES, SYSTEM_RELATIONS, VIEW_VALIDATORS +from cubicweb.devtools import fake, htmlparser -## TODO ############### -# creation tests: make sure an entity was actually created -# Existing Test Environment +# low-level utilities ########################################################## class CubicWebDebugger(Debugger): - + """special debugger class providing a 'view' function which saves some + html into a temporary file and open a web browser to examinate it. + """ def do_view(self, arg): import webbrowser data = self._getval(arg) file('/tmp/toto.html', 'w').write(data) webbrowser.open('file:///tmp/toto.html') -def how_many_dict(schema, cursor, how_many, skip): - """compute how many entities by type we need to be able to satisfy relations - cardinality - """ - # compute how many entities by type we need to be able to satisfy relation constraint - relmap = {} - for rschema in schema.relations(): - if rschema.final: - continue - for subj, obj in rschema.iter_rdefs(): - card = rschema.rproperty(subj, obj, 'cardinality') - if card[0] in '1?' and len(rschema.subjects(obj)) == 1: - relmap.setdefault((rschema, subj), []).append(str(obj)) - if card[1] in '1?' and len(rschema.objects(subj)) == 1: - relmap.setdefault((rschema, obj), []).append(str(subj)) - unprotected = unprotected_entities(schema) - for etype in skip: - unprotected.add(etype) - howmanydict = {} - for etype in unprotected_entities(schema, strict=True): - howmanydict[str(etype)] = cursor.execute('Any COUNT(X) WHERE X is %s' % etype)[0][0] - if etype in unprotected: - howmanydict[str(etype)] += how_many - for (rschema, etype), targets in relmap.iteritems(): - # XXX should 1. check no cycle 2. propagate changes - relfactor = sum(howmanydict[e] for e in targets) - howmanydict[str(etype)] = max(relfactor, howmanydict[etype]) - return howmanydict - - def line_context_filter(line_no, center, before=3, after=None): """return true if line are in context - if after is None: after = before""" + + if after is None: after = before + """ if after is None: after = before return center - before <= line_no <= center + after -## base webtest class ######################################################### -VALMAP = {None: None, 'dtd': DTDValidator, 'xml': SaxOnlyValidator} + +def unprotected_entities(schema, strict=False): + """returned a set of each non final entity type, excluding "system" entities + (eg CWGroup, CWUser...) + """ + if strict: + protected_entities = yams.schema.BASE_TYPES + else: + protected_entities = yams.schema.BASE_TYPES.union(SYSTEM_ENTITIES) + return set(schema.entities()) - protected_entities + + +def refresh_repo(repo, resetschema=False, resetvreg=False): + devtools.reset_test_database(repo.config) + for pool in repo.pools: + pool.reconnect() + repo._type_source_cache = {} + repo._extid_cache = {} + repo.querier._rql_cache = {} + for source in repo.sources: + source.reset_caches() + if resetschema: + repo.set_schema(repo.config.load_schema(), resetvreg=resetvreg) + + +# email handling, to test emails sent by an application ######################## + +MAILBOX = [] + +class Email: + """you'll get instances of Email into MAILBOX during tests that trigger + some notification. + + * `msg` is the original message object + + * `recipients` is a list of email address which are the recipients of this + message + """ + def __init__(self, recipients, msg): + self.recipients = recipients + self.msg = msg + + @property + def message(self): + return message_from_string(self.msg) + + @property + def subject(self): + return self.message.get('Subject') + + @property + def content(self): + return self.message.get_payload(decode=True) + + def __repr__(self): + return '' % (','.join(self.recipients), + self.message.get('Subject')) + +# the trick to get email into MAILBOX instead of actually sent: monkey patch +# cwconfig.SMTP object +class MockSMTP: + def __init__(self, server, port): + pass + def close(self): + pass + def sendmail(self, helo_addr, recipients, msg): + MAILBOX.append(Email(recipients, msg)) + +cwconfig.SMTP = MockSMTP + + +# base class for cubicweb tests requiring a full cw environments ############### + +class CubicWebTC(TestCase): + """abstract class for test using an apptest environment + + attributes: + `vreg`, the vregistry + `schema`, self.vreg.schema + `config`, cubicweb configuration + `cnx`, dbapi connection to the repository using an admin user + `session`, server side session associated to `cnx` + `app`, the cubicweb publisher (for web testing) + `repo`, the repository object + + `admlogin`, login of the admin user + `admpassword`, password of the admin user + + """ + appid = 'data' + configcls = devtools.ApptestConfiguration + reset_schema = reset_vreg = False # reset schema / vreg between tests + + @classproperty + def config(cls): + """return the configuration object. Configuration is cached on the test + class. + """ + try: + return cls.__dict__['_config'] + except KeyError: + config = cls._config = cls.configcls(cls.appid) + config.mode = 'test' + return config + + @classmethod + def init_config(cls, config): + """configuration initialization hooks. You may want to override this.""" + source = config.sources()['system'] + cls.admlogin = unicode(source['db-user']) + cls.admpassword = source['db-password'] + # uncomment the line below if you want rql queries to be logged + #config.global_set_option('query-log-file', + # '/tmp/test_rql_log.' + `os.getpid()`) + config.global_set_option('log-file', None) + # set default-dest-addrs to a dumb email address to avoid mailbox or + # mail queue pollution + config.global_set_option('default-dest-addrs', ['whatever']) + try: + send_to = '%s@logilab.fr' % os.getlogin() + # AttributeError since getlogin not available under all platforms + except (OSError, AttributeError): + send_to = '%s@logilab.fr' % (os.environ.get('USER') + or os.environ.get('USERNAME') + or os.environ.get('LOGNAME')) + config.global_set_option('sender-addr', send_to) + config.global_set_option('default-dest-addrs', send_to) + config.global_set_option('sender-name', 'cubicweb-test') + config.global_set_option('sender-addr', 'cubicweb-test@logilab.fr') + # web resources + config.global_set_option('base-url', devtools.BASE_URL) + try: + config.global_set_option('embed-allowed', re.compile('.*')) + except: # not in server only configuration + pass + + @classmethod + def _init_repo(cls): + """init the repository and connection to it. + + Repository and connection are cached on the test class. Once + initialized, we simply reset connections and repository caches. + """ + if not 'repo' in cls.__dict__: + cls._build_repo() + else: + cls.cnx.rollback() + cls._refresh_repo() + + @classmethod + def _build_repo(cls): + cls.repo, cls.cnx = devtools.init_test_database(config=cls.config) + cls.init_config(cls.config) + cls.vreg = cls.repo.vreg + cls._orig_cnx = cls.cnx + cls.config.repository = lambda x=None: cls.repo + # necessary for authentication tests + cls.cnx.login = cls.admlogin + cls.cnx.authinfo = {'password': cls.admpassword} + + @classmethod + def _refresh_repo(cls): + refresh_repo(cls.repo, cls.reset_schema, cls.reset_vreg) + + # global resources accessors ############################################### + + @property + def schema(self): + """return the application schema""" + return self.vreg.schema + + @property + def session(self): + """return current server side session (using default manager account)""" + return self.repo._sessions[self.cnx.sessionid] + + @property + def adminsession(self): + """return current server side session (using default manager account)""" + return self.repo._sessions[self._orig_cnx.sessionid] + + def set_option(self, optname, value): + self.config.global_set_option(optname, value) + + def set_debug(self, debugmode): + server.set_debug(debugmode) + + # default test setup and teardown ######################################### + + def setUp(self): + pause_tracing() + self._init_repo() + resume_tracing() + self.setup_database() + self.commit() + MAILBOX[:] = [] # reset mailbox + + def setup_database(self): + """add your database setup code by overriding this method""" + + # user / session management ############################################### + + def user(self, req=None): + """return the application schema""" + if req is None: + req = self.request() + return self.cnx.user(req) + else: + return req.user + + def create_user(self, login, groups=('users',), password=None, req=None, + commit=True): + """create and return a new user entity""" + if password is None: + password = login.encode('utf8') + cursor = self._orig_cnx.cursor(req or self.request()) + rset = cursor.execute('INSERT CWUser X: X login %(login)s, X upassword %(passwd)s', + {'login': unicode(login), 'passwd': password}) + user = rset.get_entity(0, 0) + cursor.execute('SET X in_group G WHERE X eid %%(x)s, G name IN(%s)' + % ','.join(repr(g) for g in groups), + {'x': user.eid}, 'x') + user.clear_related_cache('in_group', 'subject') + if commit: + self._orig_cnx.commit() + return user -class WebTest(EnvBasedTC): - """base class for web tests""" - __abstract__ = True + def login(self, login, **kwargs): + """return a connection for the given login/password""" + if login == self.admlogin: + self.restore_connection() + else: + if not kwargs: + kwargs['password'] = str(login) + self.cnx = repo_connect(self.repo, unicode(login), + cnxprops=ConnectionProperties('inmemory'), + **kwargs) + if login == self.vreg.config.anonymous_user()[0]: + self.cnx.anonymous_connection = True + return self.cnx + + def restore_connection(self): + if not self.cnx is self._orig_cnx: + try: + self.cnx.close() + except ProgrammingError: + pass # already closed + self.cnx = self._orig_cnx + + # db api ################################################################## + + @nocoverage + def cursor(self, req=None): + return self.cnx.cursor(req or self.request()) + + @nocoverage + def execute(self, rql, args=None, eidkey=None, req=None): + """executes , builds a resultset, and returns a couple (rset, req) + where req is a FakeRequest + """ + req = req or self.request(rql=rql) + return self.cnx.cursor(req).execute(unicode(rql), args, eidkey) + + @nocoverage + def commit(self): + self.cnx.commit() + + @nocoverage + def rollback(self): + try: + self.cnx.rollback() + except ProgrammingError: + pass + + # # server side db api ####################################################### + + def sexecute(self, rql, args=None, eid_key=None): + self.session.set_pool() + return self.session.execute(rql, args, eid_key) + + # other utilities ######################################################### + + def entity(self, rql, args=None, eidkey=None, req=None): + return self.execute(rql, args, eidkey, req=req).get_entity(0, 0) + + # vregistry inspection utilities ########################################### + + def pviews(self, req, rset): + return sorted((a.__regid__, a.__class__) + for a in self.vreg['views'].possible_views(req, rset=rset)) + + def pactions(self, req, rset, + skipcategories=('addrelated', 'siteactions', 'useractions', 'footer')): + return [(a.__regid__, a.__class__) + for a in self.vreg['actions'].poss_visible_objects(req, rset=rset) + if a.category not in skipcategories] + + def pactions_by_cats(self, req, rset, categories=('addrelated',)): + return [(a.__regid__, a.__class__) + for a in self.vreg['actions'].poss_visible_objects(req, rset=rset) + if a.category in categories] + + def pactionsdict(self, req, rset, + skipcategories=('addrelated', 'siteactions', 'useractions', 'footer')): + res = {} + for a in self.vreg['actions'].poss_visible_objects(req, rset=rset): + if a.category not in skipcategories: + res.setdefault(a.category, []).append(a.__class__) + return res + + def action_submenu(self, req, rset, id): + return self._test_action(self.vreg['actions'].select(id, req, rset=rset)) + + def _test_action(self, action): + class fake_menu(list): + @property + def items(self): + return self + class fake_box(object): + def mk_action(self, label, url, **kwargs): + return (label, url) + def box_action(self, action, **kwargs): + return (action.title, action.url()) + submenu = fake_menu() + action.fill_menu(fake_box(), submenu) + return submenu - pdbclass = CubicWebDebugger - # this is a hook to be able to define a list of rql queries - # that are application dependent and cannot be guessed automatically - application_rql = [] + def list_views_for(self, rset): + """returns the list of views that can be applied on `rset`""" + req = rset.req + only_once_vids = ('primary', 'secondary', 'text') + req.data['ex'] = ValueError("whatever") + viewsvreg = self.vreg['views'] + for vid, views in viewsvreg.items(): + if vid[0] == '_': + continue + if rset.rowcount > 1 and vid in only_once_vids: + continue + views = [view for view in views + if view.category != 'startupview' + and not issubclass(view, notification.NotificationView)] + if views: + try: + view = viewsvreg._select_best(views, req, rset=rset) + if view.linkable(): + yield view + else: + not_selected(self.vreg, view) + # else the view is expected to be used as subview and should + # not be tested directly + except NoSelectableObject: + continue + + def list_actions_for(self, rset): + """returns the list of actions that can be applied on `rset`""" + req = rset.req + for action in self.vreg['actions'].possible_objects(req, rset=rset): + yield action + + def list_boxes_for(self, rset): + """returns the list of boxes that can be applied on `rset`""" + req = rset.req + for box in self.vreg['boxes'].possible_objects(req, rset=rset): + yield box + + def list_startup_views(self): + """returns the list of startup views""" + req = self.request() + for view in self.vreg['views'].possible_views(req, None): + if view.category == 'startupview': + yield view.__regid__ + else: + not_selected(self.vreg, view) + + # web ui testing utilities ################################################# + + @property + @cached + def app(self): + """return a cubicweb publisher""" + publisher = application.CubicWebPublisher(self.config, vreg=self.vreg) + def raise_error_handler(*args, **kwargs): + raise + publisher.error_handler = raise_error_handler + return publisher + + requestcls = fake.FakeRequest + def request(self, *args, **kwargs): + """return a web ui request""" + req = self.requestcls(self.vreg, form=kwargs) + req.set_connection(self.cnx) + return req + + def remote_call(self, fname, *args): + """remote json call simulation""" + dump = simplejson.dumps + args = [dump(arg) for arg in args] + req = self.request(fname=fname, pageid='123', arg=args) + ctrl = self.vreg['controllers'].select('json', req) + return ctrl.publish(), req + + def app_publish(self, req, path='view'): + return self.app.publish(path, req) + + def ctrl_publish(self, req): + """call the publish method of the edit controller""" + ctrl = self.vreg['controllers'].select('edit', req) + try: + result = ctrl.publish() + req.cnx.commit() + except web.Redirect: + req.cnx.commit() + raise + return result + + def expect_redirect(self, callback, req): + """call the given callback with req as argument, expecting to get a + Redirect exception + """ + try: + callback(req) + except Redirect, ex: + try: + path, params = ex.location.split('?', 1) + except ValueError: + path = ex.location + params = {} + else: + cleanup = lambda p: (p[0], unquote(p[1])) + params = dict(cleanup(p.split('=', 1)) for p in params.split('&') if p) + path = path[len(req.base_url()):] + return path, params + else: + self.fail('expected a Redirect exception') + + def expect_redirect_publish(self, req, path='edit'): + """call the publish method of the application publisher, expecting to + get a Redirect exception + """ + return self.expect_redirect(lambda x: self.app_publish(x, path), req) + + def init_authentication(self, authmode, anonuser=None): + self.set_option('auth-mode', authmode) + self.set_option('anonymous-user', anonuser) + req = self.request() + origcnx = req.cnx + req.cnx = None + sh = self.app.session_handler + authm = sh.session_manager.authmanager + authm.anoninfo = self.vreg.config.anonymous_user() + # not properly cleaned between tests + self.open_sessions = sh.session_manager._sessions = {} + return req, origcnx + + def assertAuthSuccess(self, req, origcnx, nbsessions=1): + sh = self.app.session_handler + path, params = self.expect_redirect(lambda x: self.app.connect(x), req) + cnx = req.cnx + self.assertEquals(len(self.open_sessions), nbsessions, self.open_sessions) + self.assertEquals(cnx.login, origcnx.login) + self.assertEquals(cnx.anonymous_connection, False) + self.assertEquals(path, 'view') + self.assertEquals(params, {'__message': 'welcome %s !' % cnx.user().login}) + + def assertAuthFailure(self, req, nbsessions=0): + self.assertRaises(AuthenticationError, self.app.connect, req) + self.assertEquals(req.cnx, None) + self.assertEquals(len(self.open_sessions), nbsessions) + clear_cache(req, 'get_authorization') + + # content validation ####################################################### # validators are used to validate (XML, DTD, whatever) view's content # validators availables are : @@ -100,8 +539,8 @@ # snippets #'text/html': DTDValidator, #'application/xhtml+xml': DTDValidator, - 'application/xml': SaxOnlyValidator, - 'text/xml': SaxOnlyValidator, + 'application/xml': htmlparser.SaxOnlyValidator, + 'text/xml': htmlparser.SaxOnlyValidator, 'text/plain': None, 'text/comma-separated-values': None, 'text/x-vcard': None, @@ -110,75 +549,9 @@ 'image/png': None, } # maps vid : validator name (override content_type_validators) - vid_validators = dict((vid, VALMAP[valkey]) + vid_validators = dict((vid, htmlparser.VALMAP[valkey]) for vid, valkey in VIEW_VALIDATORS.iteritems()) - no_auto_populate = () - ignored_relations = () - - def custom_populate(self, how_many, cursor): - pass - - def post_populate(self, cursor): - pass - - @nocoverage - def auto_populate(self, how_many): - """this method populates the database with `how_many` entities - of each possible type. It also inserts random relations between them - """ - cu = self.cursor() - self.custom_populate(how_many, cu) - vreg = self.vreg - howmanydict = how_many_dict(self.schema, cu, how_many, self.no_auto_populate) - for etype in unprotected_entities(self.schema): - if etype in self.no_auto_populate: - continue - nb = howmanydict.get(etype, how_many) - for rql, args in insert_entity_queries(etype, self.schema, vreg, nb): - cu.execute(rql, args) - edict = {} - for etype in unprotected_entities(self.schema, strict=True): - rset = cu.execute('%s X' % etype) - edict[str(etype)] = set(row[0] for row in rset.rows) - existingrels = {} - ignored_relations = SYSTEM_RELATIONS + self.ignored_relations - for rschema in self.schema.relations(): - if rschema.final or rschema in ignored_relations: - continue - rset = cu.execute('DISTINCT Any X,Y WHERE X %s Y' % rschema) - existingrels.setdefault(rschema.type, set()).update((x, y) for x, y in rset) - q = make_relations_queries(self.schema, edict, cu, ignored_relations, - existingrels=existingrels) - for rql, args in q: - try: - cu.execute(rql, args) - except ValidationError, ex: - # failed to satisfy some constraint - print 'error in automatic db population', ex - self.post_populate(cu) - self.commit() - - @nocoverage - def _check_html(self, output, view, template='main-template'): - """raises an exception if the HTML is invalid""" - try: - validatorclass = self.vid_validators[view.id] - except KeyError: - if view.content_type in ('text/html', 'application/xhtml+xml'): - if template is None: - default_validator = HTMLValidator - else: - default_validator = DTDValidator - else: - default_validator = None - validatorclass = self.content_type_validators.get(view.content_type, - default_validator) - if validatorclass is None: - return None - validator = validatorclass() - return validator.parse_string(output.strip()) - def view(self, vid, rset=None, req=None, template='main-template', **kwargs): @@ -233,9 +606,9 @@ # is not an AssertionError klass, exc, tcbk = sys.exc_info() try: - msg = '[%s in %s] %s' % (klass, view.id, exc) + msg = '[%s in %s] %s' % (klass, view.__regid__, exc) except: - msg = '[%s in %s] undisplayable exception' % (klass, view.id) + msg = '[%s in %s] undisplayable exception' % (klass, view.__regid__) if output is not None: position = getattr(exc, "position", (0,))[0] if position: @@ -252,9 +625,184 @@ raise AssertionError, msg, tcbk + @nocoverage + def _check_html(self, output, view, template='main-template'): + """raises an exception if the HTML is invalid""" + try: + validatorclass = self.vid_validators[view.__regid__] + except KeyError: + if view.content_type in ('text/html', 'application/xhtml+xml'): + if template is None: + default_validator = htmlparser.HTMLValidator + else: + default_validator = htmlparser.DTDValidator + else: + default_validator = None + validatorclass = self.content_type_validators.get(view.content_type, + default_validator) + if validatorclass is None: + return None + validator = validatorclass() + if isinstance(validator, htmlparser.DTDValidator): + # XXX remove used in progress widget, unknown in html dtd + output = re.sub('', '', output) + return validator.parse_string(output.strip()) + + # deprecated ############################################################### + + @deprecated('[3.6] use self.request().create_entity(...)') + def add_entity(self, etype, req=None, **kwargs): + if req is None: + req = self.request() + return req.create_entity(etype, **kwargs) + + @deprecated('[3.4] use self.vreg["etypes"].etype_class(etype)(self.request())') + def etype_instance(self, etype, req=None): + req = req or self.request() + e = self.vreg['etypes'].etype_class(etype)(req) + e.eid = None + return e + + @nocoverage + @deprecated('[3.4] use req = self.request(); rset = req.execute()', + stacklevel=3) + def rset_and_req(self, rql, optional_args=None, args=None, eidkey=None): + """executes , builds a resultset, and returns a + couple (rset, req) where req is a FakeRequest + """ + return (self.execute(rql, args, eidkey), + self.request(rql=rql, **optional_args or {})) + + +# auto-populating test classes and utilities ################################### + +from cubicweb.devtools.fill import insert_entity_queries, make_relations_queries + +# XXX cleanup unprotected_entities & all mess + +def how_many_dict(schema, cursor, how_many, skip): + """given a schema, compute how many entities by type we need to be able to + satisfy relations cardinality. + + The `how_many` argument tells how many entities of which type we want at + least. + + Return a dictionary with entity types as key, and the number of entities for + this type as value. + """ + relmap = {} + for rschema in schema.relations(): + if rschema.final: + continue + for subj, obj in rschema.rdefs: + card = rschema.rdef(subj, obj).cardinality + # if the relation is mandatory, we'll need at least as many subj and + # obj to satisfy it + if card[0] in '1+' and card[1] in '1?': + # subj has to be linked to at least one obj, + # but obj can be linked to only one subj + # -> we need at least as many subj as obj to satisfy + # cardinalities for this relation + relmap.setdefault((rschema, subj), []).append(str(obj)) + if card[1] in '1+' and card[0] in '1?': + # reverse subj and obj in the above explanation + relmap.setdefault((rschema, obj), []).append(str(subj)) + unprotected = unprotected_entities(schema) + for etype in skip: # XXX (syt) duh? explain or kill + unprotected.add(etype) + howmanydict = {} + # step 1, compute a base number of each entity types: number of already + # existing entities of this type + `how_many` + for etype in unprotected_entities(schema, strict=True): + howmanydict[str(etype)] = cursor.execute('Any COUNT(X) WHERE X is %s' % etype)[0][0] + if etype in unprotected: + howmanydict[str(etype)] += how_many + # step 2, augment nb entity per types to satisfy cardinality constraints, + # by recomputing for each relation that constrained an entity type: + # + # new num for etype = max(current num, sum(num for possible target etypes)) + # + # XXX we should first check there is no cycle then propagate changes + for (rschema, etype), targets in relmap.iteritems(): + relfactor = sum(howmanydict[e] for e in targets) + howmanydict[str(etype)] = max(relfactor, howmanydict[etype]) + return howmanydict + + +class AutoPopulateTest(CubicWebTC): + """base class for test with auto-populating of the database""" + __abstract__ = True + + pdbclass = CubicWebDebugger + # this is a hook to be able to define a list of rql queries + # that are application dependent and cannot be guessed automatically + application_rql = [] + + no_auto_populate = () + ignored_relations = set() + def to_test_etypes(self): return unprotected_entities(self.schema, strict=True) + def custom_populate(self, how_many, cursor): + pass + + def post_populate(self, cursor): + pass + + + @nocoverage + def auto_populate(self, how_many): + """this method populates the database with `how_many` entities + of each possible type. It also inserts random relations between them + """ + cu = self.cursor() + self.custom_populate(how_many, cu) + vreg = self.vreg + howmanydict = how_many_dict(self.schema, cu, how_many, self.no_auto_populate) + for etype in unprotected_entities(self.schema): + if etype in self.no_auto_populate: + continue + nb = howmanydict.get(etype, how_many) + for rql, args in insert_entity_queries(etype, self.schema, vreg, nb): + cu.execute(rql, args) + edict = {} + for etype in unprotected_entities(self.schema, strict=True): + rset = cu.execute('%s X' % etype) + edict[str(etype)] = set(row[0] for row in rset.rows) + existingrels = {} + ignored_relations = SYSTEM_RELATIONS | self.ignored_relations + for rschema in self.schema.relations(): + if rschema.final or rschema in ignored_relations: + continue + rset = cu.execute('DISTINCT Any X,Y WHERE X %s Y' % rschema) + existingrels.setdefault(rschema.type, set()).update((x, y) for x, y in rset) + q = make_relations_queries(self.schema, edict, cu, ignored_relations, + existingrels=existingrels) + for rql, args in q: + try: + cu.execute(rql, args) + except ValidationError, ex: + # failed to satisfy some constraint + print 'error in automatic db population', ex + self.post_populate(cu) + self.commit() + + def iter_individual_rsets(self, etypes=None, limit=None): + etypes = etypes or self.to_test_etypes() + for etype in etypes: + if limit: + rql = 'Any X LIMIT %s WHERE X is %s' % (limit, etype) + else: + rql = 'Any X WHERE X is %s' % etype + rset = self.execute(rql) + for row in xrange(len(rset)): + if limit and row > limit: + break + # XXX iirk + rset2 = rset.limit(limit=1, offset=row) + yield rset2 + def iter_automatic_rsets(self, limit=10): """generates basic resultsets for each entity type""" etypes = self.to_test_etypes() @@ -275,54 +823,6 @@ for rql in self.application_rql: yield self.execute(rql) - - def list_views_for(self, rset): - """returns the list of views that can be applied on `rset`""" - req = rset.req - only_once_vids = ('primary', 'secondary', 'text') - req.data['ex'] = ValueError("whatever") - viewsvreg = self.vreg['views'] - for vid, views in viewsvreg.items(): - if vid[0] == '_': - continue - if rset.rowcount > 1 and vid in only_once_vids: - continue - views = [view for view in views - if view.category != 'startupview' - and not issubclass(view, NotificationView)] - if views: - try: - view = viewsvreg.select_best(views, req, rset=rset) - if view.linkable(): - yield view - else: - not_selected(self.vreg, view) - # else the view is expected to be used as subview and should - # not be tested directly - except NoSelectableObject: - continue - - def list_actions_for(self, rset): - """returns the list of actions that can be applied on `rset`""" - req = rset.req - for action in self.vreg['actions'].possible_objects(req, rset=rset): - yield action - - def list_boxes_for(self, rset): - """returns the list of boxes that can be applied on `rset`""" - req = rset.req - for box in self.vreg['boxes'].possible_objects(req, rset=rset): - yield box - - def list_startup_views(self): - """returns the list of startup views""" - req = self.request() - for view in self.vreg['views'].possible_views(req, None): - if view.category == 'startupview': - yield view.id - else: - not_selected(self.vreg, view) - def _test_everything_for(self, rset): """this method tries to find everything that can be tested for `rset` and yields a callable test (as needed in generative tests) @@ -334,24 +834,32 @@ propdefs[k]['default'] = True for view in self.list_views_for(rset): backup_rset = rset.copy(rset.rows, rset.description) - yield InnerTest(self._testname(rset, view.id, 'view'), - self.view, view.id, rset, + yield InnerTest(self._testname(rset, view.__regid__, 'view'), + self.view, view.__regid__, rset, rset.req.reset_headers(), 'main-template') # We have to do this because some views modify the # resultset's syntax tree rset = backup_rset for action in self.list_actions_for(rset): - yield InnerTest(self._testname(rset, action.id, 'action'), self._test_action, action) + yield InnerTest(self._testname(rset, action.__regid__, 'action'), self._test_action, action) for box in self.list_boxes_for(rset): - yield InnerTest(self._testname(rset, box.id, 'box'), box.render) + yield InnerTest(self._testname(rset, box.__regid__, 'box'), box.render) @staticmethod def _testname(rset, objid, objtype): return '%s_%s_%s' % ('_'.join(rset.column_types(0)), objid, objtype) -class AutomaticWebTest(WebTest): +# concrete class for automated application testing ############################ + +class AutomaticWebTest(AutoPopulateTest): """import this if you wan automatic tests to be ran""" + def setUp(self): + AutoPopulateTest.setUp(self) + # access to self.app for proper initialization of the authentication + # machinery (else some views may fail) + self.app + ## one each def test_one_each_config(self): self.auto_populate(1) @@ -373,17 +881,7 @@ yield self.view, vid, None, req -class RealDBTest(WebTest): - - def iter_individual_rsets(self, etypes=None, limit=None): - etypes = etypes or unprotected_entities(self.schema, strict=True) - for etype in etypes: - rset = self.execute('Any X WHERE X is %s' % etype) - for row in xrange(len(rset)): - if limit and row > limit: - break - rset2 = rset.limit(limit=1, offset=row) - yield rset2 +# registry instrumentization ################################################### def not_selected(vreg, appobject): try: @@ -391,16 +889,17 @@ except (KeyError, AttributeError): pass + def vreg_instrumentize(testclass): + # XXX broken from cubicweb.devtools.apptest import TestEnvironment - env = testclass._env = TestEnvironment('data', configcls=testclass.configcls, - requestcls=testclass.requestcls) + env = testclass._env = TestEnvironment('data', configcls=testclass.configcls) for reg in env.vreg.values(): reg._selected = {} try: orig_select_best = reg.__class__.__orig_select_best except: - orig_select_best = reg.__class__.select_best + orig_select_best = reg.__class__._select_best def instr_select_best(self, *args, **kwargs): selected = orig_select_best(self, *args, **kwargs) try: @@ -410,9 +909,10 @@ except AttributeError: pass # occurs on reg used to restore database return selected - reg.__class__.select_best = instr_select_best + reg.__class__._select_best = instr_select_best reg.__class__.__orig_select_best = orig_select_best + def print_untested_objects(testclass, skipregs=('hooks', 'etypes')): for regname, reg in testclass._env.vreg.iteritems(): if regname in skipregs: diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/README --- a/doc/book/README Fri Feb 12 15:18:00 2010 +0100 +++ b/doc/book/README Wed Mar 24 10:23:31 2010 +0100 @@ -9,6 +9,8 @@ Chapter ======= +.. _Level1AnchorForLaterReference: + Level 1 section --------------- @@ -24,20 +26,29 @@ inline directives: - :file: - :envvar: - :command: + :file:`directory/file` + :envvar:`AN_ENV_VARIABLE` + :command:`command --option arguments` :ref:, :mod: -XXX -* lien vers cw.cwconfig.CW_CUBES_PATH par ex. - - .. sourcecode:: python class SomePythonCode: ... -.. XXX a comment +.. XXX a comment, wont be rendered + + +a [foot note]_ + +.. [foot note] the foot note content + + + +XXX +* lien vers cw.cwconfig.CW_CUBES_PATH par ex. + + +automodule, autofunction, automethod, autofunction diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/.templates/layout.html --- a/doc/book/en/.templates/layout.html Fri Feb 12 15:18:00 2010 +0100 +++ b/doc/book/en/.templates/layout.html Wed Mar 24 10:23:31 2010 +0100 @@ -111,6 +111,7 @@ + {%- if use_opensearch %} _permission" where the subject is the user and the - object is a any variable, meaning that the user needs to have - permission to execute the action on the entities related - to this variable - -For RQL expressions on a relation type, the principles are the same except -for the following : - -* you have to use the class `RRQLExpression` in the case of a non-final relation - -* in the expression, the variables S, O and U are pre-defined references - to respectively the subject and the object of the current relation (on - which the action is being verified) and the user who executed the query - -* we can also define rights on attributes of an entity (non-final - relation), knowing that : - - - to define RQL expression, we have to use the class - `ERQLExpression` in which X represents the entity the attribute - belongs to - - - the permissions `add` and `delete` are equivalent. Only `add`/`read` - are actually taken in consideration. - -In addition to that the entity type `EPermission` from the standard library -allows to build very complex and dynamic security architecture. The schema of -this entity type is as follow : :: - - class CWPermission(EntityType): - """entity type that may be used to construct some advanced security configuration - """ - permissions = META_ETYPE_PERMS - - name = String(required=True, indexed=True, internationalizable=True, maxsize=100, - description=_('name or identifier of the permission')) - label = String(required=True, internationalizable=True, maxsize=100, - description=_('distinct label to distinguate between other permission entity of the same name')) - require_group = SubjectRelation('CWGroup', - description=_('groups to which the permission is granted')) - - # explicitly add X require_permission CWPermission for each entity that should have - # configurable security - class require_permission(RelationType): - """link a permission to the entity. This permission should be used in the - security definition of the entity's type to be useful. - """ - permissions = { - 'read': ('managers', 'users', 'guests'), - 'add': ('managers',), - 'delete': ('managers',), - } - - class require_group(RelationType): - """used to grant a permission to a group""" - permissions = { - 'read': ('managers', 'users', 'guests'), - 'add': ('managers',), - 'delete': ('managers',), - } - - -Example of configuration :: - - - ... - - class Version(EntityType): - """a version is defining the content of a particular project's release""" - - permissions = {'read': ('managers', 'users', 'guests',), - 'update': ('managers', 'logilab', 'owners',), - 'delete': ('managers', ), - 'add': ('managers', 'logilab', - ERQLExpression('X version_of PROJ, U in_group G,' - 'PROJ require_permission P, P name "add_version",' - 'P require_group G'),)} - - ... - - class version_of(RelationType): - """link a version to its project. A version is necessarily linked to one and only one project. - """ - permissions = {'read': ('managers', 'users', 'guests',), - 'delete': ('managers', ), - 'add': ('managers', 'logilab', - RRQLExpression('O require_permission P, P name "add_version",' - 'U in_group G, P require_group G'),) - } - inlined = True - -This configuration indicates that an entity `CWPermission` named -"add_version" can be associated to a project and provides rights to create -new versions on this project to specific groups. It is important to notice that : - -* in such case, we have to protect both the entity type "Version" and the relation - associating a version to a project ("version_of") - -* because of the genericity of the entity type `CWPermission`, we have to execute - a unification with the groups and/or the states if necessary in the expression - ("U in_group G, P require_group G" in the above example) - -Use of RQL expression for reading rights -```````````````````````````````````````` - -The principles are the same but with the following restrictions : - -* we can not use `RRQLExpression` on relation types for reading - -* special relations "has__permission" can not be used - - -Note on the use of RQL expression for `add` permission -`````````````````````````````````````````````````````` -Potentially, the use of an RQL expression to add an entity or a relation -can cause problems for the user interface, because if the expression uses -the entity or the relation to create, then we are not able to verify the -permissions before we actually add the entity (please note that this is -not a problem for the RQL server at all, because the permissions checks are -done after the creation). In such case, the permission check methods -(check_perm, has_perm) can indicate that the user is not allowed to create -this entity but can obtain the permission. - -To compensate this problem, it is usually necessary, for such case, -to use an action that reflects the schema permissions but which enables -to check properly the permissions so that it would show up if necessary. - diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/B4040-rss-xml.en.txt --- a/doc/book/en/B4040-rss-xml.en.txt Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,48 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. _rss: - -RSS Channel ------------ - -Assuming you have several blog entries, click on the title of the -search box in the left column. A larger search box should appear. Enter:: - - Any X ORDERBY D WHERE X is BlogEntry, X creation_date D - -and you get a list of blog entries. - -Click on your login at the top right corner. Chose "user preferences", -then "boxes", then "possible views box" and check "visible = yes" -before validating your changes. - -Enter the same query in the search box and you will see the same list, -plus a box titled "possible views" in the left column. Click on -"entityview", then "RSS". - -You just applied the "RSS" view to the RQL selection you requested. - -That's it, you have a RSS channel for your blog. - -Try again with:: - - Any X ORDERBY D WHERE X is BlogEntry, X creation_date D, - X entry_of B, B title "MyLife" - -Another RSS channel, but a bit more focused. - -A last one for the road:: - - Any C ORDERBY D WHERE C is Comment, C creation_date D LIMIT 15 - -displayed with the RSS view, that's a channel for the last fifteen -comments posted. - -[WRITE ME] - -* show that the RSS view can be used to display an ordered selection - of blog entries, thus providing a RSS channel - -* show that a different selection (by category) means a different channel - - diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/D060-modules-stdlib.en.txt --- a/doc/book/en/D060-modules-stdlib.en.txt Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,158 +0,0 @@ -.. -*- coding: utf-8 -*- - -================ -Standard library -================ - -:mod:`cubes.addressbook` -======================== - -.. automodule:: cubes.addressbook - :members: - -:mod:`cubes.basket` -======================== - -.. automodule:: cubes.basket - :members: - -:mod:`cubes.blog` -======================== - -.. automodule:: cubes.blog - :members: - -:mod:`cubes.book` -======================== - -.. automodule:: cubes.book - :members: - -:mod:`cubes.comment` -======================== - -.. automodule:: cubes.comment - :members: - -:mod:`cubes.company` -======================== - -.. automodule:: cubes.company - :members: - - -:mod:`cubes.conference` -======================== - -.. automodule:: cubes.conference - :members: - -:mod:`cubes.email` -======================== - -.. automodule:: cubes.email - :members: - -:mod:`cubes.event` -======================== - -.. automodule:: cubes.event - :members: - -:mod:`cubes.expense` -======================== - -.. automodule:: cubes.expense - :members: - - -:mod:`cubes.file` -======================== - -.. automodule:: cubes.file - :members: - -:mod:`cubes.folder` -======================== - -.. automodule:: cubes.folder - :members: - -:mod:`cubes.i18ncontent` -======================== - -.. automodule:: cubes.i18ncontent - :members: - -:mod:`cubes.invoice` -======================== - -.. automodule:: cubes.invoice - :members: - -:mod:`cubes.keyword` -======================== - -.. automodule:: cubes.keyword - :members: - -:mod:`cubes.link` -======================== - -.. automodule:: cubes.link - :members: - -:mod:`cubes.mailinglist` -======================== - -.. automodule:: cubes.mailinglist - :members: - -:mod:`cubes.person` -======================== - -.. automodule:: cubes.person - :members: - -:mod:`cubes.shopcart` -======================== - -.. automodule:: cubes.shopcart - :members: - -:mod:`cubes.skillmat` -======================== - -.. automodule:: cubes.skillmat - :members: - -:mod:`cubes.tag` -======================== - -.. automodule:: cubes.tag - :members: - -:mod:`cubes.task` -======================== - -.. automodule:: cubes.task - :members: - -:mod:`cubes.workcase` -======================== - -.. automodule:: cubes.workcase - :members: - -:mod:`cubes.workorder` -======================== - -.. automodule:: cubes.workorder - :members: - -:mod:`cubes.zone` -======================== - -.. automodule:: cubes.zone - :members: - diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/D070-modules-cbw-api.en.txt --- a/doc/book/en/D070-modules-cbw-api.en.txt Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1961 +0,0 @@ -.. -*- coding: utf-8 -*- - -============ -CubicWeb API -============ - -:mod:`cubicweb.hercule` -======================= - -.. automodule:: cubicweb.hercule - :members: - -:mod:`cubicweb.cwctl` -===================== - -.. automodule:: cubicweb.cwctl - :members: - -:mod:`cubicweb.schema` -====================== - -.. automodule:: cubicweb.schema - :members: - -:mod:`cubicweb.cwconfig` -======================== - -.. automodule:: cubicweb.cwconfig - :members: - -:mod:`cubicweb.schemaviewer` -============================ - -.. automodule:: cubicweb.schemaviewer - :members: - -:mod:`cubicweb._exceptions` -=========================== - -.. automodule:: cubicweb._exceptions - :members: - -:mod:`cubicweb.dbapi` -===================== - -.. automodule:: cubicweb.dbapi - :members: - -:mod:`cubicweb.toolsutils` -========================== - -.. automodule:: cubicweb.toolsutils - :members: - -:mod:`cubicweb.cwvreg` -====================== - -.. automodule:: cubicweb.cwvreg - :members: - -:mod:`cubicweb.md5crypt` -======================== - -.. automodule:: cubicweb.md5crypt - :members: - -:mod:`cubicweb.rset` -==================== - -.. automodule:: cubicweb.rset - :members: - -:mod:`cubicweb` -=============== - -.. automodule:: cubicweb - :members: - -:mod:`cubicweb.setup` -===================== - -.. automodule:: cubicweb.setup - :members: - -:mod:`cubicweb.gettext` -======================= - -.. automodule:: cubicweb.gettext - :members: - -:mod:`cubicweb.interfaces` -========================== - -.. automodule:: cubicweb.interfaces - :members: - -:mod:`cubicweb.vregistry` -========================= - -.. automodule:: cubicweb.vregistry - :members: - -:mod:`cubicweb.web.httpcache` -============================= - -.. automodule:: cubicweb.web.httpcache - :members: - -:mod:`cubicweb.web.webconfig` -============================= - -.. automodule:: cubicweb.web.webconfig - :members: - -:mod:`cubicweb.web.request` -=========================== - -.. automodule:: cubicweb.web.request - :members: - -:mod:`cubicweb.web._exceptions` -=============================== - -.. automodule:: cubicweb.web._exceptions - :members: - -:mod:`cubicweb.web.webctl` -========================== - -.. automodule:: cubicweb.web.webctl - :members: - -:mod:`cubicweb.web.application` -=============================== - -.. automodule:: cubicweb.web.application - :members: - -:mod:`cubicweb.web.controller` -============================== - -.. automodule:: cubicweb.web.controller - :members: - -:mod:`cubicweb.web.widgets` -=========================== - -.. automodule:: cubicweb.web.widgets - :members: - -:mod:`cubicweb.web.htmlwidgets` -=============================== - -.. automodule:: cubicweb.web.htmlwidgets - :members: - -:mod:`cubicweb.web` -=================== - -.. automodule:: cubicweb.web - :members: - -:mod:`cubicweb.web.form` -======================== - -.. automodule:: cubicweb.web.form - :members: - -:mod:`cubicweb.web.box` -======================= - -.. automodule:: cubicweb.web.box - :members: - -:mod:`cubicweb.web.component` -============================= - -.. automodule:: cubicweb.web.component - :members: - -:mod:`cubicweb.web.action` -========================== - -.. automodule:: cubicweb.web.action - :members: - -:mod:`cubicweb.web.facet` -========================= - -.. automodule:: cubicweb.web.facet - :members: - -:mod:`cubicweb.web.views.plots` -=============================== - -.. automodule:: cubicweb.web.views.plots - :members: - -:mod:`cubicweb.web.views.error` -=============================== - -.. automodule:: cubicweb.web.views.error - :members: - -:mod:`cubicweb.web.views.magicsearch` -===================================== - -.. automodule:: cubicweb.web.views.magicsearch - :members: - -:mod:`cubicweb.web.views.basetemplates` -======================================= - -.. automodule:: cubicweb.web.views.basetemplates - :members: - -:mod:`cubicweb.web.views.idownloadable` -======================================= - -.. automodule:: cubicweb.web.views.idownloadable - :members: - -:mod:`cubicweb.web.views.ajaxedit` -================================== - -.. automodule:: cubicweb.web.views.ajaxedit - :members: - -:mod:`cubicweb.web.views.wfentities` -==================================== - -.. automodule:: cubicweb.web.views.wfentities - :members: - -:mod:`cubicweb.web.views.navigation` -==================================== - -.. automodule:: cubicweb.web.views.navigation - :members: - -:mod:`cubicweb.web.views.schemaentities` -======================================== - -.. automodule:: cubicweb.web.views.schemaentities - :members: - -:mod:`cubicweb.web.views.treeview` -================================== - -.. automodule:: cubicweb.web.views.treeview - :members: - -:mod:`cubicweb.web.views.startup` -================================= - -.. automodule:: cubicweb.web.views.startup - :members: - -:mod:`cubicweb.web.views.iprogress` -=================================== - -.. automodule:: cubicweb.web.views.iprogress - :members: - -:mod:`cubicweb.web.views.euser` -=============================== - -.. automodule:: cubicweb.web.views.euser - :members: - -:mod:`cubicweb.web.views.facets` -================================ - -.. automodule:: cubicweb.web.views.facets - :members: - -:mod:`cubicweb.web.views.emailaddress` -====================================== - -.. automodule:: cubicweb.web.views.emailaddress - :members: - -:mod:`cubicweb.web.views.sessions` -================================== - -.. automodule:: cubicweb.web.views.sessions - :members: - -:mod:`cubicweb.web.views.timetable` -=================================== - -.. automodule:: cubicweb.web.views.timetable - :members: - -:mod:`cubicweb.web.views.timeline` -================================== - -.. automodule:: cubicweb.web.views.timeline - :members: - -:mod:`cubicweb.web.views.baseviews` -=================================== - -.. automodule:: cubicweb.web.views.baseviews - :members: - -:mod:`cubicweb.web.views.boxes` -=============================== - -.. automodule:: cubicweb.web.views.boxes - :members: - -:mod:`cubicweb.web.views.old_calendar` -====================================== - -.. automodule:: cubicweb.web.views.old_calendar - :members: - -:mod:`cubicweb.web.views.card` -============================== - -.. automodule:: cubicweb.web.views.card - :members: - -:mod:`cubicweb.web.views.ibreadcrumbs` -====================================== - -.. automodule:: cubicweb.web.views.ibreadcrumbs - :members: - -:mod:`cubicweb.web.views.basecontrollers` -========================================= - -.. automodule:: cubicweb.web.views.basecontrollers - :members: - -:mod:`cubicweb.web.views.embedding` -=================================== - -.. automodule:: cubicweb.web.views.embedding - :members: - -:mod:`cubicweb.web.views.actions` -================================= - -.. automodule:: cubicweb.web.views.actions - :members: - -:mod:`cubicweb.web.views.editcontroller` -======================================== - -.. automodule:: cubicweb.web.views.editcontroller - :members: - -:mod:`cubicweb.web.views.debug` -=============================== - -.. automodule:: cubicweb.web.views.debug - :members: - -:mod:`cubicweb.web.views.urlpublishing` -======================================= - -.. automodule:: cubicweb.web.views.urlpublishing - :members: - -:mod:`cubicweb.web.views.baseforms` -=================================== - -.. automodule:: cubicweb.web.views.baseforms - :members: - -:mod:`cubicweb.web.views.urlrewrite` -==================================== - -.. automodule:: cubicweb.web.views.urlrewrite - :members: - -:mod:`cubicweb.web.views.massmailing` -===================================== - -.. automodule:: cubicweb.web.views.massmailing - :members: - -:mod:`cubicweb.web.views` -========================= - -.. automodule:: cubicweb.web.views - :members: - -:mod:`cubicweb.web.views.eproperties` -===================================== - -.. automodule:: cubicweb.web.views.eproperties - :members: - -:mod:`cubicweb.web.views.tabs` -============================== - -.. automodule:: cubicweb.web.views.tabs - :members: - -:mod:`cubicweb.web.views.vcard` -=============================== - -.. automodule:: cubicweb.web.views.vcard - :members: - -:mod:`cubicweb.web.views.wdoc` -============================== - -.. automodule:: cubicweb.web.views.wdoc - :members: - -:mod:`cubicweb.web.views.authentication` -======================================== - -.. automodule:: cubicweb.web.views.authentication - :members: - -:mod:`cubicweb.web.views.tableview` -=================================== - -.. automodule:: cubicweb.web.views.tableview - :members: - -:mod:`cubicweb.web.views.management` -==================================== - -.. automodule:: cubicweb.web.views.management - :members: - -:mod:`cubicweb.web.views.igeocodable` -===================================== - -.. automodule:: cubicweb.web.views.igeocodable - :members: - -:mod:`cubicweb.web.views.xbel` -============================== - -.. automodule:: cubicweb.web.views.xbel - :members: - -:mod:`cubicweb.web.views.bookmark` -================================== - -.. automodule:: cubicweb.web.views.bookmark - :members: - -:mod:`cubicweb.web.views.apacherewrite` -======================================= - -.. automodule:: cubicweb.web.views.apacherewrite - :members: - -:mod:`cubicweb.web.views.dynimages` -=================================== - -.. automodule:: cubicweb.web.views.dynimages - :members: - -:mod:`cubicweb.web.views.searchrestriction` -=========================================== - -.. automodule:: cubicweb.web.views.searchrestriction - :members: - -:mod:`cubicweb.web.views.basecomponents` -======================================== - -.. automodule:: cubicweb.web.views.basecomponents - :members: - -:mod:`cubicweb.web.views.calendar` -================================== - -.. automodule:: cubicweb.web.views.calendar - :members: - -:mod:`cubicweb.sobjects.supervising` -==================================== - -.. automodule:: cubicweb.sobjects.supervising - :members: - -:mod:`cubicweb.sobjects.hooks` -============================== - -.. automodule:: cubicweb.sobjects.hooks - :members: - -:mod:`cubicweb.sobjects.email` -============================== - -.. automodule:: cubicweb.sobjects.email - :members: - -:mod:`cubicweb.sobjects` -======================== - -.. automodule:: cubicweb.sobjects - :members: - -:mod:`cubicweb.sobjects.notification` -===================================== - -.. automodule:: cubicweb.sobjects.notification - :members: - -:mod:`cubicweb.wsgi.request` -============================ - -.. automodule:: cubicweb.wsgi.request - :members: - -:mod:`cubicweb.wsgi` -==================== - -.. automodule:: cubicweb.wsgi - :members: - -:mod:`cubicweb.wsgi.handler` -============================ - -.. automodule:: cubicweb.wsgi.handler - :members: - -:mod:`cubicweb.etwist.server` -============================= - -.. automodule:: cubicweb.etwist.server - :members: - -:mod:`cubicweb.etwist.request` -============================== - -.. automodule:: cubicweb.etwist.request - :members: - -:mod:`cubicweb.etwist.twconfig` -=============================== - -.. automodule:: cubicweb.etwist.twconfig - :members: - -:mod:`cubicweb.etwist` -====================== - -.. automodule:: cubicweb.etwist - :members: - -:mod:`cubicweb.etwist.twctl` -============================ - -.. automodule:: cubicweb.etwist.twctl - :members: - -:mod:`cubicweb.goa.goaconfig` -============================= - -.. automodule:: cubicweb.goa.goaconfig - :members: - -:mod:`cubicweb.goa.rqlinterpreter` -================================== - -.. automodule:: cubicweb.goa.rqlinterpreter - :members: - -:mod:`cubicweb.goa.dbmyams` -=========================== - -.. automodule:: cubicweb.goa.dbmyams - :members: - -:mod:`cubicweb.goa.db` -====================== - -.. automodule:: cubicweb.goa.db - :members: - -:mod:`cubicweb.goa.goactl` -========================== - -.. automodule:: cubicweb.goa.goactl - :members: - -:mod:`cubicweb.goa.goavreg` -=========================== - -.. automodule:: cubicweb.goa.goavreg - :members: - -:mod:`cubicweb.goa` -=================== - -.. automodule:: cubicweb.goa - :members: - -:mod:`cubicweb.goa.gaesource` -============================= - -.. automodule:: cubicweb.goa.gaesource - :members: - -:mod:`cubicweb.goa.dbinit` -========================== - -.. automodule:: cubicweb.goa.dbinit - :members: - -:mod:`cubicweb.goa.testlib` -=========================== - -.. automodule:: cubicweb.goa.testlib - :members: - -:mod:`cubicweb.goa.appobjects.dbmgmt` -===================================== - -.. automodule:: cubicweb.goa.appobjects.dbmgmt - :members: - -:mod:`cubicweb.goa.appobjects.gauthservice` -=========================================== - -.. automodule:: cubicweb.goa.appobjects.gauthservice - :members: - -:mod:`cubicweb.goa.appobjects.sessions` -======================================= - -.. automodule:: cubicweb.goa.appobjects.sessions - :members: - -:mod:`cubicweb.goa.appobjects` -============================== - -.. automodule:: cubicweb.goa.appobjects - :members: - -:mod:`cubicweb.goa.appobjects.components` -========================================= - -.. automodule:: cubicweb.goa.appobjects.components - :members: - -:mod:`cubicweb.goa.tools.laxctl` -================================ - -.. automodule:: cubicweb.goa.tools.laxctl - :members: - -:mod:`cubicweb.goa.tools.generate_schema_img` -============================================= - -.. automodule:: cubicweb.goa.tools.generate_schema_img - :members: - -:mod:`cubicweb.goa.tools` -========================= - -.. automodule:: cubicweb.goa.tools - :members: - -:mod:`cubicweb.goa.tools.i18n` -============================== - -.. automodule:: cubicweb.goa.tools.i18n - :members: - -:mod:`cubicweb.goa.overrides.mttransforms` -========================================== - -.. automodule:: cubicweb.goa.overrides.mttransforms - :members: - -:mod:`cubicweb.goa.overrides.rqlannotation` -=========================================== - -.. automodule:: cubicweb.goa.overrides.rqlannotation - :members: - -:mod:`cubicweb.goa.overrides.toolsutils` -======================================== - -.. automodule:: cubicweb.goa.overrides.toolsutils - :members: - -:mod:`cubicweb.goa.overrides` -============================= - -.. automodule:: cubicweb.goa.overrides - :members: - -:mod:`cubicweb.goa.overrides.server__init__` -============================================ - -.. automodule:: cubicweb.goa.overrides.server__init__ - :members: - -:mod:`cubicweb.goa.overrides.server_utils` -========================================== - -.. automodule:: cubicweb.goa.overrides.server_utils - :members: - -:mod:`cubicweb.common.mttransforms` -=================================== - -.. automodule:: cubicweb.common.mttransforms - :members: - -:mod:`cubicweb.common.utils` -============================ - -.. automodule:: cubicweb.common.utils - :members: - -:mod:`cubicweb.common.schema` -============================= - -.. automodule:: cubicweb.common.schema - :members: - -:mod:`cubicweb.common.tal` -========================== - -.. automodule:: cubicweb.common.tal - :members: - -:mod:`cubicweb.common.appobject` -================================ - -.. automodule:: cubicweb.common.appobject - :members: - -:mod:`cubicweb.common.migration` -================================ - -.. automodule:: cubicweb.common.migration - :members: - -:mod:`cubicweb.common.rest` -=========================== - -.. automodule:: cubicweb.common.rest - :members: - -:mod:`cubicweb.common.html4zope` -================================ - -.. automodule:: cubicweb.common.html4zope - :members: - -:mod:`cubicweb.common.view` -=========================== - -.. automodule:: cubicweb.common.view - :members: - -:mod:`cubicweb.common.selectors` -================================ - -.. automodule:: cubicweb.common.selectors - :members: - -:mod:`cubicweb.common.entity` -============================= - -.. automodule:: cubicweb.common.entity - :members: - -:mod:`cubicweb.common.mail` -=========================== - -.. automodule:: cubicweb.common.mail - :members: - -:mod:`cubicweb.common.mixins` -============================= - -.. automodule:: cubicweb.common.mixins - :members: - -:mod:`cubicweb.common` -====================== - -.. automodule:: cubicweb.common - :members: - -:mod:`cubicweb.common.uilib` -============================ - -.. automodule:: cubicweb.common.uilib - :members: - -:mod:`cubicweb.common.registerers` -================================== - -.. automodule:: cubicweb.common.registerers - :members: - -:mod:`cubicweb.common.i18n` -=========================== - -.. automodule:: cubicweb.common.i18n - :members: - -:mod:`cubicweb.entities.schemaobjs` -=================================== - -.. automodule:: cubicweb.entities.schemaobjs - :members: - -:mod:`cubicweb.entities.wfobjs` -=============================== - -.. automodule:: cubicweb.entities.wfobjs - :members: - -:mod:`cubicweb.entities` -======================== - -.. automodule:: cubicweb.entities - :members: - -:mod:`cubicweb.entities.authobjs` -================================= - -.. automodule:: cubicweb.entities.authobjs - :members: - -:mod:`cubicweb.entities.lib` -============================ - -.. automodule:: cubicweb.entities.lib - :members: - -:mod:`cubicweb.server.server` -============================= - -.. automodule:: cubicweb.server.server - :members: - -:mod:`cubicweb.server.utils` -============================ - -.. automodule:: cubicweb.server.utils - :members: - -:mod:`cubicweb.server.checkintegrity` -===================================== - -.. automodule:: cubicweb.server.checkintegrity - :members: - -:mod:`cubicweb.server.rqlrewrite` -================================= - -.. automodule:: cubicweb.server.rqlrewrite - :members: - -:mod:`cubicweb.server.rqlannotation` -==================================== - -.. automodule:: cubicweb.server.rqlannotation - :members: - -:mod:`cubicweb.server.hooks` -============================ - -.. automodule:: cubicweb.server.hooks - :members: - -:mod:`cubicweb.server.hooksmanager` -=================================== - -.. automodule:: cubicweb.server.hooksmanager - :members: - -:mod:`cubicweb.server.securityhooks` -==================================== - -.. automodule:: cubicweb.server.securityhooks - :members: - -:mod:`cubicweb.server.schemahooks` -================================== - -.. automodule:: cubicweb.server.schemahooks - :members: - -:mod:`cubicweb.server.session` -============================== - -.. automodule:: cubicweb.server.session - :members: - -:mod:`cubicweb.server.serverctl` -================================ - -.. automodule:: cubicweb.server.serverctl - :members: - -:mod:`cubicweb.server.serverconfig` -=================================== - -.. automodule:: cubicweb.server.serverconfig - :members: - -:mod:`cubicweb.server.pool` -=========================== - -.. automodule:: cubicweb.server.pool - :members: - -:mod:`cubicweb.server.mssteps` -============================== - -.. automodule:: cubicweb.server.mssteps - :members: - -:mod:`cubicweb.server.hookhelper` -================================= - -.. automodule:: cubicweb.server.hookhelper - :members: - -:mod:`cubicweb.server` -====================== - -.. automodule:: cubicweb.server - :members: - -:mod:`cubicweb.server.sqlutils` -=============================== - -.. automodule:: cubicweb.server.sqlutils - :members: - -:mod:`cubicweb.server.schemaserial` -=================================== - -.. automodule:: cubicweb.server.schemaserial - :members: - -:mod:`cubicweb.server.repository` -================================= - -.. automodule:: cubicweb.server.repository - :members: - -:mod:`cubicweb.server.ssplanner` -================================ - -.. automodule:: cubicweb.server.ssplanner - :members: - -:mod:`cubicweb.server.msplanner` -================================ - -.. automodule:: cubicweb.server.msplanner - :members: - -:mod:`cubicweb.server.querier` -============================== - -.. automodule:: cubicweb.server.querier - :members: - -:mod:`cubicweb.server.migractions` -================================== - -.. automodule:: cubicweb.server.migractions - :members: - -:mod:`cubicweb.server.sources.rql2sql` -====================================== - -.. automodule:: cubicweb.server.sources.rql2sql - :members: - -:mod:`cubicweb.server.sources.ldapuser` -======================================= - -.. automodule:: cubicweb.server.sources.ldapuser - :members: - -:mod:`cubicweb.server.sources` -============================== - -.. automodule:: cubicweb.server.sources - :members: - -:mod:`cubicweb.server.sources.pyrorql` -====================================== - -.. automodule:: cubicweb.server.sources.pyrorql - :members: - -:mod:`cubicweb.server.sources.native` -===================================== - -.. automodule:: cubicweb.server.sources.native - :members: - -:mod:`cubicweb.server.sources.extlite` -====================================== - -.. automodule:: cubicweb.server.sources.extlite - :members: - -:mod:`cubicweb.devtools.devctl` -=============================== - -.. automodule:: cubicweb.devtools.devctl - :members: - -:mod:`cubicweb.devtools.pkginfo` -================================ - -.. automodule:: cubicweb.devtools.pkginfo - :members: - -:mod:`cubicweb.devtools.migrtest` -================================= - -.. automodule:: cubicweb.devtools.migrtest - :members: - -:mod:`cubicweb.devtools.htmlparser` -=================================== - -.. automodule:: cubicweb.devtools.htmlparser - :members: - -:mod:`cubicweb.devtools` -======================== - -.. automodule:: cubicweb.devtools - :members: - -:mod:`cubicweb.devtools.fill` -============================= - -.. automodule:: cubicweb.devtools.fill - :members: - -:mod:`cubicweb.devtools._apptest` -================================= - -.. automodule:: cubicweb.devtools._apptest - :members: - -:mod:`cubicweb.devtools.stresstester` -===================================== - -.. automodule:: cubicweb.devtools.stresstester - :members: - -:mod:`cubicweb.devtools.fake` -============================= - -.. automodule:: cubicweb.devtools.fake - :members: - -:mod:`cubicweb.devtools.apptest` -================================ - -.. automodule:: cubicweb.devtools.apptest - :members: - -:mod:`cubicweb.devtools.livetest` -================================= - -.. automodule:: cubicweb.devtools.livetest - :members: - -:mod:`cubicweb.devtools.testlib` -================================ - -.. automodule:: cubicweb.devtools.testlib - :members: - -:mod:`cubicweb.devtools.repotest` -================================= - -.. automodule:: cubicweb.devtools.repotest - :members: - -:mod:`cubicweb.devtools.cwtwill` -================================ - -.. automodule:: cubicweb.devtools.cwtwill - :members: - -:mod:`cubicweb.misc.cwdesklets.rqlsensor` -========================================= - -.. automodule:: cubicweb.misc.cwdesklets.rqlsensor - :members: - -:mod:`cubicweb.embedded.mx` -=========================== - -.. automodule:: cubicweb.embedded.mx - :members: - -:mod:`cubicweb.embedded.mx.DateTime.mxDateTime_python` -====================================================== - -.. automodule:: cubicweb.embedded.mx.DateTime.mxDateTime_python - :members: - -:mod:`cubicweb.embedded.mx.DateTime.ARPA` -========================================= - -.. automodule:: cubicweb.embedded.mx.DateTime.ARPA - :members: - -:mod:`cubicweb.embedded.mx.DateTime.ISO` -======================================== - -.. automodule:: cubicweb.embedded.mx.DateTime.ISO - :members: - -:mod:`cubicweb.embedded.mx.DateTime.Parser` -=========================================== - -.. automodule:: cubicweb.embedded.mx.DateTime.Parser - :members: - -:mod:`cubicweb.embedded.mx.DateTime` -==================================== - -.. automodule:: cubicweb.embedded.mx.DateTime - :members: - -:mod:`cubicweb.embedded.mx.DateTime.Timezone` -============================================= - -.. automodule:: cubicweb.embedded.mx.DateTime.Timezone - :members: - -:mod:`cubicweb.embedded.mx.DateTime.DateTime` -============================================= - -.. automodule:: cubicweb.embedded.mx.DateTime.DateTime - :members: - -:mod:`indexer` -============== - -.. automodule:: indexer - :members: - -:mod:`indexer.indexable_objects` -================================ - -.. automodule:: indexer.indexable_objects - :members: - -:mod:`indexer.search` -===================== - -.. automodule:: indexer.search - :members: - -:mod:`indexer.query_objects` -============================ - -.. automodule:: indexer.query_objects - :members: - -:mod:`indexer._exceptions` -========================== - -.. automodule:: indexer._exceptions - :members: - -:mod:`indexer.setup` -==================== - -.. automodule:: indexer.setup - :members: - -:mod:`indexer.query` -==================== - -.. automodule:: indexer.query - :members: - -:mod:`logilab` -============== - -.. automodule:: logilab - :members: - -:mod:`logilab.constraint.propagation` -===================================== - -.. automodule:: logilab.constraint.propagation - :members: - -:mod:`logilab.constraint.psyco_wrapper` -======================================= - -.. automodule:: logilab.constraint.psyco_wrapper - :members: - -:mod:`logilab.constraint.fd` -============================ - -.. automodule:: logilab.constraint.fd - :members: - -:mod:`logilab.constraint.fi` -============================ - -.. automodule:: logilab.constraint.fi - :members: - -:mod:`logilab.constraint` -========================= - -.. automodule:: logilab.constraint - :members: - -:mod:`logilab.constraint.setup` -=============================== - -.. automodule:: logilab.constraint.setup - :members: - -:mod:`logilab.constraint.interfaces` -==================================== - -.. automodule:: logilab.constraint.interfaces - :members: - -:mod:`logilab.constraint.distributors` -====================================== - -.. automodule:: logilab.constraint.distributors - :members: - -:mod:`logilab.common.clcommands` -================================ - -.. automodule:: logilab.common.clcommands - :members: - -:mod:`logilab.common.table` -=========================== - -.. automodule:: logilab.common.table - :members: - -:mod:`logilab.common.interface` -=============================== - -.. automodule:: logilab.common.interface - :members: - -:mod:`logilab.common.logger` -============================ - -.. automodule:: logilab.common.logger - :members: - -:mod:`logilab.common.cli` -========================= - -.. automodule:: logilab.common.cli - :members: - -:mod:`logilab.common.xmlrpcutils` -================================= - -.. automodule:: logilab.common.xmlrpcutils - :members: - -:mod:`logilab.common.corbautils` -================================ - -.. automodule:: logilab.common.corbautils - :members: - -:mod:`logilab.common.cache` -=========================== - -.. automodule:: logilab.common.cache - :members: - -:mod:`logilab.common.astutils` -============================== - -.. automodule:: logilab.common.astutils - :members: - -:mod:`logilab.common.daemon` -============================ - -.. automodule:: logilab.common.daemon - :members: - -:mod:`logilab.common.tree` -========================== - -.. automodule:: logilab.common.tree - :members: - -:mod:`logilab.common.textutils` -=============================== - -.. automodule:: logilab.common.textutils - :members: - -:mod:`logilab.common.modutils` -============================== - -.. automodule:: logilab.common.modutils - :members: - -:mod:`logilab.common.fileutils` -=============================== - -.. automodule:: logilab.common.fileutils - :members: - -:mod:`logilab.common.patricia` -============================== - -.. automodule:: logilab.common.patricia - :members: - -:mod:`logilab.common.date` -========================== - -.. automodule:: logilab.common.date - :members: - -:mod:`logilab.common.optparser` -=============================== - -.. automodule:: logilab.common.optparser - :members: - -:mod:`logilab.common.twisted_distutils` -======================================= - -.. automodule:: logilab.common.twisted_distutils - :members: - -:mod:`logilab.common.decorators` -================================ - -.. automodule:: logilab.common.decorators - :members: - -:mod:`logilab.common.db` -======================== - -.. automodule:: logilab.common.db - :members: - -:mod:`logilab.common.deprecation` -================================= - -.. automodule:: logilab.common.deprecation - :members: - -:mod:`logilab.common.tasksqueue` -================================ - -.. automodule:: logilab.common.tasksqueue - :members: - -:mod:`logilab.common.changelog` -=============================== - -.. automodule:: logilab.common.changelog - :members: - -:mod:`logilab.common.shellutils` -================================ - -.. automodule:: logilab.common.shellutils - :members: - -:mod:`logilab.common.sqlgen` -============================ - -.. automodule:: logilab.common.sqlgen - :members: - -:mod:`logilab.common.optik_ext` -=============================== - -.. automodule:: logilab.common.optik_ext - :members: - -:mod:`logilab.common.configuration` -=================================== - -.. automodule:: logilab.common.configuration - :members: - -:mod:`logilab.common.visitor` -============================= - -.. automodule:: logilab.common.visitor - :members: - -:mod:`logilab.common.pytest` -============================ - -.. automodule:: logilab.common.pytest - :members: - -:mod:`logilab.common` -===================== - -.. automodule:: logilab.common - :members: - -:mod:`logilab.common.setup` -=========================== - -.. automodule:: logilab.common.setup - :members: - -:mod:`logilab.common.logservice` -================================ - -.. automodule:: logilab.common.logservice - :members: - -:mod:`logilab.common.debugger` -============================== - -.. automodule:: logilab.common.debugger - :members: - -:mod:`logilab.common.html` -========================== - -.. automodule:: logilab.common.html - :members: - -:mod:`logilab.common.vcgutils` -============================== - -.. automodule:: logilab.common.vcgutils - :members: - -:mod:`logilab.common.compat` -============================ - -.. automodule:: logilab.common.compat - :members: - -:mod:`logilab.common.logging_ext` -================================= - -.. automodule:: logilab.common.logging_ext - :members: - -:mod:`logilab.common.umessage` -============================== - -.. automodule:: logilab.common.umessage - :members: - -:mod:`logilab.common.proc` -========================== - -.. automodule:: logilab.common.proc - :members: - -:mod:`logilab.common.monclient` -=============================== - -.. automodule:: logilab.common.monclient - :members: - -:mod:`logilab.common.bind` -========================== - -.. automodule:: logilab.common.bind - :members: - -:mod:`logilab.common.graph` -=========================== - -.. automodule:: logilab.common.graph - :members: - -:mod:`logilab.common.testlib` -============================= - -.. automodule:: logilab.common.testlib - :members: - -:mod:`logilab.common.contexts` -============================== - -.. automodule:: logilab.common.contexts - :members: - -:mod:`logilab.common.adbh` -========================== - -.. automodule:: logilab.common.adbh - :members: - -:mod:`logilab.common.pdf_ext` -============================= - -.. automodule:: logilab.common.pdf_ext - :members: - -:mod:`logilab.common.monserver` -=============================== - -.. automodule:: logilab.common.monserver - :members: - -:mod:`logilab.common.ureports.nodes` -==================================== - -.. automodule:: logilab.common.ureports.nodes - :members: - -:mod:`logilab.common.ureports` -============================== - -.. automodule:: logilab.common.ureports - :members: - -:mod:`logilab.common.ureports.html_writer` -========================================== - -.. automodule:: logilab.common.ureports.html_writer - :members: - -:mod:`logilab.common.ureports.text_writer` -========================================== - -.. automodule:: logilab.common.ureports.text_writer - :members: - -:mod:`logilab.common.ureports.docbook_writer` -============================================= - -.. automodule:: logilab.common.ureports.docbook_writer - :members: - -:mod:`logilab.mtconverter.engine` -================================= - -.. automodule:: logilab.mtconverter.engine - :members: - -:mod:`logilab.mtconverter.transform` -==================================== - -.. automodule:: logilab.mtconverter.transform - :members: - -:mod:`logilab.mtconverter` -========================== - -.. automodule:: logilab.mtconverter - :members: - -:mod:`logilab.mtconverter.setup` -================================ - -.. automodule:: logilab.mtconverter.setup - :members: - -:mod:`logilab.mtconverter.transforms.html2text` -=============================================== - -.. automodule:: logilab.mtconverter.transforms.html2text - :members: - -:mod:`logilab.mtconverter.transforms.cmdtransforms` -=================================================== - -.. automodule:: logilab.mtconverter.transforms.cmdtransforms - :members: - -:mod:`logilab.mtconverter.transforms.python` -============================================ - -.. automodule:: logilab.mtconverter.transforms.python - :members: - -:mod:`logilab.mtconverter.transforms.pygmentstransforms` -======================================================== - -.. automodule:: logilab.mtconverter.transforms.pygmentstransforms - :members: - -:mod:`logilab.mtconverter.transforms` -===================================== - -.. automodule:: logilab.mtconverter.transforms - :members: - -:mod:`logilab.mtconverter.transforms.piltransforms` -=================================================== - -.. automodule:: logilab.mtconverter.transforms.piltransforms - :members: - -:mod:`logilab.devtools.cvstatus` -================================ - -.. automodule:: logilab.devtools.cvstatus - :members: - -:mod:`logilab.devtools.changelog` -================================= - -.. automodule:: logilab.devtools.changelog - :members: - -:mod:`logilab.devtools` -======================= - -.. automodule:: logilab.devtools - :members: - -:mod:`logilab.devtools.setup` -============================= - -.. automodule:: logilab.devtools.setup - :members: - -:mod:`logilab.devtools.cvslog` -============================== - -.. automodule:: logilab.devtools.cvslog - :members: - -:mod:`logilab.devtools.lgp.utils` -================================= - -.. automodule:: logilab.devtools.lgp.utils - :members: - -:mod:`logilab.devtools.lgp.tag` -=============================== - -.. automodule:: logilab.devtools.lgp.tag - :members: - -:mod:`logilab.devtools.lgp.setupinfo` -===================================== - -.. automodule:: logilab.devtools.lgp.setupinfo - :members: - -:mod:`logilab.devtools.lgp.changelog` -===================================== - -.. automodule:: logilab.devtools.lgp.changelog - :members: - -:mod:`logilab.devtools.lgp.preparedist` -======================================= - -.. automodule:: logilab.devtools.lgp.preparedist - :members: - -:mod:`logilab.devtools.lgp.build` -================================= - -.. automodule:: logilab.devtools.lgp.build - :members: - -:mod:`logilab.devtools.lgp.clean` -================================= - -.. automodule:: logilab.devtools.lgp.clean - :members: - -:mod:`logilab.devtools.lgp` -=========================== - -.. automodule:: logilab.devtools.lgp - :members: - -:mod:`logilab.devtools.lgp.setup` -================================= - -.. automodule:: logilab.devtools.lgp.setup - :members: - -:mod:`logilab.devtools.lgp.check` -================================= - -.. automodule:: logilab.devtools.lgp.check - :members: - -:mod:`logilab.devtools.lgp.exceptions` -====================================== - -.. automodule:: logilab.devtools.lgp.exceptions - :members: - -:mod:`logilab.devtools.templates` -================================= - -.. automodule:: logilab.devtools.templates - :members: - -:mod:`logilab.devtools.templates.setup` -======================================= - -.. automodule:: logilab.devtools.templates.setup - :members: - -:mod:`logilab.devtools.lib.coverage` -==================================== - -.. automodule:: logilab.devtools.lib.coverage - :members: - -:mod:`logilab.devtools.lib.manifest` -==================================== - -.. automodule:: logilab.devtools.lib.manifest - :members: - -:mod:`logilab.devtools.lib.pkginfo` -=================================== - -.. automodule:: logilab.devtools.lib.pkginfo - :members: - -:mod:`logilab.devtools.lib` -=========================== - -.. automodule:: logilab.devtools.lib - :members: - -:mod:`logilab.devtools.vcslib.cvsparse` -======================================= - -.. automodule:: logilab.devtools.vcslib.cvsparse - :members: - -:mod:`logilab.devtools.vcslib.svn` -================================== - -.. automodule:: logilab.devtools.vcslib.svn - :members: - -:mod:`logilab.devtools.vcslib.node` -=================================== - -.. automodule:: logilab.devtools.vcslib.node - :members: - -:mod:`logilab.devtools.vcslib` -============================== - -.. automodule:: logilab.devtools.vcslib - :members: - -:mod:`logilab.devtools.vcslib.interfaces` -========================================= - -.. automodule:: logilab.devtools.vcslib.interfaces - :members: - -:mod:`logilab.devtools.vcslib.cvs` -================================== - -.. automodule:: logilab.devtools.vcslib.cvs - :members: - -:mod:`logilab.devtools.vcslib.hg` -================================= - -.. automodule:: logilab.devtools.vcslib.hg - :members: - -:mod:`rql.nodes` -================ - -.. automodule:: rql.nodes - :members: - -:mod:`rql.undo` -=============== - -.. automodule:: rql.undo - :members: - -:mod:`rql.utils` -================ - -.. automodule:: rql.utils - :members: - -:mod:`rql.base` -=============== - -.. automodule:: rql.base - :members: - -:mod:`rql.analyze` -================== - -.. automodule:: rql.analyze - :members: - -:mod:`rql._exceptions` -====================== - -.. automodule:: rql._exceptions - :members: - -:mod:`rql.compare` -================== - -.. automodule:: rql.compare - :members: - -:mod:`rql.stmts` -================ - -.. automodule:: rql.stmts - :members: - -:mod:`rql.parser_main` -====================== - -.. automodule:: rql.parser_main - :members: - -:mod:`rql.stcheck` -================== - -.. automodule:: rql.stcheck - :members: - -:mod:`rql.parser` -================= - -.. automodule:: rql.parser - :members: - -:mod:`rql` -========== - -.. automodule:: rql - :members: - -:mod:`rql.setup` -================ - -.. automodule:: rql.setup - :members: - -:mod:`rql.interfaces` -===================== - -.. automodule:: rql.interfaces - :members: - -:mod:`rql.editextensions` -========================= - -.. automodule:: rql.editextensions - :members: - -:mod:`rql.fol` -============== - -.. automodule:: rql.fol - :members: - -:mod:`rqlgen` -============= - -.. automodule:: rqlgen - :members: - -:mod:`yams.schema` -================== - -.. automodule:: yams.schema - :members: - -:mod:`yams.reader` -================== - -.. automodule:: yams.reader - :members: - -:mod:`yams.schema2sql` -====================== - -.. automodule:: yams.schema2sql - :members: - -:mod:`yams._exceptions` -======================= - -.. automodule:: yams._exceptions - :members: - -:mod:`yams.sqlreader` -===================== - -.. automodule:: yams.sqlreader - :members: - -:mod:`yams.schema2dot` -====================== - -.. automodule:: yams.schema2dot - :members: - -:mod:`yams` -=========== - -.. automodule:: yams - :members: - -:mod:`yams.setup` -================= - -.. automodule:: yams.setup - :members: - -:mod:`yams.interfaces` -====================== - -.. automodule:: yams.interfaces - :members: - -:mod:`yams.buildobjs` -===================== - -.. automodule:: yams.buildobjs - :members: - -:mod:`yams.constraints` -======================= - -.. automodule:: yams.constraints - :members: diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/Z010-beginners.en.txt --- a/doc/book/en/Z010-beginners.en.txt Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. _QuickInstall: - -Quick Installation of a *CubicWeb* instance -=========================================== - -.. include:: C010-setup.en.txt -.. include:: Z012-create-instance.en.txt - - diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/Z012-create-instance.en.txt --- a/doc/book/en/Z012-create-instance.en.txt Fri Feb 12 15:18:00 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,80 +0,0 @@ -.. -*- coding: utf-8 -*- - -=============================== -Creation of your first instance -=============================== - -What is an instance? --------------------- - -A *CubicWeb* instance is a container that -refers to cubes and configuration parameters for your web instance. -Each instance is stored as a directory in ``~/etc/cubicweb.d`` which enables -us to run your instance. - -What is a cube? ---------------- - -Cubes represent data and basic building bricks of your web instances : -blogs, person, date, addressbook and a lot more. - -.. XXX They related to each other by a 'Schema' which is also the PostGres representation. - -Each cube defines entities, their views, their schemas and workflows -in an independant directory located in ``/path/to/forest/cubicweb/cubes/`` -for a Mercurial installation or in ``/usr/share/cubicweb/cubes`` for -a debian package installation. For example, the 'blog' cube defines the entities -blogs and blogentries. - -When an *CubicWeb* instance is created, you list the cubes that you want to use. -Using a cube means having the entities defined in your cube's schema -available in your instance as well as their views and workflows. - - -Creating a basic *CubicWeb* Instance ------------------------------------- - -We can create an instance to view our -instance in a web browser. :: - - cubicweb-ctl create blog myblog - -.. XXX or :: - -.. XXX cubicweb-ctl create forge myforge - - -.. note:: - The commands used below are more detailled in the section dedicated to - :ref:`cubicweb-ctl`. - -A series of questions will be prompted to you, the default answer is usually -sufficient. You can allways modify the parameters later by editing -configuration files. When a user/psswd is requested to access the database -please use the login you create at the time you configured the database -(:ref:`ConfigurationPostgresql`). - -It is important to distinguish here the user used to access the database and -the user used to login to the cubicweb instance. When a *CubicWeb* instance -starts, it uses the login/psswd for the database to get the schema and handle -low level transaction. But, when ``cubicweb-ctl create`` asks for -a manager login/psswd of *CubicWeb*, it refers to an instance user -to administrate your web instance. -The configuration files are stored in *~/etc/cubicweb.d/myblog/*. - -To launch the web instance, you just type :: - - cubicweb-ctl start myblog - -You can see how it looks by -visiting the URL `http://localhost:8080`. -To login, please use the cubicweb administrator login/psswd you -defined when you created the instance. - -To shutdown the instance :: - - cubicweb-ctl stop myinstance - -.. XXX something like `cubicweb-ctl live-server intra` would be nice - - diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/admin/additional-tips.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/book/en/admin/additional-tips.rst Wed Mar 24 10:23:31 2010 +0100 @@ -0,0 +1,55 @@ + +.. _Additional Tips: + +Additional Tips +--------------- + +Here are some additional tips as far as administration of a CubicWeb is concerned. + +Backup, backup, backup +`````````````````````` + +It is always a good idea to backup. If your system does not do that, +you should set it up. Note that whenever you do an upgrade, +`cubicweb-ctl` offers you to backup your database. + +There are a number of ways for doing backups. Before you go ahead, +make sure the following permissions are correct :: + + # chgrp postgres /var/lib/cubicweb/backup + + # chmod g+ws /var/lib/cubicweb/backup + + # chgrp postgres /etc/cubicweb.d/**/sources + + # chmod g+r /etc/cubicweb.d/**/sources + +**Classic way** + +Simply use the pg_dump in a cron :: + + su -c "pg_dump -Fc --username=cubicweb --no-owner" postgres > -$(date '+%Y-%m-%d_%H:%M:%S').dump + +**CubicWeb way** + +The CubicWeb way is to use the `db-dump` command. For that, you have to put your passwords in a user-only-readable file at the +root of the postgres user. The file is `.pgpass` (`chmod 0600`), in this case for a socket run connection to postgres :: + + /var/run/postgresql:5432::cubicweb: + +The postgres documentation for the `.pgpass` format can be found `here`_ + +Then add the following command to the crontab of the postgres user (`su posgres 'crontab -e'`):: + + # m h dom mon dow command + 0 2 * * * cubicweb-ctl db-dump + +**The automated sysadmin way** + +You can use a combination `backup-ninja`_ (which has a postgres script in the example directory), `backuppc`)_ (for versionning). + +Please note that in the *CubicWeb way* it adds a second location for your password which is error-prone. + +.. _`here` : http://www.postgresql.org/docs/current/static/libpq-pgpass.html +.. _`backup-ninja` : https://labs.riseup.net/code/projects/show/backupninja/ +.. _`backuppc` : http://backuppc.sourceforge.net/ diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/admin/gae.rst --- a/doc/book/en/admin/gae.rst Fri Feb 12 15:18:00 2010 +0100 +++ b/doc/book/en/admin/gae.rst Wed Mar 24 10:23:31 2010 +0100 @@ -132,7 +132,7 @@ Otherwise, please skip it. You will never need to add new entries in the translation catalog. Instead we -would recommand you to use ``self.req._("msgId")`` in your code to flag new +would recommand you to use ``self._cw._("msgId")`` in your code to flag new message id to add to the catalog, where ``_`` refers to xgettext that is used to collect new strings to translate. While running ``laxctl i18ncube``, new string will be added to the catalogs. diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/admin/index.rst --- a/doc/book/en/admin/index.rst Fri Feb 12 15:18:00 2010 +0100 +++ b/doc/book/en/admin/index.rst Wed Mar 24 10:23:31 2010 +0100 @@ -19,7 +19,9 @@ site-config multisources ldap + pyro gae + additional-tips RQL logs -------- diff -r c25da7573ebd -r 02b52bf9f5f8 doc/book/en/admin/instance-config.rst --- a/doc/book/en/admin/instance-config.rst Fri Feb 12 15:18:00 2010 +0100 +++ b/doc/book/en/admin/instance-config.rst Wed Mar 24 10:23:31 2010 +0100 @@ -12,7 +12,8 @@ /etc/cubicweb.d/myblog/all-in-one.conf -It is a simple text file format INI. In the following description, +It is a simple text file in the INI format +(http://en.wikipedia.org/wiki/INI_file). In the following description, each option name is prefixed with its own section and followed by its default value if necessary, e.g. "`
    .