# HG changeset patch # User Nicolas Chauvat # Date 1299513735 -3600 # Node ID 9b1f9bc74f5d99c506629fbfff1d3fe0933ab7ff # Parent fee3a1f28ed7b81b470f14fb3d1978d7a523eed0# Parent df0e8581b06f99c28e10668427a1e02b2344756f merge stable changes into default diff -r fee3a1f28ed7 -r 9b1f9bc74f5d __pkginfo__.py diff -r fee3a1f28ed7 -r 9b1f9bc74f5d cwconfig.py --- a/cwconfig.py Fri Feb 18 17:11:45 2011 +0100 +++ b/cwconfig.py Mon Mar 07 17:02:15 2011 +0100 @@ -771,7 +771,7 @@ self.debug('%s loaded', sitefile) return module - def eproperty_definitions(self): + def cwproperty_definitions(self): cfg = self.persistent_options_configuration() for section, options in cfg.options_by_section(): section = section.lower() diff -r fee3a1f28ed7 -r 9b1f9bc74f5d cwctl.py --- a/cwctl.py Fri Feb 18 17:11:45 2011 +0100 +++ b/cwctl.py Mon Mar 07 17:02:15 2011 +0100 @@ -664,10 +664,11 @@ name = 'upgrade' actionverb = 'upgraded' options = InstanceCommand.options + ( - ('force-componant-version', - {'short': 't', 'type' : 'csv', 'metavar': 'cube1=X.Y.Z,cube2=X.Y.Z', + ('force-cube-version', + {'short': 't', 'type' : 'named', 'metavar': 'cube1:X.Y.Z,cube2:X.Y.Z', 'default': None, - 'help': 'force migration from the indicated version for the specified cube.'}), + 'help': 'force migration from the indicated version for the specified cube(s).'}), + ('force-cubicweb-version', {'short': 'e', 'type' : 'string', 'metavar': 'X.Y.Z', 'default': None, @@ -721,12 +722,9 @@ mih = config.migration_handler() repo = mih.repo_connect() vcconf = repo.get_versions() - if self.config.force_componant_version: - packversions = {} - for vdef in self.config.force_componant_version: - componant, version = vdef.split('=') - packversions[componant] = Version(version) - vcconf.update(packversions) + if self.config.force_cube_version: + for cube, version in self.config.force_cube_version.iteritems(): + vcconf[cube] = Version(version) toupgrade = [] for cube in config.cubes(): installedversion = config.cube_version(cube) diff -r fee3a1f28ed7 -r 9b1f9bc74f5d cwvreg.py --- a/cwvreg.py Fri Feb 18 17:11:45 2011 +0100 +++ b/cwvreg.py Mon Mar 07 17:02:15 2011 +0100 @@ -554,7 +554,7 @@ if not self.initialized: self['propertydefs'] = {} self['propertyvalues'] = self.eprop_values = {} - for key, propdef in self.config.eproperty_definitions(): + for key, propdef in self.config.cwproperty_definitions(): self.register_property(key, **propdef) CW_EVENT_MANAGER.emit('after-registry-reset', self) diff -r fee3a1f28ed7 -r 9b1f9bc74f5d dataimport.py --- a/dataimport.py Fri Feb 18 17:11:45 2011 +0100 +++ b/dataimport.py Mon Mar 07 17:02:15 2011 +0100 @@ -339,6 +339,7 @@ def rql(self, *args): if self._rql is not None: return self._rql(*args) + return [] @property def nb_inserted_entities(self): @@ -470,7 +471,7 @@ eid_from, rtype, eid_to = super(RQLObjectStore, self).relate( eid_from, rtype, eid_to) self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, - {'x': int(eid_from), 'y': int(eid_to)}, ('x', 'y')) + {'x': int(eid_from), 'y': int(eid_to)}) # the import controller ######################################################## @@ -513,7 +514,6 @@ traceback.print_exc(file=tmp) else: traceback.print_exception(type, value, tb, file=tmp) - print tmp.getvalue() # use a list to avoid counting a errors instead of one errorlog = self.errors.setdefault(key, []) if msg is None: diff -r fee3a1f28ed7 -r 9b1f9bc74f5d devtools/__init__.py --- a/devtools/__init__.py Fri Feb 18 17:11:45 2011 +0100 +++ b/devtools/__init__.py Mon Mar 07 17:02:15 2011 +0100 @@ -85,20 +85,10 @@ read_instance_schema = False init_repository = True db_require_setup = True - options = cwconfig.merge_options(ServerConfiguration.options + ( - ('anonymous-user', - {'type' : 'string', - 'default': None, - 'help': 'login of the CubicWeb user account to use for anonymous user (if you want to allow anonymous)', - 'group': 'main', 'level': 1, - }), - ('anonymous-password', - {'type' : 'string', - 'default': None, - 'help': 'password of the CubicWeb user account matching login', - 'group': 'main', 'level': 1, - }), - )) + options = cwconfig.merge_options( + ServerConfiguration.options + + tuple((opt, optdict) for opt, optdict in TwistedConfiguration.options + if opt in ('anonymous-user', 'anonymous-password'))) def __init__(self, appid, apphome=None, log_threshold=logging.CRITICAL+10): # must be set before calling parent __init__ diff -r fee3a1f28ed7 -r 9b1f9bc74f5d devtools/testlib.py --- a/devtools/testlib.py Fri Feb 18 17:11:45 2011 +0100 +++ b/devtools/testlib.py Mon Mar 07 17:02:15 2011 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -355,7 +355,7 @@ user = req.create_entity('CWUser', login=unicode(login), upassword=password, **kwargs) req.execute('SET X in_group G WHERE X eid %%(x)s, G name IN(%s)' - % ','.join(repr(g) for g in groups), + % ','.join(repr(str(g)) for g in groups), {'x': user.eid}) user.cw_clear_relation_cache('in_group', 'subject') if commit: @@ -434,6 +434,21 @@ # other utilities ######################################################### + def grant_permission(self, entity, group, pname, plabel=None): + """insert a permission on an entity. Will have to commit the main + connection to be considered + """ + pname = unicode(pname) + plabel = plabel and unicode(plabel) or unicode(group) + e = entity.eid + with security_enabled(self.session, False, False): + peid = self.execute( + 'INSERT CWPermission X: X name %(pname)s, X label %(plabel)s,' + 'X require_group G, E require_permission X ' + 'WHERE G name %(group)s, E eid %(e)s', + locals())[0][0] + return peid + @contextmanager def temporary_appobjects(self, *appobjects): self.vreg._loadedmods.setdefault(self.__module__, {}) @@ -445,7 +460,20 @@ for obj in appobjects: self.vreg.unregister(obj) - # vregistry inspection utilities ########################################### + def assertModificationDateGreater(self, entity, olddate): + entity.cw_attr_cache.pop('modification_date', None) + self.failUnless(entity.modification_date > olddate) + + + # workflow utilities ####################################################### + + def assertPossibleTransitions(self, entity, expected): + transitions = entity.cw_adapt_to('IWorkflowable').possible_transitions() + self.assertListEqual(sorted(tr.name for tr in transitions), + sorted(expected)) + + + # views and actions registries inspection ################################## def pviews(self, req, rset): return sorted((a.__regid__, a.__class__) diff -r fee3a1f28ed7 -r 9b1f9bc74f5d misc/migration/3.10.9_Any.py --- a/misc/migration/3.10.9_Any.py Fri Feb 18 17:11:45 2011 +0100 +++ b/misc/migration/3.10.9_Any.py Mon Mar 07 17:02:15 2011 +0100 @@ -1,3 +1,10 @@ +from __future__ import with_statement + +# fix some corrupted entities noticed on several instances +rql('DELETE CWConstraint X WHERE NOT E constrained_by X') +rql('SET X is_instance_of Y WHERE X is Y, NOT X is_instance_of Y') +commit() + if confirm('fix existing cwuri?'): from logilab.common.shellutils import ProgressBar from cubicweb.server.session import hooks_control @@ -10,3 +17,11 @@ commit(ask_confirm=False) pb.update() commit(ask_confirm=False) + +try: + from cubicweb import devtools + option_group_changed('anonymous-user', 'main', 'web') + option_group_changed('anonymous-password', 'main', 'web') +except ImportError: + # cubicweb-dev unavailable, nothing needed + pass diff -r fee3a1f28ed7 -r 9b1f9bc74f5d server/checkintegrity.py --- a/server/checkintegrity.py Fri Feb 18 17:11:45 2011 +0100 +++ b/server/checkintegrity.py Mon Mar 07 17:02:15 2011 +0100 @@ -34,6 +34,12 @@ from cubicweb.server.sqlutils import SQL_PREFIX from cubicweb.server.session import security_enabled +def notify_fixed(fix): + if fix: + print >> sys.stderr, ' [FIXED]' + else: + print >> sys.stderr + def has_eid(session, sqlcursor, eid, eids): """return true if the eid is a valid eid""" if eid in eids: @@ -167,9 +173,7 @@ print >> sys.stderr, msg % eid, if fix: session.system_sql('DELETE FROM appears WHERE uid=%s;' % eid) - print >> sys.stderr, ' [FIXED]' - else: - print >> sys.stderr + notify_fixed(fix) def check_entities(schema, session, eids, fix=1): @@ -183,9 +187,7 @@ print >> sys.stderr, msg % eid, if fix: session.system_sql('DELETE FROM entities WHERE eid=%s;' % eid) - print >> sys.stderr, ' [FIXED]' - else: - print >> sys.stderr + notify_fixed(fix) print 'Checking entities tables' for eschema in schema.entities(): if eschema.final: @@ -202,22 +204,19 @@ print >> sys.stderr, msg % (eid, eschema.type), if fix: session.system_sql('DELETE FROM %s WHERE %s=%s;' % (table, column, eid)) - print >> sys.stderr, ' [FIXED]' - else: - print >> sys.stderr + notify_fixed(fix) def bad_related_msg(rtype, target, eid, fix): msg = ' A relation %s with %s eid %s exists but no such entity in sources' print >> sys.stderr, msg % (rtype, target, eid), - if fix: - print >> sys.stderr, ' [FIXED]' - else: - print >> sys.stderr + notify_fixed(fix) def check_relations(schema, session, eids, fix=1): - """check all relations registered in the repo system table""" + """check that eids referenced by relations are registered in the repo system + table + """ print 'Checking relations' for rschema in schema.relations(): if rschema.final or rschema in PURE_VIRTUAL_RTYPES: @@ -263,6 +262,54 @@ session.system_sql(sql) +def check_mandatory_relations(schema, session, eids, fix=1): + """check entities missing some mandatory relation""" + print 'Checking mandatory relations' + for rschema in schema.relations(): + if rschema.final or rschema in PURE_VIRTUAL_RTYPES: + continue + smandatory = set() + omandatory = set() + for rdef in rschema.rdefs.values(): + if rdef.cardinality[0] in '1+': + smandatory.add(rdef.subject) + if rdef.cardinality[1] in '1+': + omandatory.add(rdef.object) + for role, etypes in (('subject', smandatory), ('object', omandatory)): + for etype in etypes: + if role == 'subject': + rql = 'Any X WHERE NOT X %s Y, X is %s' % (rschema, etype) + else: + rql = 'Any X WHERE NOT Y %s X, X is %s' % (rschema, etype) + for entity in session.execute(rql).entities(): + print >> sys.stderr, '%s #%s is missing mandatory %s relation %s' % ( + entity.__regid__, entity.eid, role, rschema) + if fix: + #if entity.cw_describe()['source']['uri'] == 'system': XXX + entity.delete() + notify_fixed(fix) + + +def check_mandatory_attributes(schema, session, eids, fix=1): + """check for entities stored in the system source missing some mandatory + attribute + """ + print 'Checking mandatory attributes' + for rschema in schema.relations(): + if not rschema.final or rschema in VIRTUAL_RTYPES: + continue + for rdef in rschema.rdefs.values(): + if rdef.cardinality[0] in '1+': + rql = 'Any X WHERE X %s NULL, X is %s, X cw_source S, S name "system"' % ( + rschema, rdef.subject) + for entity in session.execute(rql).entities(): + print >> sys.stderr, '%s #%s is missing mandatory attribute %s' % ( + entity.__regid__, entity.eid, rschema) + if fix: + entity.delete() + notify_fixed(fix) + + def check_metadata(schema, session, eids, fix=1): """check entities has required metadata @@ -285,9 +332,7 @@ session.system_sql("UPDATE %s SET %s=%%(v)s WHERE %s=%s ;" % (table, column, eidcolumn, eid), {'v': default}) - print >> sys.stderr, ' [FIXED]' - else: - print >> sys.stderr + notify_fixed(fix) cursor = session.system_sql('SELECT MIN(%s) FROM %sCWUser;' % (eidcolumn, SQL_PREFIX)) default_user_eid = cursor.fetchone()[0] @@ -303,9 +348,7 @@ if fix: session.system_sql('INSERT INTO %s_relation VALUES (%s, %s) ;' % (rel, eid, default)) - print >> sys.stderr, ' [FIXED]' - else: - print >> sys.stderr + notify_fixed(fix) def check(repo, cnx, checks, reindex, fix, withpb=True): diff -r fee3a1f28ed7 -r 9b1f9bc74f5d server/querier.py --- a/server/querier.py Fri Feb 18 17:11:45 2011 +0100 +++ b/server/querier.py Mon Mar 07 17:02:15 2011 +0100 @@ -586,12 +586,12 @@ def set_schema(self, schema): self.schema = schema repo = self._repo - # rql st and solution cache. Don't bother using a Cache instance: we - # should have a limited number of queries in there, since there are no - # entries in this cache for user queries (which have no args) - self._rql_cache = {} - # rql cache key cache - self._rql_ck_cache = Cache(repo.config['rql-cache-size']) + # rql st and solution cache. + self._rql_cache = Cache(repo.config['rql-cache-size']) + # rql cache key cache. Don't bother using a Cache instance: we should + # have a limited number of queries in there, since there are no entries + # in this cache for user queries (which have no args) + self._rql_ck_cache = {} # some cache usage stats self.cache_hit, self.cache_miss = 0, 0 # rql parsing / analysing helper @@ -656,11 +656,15 @@ print '*'*80 print 'querier input', rql, args # parse the query and binds variables + cachekey = rql try: - cachekey = rql if args: + # search for named args in query which are eids (hence + # influencing query's solutions) eidkeys = self._rql_ck_cache[rql] if eidkeys: + # if there are some, we need a better cache key, eg (rql + + # entity type of each eid) try: cachekey = self._repo.querier_cache_key(session, rql, args, eidkeys) @@ -674,15 +678,20 @@ self.cache_miss += 1 rqlst = self.parse(rql) try: + # compute solutions for rqlst and return named args in query + # which are eids. Notice that if you may not need `eidkeys`, we + # have to compute solutions anyway (kept as annotation on the + # tree) eidkeys = self.solutions(session, rqlst, args) except UnknownEid: # we want queries such as "Any X WHERE X eid 9999" return an # empty result instead of raising UnknownEid return empty_rset(rql, args, rqlst) - self._rql_ck_cache[rql] = eidkeys - if eidkeys: - cachekey = self._repo.querier_cache_key(session, rql, args, - eidkeys) + if args and not rql in self._rql_ck_cache: + self._rql_ck_cache[rql] = eidkeys + if eidkeys: + cachekey = self._repo.querier_cache_key(session, rql, args, + eidkeys) self._rql_cache[cachekey] = rqlst orig_rqlst = rqlst if rqlst.TYPE != 'select': diff -r fee3a1f28ed7 -r 9b1f9bc74f5d server/serverctl.py --- a/server/serverctl.py Fri Feb 18 17:11:45 2011 +0100 +++ b/server/serverctl.py Mon Mar 07 17:02:15 2011 +0100 @@ -388,6 +388,10 @@ 'default': False, 'help': 'insert drop statements to remove previously existant \ tables, indexes... (no by default)'}), + ('config-level', + {'short': 'l', 'type': 'int', 'default': 1, + 'help': 'level threshold for questions asked when configuring another source' + }), ) def run(self, args): @@ -412,7 +416,7 @@ % (config.sources_file(), str(ex).strip())) init_repository(config, drop=self.config.drop) while ASK.confirm('Enter another source ?', default_is_yes=False): - CWCTL.run(['add-source', config.appid]) + CWCTL.run(['add-source', '--config-level', self.config.config_level, config.appid]) class AddSourceCommand(Command): @@ -424,7 +428,12 @@ name = 'add-source' arguments = '' min_args = max_args = 1 - options = () + options = ( + ('config-level', + {'short': 'l', 'type': 'int', 'default': 1, + 'help': 'level threshold for questions asked when configuring another source' + }), + ) def run(self, args): appid = args[0] @@ -463,7 +472,7 @@ else: break # XXX configurable inputlevel - sconfig = ask_source_config(config, type, inputlevel=0) + sconfig = ask_source_config(config, type, inputlevel=self.config.config_level) cfgstr = unicode(generate_source_config(sconfig), sys.stdin.encoding) req.create_entity('CWSource', name=sourceuri, type=unicode(type), config=cfgstr) @@ -840,9 +849,12 @@ options = ( ('checks', {'short': 'c', 'type' : 'csv', 'metavar' : '', - 'default' : ('entities', 'relations', 'metadata', 'schema', 'text_index'), + 'default' : ('entities', 'relations', + 'mandatory_relations', 'mandatory_attributes', + 'metadata', 'schema', 'text_index'), 'help': 'Comma separated list of check to run. By default run all \ -checks, i.e. entities, relations, text_index and metadata.'} +checks, i.e. entities, relations, mandatory_relations, mandatory_attributes, \ +metadata, text_index and schema.'} ), ('autofix', diff -r fee3a1f28ed7 -r 9b1f9bc74f5d server/sources/__init__.py --- a/server/sources/__init__.py Fri Feb 18 17:11:45 2011 +0100 +++ b/server/sources/__init__.py Mon Mar 07 17:02:15 2011 +0100 @@ -572,7 +572,9 @@ pass def cursor(self): return None # no actual cursor support - + def close(self): + if hasattr(self.cnx, 'close'): + self.cnx.close() from cubicweb.server import SOURCE_TYPES diff -r fee3a1f28ed7 -r 9b1f9bc74f5d server/sources/ldapuser.py --- a/server/sources/ldapuser.py Fri Feb 18 17:11:45 2011 +0100 +++ b/server/sources/ldapuser.py Mon Mar 07 17:02:15 2011 +0100 @@ -95,13 +95,13 @@ {'type' : 'string', 'default': '', 'help': 'user dn to use to open data connection to the ldap (eg used \ -to respond to rql queries).', +to respond to rql queries). Leave empty for anonymous bind', 'group': 'ldap-source', 'level': 1, }), ('data-cnx-password', {'type' : 'string', 'default': '', - 'help': 'password to use to open data connection to the ldap (eg used to respond to rql queries).', + 'help': 'password to use to open data connection to the ldap (eg used to respond to rql queries). Leave empty for anonymous bind.', 'group': 'ldap-source', 'level': 1, }), @@ -109,19 +109,19 @@ {'type' : 'string', 'default': 'ou=People,dc=logilab,dc=fr', 'help': 'base DN to lookup for users', - 'group': 'ldap-source', 'level': 0, + 'group': 'ldap-source', 'level': 1, }), ('user-scope', {'type' : 'choice', 'default': 'ONELEVEL', 'choices': ('BASE', 'ONELEVEL', 'SUBTREE'), - 'help': 'user search scope', + 'help': 'user search scope (valid values: "BASE", "ONELEVEL", "SUBTREE")', 'group': 'ldap-source', 'level': 1, }), ('user-classes', {'type' : 'csv', 'default': ('top', 'posixAccount'), - 'help': 'classes of user', + 'help': 'classes of user (with Active Directory, you want to say "user" here)', 'group': 'ldap-source', 'level': 1, }), ('user-filter', @@ -133,7 +133,7 @@ ('user-login-attr', {'type' : 'string', 'default': 'uid', - 'help': 'attribute used as login on authentication', + 'help': 'attribute used as login on authentication (with Active Directory, you want to use "sAMAccountName" here)', 'group': 'ldap-source', 'level': 1, }), ('user-default-group', @@ -146,7 +146,7 @@ ('user-attrs-map', {'type' : 'named', 'default': {'uid': 'login', 'gecos': 'email'}, - 'help': 'map from ldap user attributes to cubicweb attributes', + 'help': 'map from ldap user attributes to cubicweb attributes (with Active Directory, you want to use sAMAccountName:login,mail:email,givenName:firstname,sn:surname)', 'group': 'ldap-source', 'level': 1, }), @@ -273,7 +273,7 @@ try: self._connect() except: - self.exception('cant connect to ldap') + self.exception('unable to connect to ldap:') return ConnectionWrapper(self._conn) def authenticate(self, session, login, password=None, **kwargs): diff -r fee3a1f28ed7 -r 9b1f9bc74f5d web/webconfig.py --- a/web/webconfig.py Fri Feb 18 17:11:45 2011 +0100 +++ b/web/webconfig.py Mon Mar 07 17:02:15 2011 +0100 @@ -207,8 +207,8 @@ def fckeditor_installed(self): return exists(self.uiprops['FCKEDITOR_PATH']) - def eproperty_definitions(self): - for key, pdef in super(WebConfiguration, self).eproperty_definitions(): + def cwproperty_definitions(self): + for key, pdef in super(WebConfiguration, self).cwproperty_definitions(): if key == 'ui.fckeditor' and not self.fckeditor_installed(): continue yield key, pdef