# HG changeset patch # User Sylvain Thénault # Date 1350050716 -7200 # Node ID ae0a567dff304cd6cb9f5c4645a7387225c5cd8c # Parent e20057a9ceea7740f65c6e3a92e6084bdd1f4e70# Parent e54b3bc39011f428ad6c868920c364448065dc29 backport stable diff -r e20057a9ceea -r ae0a567dff30 .hgtags --- a/.hgtags Fri Oct 12 15:38:58 2012 +0200 +++ b/.hgtags Fri Oct 12 16:05:16 2012 +0200 @@ -264,3 +264,5 @@ 9aa5553b26520ceb68539e7a32721b5cd5393e16 cubicweb-debian-version-3.15.2-1 0e012eb80990ca6f91aa9a8ad3324fbcf51435b1 cubicweb-version-3.15.3 7ad423a5b6a883dbdf00e6c87a5f8ab121041640 cubicweb-debian-version-3.15.3-1 +63260486de89a9dc32128cd0eacef891a668977b cubicweb-version-3.15.4 +70cb36c826df86de465f9b69647cef7096dcf12c cubicweb-debian-version-3.15.4-1 diff -r e20057a9ceea -r ae0a567dff30 cwctl.py --- a/cwctl.py Fri Oct 12 15:38:58 2012 +0200 +++ b/cwctl.py Fri Oct 12 16:05:16 2012 +0200 @@ -917,7 +917,7 @@ break cnx.load_appobjects() repo = cnx._repo - mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, + mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0, # hack so it don't try to load fs schema schema=1) else: diff -r e20057a9ceea -r ae0a567dff30 dbapi.py --- a/dbapi.py Fri Oct 12 15:38:58 2012 +0200 +++ b/dbapi.py Fri Oct 12 16:05:16 2012 +0200 @@ -103,7 +103,7 @@ return Repository(config, TasksManager(), vreg=vreg) elif method == 'zmq': from cubicweb.zmqclient import ZMQRepositoryClient - return ZMQRepositoryClient(config, vreg=vreg) + return ZMQRepositoryClient(database) else: # method == 'pyro' # resolve the Pyro object from logilab.common.pyro_ext import ns_get_proxy, get_proxy @@ -592,7 +592,12 @@ esubpath = list(subpath) esubpath.remove('views') esubpath.append(join('web', 'views')) + # first load available configs, necessary for proper persistent + # properties initialization + config.load_available_configs() + # then init cubes config.init_cubes(cubes) + # then load appobjects into the registry vpath = config.build_appobjects_path(reversed(config.cubes_path()), evobjpath=esubpath, tvobjpath=subpath) diff -r e20057a9ceea -r ae0a567dff30 misc/scripts/repair_splitbrain_ldapuser_source.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/scripts/repair_splitbrain_ldapuser_source.py Fri Oct 12 16:05:16 2012 +0200 @@ -0,0 +1,108 @@ +""" +CAUTION: READ THIS CAREFULLY + +Sometimes it happens that ldap (specifically ldapuser type) source +yield "ghost" users. The reasons may vary (server upgrade while some +instances are still running & syncing with the ldap source, unmanaged +updates to the upstream ldap, etc.). + +This script was written and refined enough times that we are confident +in that it does something reasonnable (at least it did for the +target application). + +However you should really REALLY understand what it does before +deciding to apply it for you. And then ADAPT it tou your needs. + +""" + +import base64 +from collections import defaultdict + +from cubicweb.server.session import hooks_control + +try: + source_name, = __args__ + source = repo.sources_by_uri[source_name] +except ValueError: + print('you should specify the source name as script argument (i.e. after --' + ' on the command line)') + sys.exit(1) +except KeyError: + print '%s is not an active source' % source_name + sys.exit(1) + +# check source is reachable before doing anything +if not source.get_connection().cnx: + print '%s is not reachable. Fix this before running this script' % source_name + sys.exit(1) + +def find_dupes(): + # XXX this retrieves entities from a source name "ldap" + # you will want to adjust + rset = sql("SELECT eid, extid FROM entities WHERE source='%s'" % source_name) + extid2eids = defaultdict(list) + for eid, extid in rset: + extid2eids[extid].append(eid) + return dict((base64.b64decode(extid).lower(), eids) + for extid, eids in extid2eids.items() + if len(eids) > 1) + +def merge_dupes(dupes, docommit=False): + gone_eids = [] + CWUser = schema['CWUser'] + for extid, eids in dupes.items(): + newest = eids.pop() # we merge everything on the newest + print 'merging ghosts of', extid, 'into', newest + # now we merge pairwise into the newest + for old in eids: + subst = {'old': old, 'new': newest} + print ' merging', old + gone_eids.append(old) + for rschema in CWUser.subject_relations(): + if rschema.final or rschema == 'identity': + continue + if CWUser.rdef(rschema, 'subject').composite == 'subject': + # old 'composite' property is wiped ... + # think about email addresses, excel preferences + for eschema in rschema.objects(): + rql('DELETE %s X WHERE U %s X, U eid %%(old)s' % (eschema, rschema), subst) + else: + # relink the new user to its old relations + rql('SET NU %s X WHERE NU eid %%(new)s, NOT NU %s X, OU %s X, OU eid %%(old)s' % + (rschema, rschema, rschema), subst) + # delete the old relations + rql('DELETE U %s X WHERE U eid %%(old)s' % rschema, subst) + # same thing ... + for rschema in CWUser.object_relations(): + if rschema.final or rschema == 'identity': + continue + rql('SET X %s NU WHERE NU eid %%(new)s, NOT X %s NU, X %s OU, OU eid %%(old)s' % + (rschema, rschema, rschema), subst) + rql('DELETE X %s U WHERE U eid %%(old)s' % rschema, subst) + if not docommit: + rollback() + return + commit() # XXX flushing operations is wanted rather than really committing + print 'clean up entities table' + sql('DELETE FROM entities WHERE eid IN (%s)' % (', '.join(str(x) for x in gone_eids))) + commit() + +def main(): + dupes = find_dupes() + if not dupes: + print 'No duplicate user' + return + + print 'Found %s duplicate user instances' % len(dupes) + + while True: + print 'Fix or dry-run? (f/d) ... or Ctrl-C to break out' + answer = raw_input('> ') + if answer.lower() not in 'fd': + continue + print 'Please STOP THE APPLICATION INSTANCES (service or interactive), and press Return when done.' + raw_input('') + with hooks_control(session, session.HOOKS_DENY_ALL): + merge_dupes(dupes, docommit=answer=='f') + +main() diff -r e20057a9ceea -r ae0a567dff30 server/repository.py --- a/server/repository.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/repository.py Fri Oct 12 16:05:16 2012 +0200 @@ -119,6 +119,26 @@ {'x': eidfrom, 'y': eidto}) +def preprocess_inlined_relations(session, entity): + """when an entity is added, check if it has some inlined relation which + requires to be extrated for proper call hooks + """ + relations = [] + activeintegrity = session.is_hook_category_activated('activeintegrity') + eschema = entity.e_schema + for attr in entity.cw_edited.iterkeys(): + rschema = eschema.subjrels[attr] + if not rschema.final: # inlined relation + value = entity.cw_edited[attr] + relations.append((attr, value)) + session.update_rel_cache_add(entity.eid, attr, value) + rdef = session.rtype_eids_rdef(attr, entity.eid, value) + if rdef.cardinality[1] in '1?' and activeintegrity: + with session.security_enabled(read=False): + session.execute('DELETE X %s Y WHERE Y eid %%(y)s' % attr, + {'x': entity.eid, 'y': value}) + return relations + class NullEventBus(object): def publish(self, msg): @@ -1333,7 +1353,6 @@ entity._cw_is_saved = False # entity has an eid but is not yet saved # init edited_attributes before calling before_add_entity hooks entity.cw_edited = edited - eschema = entity.e_schema source = self.locate_etype_source(entity.__regid__) # allocate an eid to the entity before calling hooks entity.eid = self.system_source.create_eid(session) @@ -1344,19 +1363,7 @@ prefill_entity_caches(entity) if source.should_call_hooks: self.hm.call_hooks('before_add_entity', session, entity=entity) - relations = [] - activeintegrity = session.is_hook_category_activated('activeintegrity') - for attr in edited.iterkeys(): - rschema = eschema.subjrels[attr] - if not rschema.final: # inlined relation - value = edited[attr] - relations.append((attr, value)) - session.update_rel_cache_add(entity.eid, attr, value) - rdef = session.rtype_eids_rdef(attr, entity.eid, value) - if rdef.cardinality[1] in '1?' and activeintegrity: - with session.security_enabled(read=False): - session.execute('DELETE X %s Y WHERE Y eid %%(y)s' % attr, - {'x': entity.eid, 'y': value}) + relations = preprocess_inlined_relations(session, entity) edited.set_defaults() if session.is_hook_category_activated('integrity'): edited.check(creation=True) @@ -1519,7 +1526,7 @@ activintegrity = session.is_hook_category_activated('activeintegrity') for rtype, eids_subj_obj in relations.iteritems(): if server.DEBUG & server.DBG_REPO: - for subjeid, objeid in relations: + for subjeid, objeid in eids_subj_obj: print 'ADD relation', subjeid, rtype, objeid for subjeid, objeid in eids_subj_obj: source = self.locate_relation_source(session, subjeid, rtype, objeid) diff -r e20057a9ceea -r ae0a567dff30 server/sources/datafeed.py --- a/server/sources/datafeed.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/sources/datafeed.py Fri Oct 12 16:05:16 2012 +0200 @@ -30,6 +30,7 @@ from lxml import etree from cubicweb import RegistryNotFound, ObjectNotFound, ValidationError, UnknownEid +from cubicweb.server.repository import preprocess_inlined_relations from cubicweb.server.sources import AbstractSource from cubicweb.appobject import AppObject @@ -254,11 +255,20 @@ """called by the repository after an entity stored here has been inserted in the system table. """ + relations = preprocess_inlined_relations(session, entity) if session.is_hook_category_activated('integrity'): entity.cw_edited.check(creation=True) self.repo.system_source.add_entity(session, entity) entity.cw_edited.saved = entity._cw_is_saved = True sourceparams['parser'].after_entity_copy(entity, sourceparams) + # call hooks for inlined relations + call_hooks = self.repo.hm.call_hooks + if self.should_call_hooks: + for attr, value in relations: + call_hooks('before_add_relation', session, + eidfrom=entity.eid, rtype=attr, eidto=value) + call_hooks('after_add_relation', session, + eidfrom=entity.eid, rtype=attr, eidto=value) def source_cwuris(self, session): sql = ('SELECT extid, eid, type FROM entities, cw_source_relation ' @@ -399,6 +409,7 @@ entity.cw_set(**attrs) self.notify_updated(entity) + class DataFeedXMLParser(DataFeedParser): def process(self, url, raise_on_error=False): diff -r e20057a9ceea -r ae0a567dff30 server/sources/native.py --- a/server/sources/native.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/sources/native.py Fri Oct 12 16:05:16 2012 +0200 @@ -61,7 +61,7 @@ from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.cwconfig import CubicWebNoAppConfiguration from cubicweb.server import hook -from cubicweb.server.utils import crypt_password, eschema_eid +from cubicweb.server.utils import crypt_password, eschema_eid, verify_and_update from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn from cubicweb.server.rqlannotation import set_qdata from cubicweb.server.hook import CleanupDeletedEidsCacheOp @@ -1629,7 +1629,22 @@ # get eid from login and (crypted) password rset = self.source.syntax_tree_search(session, self._auth_rqlst, args) try: - return rset[0][0] + user = rset[0][0] + # If the stored hash uses a deprecated scheme (e.g. DES or MD5 used + # before 3.14.7), update with a fresh one + if pwd.getvalue(): + verify, newhash = verify_and_update(password, pwd.getvalue()) + if not verify: # should not happen, but... + raise AuthenticationError('bad password') + if newhash: + session.system_sql("UPDATE %s SET %s=%%(newhash)s WHERE %s=%%(login)s" % ( + SQL_PREFIX + 'CWUser', + SQL_PREFIX + 'upassword', + SQL_PREFIX + 'login'), + {'newhash': self.source._binary(newhash), + 'login': login}) + session.commit(free_cnxset=False) + return user except IndexError: raise AuthenticationError('bad password') diff -r e20057a9ceea -r ae0a567dff30 server/sources/pyrorql.py --- a/server/sources/pyrorql.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/sources/pyrorql.py Fri Oct 12 16:05:16 2012 +0200 @@ -91,5 +91,9 @@ except AttributeError: # inmemory connection pass - return super(PyroRQLSource, self).check_connection(cnx) + try: + return super(PyroRQLSource, self).check_connection(cnx) + except ConnectionClosedError: + # try to reconnect + return self.get_connection() diff -r e20057a9ceea -r ae0a567dff30 server/sources/remoterql.py --- a/server/sources/remoterql.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/sources/remoterql.py Fri Oct 12 16:05:16 2012 +0200 @@ -302,7 +302,7 @@ try: cnx.check() return # ok - except (BadConnectionId, ConnectionClosedError): + except BadConnectionId: pass # try to reconnect return self.get_connection() diff -r e20057a9ceea -r ae0a567dff30 server/test/unittest_repository.py --- a/server/test/unittest_repository.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/test/unittest_repository.py Fri Oct 12 16:05:16 2012 +0200 @@ -416,8 +416,7 @@ def _zmq_client(self, done): cnxprops = ConnectionProperties('zmq') try: - cnx = connect(self.repo.config.appid, u'admin', password=u'gingkow', - host='tcp://127.0.0.1:41415', + cnx = connect('tcp://127.0.0.1:41415', u'admin', password=u'gingkow', cnxprops=cnxprops, initlog=False) # don't reset logging configuration try: diff -r e20057a9ceea -r ae0a567dff30 server/test/unittest_security.py --- a/server/test/unittest_security.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/test/unittest_security.py Fri Oct 12 16:05:16 2012 +0200 @@ -25,9 +25,10 @@ from rql import RQLException from cubicweb.devtools.testlib import CubicWebTC -from cubicweb import Unauthorized, ValidationError, QueryError +from cubicweb import Unauthorized, ValidationError, QueryError, Binary from cubicweb.schema import ERQLExpression from cubicweb.server.querier import check_read_access +from cubicweb.server.utils import _CRYPTO_CTX class BaseSecurityTC(CubicWebTC): @@ -35,7 +36,8 @@ def setup_database(self): super(BaseSecurityTC, self).setup_database() self.create_user(self.request(), 'iaminusersgrouponly') - + hash = _CRYPTO_CTX.encrypt('oldpassword', scheme='des_crypt') + self.create_user(self.request(), 'oldpassword', password=Binary(hash)) class LowLevelSecurityFunctionTC(BaseSecurityTC): @@ -60,6 +62,18 @@ self.assertRaises(Unauthorized, cu.execute, 'Any X,P WHERE X is CWUser, X upassword P') + def test_update_password(self): + """Ensure that if a user's password is stored with a deprecated hash, it will be updated on next login""" + oldhash = str(self.session.system_sql("SELECT cw_upassword FROM cw_CWUser WHERE cw_login = 'oldpassword'").fetchone()[0]) + with self.login('oldpassword') as cu: + pass + newhash = str(self.session.system_sql("SELECT cw_upassword FROM cw_CWUser WHERE cw_login = 'oldpassword'").fetchone()[0]) + self.assertNotEqual(oldhash, newhash) + self.assertTrue(newhash.startswith('$6$')) + with self.login('oldpassword') as cu: + pass + self.assertEqual(newhash, str(self.session.system_sql("SELECT cw_upassword FROM cw_CWUser WHERE cw_login = 'oldpassword'").fetchone()[0])) + class SecurityRewritingTC(BaseSecurityTC): def hijack_source_execute(self): diff -r e20057a9ceea -r ae0a567dff30 server/utils.py --- a/server/utils.py Fri Oct 12 15:38:58 2012 +0200 +++ b/server/utils.py Fri Oct 12 16:05:16 2012 +0200 @@ -52,7 +52,9 @@ return md5crypt(secret, self.salt.encode('ascii')).decode('utf-8') _calc_checksum = calc_checksum -_CRYPTO_CTX = CryptContext(['sha512_crypt', CustomMD5Crypt, 'des_crypt', 'ldap_salted_sha1']) +_CRYPTO_CTX = CryptContext(['sha512_crypt', CustomMD5Crypt, 'des_crypt', 'ldap_salted_sha1'], + deprecated=['cubicwebmd5crypt', 'des_crypt']) +verify_and_update = _CRYPTO_CTX.verify_and_update def crypt_password(passwd, salt=None): """return the encrypted password using the given salt or a generated one @@ -62,8 +64,11 @@ # empty hash, accept any password for backwards compat if salt == '': return salt - if _CRYPTO_CTX.verify(passwd, salt): - return salt + try: + if _CRYPTO_CTX.verify(passwd, salt): + return salt + except ValueError: # e.g. couldn't identify hash + pass # wrong password return '' diff -r e20057a9ceea -r ae0a567dff30 skeleton/__pkginfo__.py.tmpl --- a/skeleton/__pkginfo__.py.tmpl Fri Oct 12 15:38:58 2012 +0200 +++ b/skeleton/__pkginfo__.py.tmpl Fri Oct 12 16:05:16 2012 +0200 @@ -16,6 +16,12 @@ __depends__ = %(dependencies)s __recommends__ = {} +classifiers = [ + 'Environment :: Web Environment', + 'Framework :: CubicWeb', + 'Programming Language :: Python', + 'Programming Language :: JavaScript', + ] from os import listdir as _listdir from os.path import join, isdir diff -r e20057a9ceea -r ae0a567dff30 skeleton/setup.py --- a/skeleton/setup.py Fri Oct 12 15:38:58 2012 +0200 +++ b/skeleton/setup.py Fri Oct 12 16:05:16 2012 +0200 @@ -41,7 +41,7 @@ # import required features from __pkginfo__ import modname, version, license, description, web, \ - author, author_email + author, author_email, classifiers if exists('README'): long_description = file('README').read() @@ -193,6 +193,7 @@ data_files = data_files, ext_modules = ext_modules, cmdclass = cmdclass, + classifiers = classifiers, **kwargs ) diff -r e20057a9ceea -r ae0a567dff30 sobjects/ldapparser.py --- a/sobjects/ldapparser.py Fri Oct 12 15:38:58 2012 +0200 +++ b/sobjects/ldapparser.py Fri Oct 12 16:05:16 2012 +0200 @@ -25,7 +25,7 @@ from logilab.common.decorators import cached from logilab.common.shellutils import generate_password -from cubicweb import Binary +from cubicweb import Binary, ConfigurationError from cubicweb.server.utils import crypt_password from cubicweb.server.sources import datafeed @@ -92,7 +92,12 @@ tdict = {} for sattr, tattr in self.source.user_attrs.iteritems(): if tattr not in self.non_attribute_keys: - tdict[tattr] = sdict[sattr] + try: + tdict[tattr] = sdict[sattr] + except KeyError: + raise ConfigurationError('source attribute %s is not present ' + 'in the source, please check the ' + 'user-attrs-map field' % sattr) return tdict def before_entity_copy(self, entity, sourceparams): diff -r e20057a9ceea -r ae0a567dff30 web/views/boxes.py --- a/web/views/boxes.py Fri Oct 12 15:38:58 2012 +0200 +++ b/web/views/boxes.py Fri Oct 12 16:05:16 2012 +0200 @@ -48,7 +48,7 @@ BoxTemplate = box.BoxTemplate BoxHtml = htmlwidgets.BoxHtml -class EditBox(component.CtxComponent): # XXX rename to ActionsBox +class EditBox(component.CtxComponent): """ box with all actions impacting the entity displayed: edit, copy, delete change state, add related entities... @@ -58,6 +58,7 @@ title = _('actions') order = 2 contextual = True + __select__ = component.CtxComponent.__select__ & non_final_entity() def init_rendering(self): super(EditBox, self).init_rendering() diff -r e20057a9ceea -r ae0a567dff30 zmqclient.py --- a/zmqclient.py Fri Oct 12 15:38:58 2012 +0200 +++ b/zmqclient.py Fri Oct 12 16:05:16 2012 +0200 @@ -43,12 +43,9 @@ ZMQ is used as the transport layer and cPickle is used to serialize data. """ - def __init__(self, config, vreg=None): - self.config = config - self.vreg = vreg + def __init__(self, zmq_address): self.socket = ctx.socket(zmq.REQ) - self.host = config.get('base-url') - self.socket.connect(self.host) + self.socket.connect(zmq_address) def __zmqcall__(self, name, *args, **kwargs): self.socket.send_pyobj([name, args, kwargs])