# HG changeset patch # User Aurelien Campeas # Date 1338472581 -7200 # Node ID 5bee87a14bb1299cf3e002321f929395f9c34f6f # Parent cad2d8e03b330adfc1519b871e5c983969f3e429 fix ldap removal handling in ldapfeed (closes #2376625 and #2385133) diff -r cad2d8e03b33 -r 5bee87a14bb1 server/ldaputils.py --- a/server/ldaputils.py Fri May 25 17:18:00 2012 +0200 +++ b/server/ldaputils.py Thu May 31 15:56:21 2012 +0200 @@ -225,11 +225,12 @@ def object_exists_in_ldap(self, dn): cnx = self.get_connection().cnx #session.cnxset.connection(self.uri).cnx if cnx is None: - return True # ldap unreachable, suppose it exists + self.warning('Could not establish connexion with LDAP server, assuming dn %s exists', dn) + return True # ldap unreachable, let's not touch it try: cnx.search_s(dn, self.user_base_scope) except ldap.PARTIAL_RESULTS: - pass + self.warning('PARTIAL RESULTS for dn %s', dn) except ldap.NO_SUCH_OBJECT: return False return True diff -r cad2d8e03b33 -r 5bee87a14bb1 server/sources/datafeed.py --- a/server/sources/datafeed.py Fri May 25 17:18:00 2012 +0200 +++ b/server/sources/datafeed.py Thu May 31 15:56:21 2012 +0200 @@ -28,7 +28,6 @@ from cookielib import CookieJar from lxml import etree -from logilab.mtconverter import xml_escape from cubicweb import RegistryNotFound, ObjectNotFound, ValidationError, UnknownEid from cubicweb.server.sources import AbstractSource @@ -68,7 +67,7 @@ }), ('delete-entities', {'type' : 'yn', - 'default': True, + 'default': False, 'help': ('Should already imported entities not found anymore on the ' 'external source be deleted?'), 'group': 'datafeed-source', 'level': 2, @@ -80,6 +79,7 @@ 'group': 'datafeed-source', 'level': 2, }), ) + def __init__(self, repo, source_config, eid=None): AbstractSource.__init__(self, repo, source_config, eid) self.update_config(None, self.check_conf_dict(eid, source_config, @@ -192,23 +192,13 @@ self.release_synchronization_lock(session) def _pull_data(self, session, force=False, raise_on_error=False): - if self.config['delete-entities']: - myuris = self.source_cwuris(session) - else: - myuris = None importlog = self.init_import_log(session) + myuris = self.source_cwuris(session) parser = self._get_parser(session, sourceuris=myuris, import_log=importlog) if self.process_urls(parser, self.urls, raise_on_error): self.warning("some error occured, don't attempt to delete entities") - elif self.config['delete-entities'] and myuris: - byetype = {} - for extid, (eid, etype) in myuris.iteritems(): - if parser.is_deleted(extid, etype, eid): - byetype.setdefault(etype, []).append(str(eid)) - for etype, eids in byetype.iteritems(): - self.warning('delete %s %s entities', len(eids), etype) - session.execute('DELETE %s X WHERE X eid IN (%s)' - % (etype, ','.join(eids))) + else: + parser.handle_deletion(self.config, session, myuris) self.update_latest_retrieval(session) stats = parser.stats if stats.get('created'): @@ -376,6 +366,17 @@ """ return True + def handle_deletion(self, config, session, myuris): + if config['delete-entities'] and myuris: + byetype = {} + for extid, (eid, etype) in myuris.iteritems(): + if self.is_deleted(extid, etype, eid): + byetype.setdefault(etype, []).append(str(eid)) + for etype, eids in byetype.iteritems(): + self.warning('delete %s %s entities', len(eids), etype) + session.execute('DELETE %s X WHERE X eid IN (%s)' + % (etype, ','.join(eids))) + def update_if_necessary(self, entity, attrs): self.notify_updated(entity) entity.complete(tuple(attrs)) diff -r cad2d8e03b33 -r 5bee87a14bb1 server/sources/ldapfeed.py --- a/server/sources/ldapfeed.py Fri May 25 17:18:00 2012 +0200 +++ b/server/sources/ldapfeed.py Thu May 31 15:56:21 2012 +0200 @@ -43,4 +43,3 @@ def _entity_update(self, source_entity): datafeed.DataFeedSource._entity_update(self, source_entity) ldaputils.LDAPSourceMixIn._entity_update(self, source_entity) - diff -r cad2d8e03b33 -r 5bee87a14bb1 server/test/data/slapd.conf.in --- a/server/test/data/slapd.conf.in Fri May 25 17:18:00 2012 +0200 +++ b/server/test/data/slapd.conf.in Thu May 31 15:56:21 2012 +0200 @@ -45,9 +45,9 @@ suffix "dc=cubicweb,dc=test" # rootdn directive for specifying a superuser on the database. This is needed -# for syncrepl. -#rootdn "cn=admin,dc=cubicweb,dc=test" -#rootpw "cubicwebrocks" +# for syncrepl. and ldapdelete easyness +rootdn "cn=admin,dc=cubicweb,dc=test" +rootpw "cw" # Where the database file are physically stored for database #1 directory "%(apphome)s/ldapdb" diff -r cad2d8e03b33 -r 5bee87a14bb1 server/test/unittest_ldapuser.py --- a/server/test/unittest_ldapuser.py Fri May 25 17:18:00 2012 +0200 +++ b/server/test/unittest_ldapuser.py Thu May 31 15:56:21 2012 +0200 @@ -37,12 +37,6 @@ CONFIG = u'user-base-dn=ou=People,dc=cubicweb,dc=test' URL = None -def setUpModule(*args): - create_slapd_configuration(LDAPUserSourceTC.config) - -def tearDownModule(*args): - terminate_slapd() - def create_slapd_configuration(config): global slapd_process, URL basedir = join(config.apphome, "ldapdb") @@ -51,47 +45,89 @@ confstream = file(slapdconf, 'w') confstream.write(confin % {'apphome': config.apphome}) confstream.close() - if not exists(basedir): - os.makedirs(basedir) - # fill ldap server with some data - ldiffile = join(config.apphome, "ldap_test.ldif") - print "Initing ldap database" - cmdline = "/usr/sbin/slapadd -f %s -l %s -c" % (slapdconf, ldiffile) - subprocess.call(cmdline, shell=True) - + if exists(basedir): + shutil.rmtree(basedir) + os.makedirs(basedir) + # fill ldap server with some data + ldiffile = join(config.apphome, "ldap_test.ldif") + config.info('Initing ldap database') + cmdline = "/usr/sbin/slapadd -f %s -l %s -c" % (slapdconf, ldiffile) + subprocess.call(cmdline, shell=True) #ldapuri = 'ldapi://' + join(basedir, "ldapi").replace('/', '%2f') port = get_available_port(xrange(9000, 9100)) host = 'localhost:%s' % port ldapuri = 'ldap://%s' % host cmdline = ["/usr/sbin/slapd", "-f", slapdconf, "-h", ldapuri, "-d", "0"] - print 'Starting slapd:', ' '.join(cmdline) + config.info('Starting slapd:', ' '.join(cmdline)) slapd_process = subprocess.Popen(cmdline) time.sleep(0.2) if slapd_process.poll() is None: - print "slapd started with pid %s" % slapd_process.pid + config.info('slapd started with pid %s' % slapd_process.pid) else: raise EnvironmentError('Cannot start slapd with cmdline="%s" (from directory "%s")' % (" ".join(cmdline), os.getcwd())) URL = u'ldap://%s' % host -def terminate_slapd(): +def terminate_slapd(config): global slapd_process if slapd_process.returncode is None: - print "terminating slapd" + config.info('terminating slapd') if hasattr(slapd_process, 'terminate'): slapd_process.terminate() else: import os, signal os.kill(slapd_process.pid, signal.SIGTERM) slapd_process.wait() - print "DONE" + config.info('DONE') del slapd_process +class LDAPTestBase(CubicWebTC): + loglevel = 'ERROR' + + @classmethod + def setUpClass(cls): + from cubicweb.cwctl import init_cmdline_log_threshold + init_cmdline_log_threshold(cls.config, cls.loglevel) + create_slapd_configuration(cls.config) + + @classmethod + def tearDownClass(cls): + terminate_slapd(cls.config) + +class DeleteStuffFromLDAPFeedSourceTC(LDAPTestBase): + test_db_id = 'ldap-feed' + + @classmethod + def pre_setup_database(cls, session, config): + session.create_entity('CWSource', name=u'ldapuser', type=u'ldapfeed', parser=u'ldapfeed', + url=URL, config=CONFIG) + session.commit() + isession = session.repo.internal_session(safe=True) + lfsource = isession.repo.sources_by_uri['ldapuser'] + stats = lfsource.pull_data(isession, force=True, raise_on_error=True) + + def test_delete(self): + uri = self.repo.sources_by_uri['ldapuser'].urls[0] + from subprocess import call + deletecmd = ("ldapdelete -H %s 'uid=syt,ou=People,dc=cubicweb,dc=test' " + "-v -x -D cn=admin,dc=cubicweb,dc=test -w'cw'" % uri) + os.system(deletecmd) + isession = self.session.repo.internal_session(safe=False) + from cubicweb.server.session import security_enabled + with security_enabled(isession, read=False, write=False): + lfsource = isession.repo.sources_by_uri['ldapuser'] + stats = lfsource.pull_data(isession, force=True, raise_on_error=True) + isession.commit() + self.assertRaises(AuthenticationError, self.repo.connect, 'syt', password='syt') + self.assertEqual(self.execute('Any N WHERE U login "syt", ' + 'U in_state S, S name N').rows[0][0], + 'deactivated') -class LDAPFeedSourceTC(CubicWebTC): + +class LDAPFeedSourceTC(LDAPTestBase): test_db_id = 'ldap-feed' @classmethod @@ -150,7 +186,9 @@ self.assertEqual(len(rset), 1) e = rset.get_entity(0, 0) self.assertEqual(e.eid, eid) - self.assertEqual(e.cw_metainformation(), {'source': {'type': u'native', 'uri': u'system', 'use-cwuri-as-url': False}, + self.assertEqual(e.cw_metainformation(), {'source': {'type': u'native', + 'uri': u'system', + 'use-cwuri-as-url': False}, 'type': 'CWUser', 'extid': None}) self.assertEqual(e.cw_source[0].name, 'system') diff -r cad2d8e03b33 -r 5bee87a14bb1 sobjects/ldapparser.py --- a/sobjects/ldapparser.py Fri May 25 17:18:00 2012 +0200 +++ b/sobjects/ldapparser.py Thu May 31 15:56:21 2012 +0200 @@ -29,7 +29,7 @@ from cubicweb.server.sources import datafeed -class DataFeedlDAPParser(datafeed.DataFeedParser): +class DataFeedLDAPAdapter(datafeed.DataFeedParser): __regid__ = 'ldapfeed' # attributes that may appears in source user_attrs dict which are not # attributes of the cw user @@ -50,6 +50,26 @@ self.update_if_necessary(entity, attrs) self._process_email(entity, userdict) + + def handle_deletion(self, config, session, myuris): + if config['delete-entities']: + print 'DELETE' + super(DataFeedLDAPAdapter, self).handle_deletion(config, session, myuris) + if myuris: + byetype = {} + for extid, (eid, etype) in myuris.iteritems(): + if self.is_deleted(extid, etype, eid): + byetype.setdefault(etype, []).append(str(eid)) + + for etype, eids in byetype.iteritems(): + if etype != 'CWUser': + continue + self.warning('deactivate %s %s entities', len(eids), etype) + for eid in eids: + wf = session.entity_from_eid(eid).cw_adapt_to('IWorkflowable') + wf.fire_transition('deactivate') + session.commit(free_cnxset=False) + def ldap2cwattrs(self, sdict, tdict=None): if tdict is None: tdict = {} @@ -72,7 +92,7 @@ return entity def after_entity_copy(self, entity, sourceparams): - super(DataFeedlDAPParser, self).after_entity_copy(entity, sourceparams) + super(DataFeedLDAPAdapter, self).after_entity_copy(entity, sourceparams) if entity.__regid__ == 'EmailAddress': return groups = [self._get_group(n) for n in self.source.user_default_groups] @@ -84,7 +104,7 @@ extid, _ = extid.rsplit('@@', 1) except ValueError: pass - return self.source.object_exists_in_ldap(extid) + return not self.source.object_exists_in_ldap(extid) def _process_email(self, entity, userdict): try: