misc/scripts/ldapuser2ldapfeed.py
author Rémi Cardona <remi.cardona@logilab.fr>
Wed, 20 May 2015 16:17:28 +0200
changeset 10430 2b1ea3e8e080
parent 10366 38c7598b5c61
child 10589 7c23b7de2b8d
permissions -rw-r--r--
[devtools] qunit: use new async testing APIs http://qunitjs.com/cookbook/#asynchronous-callbacks QUnit keeps track of all the assert.async() objects created inside the test functions and expects all done() functions to be called. Failure to do so will result in the test being failed. Unlike .start and .stop which were internal APIs, assert.async() is stricter and fails tests if assert methods are used *after* all done() functions are called (see "test callback execution order"). Related to #5533333.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     1
"""turn a pyro source into a datafeed source
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
Once this script is run, execute c-c db-check to cleanup relation tables.
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     4
"""
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
import sys
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
     6
from collections import defaultdict
8521
dfdffebce8a4 ldapuser2ldapfeed: create CWUsers with random passwords, not empty ones
Julien Cristau <julien.cristau@logilab.fr>
parents: 8468
diff changeset
     7
from logilab.common.shellutils import generate_password
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
try:
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
    source_name, = __args__
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
    source = repo.sources_by_uri[source_name]
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    12
except ValueError:
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    13
    print('you should specify the source name as script argument (i.e. after --'
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    14
          ' on the command line)')
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    15
    sys.exit(1)
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    16
except KeyError:
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
    print '%s is not an active source' % source_name
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
    sys.exit(1)
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
# check source is reachable before doing anything
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
if not source.get_connection().cnx:
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
    print '%s is not reachable. Fix this before running this script' % source_name
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
    sys.exit(1)
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    25
raw_input('Ensure you have shutdown all instances of this application before continuing.'
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    26
          ' Type enter when ready.')
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
system_source = repo.system_source
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    30
from datetime import datetime
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    31
from cubicweb.server.edition import EditedEntity
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    32
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    33
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    34
print '******************** backport entity content ***************************'
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    36
todelete = defaultdict(list)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    37
extids = set()
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    38
duplicates = []
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
for entity in rql('Any X WHERE X cw_source S, S eid %(s)s', {'s': source.eid}).entities():
8900
010a59e12d89 use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 8645
diff changeset
    40
    etype = entity.cw_etype
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    41
    if not source.support_entity(etype):
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    42
        print "source doesn't support %s, delete %s" % (etype, entity.eid)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    43
        todelete[etype].append(entity)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    44
        continue
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    45
    try:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    46
        entity.complete()
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    47
    except Exception:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    48
        print '%s %s much probably deleted, delete it (extid %s)' % (
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    49
            etype, entity.eid, entity.cw_metainformation()['extid'])
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    50
        todelete[etype].append(entity)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    51
        continue
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    52
    print 'get back', etype, entity.eid
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    53
    entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    54
    if not entity.creation_date:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    55
        entity.cw_edited['creation_date'] = datetime.now()
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    56
    if not entity.modification_date:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    57
        entity.cw_edited['modification_date'] = datetime.now()
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    58
    if not entity.upassword:
8521
dfdffebce8a4 ldapuser2ldapfeed: create CWUsers with random passwords, not empty ones
Julien Cristau <julien.cristau@logilab.fr>
parents: 8468
diff changeset
    59
        entity.cw_edited['upassword'] = generate_password()
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    60
    extid = entity.cw_metainformation()['extid']
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    61
    if not entity.cwuri:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    62
        entity.cw_edited['cwuri'] = '%s/?dn=%s' % (
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    63
            source.urls[0], extid.decode('utf-8', 'ignore'))
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    64
    print entity.cw_edited
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    65
    if extid in extids:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    66
        duplicates.append(extid)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    67
        continue
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    68
    extids.add(extid)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    69
    system_source.add_entity(session, entity)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    70
    sql("UPDATE entities SET source='system' "
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    71
        "WHERE eid=%(eid)s", {'eid': entity.eid})
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    72
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    73
# only cleanup entities table, remaining stuff should be cleaned by a c-c
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
# db-check to be run after this script
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    75
if duplicates:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    76
    print 'found %s duplicate entries' % len(duplicates)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    77
    from pprint import pprint
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    78
    pprint(duplicates)
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    79
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    80
print len(todelete), 'entities will be deleted'
8635
c261d9465e65 [ldapuser2ldapfeed] .values -> .iteritems (closes #2542872)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8521
diff changeset
    81
for etype, entities in todelete.iteritems():
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    82
    print 'deleting', etype, [e.login for e in entities]
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
    system_source.delete_info_multi(session, entities, source_name)
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    86
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
source_ent = rql('CWSource S WHERE S eid %(s)s', {'s': source.eid}).get_entity(0, 0)
8483
4ba11607d84a [entity api] unify set_attributes / set_relations into a cw_set method. Closes #2423719
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8188
diff changeset
    88
source_ent.cw_set(type=u"ldapfeed", parser=u"ldapfeed")
8188
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    89
1867e252e487 [repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
9492
c7fc56eecd1a English typography
Dimitri Papadopoulos <dimitri.papadopoulos@cea.fr>
parents: 9267
diff changeset
    91
if raw_input('Commit?') in 'yY':
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    92
    print 'committing'
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    93
    commit()
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    94
else:
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    95
    rollback()
9267
24d9b86dfa54 spelling: rollbacked -> rolled back
Rémi Cardona <remi.cardona@logilab.fr>
parents: 8900
diff changeset
    96
    print 'rolled back'
8468
f52bb4226020 [ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 8188
diff changeset
    97