author | Rémi Cardona <remi.cardona@logilab.fr> |
Wed, 20 May 2015 17:25:40 +0200 | |
changeset 10431 | 7628a14d2209 |
parent 10366 | 38c7598b5c61 |
child 10589 | 7c23b7de2b8d |
permissions | -rw-r--r-- |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
1 |
"""turn a pyro source into a datafeed source |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
2 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
Once this script is run, execute c-c db-check to cleanup relation tables. |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
""" |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
5 |
import sys |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
6 |
from collections import defaultdict |
8521
dfdffebce8a4
ldapuser2ldapfeed: create CWUsers with random passwords, not empty ones
Julien Cristau <julien.cristau@logilab.fr>
parents:
8468
diff
changeset
|
7 |
from logilab.common.shellutils import generate_password |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
9 |
try: |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
source_name, = __args__ |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
source = repo.sources_by_uri[source_name] |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
except ValueError: |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
print('you should specify the source name as script argument (i.e. after --' |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
' on the command line)') |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
sys.exit(1) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
except KeyError: |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
print '%s is not an active source' % source_name |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
18 |
sys.exit(1) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
# check source is reachable before doing anything |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
21 |
if not source.get_connection().cnx: |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
print '%s is not reachable. Fix this before running this script' % source_name |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
sys.exit(1) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
24 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
25 |
raw_input('Ensure you have shutdown all instances of this application before continuing.' |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
26 |
' Type enter when ready.') |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
27 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
system_source = repo.system_source |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
from datetime import datetime |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
from cubicweb.server.edition import EditedEntity |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
32 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
print '******************** backport entity content ***************************' |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
|
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
36 |
todelete = defaultdict(list) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
37 |
extids = set() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
38 |
duplicates = [] |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
39 |
for entity in rql('Any X WHERE X cw_source S, S eid %(s)s', {'s': source.eid}).entities(): |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8645
diff
changeset
|
40 |
etype = entity.cw_etype |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
41 |
if not source.support_entity(etype): |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
42 |
print "source doesn't support %s, delete %s" % (etype, entity.eid) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
43 |
todelete[etype].append(entity) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
44 |
continue |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
45 |
try: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
46 |
entity.complete() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
47 |
except Exception: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
48 |
print '%s %s much probably deleted, delete it (extid %s)' % ( |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
49 |
etype, entity.eid, entity.cw_metainformation()['extid']) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
50 |
todelete[etype].append(entity) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
51 |
continue |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
52 |
print 'get back', etype, entity.eid |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
53 |
entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
54 |
if not entity.creation_date: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
55 |
entity.cw_edited['creation_date'] = datetime.now() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
56 |
if not entity.modification_date: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
57 |
entity.cw_edited['modification_date'] = datetime.now() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
58 |
if not entity.upassword: |
8521
dfdffebce8a4
ldapuser2ldapfeed: create CWUsers with random passwords, not empty ones
Julien Cristau <julien.cristau@logilab.fr>
parents:
8468
diff
changeset
|
59 |
entity.cw_edited['upassword'] = generate_password() |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
60 |
extid = entity.cw_metainformation()['extid'] |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
61 |
if not entity.cwuri: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
62 |
entity.cw_edited['cwuri'] = '%s/?dn=%s' % ( |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
63 |
source.urls[0], extid.decode('utf-8', 'ignore')) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
64 |
print entity.cw_edited |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
65 |
if extid in extids: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
66 |
duplicates.append(extid) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
67 |
continue |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
68 |
extids.add(extid) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
69 |
system_source.add_entity(session, entity) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
70 |
sql("UPDATE entities SET source='system' " |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
71 |
"WHERE eid=%(eid)s", {'eid': entity.eid}) |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
72 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
73 |
# only cleanup entities table, remaining stuff should be cleaned by a c-c |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
# db-check to be run after this script |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
75 |
if duplicates: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
76 |
print 'found %s duplicate entries' % len(duplicates) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
77 |
from pprint import pprint |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
78 |
pprint(duplicates) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
79 |
|
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
80 |
print len(todelete), 'entities will be deleted' |
8635
c261d9465e65
[ldapuser2ldapfeed] .values -> .iteritems (closes #2542872)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8521
diff
changeset
|
81 |
for etype, entities in todelete.iteritems(): |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
82 |
print 'deleting', etype, [e.login for e in entities] |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
83 |
system_source.delete_info_multi(session, entities, source_name) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
84 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
|
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
86 |
|
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
source_ent = rql('CWSource S WHERE S eid %(s)s', {'s': source.eid}).get_entity(0, 0) |
8483
4ba11607d84a
[entity api] unify set_attributes / set_relations into a cw_set method. Closes #2423719
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8188
diff
changeset
|
88 |
source_ent.cw_set(type=u"ldapfeed", parser=u"ldapfeed") |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
90 |
|
9492
c7fc56eecd1a
English typography
Dimitri Papadopoulos <dimitri.papadopoulos@cea.fr>
parents:
9267
diff
changeset
|
91 |
if raw_input('Commit?') in 'yY': |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
92 |
print 'committing' |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
93 |
commit() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
94 |
else: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
95 |
rollback() |
9267
24d9b86dfa54
spelling: rollbacked -> rolled back
Rémi Cardona <remi.cardona@logilab.fr>
parents:
8900
diff
changeset
|
96 |
print 'rolled back' |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
97 |