author | Denis Laxalde <denis.laxalde@logilab.fr> |
Tue, 21 Jun 2016 16:21:07 +0200 | |
changeset 11299 | f94f5e81fb9d |
parent 11033 | 63d860a14a17 |
permissions | -rw-r--r-- |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
1 |
"""turn a pyro source into a datafeed source |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
2 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
Once this script is run, execute c-c db-check to cleanup relation tables. |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
""" |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
5 |
from __future__ import print_function |
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
6 |
|
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
7 |
import sys |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
8 |
from collections import defaultdict |
8521
dfdffebce8a4
ldapuser2ldapfeed: create CWUsers with random passwords, not empty ones
Julien Cristau <julien.cristau@logilab.fr>
parents:
8468
diff
changeset
|
9 |
from logilab.common.shellutils import generate_password |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
try: |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
source_name, = __args__ |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
source = repo.sources_by_uri[source_name] |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
except ValueError: |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
print('you should specify the source name as script argument (i.e. after --' |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
' on the command line)') |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
sys.exit(1) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
18 |
except KeyError: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
19 |
print('%s is not an active source' % source_name) |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
sys.exit(1) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
21 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
# check source is reachable before doing anything |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
if not source.get_connection().cnx: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
24 |
print('%s is not reachable. Fix this before running this script' % source_name) |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
25 |
sys.exit(1) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
26 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
27 |
raw_input('Ensure you have shutdown all instances of this application before continuing.' |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
' Type enter when ready.') |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
system_source = repo.system_source |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
32 |
from datetime import datetime |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
from cubicweb.server.edition import EditedEntity |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
|
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
36 |
print('******************** backport entity content ***************************') |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
|
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
38 |
todelete = defaultdict(list) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
39 |
extids = set() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
40 |
duplicates = [] |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
41 |
for entity in rql('Any X WHERE X cw_source S, S eid %(s)s', {'s': source.eid}).entities(): |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8645
diff
changeset
|
42 |
etype = entity.cw_etype |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
43 |
if not source.support_entity(etype): |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
44 |
print("source doesn't support %s, delete %s" % (etype, entity.eid)) |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
45 |
todelete[etype].append(entity) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
46 |
continue |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
47 |
try: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
48 |
entity.complete() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
49 |
except Exception: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
50 |
print('%s %s much probably deleted, delete it (extid %s)' % ( |
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
51 |
etype, entity.eid, entity.cw_metainformation()['extid'])) |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
52 |
todelete[etype].append(entity) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
53 |
continue |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
54 |
print('get back', etype, entity.eid) |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
55 |
entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
56 |
if not entity.creation_date: |
11033
63d860a14a17
[schema] Use TZDatetime for creation_date and modification_date
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10907
diff
changeset
|
57 |
entity.cw_edited['creation_date'] = datetime.utcnow() |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
58 |
if not entity.modification_date: |
11033
63d860a14a17
[schema] Use TZDatetime for creation_date and modification_date
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10907
diff
changeset
|
59 |
entity.cw_edited['modification_date'] = datetime.utcnow() |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
60 |
if not entity.upassword: |
8521
dfdffebce8a4
ldapuser2ldapfeed: create CWUsers with random passwords, not empty ones
Julien Cristau <julien.cristau@logilab.fr>
parents:
8468
diff
changeset
|
61 |
entity.cw_edited['upassword'] = generate_password() |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
62 |
extid = entity.cw_metainformation()['extid'] |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
63 |
if not entity.cwuri: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
64 |
entity.cw_edited['cwuri'] = '%s/?dn=%s' % ( |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
65 |
source.urls[0], extid.decode('utf-8', 'ignore')) |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
66 |
print(entity.cw_edited) |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
67 |
if extid in extids: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
68 |
duplicates.append(extid) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
69 |
continue |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
70 |
extids.add(extid) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
71 |
system_source.add_entity(session, entity) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
72 |
sql("UPDATE entities SET source='system' " |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
73 |
"WHERE eid=%(eid)s", {'eid': entity.eid}) |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
75 |
# only cleanup entities table, remaining stuff should be cleaned by a c-c |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
76 |
# db-check to be run after this script |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
77 |
if duplicates: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
78 |
print('found %s duplicate entries' % len(duplicates)) |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
79 |
from pprint import pprint |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
80 |
pprint(duplicates) |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
81 |
|
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
82 |
print(len(todelete), 'entities will be deleted') |
10662
10942ed172de
[py3k] dict.iteritems → dict.items
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
83 |
for etype, entities in todelete.items(): |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
84 |
print('deleting', etype, [e.login for e in entities]) |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
system_source.delete_info_multi(session, entities, source_name) |
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
|
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
88 |
|
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
source_ent = rql('CWSource S WHERE S eid %(s)s', {'s': source.eid}).get_entity(0, 0) |
8483
4ba11607d84a
[entity api] unify set_attributes / set_relations into a cw_set method. Closes #2423719
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8188
diff
changeset
|
90 |
source_ent.cw_set(type=u"ldapfeed", parser=u"ldapfeed") |
8188
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
91 |
|
1867e252e487
[repository] ldap-feed source. Closes #2086984
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
|
9492
c7fc56eecd1a
English typography
Dimitri Papadopoulos <dimitri.papadopoulos@cea.fr>
parents:
9267
diff
changeset
|
93 |
if raw_input('Commit?') in 'yY': |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
94 |
print('committing') |
8468
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
95 |
commit() |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
96 |
else: |
f52bb4226020
[ldapuser2ldapfeed] fix confusing script structure and decode the extid to avoid an UnicodeDecodeError (closes #2413437)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8188
diff
changeset
|
97 |
rollback() |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10366
diff
changeset
|
98 |
print('rolled back') |