author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Tue, 11 Sep 2012 12:44:33 +0200 | |
changeset 8541 | 5b6bc27ece6e |
parent 8446 | cae198371548 |
child 8546 | 3d2038d6f20d |
permissions | -rw-r--r-- |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7573
diff
changeset
|
1 |
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5376
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6128 | 18 |
"""Some utilities for the CubicWeb server.""" |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7573
diff
changeset
|
19 |
|
0 | 20 |
__docformat__ = "restructuredtext en" |
21 |
||
22 |
import sys |
|
6765
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
23 |
import logging |
0 | 24 |
from threading import Timer, Thread |
25 |
from getpass import getpass |
|
2105
92ea410806fe
refactor sources configuration, add source to sources when using a cube defining
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
26 |
|
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
27 |
from passlib.utils import handlers as uh, to_hash_str |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
28 |
from passlib.context import CryptContext |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
29 |
|
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
30 |
from cubicweb.md5crypt import crypt as md5crypt |
0 | 31 |
|
32 |
||
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
33 |
class CustomMD5Crypt(uh.HasSalt, uh.GenericHandler): |
8398
a9fe30c953be
[server/utils] passlib 1.6 is now less tolerant wrt handler names (closes #2349330)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
34 |
name = 'cubicwebmd5crypt' |
a9fe30c953be
[server/utils] passlib 1.6 is now less tolerant wrt handler names (closes #2349330)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
35 |
setting_kwds = ('salt',) |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
36 |
min_salt_size = 0 |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
37 |
max_salt_size = 8 |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
38 |
salt_chars = uh.H64_CHARS |
0 | 39 |
|
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
40 |
@classmethod |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
41 |
def from_string(cls, hash): |
8414
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
42 |
salt, chk = uh.parse_mc2(hash, u'') |
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
43 |
if chk is None: |
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
44 |
raise ValueError('missing checksum') |
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
45 |
return cls(salt=salt, checksum=chk) |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
46 |
|
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
47 |
def to_string(self): |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
48 |
return to_hash_str(u'%s$%s' % (self.salt, self.checksum or u'')) |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
49 |
|
8414
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
50 |
# passlib 1.5 wants calc_checksum, 1.6 wants _calc_checksum |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
51 |
def calc_checksum(self, secret): |
8414
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
52 |
return md5crypt(secret, self.salt.encode('ascii')).decode('utf-8') |
e7243ed7bb0a
Fixes for compatibility with passlib 1.6 (closes #2356393)
Julien Cristau <julien.cristau@logilab.fr>
parents:
8398
diff
changeset
|
53 |
_calc_checksum = calc_checksum |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
54 |
|
8399 | 55 |
_CRYPTO_CTX = CryptContext(['sha512_crypt', CustomMD5Crypt, 'des_crypt', 'ldap_salted_sha1']) |
0 | 56 |
|
57 |
def crypt_password(passwd, salt=None): |
|
58 |
"""return the encrypted password using the given salt or a generated one |
|
59 |
""" |
|
60 |
if salt is None: |
|
8398
a9fe30c953be
[server/utils] passlib 1.6 is now less tolerant wrt handler names (closes #2349330)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
61 |
return _CRYPTO_CTX.encrypt(passwd) |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
62 |
# empty hash, accept any password for backwards compat |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
63 |
if salt == '': |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
64 |
return salt |
8398
a9fe30c953be
[server/utils] passlib 1.6 is now less tolerant wrt handler names (closes #2349330)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
65 |
if _CRYPTO_CTX.verify(passwd, salt): |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
66 |
return salt |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
67 |
# wrong password |
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7823
diff
changeset
|
68 |
return '' |
0 | 69 |
|
70 |
def cartesian_product(seqin): |
|
71 |
"""returns a generator which returns the cartesian product of `seqin` |
|
72 |
||
73 |
for more details, see : |
|
74 |
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302478 |
|
75 |
""" |
|
76 |
def rloop(seqin, comb): |
|
77 |
"""recursive looping function""" |
|
78 |
if seqin: # any more sequences to process? |
|
79 |
for item in seqin[0]: |
|
6128 | 80 |
newcomb = comb + [item] # add next item to current combination |
0 | 81 |
# call rloop w/ remaining seqs, newcomb |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1138
diff
changeset
|
82 |
for item in rloop(seqin[1:], newcomb): |
0 | 83 |
yield item # seqs and newcomb |
84 |
else: # processing last sequence |
|
85 |
yield comb # comb finished, add to list |
|
86 |
return rloop(seqin, []) |
|
87 |
||
88 |
||
89 |
def cleanup_solutions(rqlst, solutions): |
|
90 |
for sol in solutions: |
|
91 |
for vname in sol.keys(): |
|
92 |
if not (vname in rqlst.defined_vars or vname in rqlst.aliases): |
|
93 |
del sol[vname] |
|
94 |
||
95 |
||
5066
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
96 |
def eschema_eid(session, eschema): |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
97 |
"""get eid of the CWEType entity for the given yams type. You should use |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
98 |
this because when schema has been loaded from the file-system, not from the |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
99 |
database, (e.g. during tests), eschema.eid is not set. |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
100 |
""" |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
101 |
if eschema.eid is None: |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
102 |
eschema.eid = session.execute( |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
103 |
'Any X WHERE X is CWEType, X name %(name)s', |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
104 |
{'name': str(eschema)})[0][0] |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
105 |
return eschema.eid |
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
106 |
|
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4714
diff
changeset
|
107 |
|
0 | 108 |
DEFAULT_MSG = 'we need a manager connection on the repository \ |
109 |
(the server doesn\'t have to run, even should better not)' |
|
110 |
||
1910
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
111 |
def manager_userpasswd(user=None, msg=DEFAULT_MSG, confirm=False, |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
112 |
passwdmsg='password'): |
0 | 113 |
if not user: |
3701
104b7c326172
check we've some message to display
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3585
diff
changeset
|
114 |
if msg: |
104b7c326172
check we've some message to display
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3585
diff
changeset
|
115 |
print msg |
0 | 116 |
while not user: |
117 |
user = raw_input('login: ') |
|
118 |
user = unicode(user, sys.stdin.encoding) |
|
1910
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
119 |
passwd = getpass('%s: ' % passwdmsg) |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
120 |
if confirm: |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
121 |
while True: |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
122 |
passwd2 = getpass('confirm password: ') |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
123 |
if passwd == passwd2: |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
124 |
break |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
125 |
print 'password doesn\'t match' |
864aa3ea0db5
[server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
126 |
passwd = getpass('password: ') |
0 | 127 |
# XXX decode password using stdin encoding then encode it using appl'encoding |
128 |
return user, passwd |
|
129 |
||
130 |
||
6381
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
131 |
_MARKER=object() |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
132 |
def func_name(func): |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
133 |
name = getattr(func, '__name__', _MARKER) |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
134 |
if name is _MARKER: |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
135 |
name = getattr(func, 'func_name', _MARKER) |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
136 |
if name is _MARKER: |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
137 |
name = repr(func) |
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
138 |
return name |
2105
92ea410806fe
refactor sources configuration, add source to sources when using a cube defining
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
139 |
|
0 | 140 |
class LoopTask(object): |
141 |
"""threaded task restarting itself once executed""" |
|
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
142 |
def __init__(self, tasks_manager, interval, func, args): |
5602
277b15d6d3ed
forbid looping tasks with an interval of 0
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5601
diff
changeset
|
143 |
if interval <= 0: |
277b15d6d3ed
forbid looping tasks with an interval of 0
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5601
diff
changeset
|
144 |
raise ValueError('Loop task interval must be > 0 ' |
277b15d6d3ed
forbid looping tasks with an interval of 0
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5601
diff
changeset
|
145 |
'(current value: %f for %s)' % \ |
6381
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
146 |
(interval, func_name(func))) |
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
147 |
self._tasks_manager = tasks_manager |
0 | 148 |
self.interval = interval |
2708
60d728bdcba5
allow to specify arbitrary argument when recording a looping task func
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2105
diff
changeset
|
149 |
def auto_restart_func(self=self, func=func, args=args): |
7573
c8f8762c986d
[repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6765
diff
changeset
|
150 |
restart = True |
0 | 151 |
try: |
2708
60d728bdcba5
allow to specify arbitrary argument when recording a looping task func
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2105
diff
changeset
|
152 |
func(*args) |
7573
c8f8762c986d
[repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6765
diff
changeset
|
153 |
except Exception: |
6765
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
154 |
logger = logging.getLogger('cubicweb.repository') |
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
155 |
logger.exception('Unhandled exception in LoopTask %s', self.name) |
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
156 |
raise |
7573
c8f8762c986d
[repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6765
diff
changeset
|
157 |
except BaseException: |
c8f8762c986d
[repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6765
diff
changeset
|
158 |
restart = False |
0 | 159 |
finally: |
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
160 |
if restart and tasks_manager.running: |
7573
c8f8762c986d
[repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6765
diff
changeset
|
161 |
self.start() |
0 | 162 |
self.func = auto_restart_func |
6381
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
163 |
self.name = func_name(func) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1138
diff
changeset
|
164 |
|
4714
fccda6dd91bf
merge debug and info views
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
165 |
def __str__(self): |
fccda6dd91bf
merge debug and info views
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
166 |
return '%s (%s seconds)' % (self.name, self.interval) |
fccda6dd91bf
merge debug and info views
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
167 |
|
0 | 168 |
def start(self): |
169 |
self._t = Timer(self.interval, self.func) |
|
5601
92cf309672ca
/siteinfo page: display information about the names of the running threads
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5424
diff
changeset
|
170 |
self._t.setName('%s-%s[%d]' % (self._t.getName(), self.name, self.interval)) |
0 | 171 |
self._t.start() |
172 |
||
173 |
def cancel(self): |
|
174 |
self._t.cancel() |
|
175 |
||
176 |
def join(self): |
|
5581
0aae5216f99e
[repo] ensure thread is alive before calling .join. Closes #963580
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
177 |
if self._t.isAlive(): |
0aae5216f99e
[repo] ensure thread is alive before calling .join. Closes #963580
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
178 |
self._t.join() |
0 | 179 |
|
180 |
||
181 |
class RepoThread(Thread): |
|
182 |
"""subclass of thread so it auto remove itself from a given list once |
|
183 |
executed |
|
184 |
""" |
|
185 |
def __init__(self, target, running_threads): |
|
186 |
def auto_remove_func(self=self, func=target): |
|
187 |
try: |
|
188 |
func() |
|
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7573
diff
changeset
|
189 |
except Exception: |
6765
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
190 |
logger = logging.getLogger('cubicweb.repository') |
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
191 |
logger.exception('Unhandled exception in RepoThread %s', self._name) |
b922e3a817e9
fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6427
diff
changeset
|
192 |
raise |
0 | 193 |
finally: |
194 |
self.running_threads.remove(self) |
|
1138
22f634977c95
make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents:
1134
diff
changeset
|
195 |
Thread.__init__(self, target=auto_remove_func) |
0 | 196 |
self.running_threads = running_threads |
6381
c9eed5037223
[repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
6128
diff
changeset
|
197 |
self._name = func_name(target) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1138
diff
changeset
|
198 |
|
0 | 199 |
def start(self): |
200 |
self.running_threads.append(self) |
|
3585
cd437d24aa65
use daemon thread
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2708
diff
changeset
|
201 |
self.daemon = True |
0 | 202 |
Thread.start(self) |
203 |
||
5376
2c3f14bc2590
[python2.6] don't add a name property on Thread
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5066
diff
changeset
|
204 |
def getName(self): |
0 | 205 |
return '%s(%s)' % (self._name, Thread.getName(self)) |
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
206 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
207 |
class TasksManager(object): |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
208 |
"""Object dedicated manage background task""" |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
209 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
210 |
def __init__(self): |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
211 |
self.running = False |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
212 |
self._tasks = [] |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
213 |
self._looping_tasks = [] |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
214 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
215 |
def add_looping_task(self, interval, func, *args): |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
216 |
"""register a function to be called every `interval` seconds. |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
217 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
218 |
looping tasks can only be registered during repository initialization, |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
219 |
once done this method will fail. |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
220 |
""" |
8446
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
221 |
task = LoopTask(self, interval, func, args) |
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
222 |
if self.running: |
8446
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
223 |
self._start_task(task) |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
224 |
else: |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
225 |
self._tasks.append(task) |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
226 |
|
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
227 |
def _start_task(self, task): |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
228 |
self._looping_tasks.append(task) |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
229 |
self.info('starting task %s with interval %.2fs', task.name, |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
230 |
task.interval) |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
231 |
task.start() |
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
232 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
233 |
def start(self): |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
234 |
"""Start running looping task""" |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
235 |
assert self.running == False # bw compat purpose maintly |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
236 |
while self._tasks: |
8446
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
237 |
task = self._tasks.pop() |
cae198371548
[server/utils] add_looping_task can be used any time (closes #2408814)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8425
diff
changeset
|
238 |
self._start_task(task) |
8320
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
239 |
self.running = True |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
240 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
241 |
def stop(self): |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
242 |
"""Stop all running task. |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
243 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
244 |
returns when all task have been cancel and none are running anymore""" |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
245 |
if self.running: |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
246 |
while self._looping_tasks: |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
247 |
looptask = self._looping_tasks.pop() |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
248 |
self.info('canceling task %s...', looptask.name) |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
249 |
looptask.cancel() |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
250 |
looptask.join() |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
251 |
self.info('task %s finished', looptask.name) |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
252 |
|
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
253 |
from logging import getLogger |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
254 |
from cubicweb import set_log_methods |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
255 |
set_log_methods(TasksManager, getLogger('cubicweb.repository')) |
cd2d332b3063
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
7823
diff
changeset
|
256 |