server/utils.py
author Pierre-Yves David <pierre-yves.david@logilab.fr>
Tue, 20 Mar 2012 18:24:35 +0100
changeset 8320 cd2d332b3063
parent 7823 5f145462e041
child 8349 fdb796435d7b
permissions -rw-r--r--
[repo looping task] move looping task logic in a dedicated object (progress #2204047)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
7815
2a164a9cf81c [exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7573
diff changeset
     1
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5376
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
6128
fbb8398f80dc cleanups
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5606
diff changeset
    18
"""Some utilities for the CubicWeb server."""
7815
2a164a9cf81c [exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7573
diff changeset
    19
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    20
__docformat__ = "restructuredtext en"
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    21
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    22
import sys
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    23
import string
6765
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
    24
import logging
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    25
from threading import Timer, Thread
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    26
from getpass import getpass
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    27
from random import choice
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    28
2105
92ea410806fe refactor sources configuration, add source to sources when using a cube defining
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
    29
from cubicweb.server import SOURCE_TYPES
92ea410806fe refactor sources configuration, add source to sources when using a cube defining
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
    30
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    31
try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    32
    from crypt import crypt
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    33
except ImportError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    34
    # crypt is not available (eg windows)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    35
    from cubicweb.md5crypt import crypt
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    36
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    37
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    38
def getsalt(chars=string.letters + string.digits):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    39
    """generate a random 2-character 'salt'"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    40
    return choice(chars) + choice(chars)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    41
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    42
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    43
def crypt_password(passwd, salt=None):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    44
    """return the encrypted password using the given salt or a generated one
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    45
    """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    46
    if passwd is None:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    47
        return None
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    48
    if salt is None:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    49
        salt = getsalt()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    50
    return crypt(passwd, salt)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    51
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    52
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    53
def cartesian_product(seqin):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    54
    """returns a generator which returns the cartesian product of `seqin`
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    55
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    56
    for more details, see :
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    57
    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302478
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    58
    """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    59
    def rloop(seqin, comb):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    60
        """recursive looping function"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    61
        if seqin:                   # any more sequences to process?
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    62
            for item in seqin[0]:
6128
fbb8398f80dc cleanups
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5606
diff changeset
    63
                newcomb = comb + [item] # add next item to current combination
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    64
                # call rloop w/ remaining seqs, newcomb
1802
d628defebc17 delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1138
diff changeset
    65
                for item in rloop(seqin[1:], newcomb):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    66
                    yield item          # seqs and newcomb
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    67
        else:                           # processing last sequence
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    68
            yield comb                  # comb finished, add to list
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    69
    return rloop(seqin, [])
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    70
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    71
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    72
def cleanup_solutions(rqlst, solutions):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    73
    for sol in solutions:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    74
        for vname in sol.keys():
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    75
            if not (vname in rqlst.defined_vars or vname in rqlst.aliases):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    76
                del sol[vname]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    77
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    78
5066
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    79
def eschema_eid(session, eschema):
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    80
    """get eid of the CWEType entity for the given yams type. You should use
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    81
    this because when schema has been loaded from the file-system, not from the
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    82
    database, (e.g. during tests), eschema.eid is not set.
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    83
    """
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    84
    if eschema.eid is None:
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    85
        eschema.eid = session.execute(
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    86
            'Any X WHERE X is CWEType, X name %(name)s',
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    87
            {'name': str(eschema)})[0][0]
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    88
    return eschema.eid
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    89
bf5cbc351e99 [repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4714
diff changeset
    90
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    91
DEFAULT_MSG = 'we need a manager connection on the repository \
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    92
(the server doesn\'t have to run, even should better not)'
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    93
1910
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
    94
def manager_userpasswd(user=None, msg=DEFAULT_MSG, confirm=False,
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
    95
                       passwdmsg='password'):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    96
    if not user:
3701
104b7c326172 check we've some message to display
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3585
diff changeset
    97
        if msg:
104b7c326172 check we've some message to display
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3585
diff changeset
    98
            print msg
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    99
        while not user:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   100
            user = raw_input('login: ')
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   101
        user = unicode(user, sys.stdin.encoding)
1910
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   102
    passwd = getpass('%s: ' % passwdmsg)
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   103
    if confirm:
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   104
        while True:
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   105
            passwd2 = getpass('confirm password: ')
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   106
            if passwd == passwd2:
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   107
                break
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   108
            print 'password doesn\'t match'
864aa3ea0db5 [server] refactor server.utils.manager_userpasswd
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1802
diff changeset
   109
            passwd = getpass('password: ')
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   110
    # XXX decode password using stdin encoding then encode it using appl'encoding
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   111
    return user, passwd
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   112
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   113
6381
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   114
_MARKER=object()
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   115
def func_name(func):
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   116
    name = getattr(func, '__name__', _MARKER)
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   117
    if name is _MARKER:
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   118
        name = getattr(func, 'func_name', _MARKER)
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   119
    if name is _MARKER:
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   120
        name = repr(func)
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   121
    return name
2105
92ea410806fe refactor sources configuration, add source to sources when using a cube defining
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
   122
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   123
class LoopTask(object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   124
    """threaded task restarting itself once executed"""
8320
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   125
    def __init__(self, tasks_manager, interval, func, args):
5602
277b15d6d3ed forbid looping tasks with an interval of 0
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5601
diff changeset
   126
        if interval <= 0:
277b15d6d3ed forbid looping tasks with an interval of 0
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5601
diff changeset
   127
            raise ValueError('Loop task interval must be > 0 '
277b15d6d3ed forbid looping tasks with an interval of 0
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5601
diff changeset
   128
                             '(current value: %f for %s)' % \
6381
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   129
                             (interval, func_name(func)))
8320
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   130
        self._tasks_manager = tasks_manager
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   131
        self.interval = interval
2708
60d728bdcba5 allow to specify arbitrary argument when recording a looping task func
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2105
diff changeset
   132
        def auto_restart_func(self=self, func=func, args=args):
7573
c8f8762c986d [repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6765
diff changeset
   133
            restart = True
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   134
            try:
2708
60d728bdcba5 allow to specify arbitrary argument when recording a looping task func
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2105
diff changeset
   135
                func(*args)
7573
c8f8762c986d [repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6765
diff changeset
   136
            except Exception:
6765
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
   137
                logger = logging.getLogger('cubicweb.repository')
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
   138
                logger.exception('Unhandled exception in LoopTask %s', self.name)
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
   139
                raise
7573
c8f8762c986d [repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6765
diff changeset
   140
            except BaseException:
c8f8762c986d [repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6765
diff changeset
   141
                restart = False
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   142
            finally:
8320
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   143
                if restart and tasks_manager.running:
7573
c8f8762c986d [repo, looping task] raise a custom exception when repository is shuting down, avoid looping task to be restarted in such case. Closes #1021276
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6765
diff changeset
   144
                    self.start()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   145
        self.func = auto_restart_func
6381
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   146
        self.name = func_name(func)
1802
d628defebc17 delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1138
diff changeset
   147
4714
fccda6dd91bf merge debug and info views
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4212
diff changeset
   148
    def __str__(self):
fccda6dd91bf merge debug and info views
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4212
diff changeset
   149
        return '%s (%s seconds)' % (self.name, self.interval)
fccda6dd91bf merge debug and info views
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4212
diff changeset
   150
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   151
    def start(self):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   152
        self._t = Timer(self.interval, self.func)
5601
92cf309672ca /siteinfo page: display information about the names of the running threads
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5424
diff changeset
   153
        self._t.setName('%s-%s[%d]' % (self._t.getName(), self.name, self.interval))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   154
        self._t.start()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   155
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   156
    def cancel(self):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   157
        self._t.cancel()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   158
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   159
    def join(self):
5581
0aae5216f99e [repo] ensure thread is alive before calling .join. Closes #963580
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
   160
        if self._t.isAlive():
0aae5216f99e [repo] ensure thread is alive before calling .join. Closes #963580
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
   161
            self._t.join()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   162
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   163
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   164
class RepoThread(Thread):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   165
    """subclass of thread so it auto remove itself from a given list once
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   166
    executed
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   167
    """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   168
    def __init__(self, target, running_threads):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   169
        def auto_remove_func(self=self, func=target):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   170
            try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   171
                func()
7815
2a164a9cf81c [exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7573
diff changeset
   172
            except Exception:
6765
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
   173
                logger = logging.getLogger('cubicweb.repository')
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
   174
                logger.exception('Unhandled exception in RepoThread %s', self._name)
b922e3a817e9 fix ticket #1382716 (problem was actually more subtle than I originally thought)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6427
diff changeset
   175
                raise
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   176
            finally:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   177
                self.running_threads.remove(self)
1138
22f634977c95 make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents: 1134
diff changeset
   178
        Thread.__init__(self, target=auto_remove_func)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   179
        self.running_threads = running_threads
6381
c9eed5037223 [repo threads] Add several safety when looking for a callable name.
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 6128
diff changeset
   180
        self._name = func_name(target)
1802
d628defebc17 delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1138
diff changeset
   181
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   182
    def start(self):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   183
        self.running_threads.append(self)
3585
cd437d24aa65 use daemon thread
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2708
diff changeset
   184
        self.daemon = True
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   185
        Thread.start(self)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   186
5376
2c3f14bc2590 [python2.6] don't add a name property on Thread
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5066
diff changeset
   187
    def getName(self):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   188
        return '%s(%s)' % (self._name, Thread.getName(self))
8320
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   189
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   190
class TasksManager(object):
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   191
    """Object dedicated manage background task"""
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   192
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   193
    def __init__(self):
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   194
        self.running = False
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   195
        self._tasks = []
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   196
        self._looping_tasks = []
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   197
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   198
    def add_looping_task(self, interval, func, *args):
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   199
        """register a function to be called every `interval` seconds.
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   200
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   201
        looping tasks can only be registered during repository initialization,
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   202
        once done this method will fail.
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   203
        """
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   204
        if self.running:
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   205
            raise RuntimeError("can't add looping task once the repository is started")
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   206
        self._tasks.append( (interval, func, args) )
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   207
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   208
    def start(self):
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   209
        """Start running looping task"""
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   210
        assert self.running == False # bw compat purpose maintly
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   211
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   212
        while self._tasks:
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   213
            interval, func, args = self._tasks.pop()
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   214
            task = LoopTask(self, interval, func, args)
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   215
            self._looping_tasks.append(task)
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   216
            self.info('starting task %s with interval %.2fs', task.name,
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   217
                      interval)
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   218
            task.start()
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   219
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   220
        self.running = True
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   221
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   222
    def stop(self):
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   223
        """Stop all running task.
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   224
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   225
        returns when all task have been cancel and none are running anymore"""
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   226
        if self.running:
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   227
            while self._looping_tasks:
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   228
                looptask = self._looping_tasks.pop()
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   229
                self.info('canceling task %s...', looptask.name)
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   230
                looptask.cancel()
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   231
                looptask.join()
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   232
                self.info('task %s finished', looptask.name)
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   233
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   234
from logging import getLogger
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   235
from cubicweb import set_log_methods
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   236
set_log_methods(TasksManager, getLogger('cubicweb.repository'))
cd2d332b3063 [repo looping task] move looping task logic in a dedicated object (progress #2204047)
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 7823
diff changeset
   237