hooks/__init__.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Fri, 16 Mar 2012 10:29:15 +0100
changeset 8307 8be58694f416
parent 8306 4da49700b06a
child 8544 3d049071957e
child 8587 bde0501e0251
permissions -rw-r--r--
backport stable
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
     1
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    18
"""core hooks registering some maintainance tasks as server startup time"""
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    19
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    20
__docformat__ = "restructuredtext en"
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    21
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    22
from datetime import timedelta, datetime
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    23
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    24
from cubicweb.server import hook
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    25
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    26
class ServerStartupHook(hook.Hook):
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    27
    """task to cleanup expirated auth cookie entities"""
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    28
    __regid__ = 'cw.start-looping-tasks'
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    29
    events = ('server_startup',)
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    30
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    31
    def __call__(self):
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    32
        # XXX use named args and inner functions to avoid referencing globals
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    33
        # which may cause reloading pb
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    34
        lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime'])
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    35
        def cleanup_old_transactions(repo=self.repo, lifetime=lifetime):
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    36
            mindate = datetime.now() - lifetime
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    37
            session = repo.internal_session()
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    38
            try:
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    39
                session.system_sql(
5101
0d01acee1f0a [undo] fix transactions table name in cleanup operation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4913
diff changeset
    40
                    'DELETE FROM transactions WHERE tx_time < %(time)s',
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    41
                    {'time': mindate})
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    42
                # cleanup deleted entities
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    43
                session.system_sql(
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    44
                    'DELETE FROM deleted_entities WHERE dtime < %(time)s',
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    45
                    {'time': mindate})
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    46
                session.commit()
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    47
            finally:
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    48
                session.close()
8306
4da49700b06a [config, undo] Fix undo-support option migration
Anthony Truchet <anthony.truchet@logilab.fr>
parents: 7995
diff changeset
    49
        if self.repo.config['undo-enabled']:
7648
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    50
            self.repo.looping_task(60*60*24, cleanup_old_transactions,
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    51
                                   self.repo)
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    52
        def update_feeds(repo):
7648
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    53
            # don't iter on repo.sources which doesn't include copy based
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    54
            # sources (the one we're looking for)
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    55
            for source in repo.sources_by_eid.itervalues():
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    56
                if (not source.copy_based_source
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    57
                    or not repo.config.source_enabled(source)
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    58
                    or not source.config['synchronize']):
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    59
                    continue
7708
45be3a9debe6 [datafeed] for datafeed source, we don't want commit in extid2eid but explicitly handled by the source. Also, we should use 'safe' internal session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7648
diff changeset
    60
                session = repo.internal_session(safe=True)
7648
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    61
                try:
7995
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    62
                    source.pull_data(session)
7648
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    63
                except Exception, exc:
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    64
                    session.exception('while trying to update feed %s', source)
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    65
                finally:
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    66
                    session.close()
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    67
        self.repo.looping_task(60, update_feeds, self.repo)
7995
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    68
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    69
        def expire_dataimports(repo=self.repo):
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    70
            for source in repo.sources_by_eid.itervalues():
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    71
                if (not source.copy_based_source
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    72
                    or not repo.config.source_enabled(source)):
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    73
                    continue
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    74
                session = repo.internal_session()
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    75
                try:
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    76
                    mindate = datetime.now() - timedelta(seconds=source.config['logs-lifetime'])
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    77
                    session.execute('DELETE CWDataImport X WHERE X start_timestamp < %(time)s', {'time': mindate})
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    78
                    session.commit()
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    79
                finally:
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    80
                    session.close()
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    81
        self.repo.looping_task(60*60*24, expire_dataimports, self.repo)