cubicweb/hooks/__init__.py
author Guillaume Vandevelde <gvandevelde@logilab.fr>
Thu, 13 Feb 2020 13:56:12 +0100
branch3.26
changeset 12876 fe9bd0e937e4
parent 12145 752b94ed9748
permissions -rw-r--r--
[pkg] version 3.26.15
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
     1
# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5101
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    18
"""core hooks registering some maintainance tasks as server startup time"""
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    19
11767
432f87a63057 flake8 and all
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11057
diff changeset
    20
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    21
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    22
from datetime import timedelta, datetime
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    23
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    24
from cubicweb.server import hook
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    25
8587
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    26
class TransactionsCleanupStartupHook(hook.Hook):
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    27
    """start task to cleanup transaction data"""
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    28
    __regid__ = 'cw.looping-tasks.transactions-cleanup'
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    29
    events = ('server_startup',)
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    30
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    31
    def __call__(self):
12112
0a54f49314f6 [server] Introduce an `has_scheduler` method on Repository
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 12012
diff changeset
    32
        if not self.repo.has_scheduler():
11986
4d414ecf8416 [hooks] Return early in server_startup hooks if the repository has not tasks manager
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 11767
diff changeset
    33
            return
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    34
        # XXX use named args and inner functions to avoid referencing globals
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    35
        # which may cause reloading pb
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    36
        lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime'])
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    37
        def cleanup_old_transactions(repo=self.repo, lifetime=lifetime):
10961
6ff786884aad [hooks] fix timestamp confusion in TransactionsCleanupStartupHook
Julien Cristau <julien.cristau@logilab.fr>
parents: 10662
diff changeset
    38
            mindate = datetime.utcnow() - lifetime
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    39
            with repo.internal_cnx() as cnx:
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    40
                cnx.system_sql(
5101
0d01acee1f0a [undo] fix transactions table name in cleanup operation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4913
diff changeset
    41
                    'DELETE FROM transactions WHERE tx_time < %(time)s',
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 2835
diff changeset
    42
                    {'time': mindate})
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    43
                cnx.commit()
8306
4da49700b06a [config, undo] Fix undo-support option migration
Anthony Truchet <anthony.truchet@logilab.fr>
parents: 7995
diff changeset
    44
        if self.repo.config['undo-enabled']:
7648
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    45
            self.repo.looping_task(60*60*24, cleanup_old_transactions,
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    46
                                   self.repo)
8587
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    47
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    48
class UpdateFeedsStartupHook(hook.Hook):
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    49
    """start task to update datafeed based sources"""
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    50
    __regid__ = 'cw.looping-tasks.update-feeds'
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    51
    events = ('server_startup',)
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    52
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    53
    def __call__(self):
12112
0a54f49314f6 [server] Introduce an `has_scheduler` method on Repository
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 12012
diff changeset
    54
        if not self.repo.has_scheduler():
11986
4d414ecf8416 [hooks] Return early in server_startup hooks if the repository has not tasks manager
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 11767
diff changeset
    55
            return
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    56
        def update_feeds(repo):
9448
3e7cad3967c5 [multi-sources-removal] Drop the "true" multi-sources planner
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9263
diff changeset
    57
            # take a list to avoid iterating on a dictionary whose size may
9263
42cc70a1b6ac Use the list of sources instead of an iterator in update-feeds looping task
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 8695
diff changeset
    58
            # change
12145
752b94ed9748 [hooks] Remove list() around repo.sources_by_uri
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 12112
diff changeset
    59
            for uri, source in repo.sources_by_uri.items():
9456
a79e88aad555 [multi-sources-removal] Kill repo.sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9450
diff changeset
    60
                if (uri == 'system'
7648
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    61
                    or not repo.config.source_enabled(source)
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    62
                    or not source.config['synchronize']):
5d5d98930247 [repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7456
diff changeset
    63
                    continue
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    64
                with repo.internal_cnx() as cnx:
9746
81b56897a377 [datafeed] update datafeed internals to use connection instead of session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9456
diff changeset
    65
                    try:
81b56897a377 [datafeed] update datafeed internals to use connection instead of session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9456
diff changeset
    66
                        source.pull_data(cnx)
81b56897a377 [datafeed] update datafeed internals to use connection instead of session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9456
diff changeset
    67
                    except Exception as exc:
81b56897a377 [datafeed] update datafeed internals to use connection instead of session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9456
diff changeset
    68
                        cnx.exception('while trying to update feed %s', source)
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5424
diff changeset
    69
        self.repo.looping_task(60, update_feeds, self.repo)
7995
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    70
8587
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    71
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    72
class DataImportsCleanupStartupHook(hook.Hook):
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    73
    """start task to cleanup old data imports (ie datafeed import logs)"""
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    74
    __regid__ = 'cw.looping-tasks.dataimports-cleanup'
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    75
    events = ('server_startup',)
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    76
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    77
    def __call__(self):
12112
0a54f49314f6 [server] Introduce an `has_scheduler` method on Repository
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 12012
diff changeset
    78
        if not self.repo.has_scheduler():
11986
4d414ecf8416 [hooks] Return early in server_startup hooks if the repository has not tasks manager
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 11767
diff changeset
    79
            return
7995
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    80
        def expire_dataimports(repo=self.repo):
10662
10942ed172de [py3k] dict.iteritems → dict.items
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9775
diff changeset
    81
            for uri, source in repo.sources_by_uri.items():
9448
3e7cad3967c5 [multi-sources-removal] Drop the "true" multi-sources planner
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9263
diff changeset
    82
                if (uri == 'system'
7995
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    83
                    or not repo.config.source_enabled(source)):
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    84
                    continue
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    85
                with repo.internal_cnx() as cnx:
10962
625b951b4a1a [hooks] fix timestamp confusion in DataImportsCleanupStartupHook
Julien Cristau <julien.cristau@logilab.fr>
parents: 10961
diff changeset
    86
                    mindate = datetime.utcnow() - timedelta(seconds=source.config['logs-lifetime'])
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    87
                    cnx.execute('DELETE CWDataImport X WHERE X start_timestamp < %(time)s',
8587
bde0501e0251 [startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8306
diff changeset
    88
                                    {'time': mindate})
9775
614f8f3d2a35 [hooks] internal_session -> internal_cnx (including a typo)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9746
diff changeset
    89
                    cnx.commit()
7995
9a9f35ef418c Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents: 7708
diff changeset
    90
        self.repo.looping_task(60*60*24, expire_dataimports, self.repo)