author | Julien Cristau <julien.cristau@logilab.fr> |
Wed, 23 Oct 2013 14:18:58 +0200 | |
branch | stable |
changeset 9305 | f7a738afc295 |
parent 9263 | 42cc70a1b6ac |
child 9448 | 3e7cad3967c5 |
permissions | -rw-r--r-- |
8587
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
1 |
# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
18 |
"""core hooks registering some maintainance tasks as server startup time""" |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
19 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
20 |
__docformat__ = "restructuredtext en" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
21 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
22 |
from datetime import timedelta, datetime |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
23 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
24 |
from cubicweb.server import hook |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
25 |
|
8587
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
26 |
class TransactionsCleanupStartupHook(hook.Hook): |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
27 |
"""start task to cleanup transaction data""" |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
28 |
__regid__ = 'cw.looping-tasks.transactions-cleanup' |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
29 |
events = ('server_startup',) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
30 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
31 |
def __call__(self): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
32 |
# XXX use named args and inner functions to avoid referencing globals |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
33 |
# which may cause reloading pb |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
34 |
lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime']) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
35 |
def cleanup_old_transactions(repo=self.repo, lifetime=lifetime): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
36 |
mindate = datetime.now() - lifetime |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
37 |
session = repo.internal_session() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
38 |
try: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
39 |
session.system_sql( |
5101
0d01acee1f0a
[undo] fix transactions table name in cleanup operation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
40 |
'DELETE FROM transactions WHERE tx_time < %(time)s', |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
41 |
{'time': mindate}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
42 |
# cleanup deleted entities |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
43 |
session.system_sql( |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
44 |
'DELETE FROM deleted_entities WHERE dtime < %(time)s', |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
45 |
{'time': mindate}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
46 |
session.commit() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
47 |
finally: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
48 |
session.close() |
8306
4da49700b06a
[config, undo] Fix undo-support option migration
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
7995
diff
changeset
|
49 |
if self.repo.config['undo-enabled']: |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
50 |
self.repo.looping_task(60*60*24, cleanup_old_transactions, |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
51 |
self.repo) |
8587
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
52 |
|
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
53 |
class UpdateFeedsStartupHook(hook.Hook): |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
54 |
"""start task to update datafeed based sources""" |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
55 |
__regid__ = 'cw.looping-tasks.update-feeds' |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
56 |
events = ('server_startup',) |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
57 |
|
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
58 |
def __call__(self): |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
59 |
def update_feeds(repo): |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
60 |
# don't iter on repo.sources which doesn't include copy based |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
61 |
# sources (the one we're looking for) |
9263
42cc70a1b6ac
Use the list of sources instead of an iterator in update-feeds looping task
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8695
diff
changeset
|
62 |
# take a list to avoid iterating on a dictionary which size may |
42cc70a1b6ac
Use the list of sources instead of an iterator in update-feeds looping task
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8695
diff
changeset
|
63 |
# change |
42cc70a1b6ac
Use the list of sources instead of an iterator in update-feeds looping task
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8695
diff
changeset
|
64 |
for source in list(repo.sources_by_eid.values()): |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
65 |
if (not source.copy_based_source |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
66 |
or not repo.config.source_enabled(source) |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
67 |
or not source.config['synchronize']): |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
68 |
continue |
7708
45be3a9debe6
[datafeed] for datafeed source, we don't want commit in extid2eid but explicitly handled by the source. Also, we should use 'safe' internal session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7648
diff
changeset
|
69 |
session = repo.internal_session(safe=True) |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
70 |
try: |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
71 |
source.pull_data(session) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8596
diff
changeset
|
72 |
except Exception as exc: |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
73 |
session.exception('while trying to update feed %s', source) |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
74 |
finally: |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
75 |
session.close() |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
76 |
self.repo.looping_task(60, update_feeds, self.repo) |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
77 |
|
8587
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
78 |
|
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
79 |
class DataImportsCleanupStartupHook(hook.Hook): |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
80 |
"""start task to cleanup old data imports (ie datafeed import logs)""" |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
81 |
__regid__ = 'cw.looping-tasks.dataimports-cleanup' |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
82 |
events = ('server_startup',) |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
83 |
|
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
84 |
def __call__(self): |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
85 |
def expire_dataimports(repo=self.repo): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
86 |
for source in repo.sources_by_eid.itervalues(): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
87 |
if (not source.copy_based_source |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
88 |
or not repo.config.source_enabled(source)): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
89 |
continue |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
90 |
session = repo.internal_session() |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
91 |
try: |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
92 |
mindate = datetime.now() - timedelta(seconds=source.config['logs-lifetime']) |
8587
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
93 |
session.execute('DELETE CWDataImport X WHERE X start_timestamp < %(time)s', |
bde0501e0251
[startup hook/looping tasks] separated hook for each looping task to ease modification from cubes. Closes #2517096
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8306
diff
changeset
|
94 |
{'time': mindate}) |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
95 |
session.commit() |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
96 |
finally: |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
97 |
session.close() |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
98 |
self.repo.looping_task(60*60*24, expire_dataimports, self.repo) |