author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Fri, 28 Oct 2011 11:33:24 +0200 | |
changeset 8035 | f98012ec7c53 |
parent 7995 | 9a9f35ef418c |
child 8306 | 4da49700b06a |
permissions | -rw-r--r-- |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
1 |
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
18 |
"""core hooks registering some maintainance tasks as server startup time""" |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
19 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
20 |
__docformat__ = "restructuredtext en" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
21 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
22 |
from datetime import timedelta, datetime |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
23 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
24 |
from cubicweb.server import hook |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
25 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
26 |
class ServerStartupHook(hook.Hook): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
27 |
"""task to cleanup expirated auth cookie entities""" |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
28 |
__regid__ = 'cw.start-looping-tasks' |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
29 |
events = ('server_startup',) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
30 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
31 |
def __call__(self): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
32 |
# XXX use named args and inner functions to avoid referencing globals |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
33 |
# which may cause reloading pb |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
34 |
lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime']) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
35 |
def cleanup_old_transactions(repo=self.repo, lifetime=lifetime): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
36 |
mindate = datetime.now() - lifetime |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
37 |
session = repo.internal_session() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
38 |
try: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
39 |
session.system_sql( |
5101
0d01acee1f0a
[undo] fix transactions table name in cleanup operation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
40 |
'DELETE FROM transactions WHERE tx_time < %(time)s', |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
41 |
{'time': mindate}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
42 |
# cleanup deleted entities |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
43 |
session.system_sql( |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
44 |
'DELETE FROM deleted_entities WHERE dtime < %(time)s', |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
45 |
{'time': mindate}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
46 |
session.commit() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
47 |
finally: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
48 |
session.close() |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
49 |
if self.repo.config['undo-support']: |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
50 |
self.repo.looping_task(60*60*24, cleanup_old_transactions, |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
51 |
self.repo) |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
52 |
def update_feeds(repo): |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
53 |
# don't iter on repo.sources which doesn't include copy based |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
54 |
# sources (the one we're looking for) |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
55 |
for source in repo.sources_by_eid.itervalues(): |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
56 |
if (not source.copy_based_source |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
57 |
or not repo.config.source_enabled(source) |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
58 |
or not source.config['synchronize']): |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
59 |
continue |
7708
45be3a9debe6
[datafeed] for datafeed source, we don't want commit in extid2eid but explicitly handled by the source. Also, we should use 'safe' internal session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7648
diff
changeset
|
60 |
session = repo.internal_session(safe=True) |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
61 |
try: |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
62 |
source.pull_data(session) |
7648
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
63 |
except Exception, exc: |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
64 |
session.exception('while trying to update feed %s', source) |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
65 |
finally: |
5d5d98930247
[repo, hooks] small task optimization: don't start cleanup_old_transactions when undo support is not activated, and retrieve a session to update a source only when some source to update is found
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7456
diff
changeset
|
66 |
session.close() |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
67 |
self.repo.looping_task(60, update_feeds, self.repo) |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
68 |
|
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
69 |
def expire_dataimports(repo=self.repo): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
70 |
for source in repo.sources_by_eid.itervalues(): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
71 |
if (not source.copy_based_source |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
72 |
or not repo.config.source_enabled(source)): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
73 |
continue |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
74 |
session = repo.internal_session() |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
75 |
try: |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
76 |
mindate = datetime.now() - timedelta(seconds=source.config['logs-lifetime']) |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
77 |
session.execute('DELETE CWDataImport X WHERE X start_timestamp < %(time)s', {'time': mindate}) |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
78 |
session.commit() |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
79 |
finally: |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
80 |
session.close() |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7708
diff
changeset
|
81 |
self.repo.looping_task(60*60*24, expire_dataimports, self.repo) |