author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Fri, 27 May 2011 11:17:02 +0200 | |
changeset 7446 | 6fba86efdd09 |
parent 7398 | 26695dd703d8 |
child 7456 | c54038622fc9 |
permissions | -rw-r--r-- |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
1 |
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5101
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
18 |
"""core hooks registering some maintainance tasks as server startup time""" |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
19 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
20 |
__docformat__ = "restructuredtext en" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
21 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
22 |
from datetime import timedelta, datetime |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
23 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
24 |
from cubicweb.server import hook |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
25 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
26 |
class ServerStartupHook(hook.Hook): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
27 |
"""task to cleanup expirated auth cookie entities""" |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
28 |
__regid__ = 'cw.start-looping-tasks' |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
29 |
events = ('server_startup',) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
30 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
31 |
def __call__(self): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
32 |
# XXX use named args and inner functions to avoid referencing globals |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
33 |
# which may cause reloading pb |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
34 |
lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime']) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
35 |
def cleanup_old_transactions(repo=self.repo, lifetime=lifetime): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
36 |
mindate = datetime.now() - lifetime |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
37 |
session = repo.internal_session() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
38 |
try: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
39 |
session.system_sql( |
5101
0d01acee1f0a
[undo] fix transactions table name in cleanup operation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
40 |
'DELETE FROM transactions WHERE tx_time < %(time)s', |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
41 |
{'time': mindate}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
42 |
# cleanup deleted entities |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
43 |
session.system_sql( |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
44 |
'DELETE FROM deleted_entities WHERE dtime < %(time)s', |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
45 |
{'time': mindate}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
46 |
session.commit() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
47 |
finally: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
48 |
session.close() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
2835
diff
changeset
|
49 |
self.repo.looping_task(60*60*24, cleanup_old_transactions, self.repo) |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
50 |
def update_feeds(repo): |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
51 |
session = repo.internal_session() |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
52 |
try: |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
53 |
# don't iter on repo.sources which doesn't include copy based |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
54 |
# sources (the one we're looking for) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
55 |
for source in repo.sources_by_eid.itervalues(): |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
56 |
if (not source.copy_based_source |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
57 |
or not repo.config.source_enabled(source) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
58 |
or not source.config['synchronize']): |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
59 |
continue |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
60 |
try: |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
61 |
stats = source.pull_data(session) |
6988
2f4a7d1e15b3
[hooks] do not break if no entity created or updated during feed update
nico
parents:
6957
diff
changeset
|
62 |
if stats.get('created'): |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
63 |
source.info('added %s entities', len(stats['created'])) |
6988
2f4a7d1e15b3
[hooks] do not break if no entity created or updated during feed update
nico
parents:
6957
diff
changeset
|
64 |
if stats.get('updated'): |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
65 |
source.info('updated %s entities', len(stats['updated'])) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
66 |
session.commit() |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
67 |
except Exception, exc: |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
68 |
session.exception('while trying to update feed %s', source) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
69 |
session.rollback() |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6988
diff
changeset
|
70 |
session.set_cnxset() |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
71 |
finally: |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
72 |
session.close() |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
73 |
self.repo.looping_task(60, update_feeds, self.repo) |