cubicweb/hooks/syncsources.py
author Philippe Pepiot <ph@itsalwaysdns.eu>
Tue, 31 Mar 2020 18:22:05 +0200
changeset 12966 6cd938c29ca3
parent 12156 0d1d22a3a48b
permissions -rw-r--r--
[server] Make connection pooler configurable and set better default values Drop the configuration connections-pool-size and add new configurations options: * connections-pool-min-size. Set to 0 by default so we open connections only when needed. This avoid opening min-size*processes connections at startup, which is, it think, a good default. * connections-pool-max-size. Set to 0 (unlimited) by default, so we move the bottleneck to postgresql. * connections-idle-timeout. Set to 10 minutes. I don't have arguments about this except that this is the default in pgbouncer.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
11767
432f87a63057 flake8 and all
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11763
diff changeset
     1
# copyright 2010-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     3
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     4
# This file is part of CubicWeb.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     5
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
     9
# any later version.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    10
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    14
# details.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    15
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    18
"""hooks for repository sources synchronization"""
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    19
10666
7f6b5f023884 [py3k] replace '_ = unicode' in global scope (closes #7589459)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9619
diff changeset
    20
from cubicweb import _
8556
bbe0d6985e59 [validation error] refactor validation error handling so translation is done on the web side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8190
diff changeset
    21
bbe0d6985e59 [validation error] refactor validation error handling so translation is done on the web side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8190
diff changeset
    22
from cubicweb import validation_error
8190
2a3c1b787688 [vreg] move base registry implementation to logilab.common. Closes #1916014
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7879
diff changeset
    23
from cubicweb.predicates import is_instance
6724
24bf6f181d0e [pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6427
diff changeset
    24
from cubicweb.server import SOURCE_TYPES, hook
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    25
11767
432f87a63057 flake8 and all
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11763
diff changeset
    26
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
class SourceHook(hook.Hook):
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
    __abstract__ = True
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
    category = 'cw.sources'
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    30
12152
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    31
    def get_source(self, source_entity):
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    32
        if source_entity.name == 'system':
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    33
            return self._cw.repo.system_source
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    34
        return self._cw.repo.get_source(source_entity.type, source_entity.name,
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    35
                                        {}, source_entity.eid)
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    36
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    38
class SourceAddedHook(SourceHook):
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
    __regid__ = 'cw.sources.added'
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
    __select__ = SourceHook.__select__ & is_instance('CWSource')
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
    events = ('after_add_entity',)
12156
0d1d22a3a48b Fix some flake8 errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12153
diff changeset
    42
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
    def __call__(self):
12152
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    44
        if self.entity.type not in SOURCE_TYPES:
8557
8756f932ca83 [source synchronization hooks] Fix/enhance system source hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8556
diff changeset
    45
            msg = _('Unknown source type')
8556
bbe0d6985e59 [validation error] refactor validation error handling so translation is done on the web side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8190
diff changeset
    46
            raise validation_error(self.entity, {('type', 'subject'): msg})
12152
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    47
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    48
        source = self.get_source(self.entity)
12153
0ff0aff4413d [sources] Check source's url attribute value on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12152
diff changeset
    49
        source.check_urls(self.entity)
12152
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    50
        source.check_config(self.entity)
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
class SourceRemovedHook(SourceHook):
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
    __regid__ = 'cw.sources.removed'
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
    __select__ = SourceHook.__select__ & is_instance('CWSource')
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
    events = ('before_delete_entity',)
12156
0d1d22a3a48b Fix some flake8 errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12153
diff changeset
    57
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
    def __call__(self):
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
        if self.entity.name == 'system':
8556
bbe0d6985e59 [validation error] refactor validation error handling so translation is done on the web side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8190
diff changeset
    60
            msg = _("You cannot remove the system source")
bbe0d6985e59 [validation error] refactor validation error handling so translation is done on the web side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8190
diff changeset
    61
            raise validation_error(self.entity, {None: msg})
7735
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7650
diff changeset
    62
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7650
diff changeset
    63
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    64
class SourceUpdatedHook(SourceHook):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    65
    __regid__ = 'cw.sources.configupdate'
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    66
    __select__ = SourceHook.__select__ & is_instance('CWSource')
7735
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7650
diff changeset
    67
    events = ('before_update_entity',)
12156
0d1d22a3a48b Fix some flake8 errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12153
diff changeset
    68
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    69
    def __call__(self):
7735
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7650
diff changeset
    70
        if 'name' in self.entity.cw_edited:
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7650
diff changeset
    71
            oldname, newname = self.entity.cw_edited.oldnewvalue('name')
8557
8756f932ca83 [source synchronization hooks] Fix/enhance system source hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8556
diff changeset
    72
            if oldname == 'system':
8756f932ca83 [source synchronization hooks] Fix/enhance system source hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8556
diff changeset
    73
                msg = _("You cannot rename the system source")
8756f932ca83 [source synchronization hooks] Fix/enhance system source hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8556
diff changeset
    74
                raise validation_error(self.entity, {('name', 'subject'): msg})
12152
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    75
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    76
        source = self.get_source(self.entity)
12153
0ff0aff4413d [sources] Check source's url attribute value on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12152
diff changeset
    77
        if 'url' in self.entity.cw_edited:
0ff0aff4413d [sources] Check source's url attribute value on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12152
diff changeset
    78
            source.check_urls(self.entity)
12152
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    79
        if 'config' in self.entity.cw_edited:
f34d18d0603f [sources] Check sources configuration is fine on creation/modification
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 12150
diff changeset
    80
            source.check_config(self.entity)