diff -r 01810941a4be -r fa0cd558d829 cubicweb/server/repository.py --- a/cubicweb/server/repository.py Mon Mar 30 15:30:02 2020 +0200 +++ b/cubicweb/server/repository.py Mon Mar 30 15:45:40 2020 +0200 @@ -30,6 +30,7 @@ from contextlib import contextmanager from logging import getLogger import queue +import threading from logilab.common.decorators import cached, clear_cache @@ -170,38 +171,53 @@ class _CnxSetPool(_BaseCnxSet): - def __init__(self, source, size): + def __init__(self, source, min_size=1, max_size=4): super().__init__(source) self._cnxsets = [] self._queue = queue.LifoQueue() + self.lock = threading.Lock() + self.min_size = min_size + self.max_size = max_size - for i in range(size): + for i in range(min_size): self._queue.put_nowait(self._new_cnxset()) def _new_cnxset(self): cnxset = super()._new_cnxset() - self._cnxsets.append(cnxset) + with self.lock: + self._cnxsets.append(cnxset) return cnxset + def size(self): + with self.lock: + return len(self._cnxsets) + def qsize(self): return self._queue.qsize() def get(self): try: - return self._queue.get(True, timeout=5) + cnxset = self._queue.get_nowait() + return cnxset except queue.Empty: - raise Exception('no connections set available after 5 secs, probably either a ' - 'bug in code (too many uncommited/rolled back ' - 'connections) or too much load on the server (in ' - 'which case you can try to set a bigger ' - 'connections pool size)') + if self.max_size and self.size() >= self.max_size: + try: + return self._queue.get(True, timeout=5) + except queue.Empty: + raise Exception('no connections set available after 5 secs, probably either a ' + 'bug in code (too many uncommited/rolled back ' + 'connections) or too much load on the server (in ' + 'which case you can try to set a bigger ' + 'connections pool size)') + return self._new_cnxset() def release(self, cnxset): self._queue.put_nowait(cnxset) def __iter__(self): - for cnxset in self._cnxsets: - yield cnxset + with self.lock: + for cnxset in self._cnxsets: + yield cnxset def close(self): while True: @@ -218,7 +234,7 @@ def get_cnxset(source, size): if not size: return _BaseCnxSet(source) - return _CnxSetPool(source, size) + return _CnxSetPool(source, min_size=1, max_size=size) class Repository(object):