[server] dynamically close idle database connections
When pool hasn't been empty for `idle_timeout` time, start closing connections.
--- a/cubicweb/server/repository.py Mon Mar 30 15:45:40 2020 +0200
+++ b/cubicweb/server/repository.py Mon Mar 30 15:46:12 2020 +0200
@@ -31,6 +31,7 @@
from logging import getLogger
import queue
import threading
+import time
from logilab.common.decorators import cached, clear_cache
@@ -171,13 +172,15 @@
class _CnxSetPool(_BaseCnxSet):
- def __init__(self, source, min_size=1, max_size=4):
+ def __init__(self, source, min_size=1, max_size=4, idle_timeout=300):
super().__init__(source)
self._cnxsets = []
self._queue = queue.LifoQueue()
self.lock = threading.Lock()
self.min_size = min_size
self.max_size = max_size
+ self.idle = time.time()
+ self.idle_timeout = idle_timeout
for i in range(min_size):
self._queue.put_nowait(self._new_cnxset())
@@ -188,6 +191,19 @@
self._cnxsets.append(cnxset)
return cnxset
+ def _close_idle_cnxset(self):
+ # close connections not being used since idle_timeout
+ if abs(time.time() - self.idle) > self.idle_timeout and self.size() > self.min_size:
+ try:
+ cnxset = self._queue.get_nowait()
+ except queue.Empty:
+ # the queue has been used since we checked it size
+ pass
+ else:
+ cnxset.close(True)
+ with self.lock:
+ self._cnxsets.remove(cnxset)
+
def size(self):
with self.lock:
return len(self._cnxsets)
@@ -198,8 +214,11 @@
def get(self):
try:
cnxset = self._queue.get_nowait()
+ self._close_idle_cnxset()
return cnxset
except queue.Empty:
+ # reset idle time
+ self.idle = time.time()
if self.max_size and self.size() >= self.max_size:
try:
return self._queue.get(True, timeout=5)
@@ -213,6 +232,7 @@
def release(self, cnxset):
self._queue.put_nowait(cnxset)
+ self._close_idle_cnxset()
def __iter__(self):
with self.lock: