14 # details. |
14 # details. |
15 # |
15 # |
16 # You should have received a copy of the GNU Lesser General Public License along |
16 # You should have received a copy of the GNU Lesser General Public License along |
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
18 """Some utilities for CubicWeb server/clients.""" |
18 """Some utilities for CubicWeb server/clients.""" |
19 |
19 from __future__ import division |
20 __docformat__ = "restructuredtext en" |
20 __docformat__ = "restructuredtext en" |
21 |
21 |
22 import os |
|
23 import sys |
22 import sys |
24 import decimal |
23 import decimal |
25 import datetime |
24 import datetime |
26 import random |
25 import random |
|
26 from operator import itemgetter |
27 from inspect import getargspec |
27 from inspect import getargspec |
28 from itertools import repeat |
28 from itertools import repeat |
29 from uuid import uuid4 |
29 from uuid import uuid4 |
30 from warnings import warn |
30 from warnings import warn |
|
31 from threading import Lock |
|
32 |
|
33 from logging import getLogger |
31 |
34 |
32 from logilab.mtconverter import xml_escape |
35 from logilab.mtconverter import xml_escape |
33 from logilab.common.deprecation import deprecated |
36 from logilab.common.deprecation import deprecated |
34 |
37 |
35 _MARKER = object() |
38 _MARKER = object() |
549 'days_in_month', 'days_in_year', 'previous_month', |
552 'days_in_month', 'days_in_year', 'previous_month', |
550 'next_month', 'first_day', 'last_day', |
553 'next_month', 'first_day', 'last_day', |
551 'strptime'): |
554 'strptime'): |
552 msg = '[3.6] %s has been moved to logilab.common.date' % funcname |
555 msg = '[3.6] %s has been moved to logilab.common.date' % funcname |
553 _THIS_MOD_NS[funcname] = deprecated(msg)(getattr(date, funcname)) |
556 _THIS_MOD_NS[funcname] = deprecated(msg)(getattr(date, funcname)) |
|
557 |
|
558 |
|
559 logger = getLogger('cubicweb.utils') |
|
560 |
|
561 class QueryCache(object): |
|
562 """ a minimalist dict-like object to be used by the querier |
|
563 and native source (replaces lgc.cache for this very usage) |
|
564 |
|
565 To be efficient it must be properly used. The usage patterns are |
|
566 quite specific to its current clients. |
|
567 |
|
568 The ceiling value should be sufficiently high, else it will be |
|
569 ruthlessly inefficient (there will be warnings when this happens). |
|
570 A good (high enough) value can only be set on a per-application |
|
571 value. A default, reasonnably high value is provided but tuning |
|
572 e.g `rql-cache-size` can certainly help. |
|
573 |
|
574 There are two kinds of elements to put in this cache: |
|
575 * frequently used elements |
|
576 * occasional elements |
|
577 |
|
578 The former should finish in the _permanent structure after some |
|
579 warmup. |
|
580 |
|
581 Occasional elements can be buggy requests (server-side) or |
|
582 end-user (web-ui provided) requests. These have to be cleaned up |
|
583 when they fill the cache, without evicting the usefull, frequently |
|
584 used entries. |
|
585 """ |
|
586 # quite arbitrary, but we want to never |
|
587 # immortalize some use-a-little query |
|
588 _maxlevel = 15 |
|
589 |
|
590 def __init__(self, ceiling=3000): |
|
591 self._max = ceiling |
|
592 # keys belonging forever to this cache |
|
593 self._permanent = set() |
|
594 # mapping of key (that can get wiped) to getitem count |
|
595 self._transient = {} |
|
596 self._data = {} |
|
597 self._lock = Lock() |
|
598 |
|
599 def __len__(self): |
|
600 with self._lock: |
|
601 return len(self._data) |
|
602 |
|
603 def __getitem__(self, k): |
|
604 with self._lock: |
|
605 if k in self._permanent: |
|
606 return self._data[k] |
|
607 v = self._transient.get(k, _MARKER) |
|
608 if v is _MARKER: |
|
609 self._transient[k] = 1 |
|
610 return self._data[k] |
|
611 if v > self._maxlevel: |
|
612 self._permanent.add(k) |
|
613 self._transient.pop(k, None) |
|
614 else: |
|
615 self._transient[k] += 1 |
|
616 return self._data[k] |
|
617 |
|
618 def __setitem__(self, k, v): |
|
619 with self._lock: |
|
620 if len(self._data) >= self._max: |
|
621 self._try_to_make_room() |
|
622 self._data[k] = v |
|
623 |
|
624 def pop(self, key, default=_MARKER): |
|
625 with self._lock: |
|
626 try: |
|
627 if default is _MARKER: |
|
628 return self._data.pop(key) |
|
629 return self._data.pop(key, default) |
|
630 finally: |
|
631 if key in self._permanent: |
|
632 self._permanent.remove(key) |
|
633 else: |
|
634 self._transient.pop(key, None) |
|
635 |
|
636 def clear(self): |
|
637 with self._lock: |
|
638 self._clear() |
|
639 |
|
640 def _clear(self): |
|
641 self._permanent = set() |
|
642 self._transient = {} |
|
643 self._data = {} |
|
644 |
|
645 def _try_to_make_room(self): |
|
646 current_size = len(self._data) |
|
647 items = sorted(self._transient.items(), key=itemgetter(1)) |
|
648 level = 0 |
|
649 for k, v in items: |
|
650 self._data.pop(k, None) |
|
651 self._transient.pop(k, None) |
|
652 if v > level: |
|
653 datalen = len(self._data) |
|
654 if datalen == 0: |
|
655 return |
|
656 if (current_size - datalen) / datalen > .1: |
|
657 break |
|
658 level = v |
|
659 else: |
|
660 # we removed cruft but everything is permanent |
|
661 if len(self._data) >= self._max: |
|
662 logger.warning('Cache %s is full.' % id(self)) |
|
663 self._clear() |
|
664 |
|
665 def _usage_report(self): |
|
666 with self._lock: |
|
667 return {'itemcount': len(self._data), |
|
668 'transientcount': len(self._transient), |
|
669 'permanentcount': len(self._permanent)} |
|
670 |
|
671 def popitem(self): |
|
672 raise NotImplementedError() |
|
673 |
|
674 def setdefault(self, key, default=None): |
|
675 raise NotImplementedError() |
|
676 |
|
677 def update(self, other): |
|
678 raise NotImplementedError() |