_gcdebug.py
author Julien Cristau <julien.cristau@logilab.fr>
Mon, 18 May 2015 16:30:32 +0200
changeset 10437 9cddd711419c
parent 10331 6f25c7e4f19b
child 10589 7c23b7de2b8d
permissions -rw-r--r--
[devtools] extract functions to start/stop a postgresql cluster
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     1
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
import gc, types, weakref
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema
10331
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    22
try:
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    23
    from cubicweb.web.request import _NeedAuthAccessMock
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    24
except ImportError:
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    25
    _NeedAuthAccessMock = None
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    26
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
listiterator = type(iter([]))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
IGNORE_CLASSES = (
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    30
    type, tuple, dict, list, set, frozenset, type(len),
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    31
    weakref.ref, weakref.WeakKeyDictionary,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    32
    listiterator,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    33
    property, classmethod,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    34
    types.ModuleType, types.FunctionType, types.MethodType,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
    types.MemberDescriptorType, types.GetSetDescriptorType,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
    )
10331
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    37
if _NeedAuthAccessMock is not None:
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    38
    IGNORE_CLASSES = IGNORE_CLASSES + (_NeedAuthAccessMock,)
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
def _get_counted_class(obj, classes):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
    for cls in classes:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    42
        if isinstance(obj, cls):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
            return cls
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
    raise AssertionError()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
def gc_info(countclasses,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
            ignoreclasses=IGNORE_CLASSES,
4870
101858d845f7 [debug] allow to specify maxlevel to gc_info
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4866
diff changeset
    48
            viewreferrersclasses=(), showobjs=False, maxlevel=1):
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
    gc.collect()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    50
    gc.collect()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
    counters = {}
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
    ocounters = {}
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
    for obj in gc.get_objects():
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
        if isinstance(obj, countclasses):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
            cls = _get_counted_class(obj, countclasses)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
                counters[cls.__name__] += 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
            except KeyError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
                counters[cls.__name__] = 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
        elif not isinstance(obj, ignoreclasses):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
                key = '%s.%s' % (obj.__class__.__module__,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
                                 obj.__class__.__name__)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
            except AttributeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
                key = str(obj)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
                ocounters[key] += 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
            except KeyError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    69
                ocounters[key] = 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
        if isinstance(obj, viewreferrersclasses):
4870
101858d845f7 [debug] allow to specify maxlevel to gc_info
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4866
diff changeset
    71
            print '   ', obj, referrers(obj, showobjs, maxlevel)
10310
b49761555ad6 [services] Fix 'repo_gc_stats' to return a list of unreachable objects' repr
Rémi Cardona <remi.cardona@logilab.fr>
parents: 5424
diff changeset
    72
    garbage = [repr(obj) for obj in gc.garbage]
b49761555ad6 [services] Fix 'repo_gc_stats' to return a list of unreachable objects' repr
Rémi Cardona <remi.cardona@logilab.fr>
parents: 5424
diff changeset
    73
    return counters, ocounters, garbage
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    75
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
def referrers(obj, showobj=False, maxlevel=1):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
    objreferrers = _referrers(obj, maxlevel)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
    try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
        return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x)))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
                          for x in objreferrers))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
    except TypeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    82
        s = set()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
        unhashable = []
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
        for x in objreferrers:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
                s.add(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
            except TypeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    88
                unhashable.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    89
        return sorted(s) + unhashable
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    91
def _referrers(obj, maxlevel, _seen=None, _level=0):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    92
    interesting = []
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
    if _seen is None:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
        _seen = set()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
    for x in gc.get_referrers(obj):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    96
        if id(x) in _seen:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
        _seen.add(id(x))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
        if isinstance(x, types.FrameType):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
        if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
        if isinstance(x, (list, tuple, set, dict, listiterator)):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
            if _level >= maxlevel:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   105
                pass
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
                #interesting.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
            else:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
                interesting += _referrers(x, maxlevel, _seen, _level+1)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
        else:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
            interesting.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
    return interesting