_gcdebug.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Thu, 15 Apr 2010 12:47:29 +0200
branchstable
changeset 5273 c4caef6f09c9
parent 4870 101858d845f7
child 5421 8167de96c523
permissions -rw-r--r--
[vreg] refactor the [re]loading process so things get clearer and fix spurious errors on edge cases
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
import gc, types, weakref
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     4
from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     6
listiterator = type(iter([]))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     7
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
IGNORE_CLASSES = (
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
    type, tuple, dict, list, set, frozenset, type(len),
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
    weakref.ref, weakref.WeakKeyDictionary,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
    listiterator,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    12
    property, classmethod,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    13
    types.ModuleType, types.FunctionType, types.MethodType,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    14
    types.MemberDescriptorType, types.GetSetDescriptorType,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    15
    )
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    16
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
def _get_counted_class(obj, classes):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
    for cls in classes:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
        if isinstance(obj, cls):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
            return cls
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
    raise AssertionError()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
def gc_info(countclasses,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
            ignoreclasses=IGNORE_CLASSES,
4870
101858d845f7 [debug] allow to specify maxlevel to gc_info
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4866
diff changeset
    25
            viewreferrersclasses=(), showobjs=False, maxlevel=1):
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    26
    gc.collect()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
    gc.collect()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
    counters = {}
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
    ocounters = {}
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    30
    for obj in gc.get_objects():
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    31
        if isinstance(obj, countclasses):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    32
            cls = _get_counted_class(obj, countclasses)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    33
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    34
                counters[cls.__name__] += 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
            except KeyError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
                counters[cls.__name__] = 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
        elif not isinstance(obj, ignoreclasses):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    38
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
                key = '%s.%s' % (obj.__class__.__module__,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
                                 obj.__class__.__name__)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
            except AttributeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    42
                key = str(obj)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
                ocounters[key] += 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
            except KeyError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
                ocounters[key] = 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
        if isinstance(obj, viewreferrersclasses):
4870
101858d845f7 [debug] allow to specify maxlevel to gc_info
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4866
diff changeset
    48
            print '   ', obj, referrers(obj, showobjs, maxlevel)
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
    return counters, ocounters, gc.garbage
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    50
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
def referrers(obj, showobj=False, maxlevel=1):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
    objreferrers = _referrers(obj, maxlevel)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
    try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
        return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x)))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
                          for x in objreferrers))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
    except TypeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
        s = set()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
        unhashable = []
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
        for x in objreferrers:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
                s.add(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
            except TypeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
                unhashable.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
        return sorted(s) + unhashable
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
def _referrers(obj, maxlevel, _seen=None, _level=0):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
    interesting = []
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    69
    if _seen is None:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
        _seen = set()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    71
    for x in gc.get_referrers(obj):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    72
        if id(x) in _seen:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    73
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
        _seen.add(id(x))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    75
        if isinstance(x, types.FrameType):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
        if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
        if isinstance(x, (list, tuple, set, dict, listiterator)):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
            if _level >= maxlevel:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
                pass
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    82
                #interesting.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
            else:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
                interesting += _referrers(x, maxlevel, _seen, _level+1)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
        else:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
            interesting.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
    return interesting