_gcdebug.py
author Rémi Cardona <remi.cardona@logilab.fr>
Tue, 15 Sep 2015 17:31:07 +0200
changeset 10678 77333ec71fab
parent 10589 7c23b7de2b8d
permissions -rw-r--r--
[server/utils] convert to py3k using six
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     1
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4870
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10331
diff changeset
    18
from __future__ import print_function
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
import gc, types, weakref
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
from cubicweb.schema import CubicWebRelationSchema, CubicWebEntitySchema
10331
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    23
try:
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    24
    from cubicweb.web.request import _NeedAuthAccessMock
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    25
except ImportError:
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    26
    _NeedAuthAccessMock = None
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
listiterator = type(iter([]))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    30
IGNORE_CLASSES = (
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    31
    type, tuple, dict, list, set, frozenset, type(len),
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    32
    weakref.ref, weakref.WeakKeyDictionary,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    33
    listiterator,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    34
    property, classmethod,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
    types.ModuleType, types.FunctionType, types.MethodType,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
    types.MemberDescriptorType, types.GetSetDescriptorType,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
    )
10331
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    38
if _NeedAuthAccessMock is not None:
6f25c7e4f19b [dbapi] remove the dbapi module and its immediate remaining users
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 10312
diff changeset
    39
    IGNORE_CLASSES = IGNORE_CLASSES + (_NeedAuthAccessMock,)
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
def _get_counted_class(obj, classes):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    42
    for cls in classes:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
        if isinstance(obj, cls):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
            return cls
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
    raise AssertionError()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
def gc_info(countclasses,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    48
            ignoreclasses=IGNORE_CLASSES,
4870
101858d845f7 [debug] allow to specify maxlevel to gc_info
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4866
diff changeset
    49
            viewreferrersclasses=(), showobjs=False, maxlevel=1):
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    50
    gc.collect()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
    gc.collect()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
    counters = {}
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
    ocounters = {}
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
    for obj in gc.get_objects():
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
        if isinstance(obj, countclasses):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
            cls = _get_counted_class(obj, countclasses)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
                counters[cls.__name__] += 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
            except KeyError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
                counters[cls.__name__] = 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
        elif not isinstance(obj, ignoreclasses):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
                key = '%s.%s' % (obj.__class__.__module__,
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
                                 obj.__class__.__name__)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
            except AttributeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
                key = str(obj)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
                ocounters[key] += 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    69
            except KeyError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
                ocounters[key] = 1
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    71
        if isinstance(obj, viewreferrersclasses):
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10331
diff changeset
    72
            print('   ', obj, referrers(obj, showobjs, maxlevel))
10310
b49761555ad6 [services] Fix 'repo_gc_stats' to return a list of unreachable objects' repr
Rémi Cardona <remi.cardona@logilab.fr>
parents: 5424
diff changeset
    73
    garbage = [repr(obj) for obj in gc.garbage]
b49761555ad6 [services] Fix 'repo_gc_stats' to return a list of unreachable objects' repr
Rémi Cardona <remi.cardona@logilab.fr>
parents: 5424
diff changeset
    74
    return counters, ocounters, garbage
4866
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    75
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
def referrers(obj, showobj=False, maxlevel=1):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
    objreferrers = _referrers(obj, maxlevel)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
    try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
        return sorted(set((type(x), showobj and x or getattr(x, '__name__', '%#x' % id(x)))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
                          for x in objreferrers))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    82
    except TypeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
        s = set()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
        unhashable = []
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
        for x in objreferrers:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
            try:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
                s.add(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    88
            except TypeError:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    89
                unhashable.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
        return sorted(s) + unhashable
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    91
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    92
def _referrers(obj, maxlevel, _seen=None, _level=0):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
    interesting = []
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
    if _seen is None:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
        _seen = set()
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    96
    for x in gc.get_referrers(obj):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
        if id(x) in _seen:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
        _seen.add(id(x))
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
        if isinstance(x, types.FrameType):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
        if isinstance(x, (CubicWebRelationSchema, CubicWebEntitySchema)):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
            continue
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
        if isinstance(x, (list, tuple, set, dict, listiterator)):
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   105
            if _level >= maxlevel:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
                pass
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
                #interesting.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
            else:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
                interesting += _referrers(x, maxlevel, _seen, _level+1)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
        else:
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
            interesting.append(x)
550e35a69b75 [debug] a new view to help debugging memory leaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   112
    return interesting