Why Gemfury? Push, build, and install  RubyGems npm packages Python packages Maven artifacts PHP packages Go Modules Debian packages RPM packages NuGet packages

Repository URL to install this package:

Details    
hub-client / dockerhub / appstatus / utils.py
Size: Mime:
# Copyright 2015 Docker, Inc. All rights reserved.

import logging
import sys
import threading
import traceback
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

import guppy

logger = logging.getLogger(__name__)


def dump_stacks():
    """
    Dump the stack traces for all threads.

    Returns:
        Dictionary describing the stacks. For example::

            {
              "MainThread": [
                "File: ./manage.py, line 19, in <module>",
                "  execute_from_command_line(sys.argv)",
                "File: /usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py, line 399, in execute_from_command_line",
                "  utility.execute()",
                "File: /usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py, line 392, in execute",
                "  self.fetch_command(subcommand).run_from_argv(self.argv)",
                "File: /usr/local/lib/python2.7/dist-packages/django/core/management/base.py, line 242, in run_from_argv",
                "  self.execute(*args, **options.__dict__)",
                "File: /usr/local/lib/python2.7/dist-packages/django/core/management/base.py, line 285, in execute",
                "  output = self.handle(*args, **options)",
                "File: /usr/local/lib/python2.7/dist-packages/django_extensions/management/utils.py, line 77, in inner",
                "  ret = func(self, *args, **kwargs)",
                "File: /usr/local/lib/python2.7/dist-packages/django_extensions/management/commands/runserver_plus.py, line 263, in handle",
                "  inner_run()",
                "File: /usr/local/lib/python2.7/dist-packages/django_extensions/management/commands/runserver_plus.py, line 261, in inner_run",
                "  ssl_context=ssl_context",
                "File: /usr/local/lib/python2.7/dist-packages/werkzeug/serving.py, line 708, in run_simple",
                "  run_with_reloader(inner, extra_files, reloader_interval)",
                "File: /usr/local/lib/python2.7/dist-packages/werkzeug/serving.py, line 613, in run_with_reloader",
                "  reloader_loop(extra_files, interval)",
                "File: /usr/local/lib/python2.7/dist-packages/werkzeug/serving.py, line 532, in _reloader_stat_loop",
                "  time.sleep(interval)"
              ]
            }
    """
    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
    res = {}
    for thread_id, stack in sys._current_frames().items():
        out = StringIO()
        traceback.print_stack(stack, file=out)
        thread_name = id2name.get(thread_id, str(thread_id))
        res[thread_name] = out.getvalue().split('\n')
        out.close()

    return res


# In order to show up in the heap report, the object type needs to be in the
# top DEFAULT_HEAP_ITEMS and occupy a fraction of total heap more than
# DEFAULT_HEAP_REPORT_THRESHOLD.
DEFAULT_HEAP_ITEMS = 10
DEFAULT_HEAP_REPORT_THRESHOLD = 0.05


def analyze_heap():
    """
    Display heap usage. This is a **very expensive** method call, and should
    be used for debugging only.

    Note that the best documentaion on heapy is the creator's master thesis:
    http://guppy-pe.sourceforge.net/heapy-thesis.pdf

    Returns:
        Dictionary describing the heap analysis. For example::

            {
              "summary": [
                "Partition of a set of 409813 objects. Total size = 58771872 bytes.",
                " Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)",
                "     0 161457  39 14332280  24  14332280  24 str",
                "     1 101937  25  8742456  15  23074736  39 tuple",
                "     2  12551   3  3967320   7  27042056  46 unicode",
                "     3   5864   1  3800000   6  30842056  52 dict (no owner)",
                "     4   1683   0  3764424   6  34606480  59 dict of module",
                "     5  28190   7  3382800   6  37989280  65 function",
                "     6   3678   1  3326952   6  41316232  70 type",
                "     7  24618   6  3151104   5  44467336  76 types.CodeType",
                "     8   3678   1  2880336   5  47347672  81 dict of type",
                "     9   7053   2   931432   2  48279104  82 list",
                "<1606 more rows. Type e.g. '_.more' to view.>"
              ]
              "object sets": [
                {
                  "reference pattern": [
                    "Reference Pattern by <[dict of] class>.",
                    " 0: _ --- [-] 161457 str: 'bool(x) -> b...e subclassed.'...",
                    " 1: a      [-] 71807 tuple: 0x257e810*46, 0x25a1870*28, 0x25e7f00*38...",
                    " 2: aa ---- [S] 24613 types.CodeType: collections.py:426:most_common...",
                    " 3: ab      [S] 147 dict of type: ..Attribute..., ..KeyedRef, ..WarningMessage",
                    " 4: ac ---- [S] 62 dict of module: ..termui..., functools, sys, sysconfig",
                    " 5: ad      [-] 1275 tuple: 0x25e7f00*38, 0x25e8620*73, 0x25ec870*45...",
                    " 6: ada ---- [S] 818 types.CodeType: core.py:41:invoke_param_callback...",
                    " 7: adb      [S] 12 dict of module: ..base..., ..global_settings, ..settings",
                    " 8: adc ---- [-] 20 dict of django.db.models.options.Options: 0x42a4750...",
                    " 9: adca      [-] 20 django.db.models.options.Options: 0x3cdb150, 0x3d24c90...",
                    "<Type e.g. '_.more' for more.>"
                  ],
                  "referrers": [
                    "Partition of a set of 148538 objects. Total size = 29333472 bytes.",
                    " Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)",
                    "     0  71807  48  6474264  22   6474264  22 tuple",
                    "     1   1683   1  3764424  13  10238688  35 dict of module",
                    "     2  28190  19  3382800  12  13621488  46 function",
                    "     3  24618  17  3151104  11  16772592  57 types.CodeType",
                    "     4   3678   2  2880336  10  19652928  67 dict of type",
                    "     5   1527   1  2339880   8  21992808  75 dict (no owner)",
                    "     6    744   1   618432   2  22611240  77 dict of class",
                    "     7    153   0   491352   2  23102592  79 dict of django.db.models.fields.CharField",
                    "     8   1843   1   353752   1  23456344  80 list",
                    "     9     80   0   268160   1  23724504  81 dict of django.db.models.fields.related.ForeignKey",
                    "<663 more rows. Type e.g. '_.more' to view.>"
                  ],
                  "referenced via": [
                    "Partition of a set of 161457 objects. Total size = 14332280 bytes.",
                    " Index  Count   %     Size   % Cumulative  % Referred Via:",
                    "     0  24618  15  3277552  23   3277552  23 '.co_code'",
                    "     1   8071   5  2291808  16   5569360  39 '.func_doc', '[0]'",
                    "     2  14646   9  1342968   9   6912328  48 '.co_filename'",
                    "     3  23646  15  1260704   9   8173032  57 '.co_lnotab'",
                    "     4   2598   2   878168   6   9051200  63 \"['__doc__']\"",
                    "     5   5667   4   336232   2   9387432  65 '[1]'",
                    "     6   3156   2   182536   1   9569968  67 '[2]'",
                    "     7   2840   2   158328   1   9728296  68 '[0]'",
                    "     8   1552   1   155400   1   9883696  69 \"['__file__']\"",
                    "     9   2285   1   136120   1  10019816  70 '[3]'",
                    "<31735 more rows. Type e.g. '_.more' to view.>"
                  ],
                  "object type": [
                    "Partition of a set of 161457 objects. Total size = 14332280 bytes.",
                    " Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)",
                    "     0 161457 100 14332280 100  14332280 100 str"
                  ]
                }
              ],
            }
    """
    # Log this call because it's so damn expensive
    logger.warn('About to run expensive heap analysis')
    def _multiline(text):
        return str(text).split('\n')

    hp = guppy.hpy()
    heap = hp.heap()
    threshold = DEFAULT_HEAP_REPORT_THRESHOLD * heap.size

    res = {}
    obj_sets = []
    res['summary'] = _multiline(heap)
    res['object sets'] = obj_sets

    for i in range(DEFAULT_HEAP_ITEMS):
        heapset = heap[i]
        if heapset.size < threshold:
            break
        heapset_res = {}
        heapset_res['object type'] = _multiline(heapset)
        heapset_res['referrers'] = _multiline(heapset.referrers)
        heapset_res['referenced via'] = _multiline(heapset.byvia)
        heapset_res['reference pattern'] = _multiline(heapset.rp)
        obj_sets.append(heapset_res)

    return res