Learn more  » Push, build, and install  RubyGems npm packages Python packages Maven artifacts PHP packages Go Modules Bower components Debian packages RPM packages NuGet packages

hemamaps / Scrapy   python

Repository URL to install this package:

Version: 1.1.1 

/ extensions / debug.py

"""
Extensions for debugging Scrapy

See documentation in docs/topics/extensions.rst
"""

import sys
import signal
import logging
import traceback
import threading
from pdb import Pdb

from scrapy.utils.engine import format_engine_status
from scrapy.utils.trackref import format_live_refs

logger = logging.getLogger(__name__)


class StackTraceDump(object):

    def __init__(self, crawler=None):
        self.crawler = crawler
        try:
            signal.signal(signal.SIGUSR2, self.dump_stacktrace)
            signal.signal(signal.SIGQUIT, self.dump_stacktrace)
        except AttributeError:
            # win32 platforms don't support SIGUSR signals
            pass

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def dump_stacktrace(self, signum, frame):
        log_args = {
            'stackdumps': self._thread_stacks(),
            'enginestatus': format_engine_status(self.crawler.engine),
            'liverefs': format_live_refs(),
        }
        logger.info("Dumping stack trace and engine status\n"
                    "%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s",
                    log_args, extra={'crawler': self.crawler})

    def _thread_stacks(self):
        id2name = dict((th.ident, th.name) for th in threading.enumerate())
        dumps = ''
        for id_, frame in sys._current_frames().items():
            name = id2name.get(id_, '')
            dump = ''.join(traceback.format_stack(frame))
            dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
        return dumps


class Debugger(object):
    def __init__(self):
        try:
            signal.signal(signal.SIGUSR2, self._enter_debugger)
        except AttributeError:
            # win32 platforms don't support SIGUSR signals
            pass

    def _enter_debugger(self, signum, frame):
        Pdb().set_trace(frame.f_back)