From ac6ca0510e1538ce5ea3683f6dd2c501517c2934 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Tue, 4 Aug 2026 18:27:20 +0200 Subject: [PATCH] Defer debugpy and psutil imports until they are actually needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both imports were paid for on every kernel startup even though most sessions never debug or ask for usage information. * `IPythonKernel.debugger` is now a lazily-created property; the debugger (and the debugpy import) is only built on the first debug request. `poll_stopped_queue` is scheduled at that point rather than in `start()`. * `debugger_class` was a `Type` trait, which traitlets resolves — and therefore imports — as soon as the kernel is instantiated. It is replaced by a plain `debugger_class_name` string; `debugger_class` remains as a deprecated property, and subclasses still overriding it keep working with a DeprecationWarning. * `psutil` is imported through `_get_psutil()`, which caches the (possibly None) result on first use. On a local test (where I have optimisation in IPython and traitlets as well), this brings the startup time from 220ms to 170ms --- ipykernel/ipkernel.py | 137 +++++++++++++++++++++++++++++------- ipykernel/kernelbase.py | 35 ++++++--- tests/test_kernel_direct.py | 2 + 3 files changed, 136 insertions(+), 38 deletions(-) diff --git a/ipykernel/ipkernel.py b/ipykernel/ipkernel.py index 7ce9cc013..e4ff5f9de 100644 --- a/ipykernel/ipkernel.py +++ b/ipykernel/ipkernel.py @@ -10,6 +10,7 @@ import sys import threading import typing as t +import warnings from contextlib import contextmanager from functools import partial @@ -17,6 +18,7 @@ from IPython.core import release from IPython.utils.tokenutil import line_at_cursor, token_at_cursor from traitlets import Any, Bool, HasTraits, Instance, List, Type, default, observe, observe_compat +from traitlets.utils.importstring import import_item from zmq.eventloop.zmqstream import ZMQStream from .comm.comm import BaseComm @@ -74,9 +76,15 @@ class IPythonKernel(KernelBase): shell = Instance("IPython.core.interactiveshell.InteractiveShellABC", allow_none=True) shell_class = Type(ZMQInteractiveShell) - # use fully-qualified name to ensure lazy import and prevent the issue from - # https://github.com/ipython/ipykernel/issues/1198 - debugger_class = Type("ipykernel.debugger.Debugger") + # Do not use a Type() trait: traitlets resolves (and thus imports) a Type + # trait's string default as soon as the owning HasTraits instance is + # created, which would force the expensive debugpy import on every + # kernel startup. Resolved lazily instead, see the `debugger` property. + debugger_class_name = "ipykernel.debugger.Debugger" + + # Set by the deprecated `debugger_class` setter, takes precedence over + # `debugger_class_name` when not None. + _debugger_class: type | None = None compiler_class = Type(XCachingCompiler) @@ -117,24 +125,14 @@ def __init__(self, **kwargs): """Initialize the kernel.""" super().__init__(**kwargs) - from .debugger import _is_debugpy_available - self._kernel_modules = [ m.__file__ for m in sys.modules.copy().values() if hasattr(m, "__file__") and m.__file__ ] - # Initialize the Debugger - if _is_debugpy_available: - self.debugger = self.debugger_class( - self.log, - self.debugpy_stream, - self._publish_debug_event, - self.debug_shell_socket, - self.session, - self._kernel_modules, - self.debug_just_my_code, - self.filter_internal_frames, - ) + # The debugger itself (and the debugpy import it requires) is + # created lazily on first use, see the `debugger` property below. + self._debugger = None + self._debugger_init_attempted = False # Initialize the InteractiveShell subclass self.shell = self.shell_class.instance( @@ -216,10 +214,101 @@ def __init__(self, **kwargs): "file_extension": ".py", } - def dispatch_debugpy(self, msg): - from .debugger import _is_debugpy_available + @property + def debugger_class(self): + """Deprecated, use :attr:`debugger_class_name` instead. + + .. deprecated:: 7.4 + Accessing this attribute imports the debugger module (and thus + debugpy), which is exactly what ``debugger_class_name`` exists to + avoid. + """ + warnings.warn( + "IPythonKernel.debugger_class is deprecated in ipykernel 7.4," + " use IPythonKernel.debugger_class_name instead.", + DeprecationWarning, + stacklevel=2, + ) + return self._resolve_debugger_class() + + @debugger_class.setter + def debugger_class(self, value): + warnings.warn( + "IPythonKernel.debugger_class is deprecated in ipykernel 7.4," + " set IPythonKernel.debugger_class_name to the fully qualified" + " name of the class instead.", + DeprecationWarning, + stacklevel=2, + ) + self._debugger_class = value + + def _resolve_debugger_class(self): + """Return the class to instantiate the debugger from. + + Honors the deprecated ``debugger_class`` attribute, whether it was set + on an instance or overridden by a subclass, before falling back to + ``debugger_class_name``. + """ + if self._debugger_class is not None: + return self._debugger_class + for klass in type(self).__mro__: + if klass is IPythonKernel: + break + if "debugger_class" in klass.__dict__: + warnings.warn( + f"{klass.__module__}.{klass.__qualname__} overrides" + " `debugger_class`, which is deprecated in ipykernel 7.4;" + " override `debugger_class_name` with the fully qualified" + " name of the class instead.", + DeprecationWarning, + stacklevel=3, + ) + # The subclass attribute shadows the property defined here, so + # this resolves the override (a plain class or a Type trait). + return self.debugger_class + return import_item(self.debugger_class_name) + + @property + def debugger(self): + """The debugger instance, created lazily on first use. - if _is_debugpy_available: + Importing debugpy is expensive, so we avoid it until a debug + request actually comes in. + """ + if self._debugger is None and not self._debugger_init_attempted: + self._debugger_init_attempted = True + from .debugger import _is_debugpy_available + + if _is_debugpy_available: + debugger_class = self._resolve_debugger_class() + self._debugger = debugger_class( + self.log, + self.debugpy_stream, + self._publish_debug_event, + self.debug_shell_socket, + self.session, + self._kernel_modules, + self.debug_just_my_code, + self.filter_internal_frames, + ) + # Mirrors the guard that used to live in `start()`: without a + # debugpy stream or a control thread there is nothing to poll + # on, and no loop to poll from. + if self.debugpy_stream is not None and self.control_thread is not None: + asyncio.run_coroutine_threadsafe( + self.poll_stopped_queue(), self.control_thread.io_loop.asyncio_loop + ) + return self._debugger + + @debugger.setter + def debugger(self, value): + # `debugger` used to be a plain instance attribute assigned in + # __init__; keep it writable for subclasses that replace it. + self._debugger = value + self._debugger_init_attempted = True + + def dispatch_debugpy(self, msg): + if self.debugger is not None: # The first frame is the socket id, we can drop it frame = msg[1].bytes.decode("utf-8") self.log.debug("Debugpy received: %s", frame) @@ -245,10 +334,6 @@ def start(self): else: self.debugpy_stream.on_recv(self.dispatch_debugpy, copy=False) super().start() - if self.debugpy_stream: - asyncio.run_coroutine_threadsafe( - self.poll_stopped_queue(), self.control_thread.io_loop.asyncio_loop - ) def set_parent(self, ident, parent, channel="shell"): """Overridden from parent to tell the display hook and output streams @@ -535,9 +620,7 @@ def do_complete(self, code, cursor_pos): async def do_debug_request(self, msg): """Handle a debug request.""" - from .debugger import _is_debugpy_available - - if _is_debugpy_available: + if self.debugger is not None: return await self.debugger.process_request(msg) return None diff --git a/ipykernel/kernelbase.py b/ipykernel/kernelbase.py index 62e4fa239..a6565e3ad 100644 --- a/ipykernel/kernelbase.py +++ b/ipykernel/kernelbase.py @@ -61,18 +61,29 @@ from .iostream import OutStream from .utils import LazyDict, _async_in_context -psutil: t.Any | None -try: - import psutil as _psutil -except ImportError: - psutil = None -else: - psutil = _psutil +psutil: t.Any | None = None +_NO_SUCH_PROCESS: tuple[type[BaseException], ...] = () +_psutil_import_attempted = False + + +def _get_psutil() -> t.Any | None: + """Import psutil on first use, caching the (possibly None) result. + + psutil is optional and its import is not cheap, so we avoid paying for + it unless something actually needs process/resource-usage information. + """ + global psutil, _NO_SUCH_PROCESS, _psutil_import_attempted # noqa: PLW0603 + if not _psutil_import_attempted: + _psutil_import_attempted = True + try: + import psutil as _psutil + except ImportError: + pass + else: + psutil = _psutil + _NO_SUCH_PROCESS = (psutil.NoSuchProcess,) + return psutil -if psutil is None: - _NO_SUCH_PROCESS: tuple[type[BaseException], ...] = () -else: - _NO_SUCH_PROCESS = (psutil.NoSuchProcess,) _AWAITABLE_MESSAGE: str = ( "For consistency across implementations, it is recommended that `{func_name}`" @@ -1184,6 +1195,7 @@ async def usage_request(self, stream, ident, parent): if not self.session: return reply_content = {"hostname": socket.gethostname(), "pid": os.getpid()} + psutil = _get_psutil() if psutil is None: reply_content["cpu_count"] = os.cpu_count() reply_msg = self.session.send(stream, "usage_reply", reply_content, parent, ident) @@ -1503,6 +1515,7 @@ def _process_children(self): - including parents and self with killpg - including all children that may have forked-off a new group """ + psutil = _get_psutil() if psutil is None: return [] diff --git a/tests/test_kernel_direct.py b/tests/test_kernel_direct.py index 146ae3ead..d8757e103 100644 --- a/tests/test_kernel_direct.py +++ b/tests/test_kernel_direct.py @@ -157,6 +157,7 @@ async def test_usage_request_without_psutil(kernel, monkeypatch): import ipykernel.kernelbase as kernelbase monkeypatch.setattr(kernelbase, "psutil", None) + monkeypatch.setattr(kernelbase, "_psutil_import_attempted", True) reply = await kernel.test_control_message("usage_request", {}) content = reply["content"] @@ -173,6 +174,7 @@ async def test_child_process_fallbacks_without_psutil(kernel, monkeypatch): import ipykernel.kernelbase as kernelbase monkeypatch.setattr(kernelbase, "psutil", None) + monkeypatch.setattr(kernelbase, "_psutil_import_attempted", True) assert kernel._process_children() == [] kernel._signal_children(signal.SIGTERM)