From 07a943efd19fc1c5fbca7989db5b9eea536c553d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 15:24:16 +0300 Subject: [PATCH 1/3] gh-69371: Fix pydoc for modules whose path contains undecodable bytes "python -m pydoc -w" and the pydoc HTTP server failed with UnicodeEncodeError if the path of the documented module contained bytes undecodable in the filesystem encoding. The file URL is now created with urllib.request.pathname2url(), which percent-encodes the path using the filesystem encoding, and characters unencodable in the generated HTML page (which is always UTF-8) are escaped with backslashes. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/pydoc.py | 9 ++++---- Lib/test/test_pydoc/test_pydoc.py | 23 +++++++++++++++++++ ...6-08-05-16-40-00.gh-issue-69371.pyDocU.rst | 5 ++++ 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 72974af26bee64c..1dd06a3afc291b1 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -71,7 +71,6 @@ class or function within a module or module in a package. If the import textwrap import time import tokenize -import urllib.parse import warnings from annotationlib import Format from collections import deque @@ -792,7 +791,8 @@ def docmodule(self, object, name=None, mod=None, *ignored): head = linkedname try: path = inspect.getabsfile(object) - url = urllib.parse.quote(path) + import urllib.request + url = urllib.request.pathname2url(path) filelink = self.filelink(url, path) except TypeError: filelink = '(built-in)' @@ -1787,7 +1787,8 @@ def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" object, name = resolve(thing, forceload) page = html.page(describe(object), html.document(object, name)) - with open(name + '.html', 'w', encoding='utf-8') as file: + with open(name + '.html', 'w', encoding='utf-8', + errors='backslashreplace') as file: file.write(page) print('wrote', name + '.html') @@ -2384,7 +2385,7 @@ def do_GET(self): self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.end_headers() self.wfile.write(self.urlhandler( - self.path, content_type).encode('utf-8')) + self.path, content_type).encode('utf-8', 'backslashreplace')) def log_message(self, *args): # Don't log messages. diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 34d30f54e9c9c7f..8db0240b17cc076 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -19,6 +19,7 @@ import unittest import unittest.mock import urllib.parse +import urllib.request import xml.etree import xml.etree.ElementTree import textwrap @@ -1024,6 +1025,28 @@ def test_synopsis_sourceless_empty_doc(self): synopsis_cached = pydoc.synopsis(cached_path, {}) self.assertIsNone(synopsis_cached) + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'requires undecodable file names') + def test_html_doc_undecodable_path(self): + # gh-69371: the path of the module is not encodable in UTF-8. + with os_helper.temp_cwd() as test_dir: + subdir = os.path.join(os.fsencode(test_dir), + os_helper.TESTFN_UNDECODABLE) + os.mkdir(subdir) + with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f: + f.write('"""Module docstring."""\n') + with import_helper.DirsOnSysPath(os.fsdecode(subdir)): + mod = import_helper.import_fresh_module('undecodable_mod') + doc = pydoc.HTMLDoc().docmodule(mod) + with captured_stdout(): + pydoc.writedoc(mod) + # The link contains the percent-encoded path... + path = os.fsdecode(os.path.join(subdir, b'undecodable_mod.py')) + self.assertIn(urllib.request.pathname2url(path), doc) + # ...and the page can be written and served as UTF-8. + with open('undecodable_mod.html', encoding='utf-8') as f: + self.assertIn('undecodable_mod', f.read()) + def test_splitdoc_with_description(self): example_string = "I Am A Doc\n\n\nHere is my description" self.assertEqual(pydoc.splitdoc(example_string), diff --git a/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst b/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst new file mode 100644 index 000000000000000..4d8f866366f5beb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst @@ -0,0 +1,5 @@ +Fix :mod:`pydoc` for modules whose path contains undecodable bytes. +:func:`!pydoc.writedoc` and the pydoc HTTP server no longer fail with +:exc:`UnicodeEncodeError`: the file URL is now percent-encoded using the +filesystem encoding, and characters unencodable in the generated HTML page +are escaped with backslashes. From 5ee9f1df36e477d3ae11b4c11ecc8bfe28099f5a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 18:31:50 +0300 Subject: [PATCH 2/3] Skip the test if undecodable paths are not supported The file system can reject a name with undecodable bytes even if TESTFN_UNDECODABLE is not None (e.g. on macOS). Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_pydoc/test_pydoc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 8db0240b17cc076..986f7aea3596f41 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1032,7 +1032,10 @@ def test_html_doc_undecodable_path(self): with os_helper.temp_cwd() as test_dir: subdir = os.path.join(os.fsencode(test_dir), os_helper.TESTFN_UNDECODABLE) - os.mkdir(subdir) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f: f.write('"""Module docstring."""\n') with import_helper.DirsOnSysPath(os.fsdecode(subdir)): From 021ca086440519bc8d2f3d7946f3e5c81d0dc5b9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 19:17:06 +0300 Subject: [PATCH 3/3] Fix the tests on Windows test_html_doc contained the URL in the old format, in which the drive letter and backslashes of a Windows path were percent-encoded. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_pydoc/test_pydoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 986f7aea3596f41..006fbc8b6968f3b 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -430,7 +430,7 @@ def test_html_doc(self): expected_lines = [line.strip() for line in expected_lines if line] self.assertEqual(text_lines, expected_lines) mod_file = inspect.getabsfile(pydoc_mod) - mod_url = urllib.parse.quote(mod_file) + mod_url = urllib.request.pathname2url(mod_file) self.assertIn(mod_url, result) self.assertIn(mod_file, result) self.assertIn(doc_loc, result) @@ -1044,7 +1044,7 @@ def test_html_doc_undecodable_path(self): with captured_stdout(): pydoc.writedoc(mod) # The link contains the percent-encoded path... - path = os.fsdecode(os.path.join(subdir, b'undecodable_mod.py')) + path = inspect.getabsfile(mod) self.assertIn(urllib.request.pathname2url(path), doc) # ...and the page can be written and served as UTF-8. with open('undecodable_mod.html', encoding='utf-8') as f: