diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 374af16b2b6f98b..b8bf09a57d2ba84 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -26,6 +26,14 @@ Comparisons are done using a matching algorithm implemented in of any type, not just text, so long as the sequence elements are :term:`hashable`. +.. versionchanged:: 3.16 + Exposed *autojunk* parameter of :class:`SequenceMatcher` in public functions + and classes of this module (:class:`Differ`, :class:`HtmlDiff`, :func:`ndiff`, + :func:`unified_diff`, :func:`context_diff`). For backward compatibility + this parameter is set everywhere to be ``True`` by default. + + See :gh:`118150` for motivation and reasons. + .. _difflib-junk: @@ -161,6 +169,8 @@ Diff generation printed as-is via the :meth:`~io.IOBase.writelines` method of a file-like object. + + .. class:: HtmlDiff This class can be used to create an HTML table (or a complete HTML file @@ -176,7 +186,7 @@ Diff generation The constructor for this class is: - .. method:: __init__(tabsize=8, wrapcolumn=None, linejunk=None, charjunk=IS_CHARACTER_JUNK) + .. method:: __init__(tabsize=8, wrapcolumn=None, linejunk=None, charjunk=IS_CHARACTER_JUNK, *, autojunk=True): Initializes instance of :class:`HtmlDiff`. @@ -187,8 +197,13 @@ Diff generation broken and wrapped, defaults to ``None`` where lines are not wrapped. *linejunk* and *charjunk* are optional keyword arguments passed into :func:`ndiff` - (used by :class:`HtmlDiff` to generate the side by side HTML differences). See - :func:`ndiff` documentation for argument default values and descriptions. + (used by :class:`HtmlDiff` to generate the side by side HTML differences). + See :func:`ndiff` documentation for argument default values and descriptions. + + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + + *autojunk* flag is for setting on/off automatic junk heuristic of :class:`SequenceMatcher`. The following methods are public: @@ -231,7 +246,7 @@ Diff generation -.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, autojunk=True) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in context diff format. @@ -249,6 +264,11 @@ Diff generation For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + + Optional *autojunk* flag sets on/off automatic junk heuristic of :class:`SequenceMatcher`. + The context diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally @@ -278,7 +298,7 @@ Diff generation See :ref:`difflib-interface` for a more detailed example. -.. function:: get_close_matches(word, possibilities, n=3, cutoff=0.6) +.. function:: get_close_matches(word, possibilities, n=3, cutoff=0.6, *, autojunk=True) Return a list of the best "good enough" matches. *word* is a sequence for which close matches are desired (typically a string), and *possibilities* is a list of @@ -290,6 +310,12 @@ Diff generation Optional argument *cutoff* (default ``0.6``) is a float in the range [0, 1]. Possibilities that don't score at least that similar to *word* are ignored. + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + + Optional *autojunk* param is a flag for turning on/off + an automatic junk heuristic of :class:`SequenceMatcher`. + The best (no more than *n*) matches among the possibilities are returned in a list, sorted by similarity score, most similar first. @@ -304,7 +330,7 @@ Diff generation ['except'] -.. function:: ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK) +.. function:: ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK, *, autojunk=True) Compare *a* and *b* (lists of strings); return a :class:`Differ`\ -style delta (a :term:`generator` generating the delta lines). @@ -325,6 +351,14 @@ Diff generation function :func:`IS_CHARACTER_JUNK`, which filters out whitespace characters (a blank or tab; it's a bad idea to include newline in this!). + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + + *autojunk*: An optional parameter for setting on/off automatic junk heuristic + of :class:`SequenceMatcher`. + + Example: + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> print(''.join(diff), end="") @@ -362,7 +396,7 @@ Diff generation emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, autojunk=True, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -410,6 +444,12 @@ Diff generation .. versionchanged:: 3.15 Added the *color* parameter. + .. versionchanged:: 3.16 + Added keyword-only *autojunk* parameter. + + Set *autojunk* to ``False`` in order to disable automatic junk heuristic + of underlying :class:`SequenceMatcher`. + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 90ab6758a264cd8..12fcd1cedcd39d3 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -276,6 +276,16 @@ ctypes (Contributed by Peter Bierma in :gh:`153903`.) +difflib +------- + +* Expose optional ``autojunk`` parameter from :class:`difflib.SequenceMatcher` + to public functions and class methods in :mod:`difflib`, + allowing to modify behavior of automatic junk heuristic in this module + in higher public class methods and functions. + (Contributed by Tomasz Kazimierczak in :gh:`118150`) + + encodings --------- diff --git a/Lib/difflib.py b/Lib/difflib.py index 95ba8fd782c6c3c..c081cd8606df5bb 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -664,7 +664,7 @@ def real_quick_ratio(self): __class_getitem__ = classmethod(GenericAlias) -def get_close_matches(word, possibilities, n=3, cutoff=0.6): +def get_close_matches(word, possibilities, n=3, cutoff=0.6, *, autojunk=True): """Use SequenceMatcher to return list of the best "good enough" matches. word is a sequence for which close matches are desired (typically a @@ -698,7 +698,7 @@ def get_close_matches(word, possibilities, n=3, cutoff=0.6): if not 0.0 <= cutoff <= 1.0: raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) result = [] - s = SequenceMatcher() + s = SequenceMatcher(autojunk=autojunk) s.set_seq2(word) for x in possibilities: s.set_seq1(x) @@ -810,7 +810,7 @@ class Differ: + 5. Flat is better than nested. """ - def __init__(self, linejunk=None, charjunk=None): + def __init__(self, linejunk=None, charjunk=None, *, autojunk=True): """ Construct a text differencer, with optional filters. @@ -828,10 +828,13 @@ def __init__(self, linejunk=None, charjunk=None): module-level function `IS_CHARACTER_JUNK` may be used to filter out whitespace characters (a blank or tab; **note**: bad idea to include newline in this!). Use of IS_CHARACTER_JUNK is recommended. + - `autojunk`: automatic junk diff heuristic + (refer to :class:`SequenceMatcher` for specifics). """ self.linejunk = linejunk self.charjunk = charjunk + self.autojunk = autojunk def compare(self, a, b): r""" @@ -859,7 +862,7 @@ def compare(self, a, b): + emu """ - cruncher = SequenceMatcher(self.linejunk, a, b) + cruncher = SequenceMatcher(self.linejunk, a, b, autojunk=self.autojunk) for tag, alo, ahi, blo, bhi in cruncher.get_opcodes(): if tag == 'replace': g = self._fancy_replace(a, alo, ahi, b, blo, bhi) @@ -920,7 +923,7 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): # Later, more pathological cases prompted removing recursion # entirely. cutoff = 0.74999 - cruncher = SequenceMatcher(self.charjunk) + cruncher = SequenceMatcher(self.charjunk, autojunk=self.autojunk) crqr = cruncher.real_quick_ratio cqr = cruncher.quick_ratio cr = cruncher.ratio @@ -1099,7 +1102,7 @@ def _format_range_unified(start, stop): return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n', *, color=False): + tofiledate='', n=3, lineterm='\n', *, autojunk=True, color=False): r""" Compare two sequences of lines; generate the delta as a unified diff. @@ -1120,6 +1123,9 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', 'git diff --color'. Even if enabled, it can be controlled using environment variables such as 'NO_COLOR'. + Set `autojunk` to False if you don't want automated junk heuristic. + See details in :class:`SequenceMatcher. + The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. @@ -1150,7 +1156,7 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False - for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): + for group in SequenceMatcher(None, a, b, autojunk=autojunk).get_grouped_opcodes(n): if not started: started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' @@ -1193,7 +1199,7 @@ def _format_range_context(start, stop): # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', - fromfiledate='', tofiledate='', n=3, lineterm='\n'): + fromfiledate='', tofiledate='', n=3, lineterm='\n', *, autojunk=True): r""" Compare two sequences of lines; generate the delta as a context diff. @@ -1216,6 +1222,10 @@ def context_diff(a, b, fromfile='', tofile='', The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. + The kwarg `autojunk` sets up automated junk heuristic with + :class:`SequenceMatcher`, which is used under the hood in this function. + See documentation of :class:`SequenceMatcher` for details. + Example: >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), @@ -1239,7 +1249,7 @@ def context_diff(a, b, fromfile='', tofile='', _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False - for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): + for group in SequenceMatcher(None, a, b, autojunk=autojunk).get_grouped_opcodes(n): if not started: started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' @@ -1321,7 +1331,7 @@ def decode(s): for line in lines: yield line.encode('ascii', 'surrogateescape') -def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): +def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK, *, autojunk=True): r""" Compare `a` and `b` (lists of strings); return a `Differ`-style delta. @@ -1339,6 +1349,8 @@ def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): whitespace characters (a blank or tab; note: it's a bad idea to include newline in this!). + - autojunk: automatic junk heuristic - refer to :class:`SequenceMatcher` for details + Tools/scripts/ndiff.py is a command-line front-end to this function. Example: @@ -1356,10 +1368,10 @@ def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): + tree + emu """ - return Differ(linejunk, charjunk).compare(a, b) + return Differ(linejunk, charjunk, autojunk=autojunk).compare(a, b) def _mdiff(fromlines, tolines, context=None, linejunk=None, - charjunk=IS_CHARACTER_JUNK): + charjunk=IS_CHARACTER_JUNK, *, autojunk=True): r"""Returns generator yielding marked up from/to side by side differences. Arguments: @@ -1369,6 +1381,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, if None, all from/to text lines will be generated. linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) + autojunk -- passed on to ndiff (see ndiff documentation) This function returns an iterator which returns a tuple: (from line tuple, to line tuple, boolean flag) @@ -1398,7 +1411,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, change_re = re.compile(r'(\++|\-+|\^+)') # create the difference iterator to generate the differences - diff_lines_iterator = ndiff(fromlines,tolines,linejunk,charjunk) + diff_lines_iterator = ndiff(fromlines, tolines, linejunk, charjunk, autojunk=autojunk) def _make_line(lines, format_key, side, num_lines=[0,0]): """Returns line of text with user's change markup and line formatting. @@ -1738,14 +1751,14 @@ class HtmlDiff(object): _default_prefix = 0 def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, - charjunk=IS_CHARACTER_JUNK): + charjunk=IS_CHARACTER_JUNK, *, autojunk=True): """HtmlDiff instance initializer Arguments: tabsize -- tab stop spacing, defaults to 8. wrapcolumn -- column number where lines are broken and wrapped, defaults to None where lines are not wrapped. - linejunk,charjunk -- keyword arguments passed into ndiff() (used by + linejunk, charjunk, autojunk -- keyword arguments passed into ndiff() (used by HtmlDiff() to generate the side by side HTML differences). See ndiff() documentation for argument default values and descriptions. """ @@ -1753,6 +1766,7 @@ def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, self._wrapcolumn = wrapcolumn self._linejunk = linejunk self._charjunk = charjunk + self._autojunk = autojunk def make_file(self, fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5, *, charset='utf-8'): @@ -2026,7 +2040,7 @@ def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, else: context_lines = None diffs = _mdiff(fromlines,tolines,context_lines,linejunk=self._linejunk, - charjunk=self._charjunk) + charjunk=self._charjunk, autojunk=self._autojunk) # set up iterator to wrap lines that exceed desired width if self._wrapcolumn: diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 4f99b7c91c654e4..5babe834e9beac2 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -56,7 +56,7 @@ def test_bjunk(self): class TestAutojunk(unittest.TestCase): - """Tests for the autojunk parameter added in 2.7""" + """Tests for the autojunk parameter added in SequenceMatcher and higher-level difflib APIs""" def test_one_insert_homogenous_sequence(self): # By default autojunk=True and the heuristic kicks in for a sequence # of length 200+ @@ -72,6 +72,88 @@ def test_one_insert_homogenous_sequence(self): self.assertAlmostEqual(sm.ratio(), 0.9975, places=3) self.assertEqual(sm.bpopular, set()) + def test_get_close_matches(self): + word = 'a' + 'b' * 200 + possibilities = ['b' * 200] + + # By default autojunk=True, so 'b' is junk -> ratio ~ 0 -> no matches + self.assertEqual(difflib.get_close_matches(word, possibilities, cutoff=0.6), []) + self.assertEqual(difflib.get_close_matches(word, possibilities, cutoff=0.6, autojunk=True), []) + + # With autojunk=False, ratio ~ 0.9975 -> match returned + self.assertEqual(difflib.get_close_matches(word, possibilities, cutoff=0.6, autojunk=False), ['b' * 200]) + + def test_differ_and_ndiff(self): + lines1 = ["x\n"] * 200 + ["a\n", "b\n", "c\n"] + ["x\n"] * 50 + lines2 = ["a\n", "b\n", "c\n"] + ["x\n"] * 250 + + # Line-level autojunk propagation + d_true = difflib.Differ(autojunk=True) + d_false = difflib.Differ(autojunk=False) + res_true = list(d_true.compare(lines1, lines2)) + res_false = list(d_false.compare(lines1, lines2)) + self.assertNotEqual(res_true, res_false) + + ndiff_true = list(difflib.ndiff(lines1, lines2, autojunk=True)) + ndiff_false = list(difflib.ndiff(lines1, lines2, autojunk=False)) + self.assertNotEqual(ndiff_true, ndiff_false) + self.assertEqual(ndiff_true, res_true) + self.assertEqual(ndiff_false, res_false) + + # Character-level autojunk propagation in Differ (_fancy_replace) + line1 = "x" * 200 + "abc" + "x" * 50 + "\n" + line2 = "abc" + "x" * 250 + "\n" + fancy_true = list(difflib.Differ(autojunk=True).compare([line1], [line2])) + fancy_false = list(difflib.Differ(autojunk=False).compare([line1], [line2])) + self.assertNotEqual(fancy_true, fancy_false) + + def test_unified_and_context_diff(self): + lines1 = ["x\n"] * 200 + ["a\n", "b\n", "c\n"] + ["x\n"] * 50 + lines2 = ["a\n", "b\n", "c\n"] + ["x\n"] * 250 + + u_true = list(difflib.unified_diff(lines1, lines2, autojunk=True)) + u_false = list(difflib.unified_diff(lines1, lines2, autojunk=False)) + self.assertNotEqual(u_true, u_false) + + c_true = list(difflib.context_diff(lines1, lines2, autojunk=True)) + c_false = list(difflib.context_diff(lines1, lines2, autojunk=False)) + self.assertNotEqual(c_true, c_false) + + def test_htmldiff(self): + lines1 = ["x\n"] * 200 + ["a\n", "b\n", "c\n"] + ["x\n"] * 50 + lines2 = ["a\n", "b\n", "c\n"] + ["x\n"] * 250 + + old_prefix = difflib.HtmlDiff._default_prefix + try: + html_true = difflib.HtmlDiff(autojunk=True).make_file(lines1, lines2) + html_false = difflib.HtmlDiff(autojunk=False).make_file(lines1, lines2) + self.assertNotEqual(html_true, html_false) + finally: + difflib.HtmlDiff._default_prefix = old_prefix + + def test_autojunk_signatures(self): + import inspect + + funcs = [ + difflib.get_close_matches, + difflib.unified_diff, + difflib.context_diff, + difflib.ndiff, + ] + for func in funcs: + sig = inspect.signature(func) + self.assertIn('autojunk', sig.parameters) + param = sig.parameters['autojunk'] + self.assertEqual(param.default, True) + self.assertEqual(param.kind, inspect.Parameter.KEYWORD_ONLY) + + for cls in [difflib.Differ, difflib.HtmlDiff]: + sig = inspect.signature(cls.__init__) + self.assertIn('autojunk', sig.parameters) + param = sig.parameters['autojunk'] + self.assertEqual(param.default, True) + + class TestSFbugs(unittest.TestCase): def test_ratio_for_null_seqn(self): diff --git a/Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst b/Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst new file mode 100644 index 000000000000000..b479302ade4064c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-16-43-02.gh-issue-118150.x0mV8P.rst @@ -0,0 +1,5 @@ +Expose automated junk heuristic kwarg-only flag ``autojunk`` from +:class:`difflib.SequenceMatcher` to the public functions +and class methods in the :mod:`difflib`. +See :class:`difflib.SequenceMatcher` documentation for details +and issue :gh:`118150` for the motivation.