From 922253329befd95afb122a264c699d2fcd69c477 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Aug 2026 22:24:14 +0300 Subject: [PATCH 1/4] gh-64502: Fix Argument Clinic support of optional groups with defaults Parameters with a default value which are not in any group were always required in the generated argument parsing code, although they were rendered as optional in the signature. They can now be omitted, and ambiguous combinations of optional groups and parameters with a default value are rejected. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/clinic.test.c | 61 ++++++++++++ Lib/test/test_clinic.py | 39 ++++++++ ...6-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst | 3 + Modules/_testclinic.c | 48 ++++++++++ Modules/clinic/_testclinic.c.h | 92 ++++++++++++++++++- Tools/clinic/libclinic/clanguage.py | 39 ++++++-- 6 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 171570588e7a2b5..870e4f6956ce4fd 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5769,6 +5769,67 @@ Test___init___impl(TestObj *self, PyObject *a, int group_right_1, /*[clinic end generated code: output=2bbb8ea60e8f57a6 input=10f5d0f1e8e466ef]*/ +/*[clinic input] +group_and_optional_parameter + [ + a: object + b: object + ] + c: object = None + / +The optional parameter can be omitted with or without the group. +[clinic start generated code]*/ + +PyDoc_STRVAR(group_and_optional_parameter__doc__, +"group_and_optional_parameter([a, b,] c=None)\n" +"The optional parameter can be omitted with or without the group."); + +#define GROUP_AND_OPTIONAL_PARAMETER_METHODDEF \ + {"group_and_optional_parameter", (PyCFunction)group_and_optional_parameter, METH_VARARGS, group_and_optional_parameter__doc__}, + +static PyObject * +group_and_optional_parameter_impl(PyObject *module, int group_left_1, + PyObject *a, PyObject *b, PyObject *c); + +static PyObject * +group_and_optional_parameter(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + PyObject *c = Py_None; + + switch (PyTuple_GET_SIZE(args)) { + case 0: + case 1: + if (!PyArg_ParseTuple(args, "|O:group_and_optional_parameter", &c)) { + goto exit; + } + break; + case 2: + case 3: + if (!PyArg_ParseTuple(args, "OO|O:group_and_optional_parameter", &a, &b, &c)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "group_and_optional_parameter requires 0 to 3 arguments"); + goto exit; + } + return_value = group_and_optional_parameter_impl(module, group_left_1, a, b, c); + +exit: + return return_value; +} + +static PyObject * +group_and_optional_parameter_impl(PyObject *module, int group_left_1, + PyObject *a, PyObject *b, PyObject *c) +/*[clinic end generated code: output=3faea69eafd5bbbe input=7f0fbb6124f5a972]*/ + + /*[clinic input] Test._pyarg_parsestackandkeywords cls: defining_class diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 93c284e58764f46..a9b3273ae2d7e13 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -330,6 +330,24 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) + def test_ambiguous_group_and_optional_parameters(self): + err = ("Function 'my_test_func' has an ambiguous group configuration: " + "a call with 2 argument(s) can be parsed in more than one way.") + block = """ + /*[clinic input] + my_test_func + + [ + a: object + b: object + ] + c: object = None + d: object = None + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err) + def test_star_after_vararg(self): err = "'my_test_func' uses '*' more than once." block = """ @@ -3865,6 +3883,27 @@ def test_varpos_kwonly_req_opt(self): self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, False)) self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) + def test_group_and_opt(self): + # fn([a, b,] c=None) + fn = ac_tester.group_and_opt + self.assertEqual(fn(), (False, None, None, None)) + self.assertEqual(fn(1), (False, None, None, 1)) + self.assertEqual(fn(1, 2), (True, 1, 2, None)) + self.assertEqual(fn(1, 2, 3), (True, 1, 2, 3)) + self.assertRaises(TypeError, fn, 1, 2, 3, 4) + self.assertRaises(TypeError, fn, c=1) + + def test_group_and_two_opt(self): + # fn([a, b, c,] d=None, e=None) + fn = ac_tester.group_and_two_opt + self.assertEqual(fn(), (False, None, None, None, None, None)) + self.assertEqual(fn(1), (False, None, None, None, 1, None)) + self.assertEqual(fn(1, 2), (False, None, None, None, 1, 2)) + self.assertEqual(fn(1, 2, 3), (True, 1, 2, 3, None, None)) + self.assertEqual(fn(1, 2, 3, 4), (True, 1, 2, 3, 4, None)) + self.assertEqual(fn(1, 2, 3, 4, 5), (True, 1, 2, 3, 4, 5)) + self.assertRaises(TypeError, fn, 1, 2, 3, 4, 5, 6) + def test_gh_32092_oob(self): ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst new file mode 100644 index 000000000000000..da9647d1fdd369b --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst @@ -0,0 +1,3 @@ +Fix Argument Clinic support of parameters with a default value used together +with optional groups. +Such parameters were always required in the generated parsing code. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 66a375589ba38e5..5742dd65f6742a7 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1237,6 +1237,52 @@ posonly_poskw_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, } +/*[clinic input] +group_and_opt + + [ + a: object + b: object + ] + c: object = None + / + +[clinic start generated code]*/ + +static PyObject * +group_and_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=23413ec545526111 input=8a84d8f44bc8bd0b]*/ +{ + return pack_arguments_newref(4, group_left_1 ? Py_True : Py_False, + a, b, c); +} + + +/*[clinic input] +group_and_two_opt + + [ + a: object + b: object + c: object + ] + d: object = None + e: object = None + / + +[clinic start generated code]*/ + +static PyObject * +group_and_two_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=1427c4b3c35f24ff input=cdda98eec1e365ea]*/ +{ + return pack_arguments_newref(6, group_left_1 ? Py_True : Py_False, + a, b, c, d, e); +} + + /*[clinic input] gh_32092_oob @@ -2455,6 +2501,8 @@ static PyMethodDef tester_methods[] = { POSONLY_VARPOS_ARRAY_METHODDEF POSONLY_REQ_OPT_VARPOS_ARRAY_METHODDEF POSONLY_POSKW_VARPOS_ARRAY_METHODDEF + GROUP_AND_OPT_METHODDEF + GROUP_AND_TWO_OPT_METHODDEF GH_32092_OOB_METHODDEF GH_32092_KW_PASS_METHODDEF diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 05615c1fdd81b9c..19b215b0cca8899 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -3477,6 +3477,96 @@ posonly_poskw_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t n return return_value; } +PyDoc_STRVAR(group_and_opt__doc__, +"group_and_opt([a, b,] c=None)"); + +#define GROUP_AND_OPT_METHODDEF \ + {"group_and_opt", (PyCFunction)group_and_opt, METH_VARARGS, group_and_opt__doc__}, + +static PyObject * +group_and_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c); + +static PyObject * +group_and_opt(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + PyObject *c = Py_None; + + switch (PyTuple_GET_SIZE(args)) { + case 0: + case 1: + if (!PyArg_ParseTuple(args, "|O:group_and_opt", &c)) { + goto exit; + } + break; + case 2: + case 3: + if (!PyArg_ParseTuple(args, "OO|O:group_and_opt", &a, &b, &c)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "group_and_opt requires 0 to 3 arguments"); + goto exit; + } + return_value = group_and_opt_impl(module, group_left_1, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(group_and_two_opt__doc__, +"group_and_two_opt([a, b, c,] d=None, e=None)"); + +#define GROUP_AND_TWO_OPT_METHODDEF \ + {"group_and_two_opt", (PyCFunction)group_and_two_opt, METH_VARARGS, group_and_two_opt__doc__}, + +static PyObject * +group_and_two_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +group_and_two_opt(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + PyObject *c = NULL; + PyObject *d = Py_None; + PyObject *e = Py_None; + + switch (PyTuple_GET_SIZE(args)) { + case 0: + case 1: + case 2: + if (!PyArg_ParseTuple(args, "|OO:group_and_two_opt", &d, &e)) { + goto exit; + } + break; + case 3: + case 4: + case 5: + if (!PyArg_ParseTuple(args, "OOO|OO:group_and_two_opt", &a, &b, &c, &d, &e)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "group_and_two_opt requires 0 to 5 arguments"); + goto exit; + } + return_value = group_and_two_opt_impl(module, group_left_1, a, b, c, d, e); + +exit: + return return_value; +} + PyDoc_STRVAR(gh_32092_oob__doc__, "gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" "--\n" @@ -4600,4 +4690,4 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=9971dbbc5f62b8d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f6a3b617130c4e3a input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 7f02c7790f015aa..ed77957dbca28bd 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -12,7 +12,7 @@ from libclinic.codegen import CRenderData, TemplateDict, CodeGen from libclinic.language import Language from libclinic.function import ( - Module, Class, Function, Parameter, + Module, Class, Function, Parameter, ParamTuple, permute_optional_groups, GETTER, SETTER, METHOD_INIT) from libclinic.converters import self_converter @@ -304,14 +304,34 @@ def render_option_group_parsing( count_min = sys.maxsize count_max = -1 + # Trailing parameters with a default value which are not in any group + # can be omitted, so a subset matches a range of argument counts. + subsets: list[tuple[ParamTuple, int]] = [] + for subset in permute_optional_groups(left, required, right): + first_optional = len(subset) + for p in reversed(subset): + if p.group or not p.is_optional(): + break + first_optional -= 1 + subsets.append((subset, first_optional)) + + seen: set[int] = set() + for subset, first_optional in subsets: + for count in range(first_optional, len(subset) + 1): + if count in seen: + fail(f"Function {f.full_name!r} has an ambiguous group " + f"configuration: a call with {count} argument(s) " + f"can be parsed in more than one way.") + seen.add(count) + if limited_capi: nargs = 'PyTuple_Size(args)' else: nargs = 'PyTuple_GET_SIZE(args)' out.append(f"switch ({nargs}) {{\n") - for subset in permute_optional_groups(left, required, right): + for subset, first_optional in subsets: count = len(subset) - count_min = min(count_min, count) + count_min = min(count_min, first_optional) count_max = max(count_max, count) if count == 0: @@ -322,9 +342,11 @@ def render_option_group_parsing( group_ids = {p.group for p in subset} # eliminate duplicates d: dict[str, str | int] = {} - d['count'] = count d['name'] = f.name - d['format_units'] = "".join(p.converter.format_unit for p in subset) + format_units = [p.converter.format_unit for p in subset] + if first_optional < count: + format_units.insert(first_optional, '|') + d['format_units'] = "".join(format_units) parse_arguments: list[str] = [] for p in subset: @@ -337,8 +359,13 @@ def render_option_group_parsing( for g in group_ids ]) + d['cases'] = "\n".join([ + f" case {n}:" + for n in range(first_optional, count + 1) + ]) + s = """\ - case {count}: +{cases} if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ goto exit; }} From 6baeaeff9b275819f614b94401fd1b8bc77711dd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Aug 2026 23:45:01 +0300 Subject: [PATCH 2/4] gh-64502: Increase the c-analyzer size limit for Modules/_testclinic.c The added test functions made the preprocessed method table exceed the default limit of the C globals checker. Co-Authored-By: Claude Opus 5 (1M context) --- Tools/c-analyzer/cpython/_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 2875f45cb8d3756..3d755765b967097 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -318,6 +318,7 @@ def format_tsv_lines(lines): _abs('Modules/_remote_debugging/debug_offsets_validation.h'): (25_000, 1000), _abs('Modules/_remote_debugging/*.h'): (20_000, 1000), _abs('Modules/_testcapimodule.c'): (20_000, 400), + _abs('Modules/_testclinic.c'): (20_000, 400), _abs('Modules/expat/expat.h'): (10_000, 400), _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (380_000, 13_000), From b4c348712d06a3648c420df14bc727a898d58373 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 08:32:21 +0300 Subject: [PATCH 3/4] gh-64502: Map argument counts to option group subsets Use a single dict which maps the number of arguments to the subset of parameters which accepts it, instead of a list of subsets and a set of already used counts. Co-Authored-By: Claude Opus 5 (1M context) --- Tools/clinic/libclinic/clanguage.py | 65 +++++++++++++++-------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index ed77957dbca28bd..23ba65018ca925f 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -1,6 +1,5 @@ from __future__ import annotations import itertools -import sys import textwrap from typing import TYPE_CHECKING, Literal, Final from operator import attrgetter @@ -21,6 +20,20 @@ from libclinic.app import Clinic +def count_required(subset: ParamTuple) -> int: + """Return the number of arguments which cannot be omitted. + + Trailing parameters with a default value which are not in an optional + group can be omitted. + """ + count = len(subset) + for p in reversed(subset): + if p.group or not p.is_optional(): + break + count -= 1 + return count + + def c_id(name: str) -> str: if len(name) == 1 and ord(name) < 256: if name.isalnum(): @@ -301,38 +314,29 @@ def render_option_group_parsing( assert group is not None group.append(p) - count_min = sys.maxsize - count_max = -1 - - # Trailing parameters with a default value which are not in any group - # can be omitted, so a subset matches a range of argument counts. - subsets: list[tuple[ParamTuple, int]] = [] + # Map the number of arguments to the subset which accepts them. + # A subset accepts a range of counts, because its trailing parameters + # with a default value which are not in any group can be omitted. + subsets: dict[int, ParamTuple] = {} for subset in permute_optional_groups(left, required, right): - first_optional = len(subset) - for p in reversed(subset): - if p.group or not p.is_optional(): - break - first_optional -= 1 - subsets.append((subset, first_optional)) - - seen: set[int] = set() - for subset, first_optional in subsets: - for count in range(first_optional, len(subset) + 1): - if count in seen: + for count in range(count_required(subset), len(subset) + 1): + if count in subsets: fail(f"Function {f.full_name!r} has an ambiguous group " f"configuration: a call with {count} argument(s) " f"can be parsed in more than one way.") - seen.add(count) + subsets[count] = subset if limited_capi: nargs = 'PyTuple_Size(args)' else: nargs = 'PyTuple_GET_SIZE(args)' out.append(f"switch ({nargs}) {{\n") - for subset, first_optional in subsets: - count = len(subset) - count_min = min(count_min, first_optional) - count_max = max(count_max, count) + for count, subset in sorted(subsets.items()): + if count < len(subset): + # Some of the trailing parameters are omitted; + # they are parsed together with the following case. + out.append(f" case {count}:\n") + continue if count == 0: out.append(""" case 0: @@ -342,10 +346,12 @@ def render_option_group_parsing( group_ids = {p.group for p in subset} # eliminate duplicates d: dict[str, str | int] = {} + d['count'] = count d['name'] = f.name format_units = [p.converter.format_unit for p in subset] - if first_optional < count: - format_units.insert(first_optional, '|') + n_required = count_required(subset) + if n_required < count: + format_units.insert(n_required, '|') d['format_units'] = "".join(format_units) parse_arguments: list[str] = [] @@ -359,13 +365,8 @@ def render_option_group_parsing( for g in group_ids ]) - d['cases'] = "\n".join([ - f" case {n}:" - for n in range(first_optional, count + 1) - ]) - s = """\ -{cases} + case {count}: if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ goto exit; }} @@ -378,7 +379,7 @@ def render_option_group_parsing( out.append(" default:\n") s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - out.append(s.format(f.full_name, count_min, count_max)) + out.append(s.format(f.full_name, min(subsets), max(subsets))) out.append(' goto exit;\n') out.append("}") From 1667e85d386d38128944a72b314598e186e31d00 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 08:38:54 +0300 Subject: [PATCH 4/4] gh-64502: Simplify comments in the option group parsing code Co-Authored-By: Claude Opus 5 (1M context) --- Tools/clinic/libclinic/clanguage.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 23ba65018ca925f..a76fddb7602001e 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -23,8 +23,8 @@ def count_required(subset: ParamTuple) -> int: """Return the number of arguments which cannot be omitted. - Trailing parameters with a default value which are not in an optional - group can be omitted. + A parameter in an optional group is passed together with its group, + so only trailing parameters with a default value can be omitted. """ count = len(subset) for p in reversed(subset): @@ -314,9 +314,7 @@ def render_option_group_parsing( assert group is not None group.append(p) - # Map the number of arguments to the subset which accepts them. - # A subset accepts a range of counts, because its trailing parameters - # with a default value which are not in any group can be omitted. + # Map the number of arguments to the subset which accepts it. subsets: dict[int, ParamTuple] = {} for subset in permute_optional_groups(left, required, right): for count in range(count_required(subset), len(subset) + 1): @@ -333,8 +331,7 @@ def render_option_group_parsing( out.append(f"switch ({nargs}) {{\n") for count, subset in sorted(subsets.items()): if count < len(subset): - # Some of the trailing parameters are omitted; - # they are parsed together with the following case. + # The omitted parameters are parsed by the following case. out.append(f" case {count}:\n") continue