From 6b5fb00c18e2b99f57804bdde7274fe0cb0f419d Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Tue, 18 Aug 2026 09:07:58 +0200 Subject: [PATCH] Security audit of the Python 3.15 changes Refs #306 - Disallow lazy import statements (PEP 810). - Disallow unpacking in comprehensions (PEP 798). - Block the attributes of async generators in INSPECT_ATTRIBUTES. --- CHANGES.rst | 13 ++ docs/conf.py | 1 + docs/contributing/ast/python3_15.ast | 196 ++++++++++++++++++ docs/contributing/changes_from314to315.rst | 48 +++++ docs/contributing/index.rst | 8 +- docs/index.rst | 2 +- pyproject.toml | 1 + src/RestrictedPython/_compat.py | 1 + src/RestrictedPython/transformer.py | 40 +++- .../test_comprehension_unpacking.py | 66 ++++++ tests/transformer/test_inspect.py | 17 ++ tests/transformer/test_lazy_import.py | 27 +++ 12 files changed, 414 insertions(+), 6 deletions(-) create mode 100644 docs/contributing/ast/python3_15.ast create mode 100644 docs/contributing/changes_from314to315.rst create mode 100644 tests/transformer/test_comprehension_unpacking.py create mode 100644 tests/transformer/test_lazy_import.py diff --git a/CHANGES.rst b/CHANGES.rst index c05a3bdf..590450cf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,19 @@ Changes 8.5 (unreleased) ---------------- +- Officially support Python 3.15 after performing a security audit of its + changes: + + - Disallow lazy import statements (PEP 810) as they bypass a guarded + ``__import__``. + + - Disallow unpacking in comprehensions (PEP 798) as it bypasses the + ``_getiter_`` guard. + +- Add the attributes of asynchronous generator objects (``ag_await``, + ``ag_frame``, ``ag_code``) to the restricted names in + ``INSPECT_ATTRIBUTES`` as they were missing there. + - Fix the combined coverage report: the ``coverage`` tox environment now combines the coverage data of all supported Python versions instead of measuring a single one, and enforces 100 % coverage. The broken diff --git a/docs/conf.py b/docs/conf.py index bb0d2ef6..8b06e17c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -117,6 +117,7 @@ 'python312': ('https://docs.python.org/3.12', None), 'python313': ('https://docs.python.org/3.13', None), 'python314': ('https://docs.python.org/3.14', None), + 'python315': ('https://docs.python.org/3.15', None), } # Options for sphinx.ext.todo: diff --git a/docs/contributing/ast/python3_15.ast b/docs/contributing/ast/python3_15.ast new file mode 100644 index 00000000..5894ea5f --- /dev/null +++ b/docs/contributing/ast/python3_15.ast @@ -0,0 +1,196 @@ +-- Python 3.15 AST +-- ASDL's 4 builtin types are: +-- identifier, int, string, constant + +module Python version "3.15" +{ + mod = Module(stmt* body, type_ignore* type_ignores) + | Interactive(stmt* body) + | Expression(expr body) + | FunctionType(expr* argtypes, expr returns) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns, + string? type_comment, + type_param* type_params) + | AsyncFunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns, + string? type_comment, + type_param* type_params) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list, + type_param* type_params) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value, string? type_comment) + | TypeAlias(expr name, type_param* type_params, expr value) + | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body, string? type_comment) + | AsyncWith(withitem* items, stmt* body, string? type_comment) + + | Match(expr subject, match_case* cases) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names, int? is_lazy) + | ImportFrom(identifier? module, alias* names, int? level, int? is_lazy) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | NamedExpr(expr target, expr value) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr?* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr? value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Await(expr value) + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords) + | FormattedValue(expr value, int conversion, expr? format_spec) + | Interpolation(expr value, constant str, int conversion, expr? format_spec) + | JoinedStr(expr* values) + | TemplateStr(expr* values) + | Constant(constant value, string? kind) + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, expr slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- can appear only in Subscript + | Slice(expr? lower, expr? upper, expr? step) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + expr_context = Load + | Store + | Del + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs, int is_async) + + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + arguments = (arg* posonlyargs, + arg* args, + arg? vararg, + arg* kwonlyargs, + expr* kw_defaults, + arg? kwarg, + expr* defaults) + + arg = (identifier arg, expr? annotation, string? type_comment) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- keyword arguments supplied to call (NULL identifier for **kwargs) + keyword = (identifier? arg, expr value) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + withitem = (expr context_expr, expr? optional_vars) + + match_case = (pattern pattern, expr? guard, stmt* body) + + pattern = MatchValue(expr value) + | MatchSingleton(constant value) + | MatchSequence(pattern* patterns) + | MatchMapping(expr* keys, pattern* patterns, identifier? rest) + | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) + + | MatchStar(identifier? name) + -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys + + | MatchAs(pattern? pattern, identifier? name) + | MatchOr(pattern* patterns) + + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) + + type_ignore = TypeIgnore(int lineno, string tag) + + type_param = TypeVar(identifier name, expr? bound, expr? default_value) + | ParamSpec(identifier name, expr? default_value) + | TypeVarTuple(identifier name, expr? default_value) + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) +} diff --git a/docs/contributing/changes_from314to315.rst b/docs/contributing/changes_from314to315.rst new file mode 100644 index 00000000..471ae18d --- /dev/null +++ b/docs/contributing/changes_from314to315.rst @@ -0,0 +1,48 @@ +Changes from Python 3.14 to Python 3.15 +--------------------------------------- + +.. literalinclude:: ast/python3_15.ast + :diff: ast/python3_14.ast + +Security audit of the Python 3.15 changes ++++++++++++++++++++++++++++++++++++++++++ + +Lazy imports (:pep:`810`) + ``lazy import`` statements do not introduce a new AST node. + They only add a new field ``is_lazy`` to the existing ``Import`` and + ``ImportFrom`` nodes, so the default-deny mechanism of + ``RestrictingNodeTransformer.generic_visit`` does **not** apply to them. + At run time a lazy import is resolved through the new ``__lazy_import__`` + builtin instead of ``__import__``, thus bypassing a guarded + ``__import__``. + Therefore lazy imports are explicitly not allowed. + Assigning ``__lazy_modules__`` was already blocked by the rule denying + names which start with an underscore. + +Unpacking in comprehensions (:pep:`798`) + ``[*x for x in seq]``, ``{*x for x in seq}``, ``(*x for x in seq)`` and + ``{**x for x in seq}`` reuse the existing ``Starred`` node (resp. a + ``DictComp`` node without a value), so they compiled silently. + The unpacked value is iterated by the bytecode without calling the + ``_getiter_`` guard — unlike the equivalent nested comprehension + ``[y for x in seq for y in x]``. + Therefore unpacking in comprehensions is explicitly not allowed. + +Unary ``+`` in ``match`` literal patterns + No action needed as the ``match`` statement is not allowed in + RestrictedPython. + +New builtins ``frozendict`` (:pep:`814`) and ``sentinel`` (:pep:`661`) + No action needed as ``safe_builtins`` is an allow list, so the new + builtins are not available in restricted code. + +New ``inspect`` attributes ``gi_state``, ``cr_state`` and ``ag_state`` + They only reveal the state of a (async) generator resp. coroutine as a + string, so they are treated like the other harmless attributes + (e. g. ``gi_running``) and remain accessible. + Reviewing ``INSPECT_ATTRIBUTES`` also revealed that the attributes of + asynchronous generator objects (``ag_await``, ``ag_frame``, ``ag_code``) + were missing from the list; they are now blocked. + +Removed ``ast`` classes (``ast.Num``, ``ast.Str``, ``ast.Bytes``, ``ast.NameConstant``, ``ast.Ellipsis``) + No action needed as they are no longer used by RestrictedPython. diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 67b4f84c..9559ae76 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -67,7 +67,7 @@ To do so: * Add a corresponding changelog entry. * Additionally modify ``.meta.toml`` and run the ``meta/config`` script (for details see: https://github.com/mgedmin/check-python-versions) to update the following files: - * ``/setup.py`` - Check that the new Python version classifier has been added ``"Programming Language :: Python :: ",``, and that the ``python_requires`` section has been updated correctly. + * ``/pyproject.toml`` - Check that the new Python version classifier has been added ``"Programming Language :: Python :: ",``, and that the ``requires-python`` value has been updated correctly. * ``/tox.ini`` - Check that a ``testenv`` entry is added to the general ``envlist`` statement. * ``/.github/workflows/tests.yml`` - Check that a corresponding Python version entry has been added to the matrix definition. * ``/docs/conf.py`` - Add the Python version to the ``intersphinx_mapping`` list. @@ -103,6 +103,7 @@ A (modified style) Copy of all Abstract Grammar Definitions for the Python versi changes_from311to312 changes_from312to313 changes_from313to314 + changes_from314to315 .. _understand: @@ -235,6 +236,7 @@ Technical Backgrounds - Links to External Documentation * AST Grammar of Python (`Status of Python Versions`_) + * `Python 3.15 AST`_ (EOL 2031-10) * `Python 3.14 AST`_ (EOL 2030-10) * `Python 3.13 AST`_ (EOL 2029-10) * `Python 3.12 AST`_ (EOL 2028-10) @@ -257,6 +259,8 @@ Todos .. _`What's new in Python`: https://docs.python.org/3/whatsnew/ +.. _`What's new in Python 3.15`: https://docs.python.org/3.15/whatsnew/3.15.html + .. _`What's new in Python 3.14`: https://docs.python.org/3.14/whatsnew/3.14.html .. _`What's new in Python 3.13`: https://docs.python.org/3.13/whatsnew/3.13.html @@ -281,6 +285,8 @@ Todos .. _`Python 3 AST`: https://docs.python.org/3/library/ast.html#abstract-grammar +.. _`Python 3.15 AST`: https://docs.python.org/3.15/library/ast.html#abstract-grammar + .. _`Python 3.14 AST`: https://docs.python.org/3.14/library/ast.html#abstract-grammar .. _`Python 3.13 AST`: https://docs.python.org/3.13/library/ast.html#abstract-grammar diff --git a/docs/index.rst b/docs/index.rst index 1ce9733a..49ba7154 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ RestrictedPython is not a sandbox system or a secured environment, but it helps Supported Python versions ========================= -RestrictedPython supports CPython 3.10 up to 3.14. +RestrictedPython supports CPython 3.10 up to 3.15. It does _not_ support PyPy or other alternative Python implementations. Contents diff --git a/pyproject.toml b/pyproject.toml index a37a3522..4fc96745 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Typing :: Typed", diff --git a/src/RestrictedPython/_compat.py b/src/RestrictedPython/_compat.py index 63c7fa1e..5ae4b847 100644 --- a/src/RestrictedPython/_compat.py +++ b/src/RestrictedPython/_compat.py @@ -7,5 +7,6 @@ IS_PY312_OR_GREATER = _version.major == 3 and _version.minor >= 12 IS_PY313_OR_GREATER = _version.major == 3 and _version.minor >= 13 IS_PY314_OR_GREATER = _version.major == 3 and _version.minor >= 14 +IS_PY315_OR_GREATER = _version.major == 3 and _version.minor >= 15 IS_CPYTHON = platform.python_implementation() == 'CPython' diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 2f7477d2..e129b631 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -105,14 +105,22 @@ "gi_frame", # "gi_running", # bool # "gi_suspended", # bool + # "gi_state", # str "gi_code", "gi_yieldfrom", # on coroutine objects: "cr_await", "cr_frame", # "cr_running", # bool + # "cr_state", # str "cr_code", "cr_origin", + # on asynchronous generator objects: + "ag_await", + "ag_frame", + # "ag_running", # bool + # "ag_state", # str + "ag_code", ]) _T_visit_return: typing.TypeAlias = ast.AST | typing.Iterable[ast.AST] | None @@ -439,6 +447,11 @@ def check_import_names(self, node: ast.ImportFrom | ast.Import) -> ast.AST: => 'from _a import x' is ok, because '_a' is not added to the scope. """ + if getattr(node, 'is_lazy', 0): + # `lazy import` (Python 3.15+) resolves through the + # `__lazy_import__` builtin at first use, thus bypassing a guarded + # `__import__`. + self.error(node, 'Lazy import statements are not allowed.') for name in node.names: if '*' in name.name: self.error(node, '"*" imports are not allowed.') @@ -950,27 +963,46 @@ def visit_Slice(self, node: ast.Slice) -> _T_visit_return: # Comprehensions def visit_ListComp(self, node: ast.ListComp) -> _T_visit_return: - """ + """Allow list comprehensions except unpacking (Python 3.15+). + Unpacking iterates the starred value without calling `_getiter_`, + unlike the equivalent nested comprehension. """ + if isinstance(node.elt, ast.Starred): + self.error(node, 'Unpacking in comprehensions is not allowed.') return self.node_contents_visit(node) def visit_SetComp(self, node: ast.SetComp) -> _T_visit_return: - """ + """Allow set comprehensions except unpacking (Python 3.15+). + Unpacking iterates the starred value without calling `_getiter_`, + unlike the equivalent nested comprehension. """ + if isinstance(node.elt, ast.Starred): + self.error(node, 'Unpacking in comprehensions is not allowed.') return self.node_contents_visit(node) def visit_GeneratorExp(self, node: ast.GeneratorExp) -> _T_visit_return: - """ + """Allow generator expressions except unpacking (Python 3.15+). + Unpacking iterates the starred value without calling `_getiter_`, + unlike the equivalent nested comprehension. """ + if isinstance(node.elt, ast.Starred): + self.error(node, 'Unpacking in comprehensions is not allowed.') return self.node_contents_visit(node) def visit_DictComp(self, node: ast.DictComp) -> _T_visit_return: - """ + """Allow dict comprehensions except unpacking (Python 3.15+). + Unpacking iterates the doubly-starred mapping without calling + `_getiter_`, unlike the equivalent nested comprehension. """ + # Since Python 3.15 `value` can be `None`, but typeshed does not know + # this, yet: + value: ast.expr | None = node.value + if value is None: + self.error(node, 'Unpacking in comprehensions is not allowed.') return self.node_contents_visit(node) def visit_comprehension(self, node: ast.comprehension) -> _T_visit_return: diff --git a/tests/transformer/test_comprehension_unpacking.py b/tests/transformer/test_comprehension_unpacking.py new file mode 100644 index 00000000..1ece45cc --- /dev/null +++ b/tests/transformer/test_comprehension_unpacking.py @@ -0,0 +1,66 @@ +import pytest + +from RestrictedPython import compile_restricted_exec +from RestrictedPython._compat import IS_PY315_OR_GREATER +from RestrictedPython.Eval import default_guarded_getiter +from tests.helper import restricted_eval + + +unpacking_errmsg = 'Line 1: Unpacking in comprehensions is not allowed.' + + +@pytest.mark.skipif( + not IS_PY315_OR_GREATER, + reason="unpacking in comprehensions was added in Python 3.15.", +) +def test_RestrictingNodeTransformer__visit_ListComp__unpacking(): + """It denies `*` unpacking in a list comprehension.""" + result = compile_restricted_exec('[*x for x in seq]') + assert result.errors == (unpacking_errmsg,) + + +@pytest.mark.skipif( + not IS_PY315_OR_GREATER, + reason="unpacking in comprehensions was added in Python 3.15.", +) +def test_RestrictingNodeTransformer__visit_SetComp__unpacking(): + """It denies `*` unpacking in a set comprehension.""" + result = compile_restricted_exec('{*x for x in seq}') + assert result.errors == (unpacking_errmsg,) + + +@pytest.mark.skipif( + not IS_PY315_OR_GREATER, + reason="unpacking in comprehensions was added in Python 3.15.", +) +def test_RestrictingNodeTransformer__visit_GeneratorExp__unpacking(): + """It denies `*` unpacking in a generator expression.""" + result = compile_restricted_exec('(*x for x in seq)') + assert result.errors == (unpacking_errmsg,) + + +@pytest.mark.skipif( + not IS_PY315_OR_GREATER, + reason="unpacking in comprehensions was added in Python 3.15.", +) +def test_RestrictingNodeTransformer__visit_DictComp__unpacking(): + """It denies `**` unpacking in a dict comprehension.""" + result = compile_restricted_exec('{**x for x in seq}') + assert result.errors == (unpacking_errmsg,) + + +def test_RestrictingNodeTransformer__visit_ListComp__no_unpacking(): + """It still allows list comprehensions without unpacking.""" + glb = {'_getiter_': default_guarded_getiter} + assert restricted_eval('[x for x in (1, 2)]', glb) == [1, 2] + + +def test_RestrictingNodeTransformer__visit_DictComp__no_unpacking(): + """It still allows dict comprehensions without unpacking.""" + glb = {'_getiter_': default_guarded_getiter} + assert restricted_eval('{x: x for x in (1, 2)}', glb) == {1: 1, 2: 2} + + +def test_RestrictingNodeTransformer__visit_List__unpacking(): + """It still allows `*` unpacking in a list display.""" + assert restricted_eval('[*(1, 2), 3]') == [1, 2, 3] diff --git a/tests/transformer/test_inspect.py b/tests/transformer/test_inspect.py index 05e7d41f..fee69158 100644 --- a/tests/transformer/test_inspect.py +++ b/tests/transformer/test_inspect.py @@ -31,6 +31,23 @@ def test_get_inspect_frame_back_on_generator(): ) +def test_get_inspect_attributes_on_async_generator(): + source_code = """ +frame = agen.ag_frame +code = agen.ag_code +awaited = agen.ag_await +""" + result = compile_restricted_exec(source_code) + assert result.errors == ( + 'Line 2: "ag_frame" is a restricted name, ' + 'that is forbidden to access in RestrictedPython.', + 'Line 3: "ag_code" is a restricted name, ' + 'that is forbidden to access in RestrictedPython.', + 'Line 4: "ag_await" is a restricted name, ' + 'that is forbidden to access in RestrictedPython.', + ) + + def test_call_inspect_frame_on_generator(): source_code = """ generator = None diff --git a/tests/transformer/test_lazy_import.py b/tests/transformer/test_lazy_import.py new file mode 100644 index 00000000..e25aef4f --- /dev/null +++ b/tests/transformer/test_lazy_import.py @@ -0,0 +1,27 @@ +import pytest + +from RestrictedPython import compile_restricted_exec +from RestrictedPython._compat import IS_PY315_OR_GREATER + + +lazy_import_errmsg = 'Line 1: Lazy import statements are not allowed.' + + +@pytest.mark.skipif( + not IS_PY315_OR_GREATER, + reason="lazy imports were added in Python 3.15.", +) +def test_RestrictingNodeTransformer__visit_Import__lazy(): + """It denies lazy importing a module.""" + result = compile_restricted_exec('lazy import a') + assert result.errors == (lazy_import_errmsg,) + + +@pytest.mark.skipif( + not IS_PY315_OR_GREATER, + reason="lazy imports were added in Python 3.15.", +) +def test_RestrictingNodeTransformer__visit_ImportFrom__lazy(): + """It denies lazy importing from a module.""" + result = compile_restricted_exec('lazy from a import m') + assert result.errors == (lazy_import_errmsg,)