diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 859687340882de9..a9dbaae9d4bf2f2 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -57,6 +57,14 @@ Examining Symbol Tables Used for the symbol table of a class. + .. attribute:: INLINED_COMPREHENSION + :value: "inlined comprehension" + + Used for the symbol table of a list, set or dict comprehension that + is inlined into the enclosing code unit (see :pep:`709`). A symbol + table of this type represents a sub-scope of the enclosing code unit's + scope, and it does not correspond to a separate compilation unit. + The following members refer to different flavors of :ref:`annotation scopes `. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 3262acd87d6d49f..dfe8d95fa1c8410 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -498,6 +498,15 @@ symtable like the builtin :func:`compile`. (Contributed by Serhiy Storchaka in :gh:`153844`.) +* Inlined list, set and dict comprehensions (:pep:`709`) are now represented + as their own symbol table entries, of type + :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. This entry type + represents a sub-scope, and holds information only on the symbols whose + scopes are different in the comprehension and the enclosing scope. + Sub-scopes are a new mechanism that can be used when a symbol's scope + changes within the same compilation unit. + (Contributed by Irit Katriel in :gh:`124697`.) + tkinter ------- diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 7e248429af8eb8a..05f6e7cbd99adfd 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -69,9 +69,8 @@ typedef struct { PyObject *u_varnames; /* local variables */ PyObject *u_cellvars; /* cell variables */ PyObject *u_freevars; /* free variables */ - PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only - temporarily within an inlined comprehension. When - value is True, treat as fast-local. */ + PyObject *u_fasthidden; /* set of names that are fast-locals only + temporarily within an inlined comprehension. */ Py_ssize_t u_argcount; /* number of arguments for block */ Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ @@ -155,7 +154,6 @@ int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope _PyCompile_optype *optype, Py_ssize_t *arg); int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c); -int _PyCompile_IsInInlinedComp(struct _PyCompiler *c); int _PyCompile_ScopeType(struct _PyCompiler *c); int _PyCompile_OptimizationLevel(struct _PyCompiler *c); int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name); @@ -179,16 +177,15 @@ enum { typedef struct { PyObject *pushed_locals; - PyObject *temp_symbols; - PyObject *fast_hidden; _PyJumpTargetLabel cleanup; + PySTEntryObject *saved_ste; } _PyCompile_InlinedComprehensionState; -int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, - PySTEntryObject *entry, - _PyCompile_InlinedComprehensionState *state); -int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, - _PyCompile_InlinedComprehensionState *state); +int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c, + _PyCompile_InlinedComprehensionState *state); int _PyCompile_AddDeferredAnnotation(struct _PyCompiler *c, stmt_ty s, PyObject **conditional_annotation_index); void _PyCompile_EnterConditionalBlock(struct _PyCompiler *c); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index c650a94a1eab2e1..db609243f2f41e7 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -33,6 +33,10 @@ typedef enum _block_type { // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two // do not support a bound or a constraint tuple). TypeVariableBlock, + // Comprehension which is inlined into the enclosing code unit (see PEP 709). + // Represents a sub-scope of the enclosing code unit's scope rather than a + // separate scope. + InlinedComprehensionBlock, } _Py_block_ty; typedef enum _comprehension_type { @@ -119,7 +123,6 @@ typedef struct _symtable_entry { should be created */ unsigned ste_needs_classdict : 1; /* for class scopes, true if a closure over the class dict should be created */ - unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */ unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ @@ -132,6 +135,7 @@ typedef struct _symtable_entry { int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ + struct _symtable_entry *ste_parent; /* st entry for the enclosing block if this entry is a sub-scope, NULL otherwise */ struct symtable *ste_table; } PySTEntryObject; diff --git a/Lib/symtable.py b/Lib/symtable.py index 18bb355d86b09e0..3d9c4f6b6ea9835 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -56,6 +56,7 @@ class SymbolTableType(StrEnum): TYPE_ALIAS = "type alias" TYPE_PARAMETERS = "type parameters" TYPE_VARIABLE = "type variable" + INLINED_COMPREHENSION = "inlined comprehension" class SymbolTable: @@ -98,6 +99,8 @@ def get_type(self): return SymbolTableType.TYPE_PARAMETERS if self._table.type == _symtable.TYPE_TYPE_VARIABLE: return SymbolTableType.TYPE_VARIABLE + if self._table.type == _symtable.TYPE_INLINED_COMPREHENSION: + return SymbolTableType.INLINED_COMPREHENSION assert False, f"unexpected type: {self._table.type}" def get_id(self): diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 99a11e99d564852..6e04df99b453ecf 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -17,8 +17,9 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault(key, key) for key in ['consts']: metadata.setdefault(key, []) - for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: + for key in ['names', 'varnames', 'cellvars', 'freevars']: metadata.setdefault(key, {}) + metadata.setdefault('fasthidden', None) for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: metadata.setdefault(key, 0) metadata.setdefault('firstlineno', 1) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ce02b27c599c420..5be898abd2d1698 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -426,11 +426,11 @@ def test_symbol_repr(self): "") st1 = symtable.symtable("[x for x in [1]]", "?", "exec") - self.assertEqual(repr(st1.lookup("x")), + self.assertEqual(repr(st1.get_children()[0].lookup("x")), "") st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") - self.assertEqual(repr(st2.lookup("x")), + self.assertEqual(repr(st2.get_children()[0].lookup("x")), "") st3 = symtable.symtable("def f():\n" @@ -502,6 +502,64 @@ def test_nested_genexpr(self): self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) self.assertEqual(st.get_children(), []) + def test_inlined_comprehension_in_genexpr(self): + st = symtable.symtable("([y for y in x] for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_inlined_listcomp(children[0], ["y"], nested=True) + + def check_inlined_listcomp(self, st, identifiers, *, nested, nchildren=0): + self.assertIs(st.get_type(), symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertEqual(st.get_name(), "") + self.assertEqual(st.is_nested(), nested) + self.assertEqual(sorted(st.get_identifiers()), identifiers) + children = st.get_children() + self.assertEqual(len(children), nchildren) + return children + + def check_nested_inlined_listcomp(self, outer, hoisted, outer_ids, inner_ids, *, nested): + # Nested namespaces of inlined comprehensions are also hoisted into + # the enclosing scope's children list. + inner, = self.check_inlined_listcomp( + outer, outer_ids, nested=nested, nchildren=1) + self.check_inlined_listcomp(inner, inner_ids, nested=True) + self.assertIs(hoisted, inner) + + def test_inlined_comprehension(self): + st = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(sorted(st.get_identifiers()), []) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_inlined_listcomp(children[0], ["x"], nested=False) + + def test_inlined_nested_comprehension(self): + st = symtable.symtable("[[y for y in x] for x in [1]]", "?", "exec") + self.assertEqual(sorted(st.get_identifiers()), []) + children = st.get_children() + self.assertEqual(len(children), 2) + self.check_nested_inlined_listcomp( + children[0], children[1], ["x"], ["y"], nested=False) + + def test_inlined_sibling_nested_comprehensions(self): + st = symtable.symtable( + "def f(): [[y for y in x] for x in [1]]; [[w for w in z] for z in [2]]", + "?", "exec") + f = find_block(st, "f") + self.assertIs(f.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(sorted(f.get_identifiers()), []) + children = f.get_children() + self.assertEqual(len(children), 4) + self.check_nested_inlined_listcomp( + children[0], children[1], ["x"], ["y"], nested=True) + self.check_nested_inlined_listcomp( + children[2], children[3], ["z"], ["w"], nested=True) + def test__symtable_refleak(self): # Regression test for reference leak in PyUnicode_FSDecoder. # See https://github.com/python/cpython/issues/139748. diff --git a/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst new file mode 100644 index 000000000000000..8dfdbeaf1856fc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst @@ -0,0 +1,3 @@ +The :mod:`symtable` module now represents inlined list, set and dict +comprehensions (:pep:`709`) as their own symbol table entries of type +:attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38e56ae70420985..beb00a8af6858b2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1357,13 +1357,16 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden"); + if (umd.u_fasthidden == Py_None) { + umd.u_fasthidden = NULL; + } assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); assert(PyDict_Check(umd.u_freevars)); - assert(PyDict_Check(umd.u_fasthidden)); + assert(umd.u_fasthidden == NULL || PySet_Check(umd.u_fasthidden)); umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 7e20b5c7173ae5d..0ce5b73add95247 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -144,6 +144,8 @@ symtable_init_constants(PyObject *m) return -1; if (PyModule_AddIntConstant(m, "TYPE_TYPE_VARIABLE", TypeVariableBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_INLINED_COMPREHENSION", InlinedComprehensionBlock) < 0) + return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, GLOBAL_EXPLICIT) < 0) return -1; diff --git a/Python/assemble.c b/Python/assemble.c index 4bbebe30299906a..486c4f83898caeb 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -520,13 +520,15 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, _PyLocals_Kind kind = CO_FAST_LOCAL | argvarkinds[i].kind; - int has_key = PyDict_Contains(umd->u_fasthidden, k); - RETURN_IF_ERROR(has_key); - if (has_key) { - kind |= CO_FAST_HIDDEN; + if (umd->u_fasthidden != NULL) { + int hidden = PySet_Contains(umd->u_fasthidden, k); + RETURN_IF_ERROR(hidden); + if (hidden) { + kind |= CO_FAST_HIDDEN; + } } - has_key = PyDict_Contains(umd->u_cellvars, k); + int has_key = PyDict_Contains(umd->u_cellvars, k); RETURN_IF_ERROR(has_key); if (has_key) { kind |= CO_FAST_CELL; diff --git a/Python/codegen.c b/Python/codegen.c index 79b84f13e629c76..90a17a90539d107 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3342,7 +3342,7 @@ codegen_nameop(compiler *c, location loc, case COMPILE_OP_DEREF: switch (ctx) { case Load: - if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && !_PyCompile_IsInInlinedComp(c)) { + if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) { op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { @@ -3395,8 +3395,9 @@ codegen_nameop(compiler *c, location loc, case COMPILE_OP_NAME: switch (ctx) { case Load: - op = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock - && _PyCompile_IsInInlinedComp(c)) + /* LOAD_NAME in a class reads the class dict; inlined comps must not. */ + op = (SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS + && SYMTABLE_ENTRY(c)->ste_type == InlinedComprehensionBlock) ? LOAD_GLOBAL : LOAD_NAME; break; @@ -4917,9 +4918,6 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, PySTEntryObject *comp, _PyCompile_InlinedComprehensionState *state) { - int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && - !_PyCompile_IsInInlinedComp(c); - PySTEntryObject *outer = SYMTABLE_ENTRY(c); // iterate over names bound in the comprehension and ensure we isolate // them from the outer scope as needed PyObject *k, *v; @@ -4930,11 +4928,7 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, RETURN_IF_ERROR(symbol); long scope = SYMBOL_TO_SCOPE(symbol); - long outsymbol = _PyST_GetSymbol(outer, k); - RETURN_IF_ERROR(outsymbol); - long outsc = SYMBOL_TO_SCOPE(outsymbol); - - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { + if ((symbol & DEF_LOCAL) && !(symbol & DEF_NONLOCAL)) { // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack @@ -4949,15 +4943,17 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, // comprehension and restore the original one after ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); if (scope == CELL) { - if (outsc == FREE) { - ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); - } else { - ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); - } + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); } if (PyList_Append(state->pushed_locals, k) < 0) { return ERROR; } + if (METADATA(c)->u_fasthidden != NULL) { + /* For Module/Class scopes, assemble needs to set CO_FAST_HIDDEN on these names */ + if (PySet_Add(METADATA(c)->u_fasthidden, k) < 0) { + return ERROR; + } + } } } if (state->pushed_locals) { @@ -4986,7 +4982,7 @@ push_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR( - _PyCompile_TweakInlinedComprehensionScopes(c, loc, comp, state)); + _PyCompile_EnterInlinedComprehensionScope(c, comp, state)); RETURN_IF_ERROR( codegen_push_inlined_comprehension_locals(c, loc, comp, state)); return SUCCESS; @@ -5044,7 +5040,7 @@ pop_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); - RETURN_IF_ERROR(_PyCompile_RevertInlinedComprehensionScopes(c, loc, state)); + RETURN_IF_ERROR(_PyCompile_ExitInlinedComprehensionScope(c, state)); return SUCCESS; } @@ -5054,13 +5050,13 @@ codegen_comprehension(compiler *c, expr_ty e, int type, expr_ty val, bool avoid_creation) { PyCodeObject *co = NULL; - _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL}; + _PyCompile_InlinedComprehensionState inline_state = {NULL, NO_LABEL, NULL}; comprehension_ty outermost; PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); if (entry == NULL) { goto error; } - int is_inlined = entry->ste_comp_inlined; + int is_inlined = (entry->ste_type == InlinedComprehensionBlock); int is_async_comprehension = entry->ste_coroutine; location loc = LOC(e); @@ -5069,7 +5065,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, IterStackPosition iter_state; if (is_inlined) { VISIT(c, expr, outermost->iter); - if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { + if (push_inlined_comprehension_state(c, loc, entry, &inline_state) < 0) { goto error; } iter_state = ITERABLE_ON_STACK; @@ -5140,8 +5136,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } if (is_inlined) { - if (pop_inlined_comprehension_state(c, loc, &inline_state)) { - goto error; + if (pop_inlined_comprehension_state(c, loc, &inline_state) < 0) { + goto error_in_scope; } return SUCCESS; } @@ -5181,15 +5177,18 @@ codegen_comprehension(compiler *c, expr_ty e, int type, return SUCCESS; error_in_scope: - if (!is_inlined) { + if (is_inlined) { + if (inline_state.saved_ste != NULL) { + pop_inlined_comprehension_state(c, loc, &inline_state); + } + } + else { _PyCompile_ExitScope(c); } error: Py_XDECREF(co); Py_XDECREF(entry); Py_XDECREF(inline_state.pushed_locals); - Py_XDECREF(inline_state.temp_symbols); - Py_XDECREF(inline_state.fast_hidden); return ERROR; } diff --git a/Python/compile.c b/Python/compile.c index f3852041bce69ca..bd8171c40e6cf4d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -68,7 +68,6 @@ struct compiler_unit { instr_sequence *u_stashed_instr_sequence; /* temporarily stashed parent instruction sequence */ int u_nfblocks; - int u_in_inlined_comp; int u_in_conditional_block; _PyCompile_FBlockInfo u_fblock[CO_MAXBLOCKS]; @@ -670,14 +669,18 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, return ERROR; } - u->u_metadata.u_fasthidden = PyDict_New(); - if (!u->u_metadata.u_fasthidden) { - compiler_unit_free(u); - return ERROR; + if (scope_type == COMPILE_SCOPE_MODULE || scope_type == COMPILE_SCOPE_CLASS) { + u->u_metadata.u_fasthidden = PySet_New(NULL); + if (!u->u_metadata.u_fasthidden) { + compiler_unit_free(u); + return ERROR; + } + } + else { + u->u_metadata.u_fasthidden = NULL; } u->u_nfblocks = 0; - u->u_in_inlined_comp = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); if (!u->u_metadata.u_consts) { @@ -1013,6 +1016,9 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, PyObject *dict = c->u->u_metadata.u_names; *optype = COMPILE_OP_NAME; + PySTEntryObject *ste = c->u->u_ste; + assert(ste != NULL); + assert(scope >= 0); switch (scope) { case FREE: @@ -1024,24 +1030,24 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, *optype = COMPILE_OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + /* Inlined comprehensions isolate their locals as FAST, even when + * nested in class or module scope. */ + if (_PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock) { *optype = COMPILE_OP_FAST; } - else { - PyObject *item; - RETURN_IF_ERROR(PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, - &item)); - if (item == Py_True) { - *optype = COMPILE_OP_FAST; - } - Py_XDECREF(item); - } break; - case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + case GLOBAL_IMPLICIT: { + /* Opcode depends on the enclosing non-inlined scope. */ + PySTEntryObject *enclosing = ste; + while (enclosing->ste_parent != NULL) { + assert(enclosing->ste_type == InlinedComprehensionBlock); + enclosing = enclosing->ste_parent; + } + if (_PyST_IsFunctionLike(enclosing)) { *optype = COMPILE_OP_GLOBAL; } break; + } case GLOBAL_EXPLICIT: *optype = COMPILE_OP_GLOBAL; break; @@ -1057,120 +1063,23 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, } int -_PyCompile_TweakInlinedComprehensionScopes(compiler *c, location loc, - PySTEntryObject *entry, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_EnterInlinedComprehensionScope(compiler *c, PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state) { - int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; - c->u->u_in_inlined_comp++; - - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { - long symbol = PyLong_AsLong(v); - assert(symbol >= 0 || PyErr_Occurred()); - RETURN_IF_ERROR(symbol); - long scope = SYMBOL_TO_SCOPE(symbol); - - long outsymbol = _PyST_GetSymbol(c->u->u_ste, k); - RETURN_IF_ERROR(outsymbol); - long outsc = SYMBOL_TO_SCOPE(outsymbol); - - // If a name has different scope inside than outside the comprehension, - // we need to temporarily handle it with the right scope while - // compiling the comprehension. If it's free in the comprehension - // scope, no special handling; it should be handled the same as the - // enclosing scope. (If it's free in outer scope and cell in inner - // scope, we can't treat it as both cell and free in the same function, - // but treating it as free throughout is fine; it's *_DEREF - // either way.) - if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) - || in_class_block) { - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - return ERROR; - } - PyObject *outv = PyLong_FromLong(outsymbol); - if (outv == NULL) { - return ERROR; - } - int res = PyDict_SetItem(state->temp_symbols, k, outv); - Py_DECREF(outv); - RETURN_IF_ERROR(res); - } - // locals handling for names bound in comprehension (DEF_LOCAL | - // DEF_NONLOCAL occurs in assignment expression to nonlocal) - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - // non-function scope: override this name to use fast locals - PyObject *orig; - if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { - return ERROR; - } - assert(orig == NULL || orig == Py_True || orig == Py_False); - if (orig != Py_True) { - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { - Py_XDECREF(orig); - return ERROR; - } - if (state->fast_hidden == NULL) { - state->fast_hidden = PySet_New(NULL); - if (state->fast_hidden == NULL) { - Py_XDECREF(orig); - return ERROR; - } - } - if (PySet_Add(state->fast_hidden, k) < 0) { - Py_XDECREF(orig); - return ERROR; - } - } - Py_XDECREF(orig); - } - } - } + assert(state->saved_ste == NULL); + state->saved_ste = c->u->u_ste; + c->u->u_ste = (PySTEntryObject *)Py_NewRef(entry); return SUCCESS; } int -_PyCompile_RevertInlinedComprehensionScopes(compiler *c, location loc, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_ExitInlinedComprehensionScope(compiler *c, + _PyCompile_InlinedComprehensionState *state) { - c->u->u_in_inlined_comp--; - if (state->temp_symbols) { - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(state->temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { - return ERROR; - } - } - Py_CLEAR(state->temp_symbols); - } - if (state->fast_hidden) { - while (PySet_Size(state->fast_hidden) > 0) { - PyObject *k = PySet_Pop(state->fast_hidden); - if (k == NULL) { - return ERROR; - } - // we set to False instead of clearing, so we can track which names - // were temporarily fast-locals and should use CO_FAST_HIDDEN - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { - Py_DECREF(k); - return ERROR; - } - Py_DECREF(k); - } - Py_CLEAR(state->fast_hidden); - } + assert(state->saved_ste != NULL); + Py_DECREF(c->u->u_ste); + c->u->u_ste = state->saved_ste; + state->saved_ste = NULL; return SUCCESS; } @@ -1362,12 +1271,6 @@ _PyCompile_ScopeType(compiler *c) return c->u->u_scope_type; } -int -_PyCompile_IsInInlinedComp(compiler *c) -{ - return c->u->u_in_inlined_comp; -} - PyObject * _PyCompile_Qualname(compiler *c) { @@ -1513,10 +1416,7 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyObject *filename = c->c_filename; int code_flags = compute_code_flags(c); - if (code_flags < 0) { - return NULL; - } - + assert(code_flags >= 0); if (_PyCodegen_AddReturnAtEnd(c, addNone) < 0) { return NULL; } diff --git a/Python/symtable.c b/Python/symtable.c index 8da04b40e8ad142..aac0c626beaff57 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -5,8 +5,10 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_symtable.h" // PySTEntryObject #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString +#include "setobject.h" #include // offsetof() +#include // Set this to 1 to dump all symtables to stdout for debugging @@ -89,6 +91,12 @@ #define IS_ASYNC_DEF(st) ((st)->st_cur->ste_type == FunctionBlock && (st)->st_cur->ste_coroutine) +static int +ste_uses_fast_locals(PySTEntryObject *ste) +{ + return _PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock; +} + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, _Py_SourceLocation loc) @@ -128,14 +136,14 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, if (st->st_cur != NULL && (st->st_cur->ste_nested || - _PyST_IsFunctionLike(st->st_cur))) + ste_uses_fast_locals(st->st_cur))) ste->ste_nested = 1; ste->ste_generator = 0; ste->ste_coroutine = 0; ste->ste_comprehension = NoComprehension; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; - ste->ste_comp_inlined = 0; + ste->ste_parent = (block == InlinedComprehensionBlock) ? st->st_cur : NULL; ste->ste_comp_iter_target = 0; ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; @@ -295,6 +303,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; + case InlinedComprehensionBlock: blocktype = "InlinedComprehensionBlock"; break; } const char *comptype = ""; switch (ste->ste_comprehension) { @@ -308,7 +317,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ( "%U=== Symtable for %U ===\n" "%U%s%s\n" - "%U%s%s%s%s%s%s%s%s%s%s%s\n" + "%U%s%s%s%s%s%s%s%s%s%s\n" "%Ulineno: %d col_offset: %d\n" "%U--- Symbols ---\n" ), @@ -326,7 +335,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ste->ste_returns_value ? " returns_value" : "", ste->ste_needs_class_closure ? " needs_class_closure" : "", ste->ste_needs_classdict ? " needs_classdict" : "", - ste->ste_comp_inlined ? " comp_inlined" : "", ste->ste_comp_iter_target ? " comp_iter_target" : "", ste->ste_can_see_class_scope ? " can_see_class_scope" : "", prefix, @@ -537,22 +545,26 @@ _PySymtable_LookupOptional(struct symtable *st, void *key, long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { - PyObject *v; - if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { - return -1; - } - if (!v) { - return 0; - } - long symbol = PyLong_AsLong(v); - Py_DECREF(v); - if (symbol < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_SystemError, "invalid symbol"); + while (ste != NULL) { + PyObject *v; + if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { + return -1; } - return -1; + if (v != NULL) { + long symbol = PyLong_AsLong(v); + Py_DECREF(v); + if (symbol < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "invalid symbol"); + } + return -1; + } + return symbol; + } + assert(ste->ste_parent == NULL || ste->ste_type == InlinedComprehensionBlock); + ste = ste->ste_parent; } - return symbol; + return 0; } int @@ -801,37 +813,102 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) return 0; } +/* True if name is FREE in the comprehension and bound in the enclosing class. + * Those names are kept in the compressed delta so lookup does not treat them + * as class locals. */ static int -inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, - PyObject *scopes, PyObject *comp_free, - PyObject *inlined_cells) +class_binds_free_name(PySTEntryObject *ste, PyObject *name, long comp_flags) { + if (SYMBOL_TO_SCOPE(comp_flags) != FREE) { + return 0; + } + if (ste->ste_type != ClassBlock) { + return 0; + } + PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name); + if (v == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + long class_flags = PyLong_AsLong(v); + if (class_flags == -1 && PyErr_Occurred()) { + return -1; + } + if (class_flags & (DEF_LOCAL | DEF_GLOBAL | DEF_FREE_CLASS | DEF_TYPE_PARAM)) + { + return 1; + } + return 0; +} + +static PyObject * +get_freevar_names(PySTEntryObject *ste) +{ + PyObject *free = PySet_New(NULL); + if (free == NULL) { + return NULL; + } PyObject *k, *v; Py_ssize_t pos = 0; - int remove_dunder_class = 0; - int remove_dunder_classdict = 0; - int remove_dunder_cond_annotations = 0; + while (PyDict_Next(ste->ste_symbols, &pos, &k, &v)) { + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + Py_DECREF(free); + return NULL; + } + if (SYMBOL_TO_SCOPE(flags) == FREE) { + if (PySet_Add(free, k) < 0) { + Py_DECREF(free); + return NULL; + } + } + } + return free; +} + +static int +finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, + PyObject *comp_free, PyObject *outer_newfree, + PyObject *inlined_cells) +{ + PyObject *k, *v; + Py_ssize_t pos = 0; + PyObject *to_remove = NULL; + + assert(comp->ste_type == InlinedComprehensionBlock); + assert(comp->ste_parent != NULL); + + to_remove = PyList_New(0); + if (to_remove == NULL) { + return 0; + } while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { - // skip comprehension parameter long comp_flags = PyLong_AsLong(v); if (comp_flags == -1 && PyErr_Occurred()) { - return 0; - } - if (comp_flags & DEF_PARAM) { - assert(_PyUnicode_EqualToASCIIString(k, ".0")); - continue; + goto error; } int scope = SYMBOL_TO_SCOPE(comp_flags); int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1); if (scope == CELL || only_flags & DEF_COMP_CELL) { if (PySet_Add(inlined_cells, k) < 0) { - return 0; + goto error; + } + if (!(only_flags & DEF_COMP_CELL)) { + comp_flags |= DEF_COMP_CELL; + PyObject *newv = PyLong_FromLong(comp_flags); + if (newv == NULL) { + goto error; + } + if (PyDict_SetItem(comp->ste_symbols, k, newv) < 0) { + Py_DECREF(newv); + goto error; + } + Py_DECREF(newv); } } PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k); if (existing == NULL && PyErr_Occurred()) { - return 0; + goto error; } // __class__, __classdict__ and __conditional_annotations__ are // not allowed to be free through a class scope (see @@ -840,71 +917,103 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, (_PyUnicode_EqualToASCIIString(k, "__class__") || _PyUnicode_EqualToASCIIString(k, "__classdict__") || _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { - scope = GLOBAL_IMPLICIT; int child_needs_free = is_free_in_any_child(comp, k); if (child_needs_free < 0) { - return 0; + goto error; } if (!child_needs_free) { if (PySet_Discard(comp_free, k) < 0) { - return 0; + goto error; } } - if (_PyUnicode_EqualToASCIIString(k, "__class__")) { - remove_dunder_class = 1; - } - else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { - remove_dunder_cond_annotations = 1; - } - else { - remove_dunder_classdict = 1; - } - } - if (!existing) { - // name does not exist in scope, copy from comprehension - assert(scope != FREE || PySet_Contains(comp_free, k) == 1); - PyObject *v_flags = PyLong_FromLong(only_flags); - if (v_flags == NULL) { - return 0; + long new_flags = only_flags | (GLOBAL_IMPLICIT << SCOPE_OFFSET); + PyObject *newv = PyLong_FromLong(new_flags); + if (newv == NULL) { + goto error; } - int ok = PyDict_SetItem(ste->ste_symbols, k, v_flags); - Py_DECREF(v_flags); - if (ok < 0) { - return 0; + if (PyDict_SetItem(comp->ste_symbols, k, newv) < 0) { + Py_DECREF(newv); + goto error; } - SET_SCOPE(scopes, k, scope); + Py_DECREF(newv); + continue; } - else { + if (existing) { long flags = PyLong_AsLong(existing); if (flags == -1 && PyErr_Occurred()) { - return 0; + goto error; } if ((flags & DEF_BOUND) && ste->ste_type != ClassBlock) { // free vars in comprehension that are locals in outer scope can // now simply be locals, unless they are free in comp children, - // or if the outer scope is a class block + // needed as cells by sibling nested scopes, or if the outer + // scope is a class block int ok = is_free_in_any_child(comp, k); if (ok < 0) { - return 0; + goto error; } if (!ok) { - if (PySet_Discard(comp_free, k) < 0) { - return 0; + int in_newfree = PySet_Contains(outer_newfree, k); + if (in_newfree < 0) { + goto error; + } + if (!in_newfree) { + if (PySet_Discard(comp_free, k) < 0) { + goto error; + } } } } } + else { + assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + } + + /* keep bindings, globals, and class-bound frees in the delta; + drop other names (typically FREE uses) so lookup climbs to parent. */ + if ((comp_flags & DEF_LOCAL) && !(comp_flags & DEF_NONLOCAL)) { + continue; + } + if (scope == GLOBAL_IMPLICIT || scope == GLOBAL_EXPLICIT) { + continue; + } + int keep = class_binds_free_name(ste, k, comp_flags); + if (keep < 0) { + goto error; + } + if (!keep) { + if (PyList_Append(to_remove, k) < 0) { + goto error; + } + } } - if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { - return 0; - } - if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { - return 0; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(to_remove); i++) { + PyObject *name = PyList_GET_ITEM(to_remove, i); + if (PyDict_DelItem(comp->ste_symbols, name) < 0) { + goto error; + } } - if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { - return 0; + Py_CLEAR(to_remove); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(comp->ste_children); i++) { + PySTEntryObject *child = (PySTEntryObject *)PyList_GET_ITEM(comp->ste_children, i); + if (child->ste_type != InlinedComprehensionBlock) { + continue; + } + PyObject *child_free = get_freevar_names(child); + if (child_free == NULL) { + return 0; + } + int ok = finalize_inlined_comprehension(ste, child, child_free, + outer_newfree, inlined_cells); + Py_DECREF(child_free); + if (!ok) { + return 0; + } } return 1; +error: + Py_XDECREF(to_remove); + return 0; } #undef SET_SCOPE @@ -1210,7 +1319,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { /* Add function locals to bound set */ - if (_PyST_IsFunctionLike(ste)) { + if (ste_uses_fast_locals(ste)) { temp = PyNumber_InPlaceOr(newbound, local); if (!temp) goto error; @@ -1262,24 +1371,18 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions, - // except those in annotation scopes that are nested in classes - int inline_comp = - entry->ste_comprehension && - !entry->ste_generator && - !ste->ste_can_see_class_scope; - + // Compress InlinedComprehensionBlocks ste_symbols to a delta (bindings + + // class FREE overrides). Nested deltas are finalized recursively. if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) { goto error; } - if (inline_comp) { - if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells)) { + if (entry->ste_type == InlinedComprehensionBlock && ste->ste_type != InlinedComprehensionBlock) { + if (!finalize_inlined_comprehension(ste, entry, child_free, newfree, inlined_cells)) { Py_DECREF(child_free); goto error; } - entry->ste_comp_inlined = 1; } temp = PyNumber_InPlaceOr(newfree, child_free); Py_DECREF(child_free); @@ -1294,8 +1397,9 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PySTEntryObject* entry; assert(c && PySTEntry_Check(c)); entry = (PySTEntryObject*)c; - if (entry->ste_comp_inlined && - PyList_SetSlice(ste->ste_children, i, i + 1, + if (entry->ste_type == InlinedComprehensionBlock && + PyList_GET_SIZE(entry->ste_children) > 0 && + PyList_SetSlice(ste->ste_children, i+1, i + 1, entry->ste_children) < 0) { goto error; @@ -1303,10 +1407,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } /* Check if any local variables must be converted to cell variables */ - if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells)) + if (ste_uses_fast_locals(ste) && !analyze_cells(scopes, newfree, inlined_cells)) { goto error; - else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) + } + else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) { goto error; + } /* Records the results of the analysis in the symbol table entry */ if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, inlined_cells, (ste->ste_type == ClassBlock) || ste->ste_can_see_class_scope)) @@ -2582,7 +2688,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) return 0; } if (!allows_top_level_await(st)) { - if (!_PyST_IsFunctionLike(st->st_cur)) { + if (!ste_uses_fast_locals(st->st_cur)) { PyErr_SetString(PyExc_SyntaxError, "'await' outside function"); SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); @@ -2660,7 +2766,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && - _PyST_IsFunctionLike(st->st_cur) && + ste_uses_fast_locals(st->st_cur) && _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e))) return 0; @@ -3103,9 +3209,16 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, outermost->iter); st->st_cur->ste_comp_iter_expr--; + + /* Non-generator comprehensions are inlined into the enclosing compilation + * unit (including generator expressions), except in annotation scopes + * that can see a class. */ + int will_inline = !is_generator && !st->st_cur->ste_can_see_class_scope; + _Py_block_ty block = will_inline ? InlinedComprehensionBlock : FunctionBlock; + /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, LOCATION(e))) { + !symtable_enter_block(st, scope_name, block, (void *)e, LOCATION(e))) { return 0; } switch(e->kind) { @@ -3126,8 +3239,8 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_coroutine = 1; } - /* Outermost iter is received as an argument */ - if (!symtable_implicit_arg(st, 0)) { + /* Outermost iter is received as an argument for non-inlined comps */ + if (!will_inline && !symtable_implicit_arg(st, 0)) { symtable_exit_block(st); return 0; }