From 022712aad938484e19aa7d135731305723a59fb7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Jun 2026 10:15:58 +0100 Subject: [PATCH 01/33] Add failing test case --- mypyc/test-data/irbuild-classes.test | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 66caf0772ec4..10c7668ed194 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -3262,3 +3262,41 @@ L5: r84 = r83 >= 0 :: signed r85 = __main__.NonExt :: type return 1 + +[case testFinalAttrBorrowed] +from typing import Final + +class C: + def __init__(self, x: str) -> None: + self.f: Final = x + self.x = x + +def f(s: str) -> None: pass + +def g(c: C) -> None: + f(c.f) + f(c.x) +[out] +def C.__init__(self, x): + self :: __main__.C + x :: str +L0: + self.f = x + self.x = x + return 1 +def f(s): + s :: str +L0: + return 1 +def g(c): + c :: __main__.C + r0 :: str + r1 :: None + r2 :: str + r3 :: None +L0: + r0 = borrow c.f + r1 = f(r0) + r2 = c.x + r3 = f(r2) + return 1 From 618e15388d0469f06bf5094eb663e425f45a245a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Jun 2026 10:55:14 +0100 Subject: [PATCH 02/33] Always borrow final attributes --- mypyc/irbuild/builder.py | 12 +++++++++++- mypyc/irbuild/expression.py | 5 +++-- mypyc/test-data/irbuild-classes.test | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 56a3f944fe77..9df4cd44e3d9 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1192,7 +1192,9 @@ def is_synthetic_type(self, typ: TypeInfo) -> bool: return typ.is_named_tuple or typ.is_newtype or typ.typeddict_type is not None def get_final_ref(self, expr: MemberExpr) -> tuple[str, Var, bool] | None: - """Check if `expr` is a final attribute. + """Check if `expr` is a final class or module attribute. + + Return False for instance attributes. This needs to be done differently for class and module attributes to correctly determine fully qualified name. Return a tuple that consists of @@ -1581,6 +1583,14 @@ def is_final_native_attr_ref(self, expr: MemberExpr) -> bool: return expr.name in ir.final_attributes return False + def is_final_instance_attr_ref(self, expr: MemberExpr) -> bool: + obj_rtype = self.node_type(expr.expr) + return ( + isinstance(obj_rtype, RInstance) + and obj_rtype.class_ir.is_ext_class + and any(expr.name in ir.final_attributes for ir in obj_rtype.class_ir.mro) + ) + def mark_block_unreachable(self) -> None: """Mark statements in the innermost block being processed as unreachable. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index b99e8161c08c..31c0b72c3a45 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -252,7 +252,7 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: if expr.fullname in ("typing.TYPE_CHECKING", "typing_extensions.TYPE_CHECKING"): return builder.false(expr.line) - # First check if this is maybe a final attribute. + # First check if this is maybe a final class/module attribute. final = builder.get_final_ref(expr) if final is not None: fullname, final_var, native = final @@ -305,7 +305,8 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) - borrow = can_borrow and builder.can_borrow + is_final = builder.is_final_instance_attr_ref(expr) + borrow = (can_borrow and builder.can_borrow) or is_final return builder.builder.get_attr(obj, expr.name, rtype, expr.line, borrow=borrow) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 10c7668ed194..afeba1ad9728 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -3296,6 +3296,7 @@ def g(c): r3 :: None L0: r0 = borrow c.f + keep_alive c r1 = f(r0) r2 = c.x r3 = f(r2) From f587fbb7358074885acbb7dcf018b40b8fa9ea31 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Jun 2026 16:03:25 +0100 Subject: [PATCH 03/33] Add scope/duration to keep alives --- mypyc/common.py | 4 ++++ mypyc/irbuild/ll_builder.py | 30 ++++++++++++++++++++---------- mypyc/irbuild/vec.py | 4 ++-- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index 382d640a8408..aa40c057bc07 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -68,6 +68,10 @@ BITMAP_TYPE: Final = "uint32_t" BITMAP_BITS: Final = 32 +# Constant for keeping a (often borrowed) Value alive for a short time (e.g. a few subexpressions), +# until flush_kee_alives() is called with this value. +KEEP_ALIVE_SHORT_LIVED: Final = 0 + # Runtime C library files that are always included (some ops may bring # extra dependencies via mypyc.ir.deps.SourceDep or mypyc.ir.deps.HeaderDep) RUNTIME_C_FILES: Final = [ diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c0ad1cf1f826..1a5b2d77cf87 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -19,6 +19,7 @@ FAST_ISINSTANCE_MAX_SUBCLASSES, FAST_PREFIX, IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, @@ -276,7 +277,9 @@ def __init__(self, errors: Errors | None, options: CompilerOptions) -> None: self.error_handlers: list[BasicBlock | None] = [None] # Values that we need to keep alive as long as we have borrowed # temporaries. Use flush_keep_alives() to mark the end of the live range. - self.keep_alives: list[Value] = [] + # The second value is the scope/duration of keep alive (KEEP_ALIVE_* constant). + # Different values must be kept alive for different durations. + self.keep_alives: list[tuple[Value, int]] = [] def set_module(self, module_name: str, module_path: str) -> None: """Set the name and path of the current module.""" @@ -314,7 +317,14 @@ def goto_and_activate(self, block: BasicBlock) -> None: self.goto(block) self.activate_block(block) - def keep_alive(self, values: list[Value], line: int, *, steal: bool = False) -> None: + def keep_alive( + self, + values: list[Value], + line: int, + *, + scope: int = KEEP_ALIVE_SHORT_LIVED, + steal: bool = False, + ) -> None: self.add(KeepAlive(values, line, steal=steal)) def load_mem(self, ptr: Value, value_type: RType, *, borrow: bool = False) -> Value: @@ -367,10 +377,10 @@ def self(self) -> Register: """ return self.args[0] - def flush_keep_alives(self, line: int) -> None: - if self.keep_alives: - self.add(KeepAlive(self.keep_alives.copy(), line)) - self.keep_alives = [] + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: + if any(s == scope for _, s in self.keep_alives): + self.add(KeepAlive([v for v, s in self.keep_alives if s == scope], line)) + self.keep_alives = [(v, s) for v, s in self.keep_alives if s != scope] def debug_print(self, toprint: str | Value) -> None: if isinstance(toprint, str): @@ -400,7 +410,7 @@ def unbox_or_cast( return self.add(Unbox(src, target_type, line)) else: if can_borrow: - self.keep_alives.append(src) + self.keep_alives.append((src, KEEP_ALIVE_SHORT_LIVED)) return self.add(Cast(src, target_type, line, borrow=can_borrow, unchecked=unchecked)) def coerce( @@ -817,7 +827,7 @@ def get_attr( # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - self.keep_alives.append(obj) + self.keep_alives.append((obj, KEEP_ALIVE_SHORT_LIVED)) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) @@ -2281,7 +2291,7 @@ def call_c( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append(arg) + self.keep_alives.append((arg, KEEP_ALIVE_SHORT_LIVED)) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE @@ -2402,7 +2412,7 @@ def primitive_op( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append(arg) + self.keep_alives.append((arg, KEEP_ALIVE_SHORT_LIVED)) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 38615ebe1602..e965ddb567e7 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Final -from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE +from mypyc.common import IS_32_BIT_PLATFORM, KEEP_ALIVE_SHORT_LIVED, PLATFORM_SIZE from mypyc.ir.ops import ( ERR_MAGIC, Assign, @@ -351,7 +351,7 @@ def vec_get_item_unsafe_lower( vtype = base.type item_addr = vec_item_ptr(builder, base, index) result = vec_load_mem_item(builder, item_addr, vtype.item_type, can_borrow=can_borrow) - builder.keep_alives.append(base) + builder.keep_alives.append((base, KEEP_ALIVE_SHORT_LIVED)) return result From 231ecf233455fd6cf6122f83cbd7c38389822bbc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Jun 2026 16:24:07 +0100 Subject: [PATCH 04/33] Actually fix keep alives to stay around long enough --- mypyc/common.py | 3 +++ mypyc/irbuild/builder.py | 5 +++-- mypyc/irbuild/expression.py | 4 +++- mypyc/irbuild/ll_builder.py | 15 +++++++++++++-- mypyc/irbuild/statement.py | 5 +++-- mypyc/test-data/irbuild-classes.test | 12 +++++++++++- 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index aa40c057bc07..0b5f0bddda72 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -71,6 +71,9 @@ # Constant for keeping a (often borrowed) Value alive for a short time (e.g. a few subexpressions), # until flush_kee_alives() is called with this value. KEEP_ALIVE_SHORT_LIVED: Final = 0 +# Keep value alive until the current top-level expression has been fully evaluated. +# This allows keeping alive values across function calls and arbitrary computation. +KEEP_ALIVE_WHOLE_EXPRESSION: Final = 1 # Runtime C library files that are always included (some ops may bring # extra dependencies via mypyc.ir.deps.SourceDep or mypyc.ir.deps.HeaderDep) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 9df4cd44e3d9..1891f0e6e92d 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -63,6 +63,7 @@ EXT_SUFFIX, GENERATOR_ATTRIBUTE_PREFIX, IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, MODULE_PREFIX, SELF_NAME, TEMP_ATTR_NAME, @@ -337,8 +338,8 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V pass return None - def flush_keep_alives(self, line: int) -> None: - self.builder.flush_keep_alives(line) + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: + self.builder.flush_keep_alives(line, scope=scope) # Pass through methods for the most common low-level builder ops, for convenience. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 31c0b72c3a45..a5b59ec304cd 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -307,7 +307,9 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: is_final = builder.is_final_instance_attr_ref(expr) borrow = (can_borrow and builder.can_borrow) or is_final - return builder.builder.get_attr(obj, expr.name, rtype, expr.line, borrow=borrow) + return builder.builder.get_attr( + obj, expr.name, rtype, expr.line, borrow=borrow, is_final=is_final + ) def check_instance_attribute_access_through_class( diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 1a5b2d77cf87..9c5960314e77 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -20,6 +20,7 @@ FAST_PREFIX, IS_FREE_THREADED, KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, @@ -815,7 +816,14 @@ def coerce_nullable(self, src: Value, target_type: RType, line: int) -> Value: # Attribute access def get_attr( - self, obj: Value, attr: str, result_type: RType, line: int, *, borrow: bool = False + self, + obj: Value, + attr: str, + result_type: RType, + line: int, + *, + borrow: bool = False, + is_final: bool = False, ) -> Value: """Get a native or Python attribute of an object.""" if ( @@ -827,7 +835,10 @@ def get_attr( # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - self.keep_alives.append((obj, KEEP_ALIVE_SHORT_LIVED)) + # Final attributes can be borrwed over arbitrary compuation, since they + # won't be rebound, as long as we keep the object alive + scope = KEEP_ALIVE_SHORT_LIVED if not is_final else KEEP_ALIVE_WHOLE_EXPRESSION + self.keep_alives.append((obj, scope)) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index e90aa089305f..8b9bc28363c3 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,7 @@ YieldExpr, YieldFromExpr, ) -from mypyc.common import TEMP_ATTR_NAME +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, TEMP_ATTR_NAME from mypyc.ir.ops import ( ERR_NEVER, NAMESPACE_MODULE, @@ -164,7 +164,8 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: # ExpressionStmts do not need to be coerced like other Expressions, so we shouldn't # call builder.accept here. stmt.expr.accept(builder.visitor) - builder.flush_keep_alives(stmt.line) + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index afeba1ad9728..fe2058b6066d 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -3276,6 +3276,7 @@ def f(s: str) -> None: pass def g(c: C) -> None: f(c.f) f(c.x) + c.f.startswith("a") [out] def C.__init__(self, x): self :: __main__.C @@ -3294,10 +3295,19 @@ def g(c): r1 :: None r2 :: str r3 :: None + r4, r5 :: str + r6 :: i32 + r7 :: bool L0: r0 = borrow c.f - keep_alive c r1 = f(r0) + keep_alive c r2 = c.x r3 = f(r2) + r4 = borrow c.f + r5 = 'a' + r6 = CPyStr_Startswith(r4, r5) + r7 = truncate r6: i32 to builtins.bool + keep_alive c return 1 + From 436c9721a0b3e57a416445c9a9d60a9afea9a8da Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Jun 2026 16:42:17 +0100 Subject: [PATCH 05/33] Add helper for transforming top-level expressions (with keep alive flushes) --- mypyc/irbuild/builder.py | 15 ++++++++++++++- mypyc/irbuild/statement.py | 9 +++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 1891f0e6e92d..fb1f32d0ab85 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -64,6 +64,7 @@ GENERATOR_ATTRIBUTE_PREFIX, IS_FREE_THREADED, KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, MODULE_PREFIX, SELF_NAME, TEMP_ATTR_NAME, @@ -308,7 +309,10 @@ def accept(self, node: Expression, *, can_borrow: bool = False) -> Value: ... def accept(self, node: Statement) -> None: ... def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> Value | None: - """Transform an expression or a statement. + """Transform a sub-expression or a statement. + + NOTE: Use accept_top_level_expr() for top-level expressions so that keep-alives + will be flushed correctly. If can_borrow is true, prefer to generate a borrowed reference. Borrowed references are faster since they don't require reference count @@ -338,6 +342,15 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V pass return None + def accept_top_level_expr(self, node: Expression, *, coerce: bool = True) -> Value: + """Transform a top-level expression to IR.""" + res = node.accept(self.visitor) + if coerce: + res = self.coerce(res, self.node_type(node), node.line) + self.flush_keep_alives(node.line, scope=KEEP_ALIVE_SHORT_LIVED) + self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) + return res + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: self.builder.flush_keep_alives(line, scope=scope) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 8b9bc28363c3..dad59325d92c 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,7 @@ YieldExpr, YieldFromExpr, ) -from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, TEMP_ATTR_NAME +from mypyc.common import TEMP_ATTR_NAME from mypyc.ir.ops import ( ERR_NEVER, NAMESPACE_MODULE, @@ -161,11 +161,8 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: if isinstance(stmt.expr, StrExpr): # Docstring. Ignore return - # ExpressionStmts do not need to be coerced like other Expressions, so we shouldn't - # call builder.accept here. - stmt.expr.accept(builder.visitor) - builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) - builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) + # ExpressionStmts do not need to be coerced like other Expressions. + builder.accept_top_level_expr(stmt.expr, coerce=False) def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: From 4253768ac1a4c34e5f8849e11025e73bf992f0be Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Jun 2026 18:07:39 +0100 Subject: [PATCH 06/33] Don't implicitly borrow top level attribute references --- mypyc/irbuild/builder.py | 14 +++++--------- mypyc/irbuild/expression.py | 2 +- mypyc/irbuild/statement.py | 11 ++++++++--- mypyc/test-data/irbuild-classes.test | 14 +++++++++++--- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index fb1f32d0ab85..c37b03b2bc61 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -286,6 +286,7 @@ def __init__( self.imports: dict[str, None] = {} self.can_borrow = False + self.expression_depth = 0 # When set, load_globals_dict uses this module instead of self.module_name. # Used by generate_attr_defaults_init for cross-module inherited defaults. @@ -320,6 +321,7 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V """ with self.catch_errors(node.line): if isinstance(node, Expression): + self.expression_depth += 1 old_can_borrow = self.can_borrow self.can_borrow = can_borrow try: @@ -334,6 +336,9 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V self.can_borrow = old_can_borrow if not can_borrow: self.flush_keep_alives(node.line) + self.expression_depth -= 1 + if self.expression_depth == 0: + self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) return res else: try: @@ -342,15 +347,6 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V pass return None - def accept_top_level_expr(self, node: Expression, *, coerce: bool = True) -> Value: - """Transform a top-level expression to IR.""" - res = node.accept(self.visitor) - if coerce: - res = self.coerce(res, self.node_type(node), node.line) - self.flush_keep_alives(node.line, scope=KEEP_ALIVE_SHORT_LIVED) - self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) - return res - def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: self.builder.flush_keep_alives(line, scope=scope) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index a5b59ec304cd..825c66bcb7fb 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -306,7 +306,7 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) is_final = builder.is_final_instance_attr_ref(expr) - borrow = (can_borrow and builder.can_borrow) or is_final + borrow = (can_borrow and builder.can_borrow) or (is_final and builder.expression_depth > 1) return builder.builder.get_attr( obj, expr.name, rtype, expr.line, borrow=borrow, is_final=is_final ) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index dad59325d92c..39398c6bc27e 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,7 @@ YieldExpr, YieldFromExpr, ) -from mypyc.common import TEMP_ATTR_NAME +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, TEMP_ATTR_NAME from mypyc.ir.ops import ( ERR_NEVER, NAMESPACE_MODULE, @@ -161,8 +161,13 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: if isinstance(stmt.expr, StrExpr): # Docstring. Ignore return - # ExpressionStmts do not need to be coerced like other Expressions. - builder.accept_top_level_expr(stmt.expr, coerce=False) + # ExpressionStmts do not need to be coerced like other Expressions, so + # we shouldn't call builder.accept here. + builder.expression_depth += 1 + stmt.expr.accept(builder.visitor) + builder.expression_depth -= 1 + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index fe2058b6066d..eee4fd6dc33f 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -3273,10 +3273,13 @@ class C: def f(s: str) -> None: pass -def g(c: C) -> None: +def calls(c: C) -> None: f(c.f) f(c.x) c.f.startswith("a") + +def ret(c: C) -> str: + return c.f [out] def C.__init__(self, x): self :: __main__.C @@ -3289,7 +3292,7 @@ def f(s): s :: str L0: return 1 -def g(c): +def calls(c): c :: __main__.C r0 :: str r1 :: None @@ -3310,4 +3313,9 @@ L0: r7 = truncate r6: i32 to builtins.bool keep_alive c return 1 - +def ret(c): + c :: __main__.C + r0 :: str +L0: + r0 = c.f + return r0 From 9cf4a3052ee85571d602fb2b5f485e75224f4912 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Jun 2026 13:06:22 +0100 Subject: [PATCH 07/33] Test inheritance --- mypyc/test-data/irbuild-classes.test | 57 ------------- mypyc/test-data/irbuild-final.test | 119 +++++++++++++++++++++++++++ mypyc/test-data/refcount.test | 48 +++++++++++ mypyc/test/test_irbuild.py | 1 + 4 files changed, 168 insertions(+), 57 deletions(-) create mode 100644 mypyc/test-data/irbuild-final.test diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index eee4fd6dc33f..66caf0772ec4 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -3262,60 +3262,3 @@ L5: r84 = r83 >= 0 :: signed r85 = __main__.NonExt :: type return 1 - -[case testFinalAttrBorrowed] -from typing import Final - -class C: - def __init__(self, x: str) -> None: - self.f: Final = x - self.x = x - -def f(s: str) -> None: pass - -def calls(c: C) -> None: - f(c.f) - f(c.x) - c.f.startswith("a") - -def ret(c: C) -> str: - return c.f -[out] -def C.__init__(self, x): - self :: __main__.C - x :: str -L0: - self.f = x - self.x = x - return 1 -def f(s): - s :: str -L0: - return 1 -def calls(c): - c :: __main__.C - r0 :: str - r1 :: None - r2 :: str - r3 :: None - r4, r5 :: str - r6 :: i32 - r7 :: bool -L0: - r0 = borrow c.f - r1 = f(r0) - keep_alive c - r2 = c.x - r3 = f(r2) - r4 = borrow c.f - r5 = 'a' - r6 = CPyStr_Startswith(r4, r5) - r7 = truncate r6: i32 to builtins.bool - keep_alive c - return 1 -def ret(c): - c :: __main__.C - r0 :: str -L0: - r0 = c.f - return r0 diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test new file mode 100644 index 000000000000..14b34d1eed96 --- /dev/null +++ b/mypyc/test-data/irbuild-final.test @@ -0,0 +1,119 @@ +[case testFinalAttrBorrowed] +from typing import Final + +class C: + def __init__(self, x: str) -> None: + self.f: Final = x + self.x = x + +def f(s: str) -> None: pass + +def calls(c: C) -> None: + f(c.f) + f(c.x) + c.f.startswith("a") + +def ret(c: C) -> str: + return c.f +[out] +def C.__init__(self, x): + self :: __main__.C + x :: str +L0: + self.f = x + self.x = x + return 1 +def f(s): + s :: str +L0: + return 1 +def calls(c): + c :: __main__.C + r0 :: str + r1 :: None + r2 :: str + r3 :: None + r4, r5 :: str + r6 :: i32 + r7 :: bool +L0: + r0 = borrow c.f + r1 = f(r0) + keep_alive c + r2 = c.x + r3 = f(r2) + r4 = borrow c.f + r5 = 'a' + r6 = CPyStr_Startswith(r4, r5) + r7 = truncate r6: i32 to builtins.bool + keep_alive c + return 1 +def ret(c): + c :: __main__.C + r0 :: str +L0: + r0 = c.f + return r0 + +[case testInheritedFinalAttrBorrowed] +from typing import Final + +class Base: + x: str + + def __init__(self, s: str) -> None: + self.s: Final = s + +class Deriv(Base): + def __init__(self, n: int) -> None: + self.n: Final = n + +def f(b: Base) -> str: + return b.s + "a" + +def g(d: Deriv) -> str: + return d.s * d.n + +def h(d: Deriv) -> str: + return d.x + "a" +[out] +def Base.__init__(self, s): + self :: __main__.Base + s :: str +L0: + self.s = s + return 1 +def Deriv.__init__(self, n): + self :: __main__.Deriv + n :: int +L0: + self.n = n + return 1 +def f(b): + b :: __main__.Base + r0, r1, r2 :: str +L0: + r0 = borrow b.s + r1 = 'a' + r2 = PyUnicode_Concat(r0, r1) + keep_alive b + return r2 +def g(d): + d :: __main__.Deriv + r0 :: str + r1 :: int + r2 :: str +L0: + r0 = borrow d.s + r1 = borrow d.n + r2 = CPyStr_Multiply(r0, r1) + keep_alive d, d + return r2 +def h(d): + d :: __main__.Deriv + r0, r1, r2 :: str +L0: + r0 = d.x + r1 = 'a' + r2 = PyUnicode_Concat(r0, r1) + return r2 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 7c7134dcfbfa..ea667da7ae55 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2366,3 +2366,51 @@ L0: r3 = CPyBytes_Concat(r2, c) dec_ref r2 return r3 + +[case testBorrowedFinalAttribute] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C, b: bool) -> str: + a = c.s if b else "x" + return a + +def g(c: C) -> C: + return C(c.s) +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(c, b): + c :: __main__.C + b :: bool + r0, r1, r2, a :: str +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = borrow c.s + inc_ref r0 + r1 = r0 + goto L3 +L2: + r2 = 'x' + inc_ref r2 + r1 = r2 +L3: + a = r1 + return a +def g(c): + c :: __main__.C + r0 :: str + r1 :: __main__.C +L0: + r0 = borrow c.s + r1 = C(r0) + return r1 diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 50ffc9004743..6608e6db8e7b 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -36,6 +36,7 @@ "irbuild-statements.test", "irbuild-nested.test", "irbuild-classes.test", + "irbuild-final.test", "irbuild-optional.test", "irbuild-any.test", "irbuild-generics.test", From e075e2cf6c72ad30fb7ad5a4ab36ecf0cdbbdeba Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Jun 2026 13:17:39 +0100 Subject: [PATCH 08/33] Add test case --- mypyc/test-data/irbuild-final.test | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test index 14b34d1eed96..2f4624495db4 100644 --- a/mypyc/test-data/irbuild-final.test +++ b/mypyc/test-data/irbuild-final.test @@ -117,3 +117,81 @@ L0: r1 = 'a' r2 = PyUnicode_Concat(r0, r1) return r2 + +[case testFinalAttrExpressionKinds] +from typing import Final + +class C: + def __init__(self, s: str, n: int, l: list[str]) -> None: + self.f: Final = s + self.n: Final = n + self.l: Final = l + self.s = s + +def f() -> C: + return C("", 0, []) + +def assign(c: C) -> None: + a = c.f + c.s = c.f + +def indexing(c: C) -> str: + return c.l[c.n] + +def call_eq(c: C) -> bool: + return f().f == c.f +[out] +def C.__init__(self, s, n, l): + self :: __main__.C + s :: str + n :: int + l :: list +L0: + self.f = s + self.n = n + self.l = l + self.s = s + return 1 +def f(): + r0 :: str + r1 :: list + r2 :: __main__.C +L0: + r0 = '' + r1 = PyList_New(0) + r2 = C(r0, 0, r1) + return r2 +def assign(c): + c :: __main__.C + r0, a, r1 :: str + r2 :: bool +L0: + r0 = c.f + a = r0 + r1 = c.f + c.s = r1; r2 = is_error + return 1 +def indexing(c): + c :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = borrow c.l + r1 = borrow c.n + r2 = CPyList_GetItem(r0, r1) + r3 = cast(str, r2) + keep_alive c, c + return r3 +def call_eq(c): + c, r0 :: __main__.C + r1, r2 :: str + r3 :: bool +L0: + r0 = f() + r1 = borrow r0.f + r2 = borrow c.f + r3 = CPyStr_Equal(r1, r2) + keep_alive r0, c + return r3 From 220313b48ecd59409d70ee3f71ad04d652c50ccb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Jun 2026 15:39:14 +0100 Subject: [PATCH 09/33] Fix expressions with mixed borrowing scopes --- mypyc/ir/ops.py | 2 + mypyc/irbuild/expression.py | 27 ++++- mypyc/irbuild/ll_builder.py | 2 +- mypyc/test-data/irbuild-final.test | 56 +++++++++++ mypyc/test-data/refcount.test | 154 +++++++++++++++++++++++++++++ 5 files changed, 238 insertions(+), 3 deletions(-) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 4bc7671b8208..d692b792f79d 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -898,6 +898,7 @@ def __init__( *, borrow: bool = False, allow_error_value: bool = False, + is_final: bool = False, ) -> None: super().__init__(line) self.obj = obj @@ -912,6 +913,7 @@ def __init__( elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted + self.is_final = is_final def sources(self) -> list[Value]: return [self.obj] diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 825c66bcb7fb..a2ad52de4a4b 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -58,7 +58,12 @@ TypeType, get_proper_type, ) -from mypyc.common import IS_FREE_THREADED, MAX_SHORT_INT +from mypyc.common import ( + IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, + MAX_SHORT_INT, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD from mypyc.ir.ops import ( @@ -66,7 +71,9 @@ Assign, BasicBlock, CallC, + Cast, ComparisonOp, + GetAttr, Integer, LoadAddress, LoadLiteral, @@ -306,12 +313,28 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) is_final = builder.is_final_instance_attr_ref(expr) - borrow = (can_borrow and builder.can_borrow) or (is_final and builder.expression_depth > 1) + borrow = (can_borrow and builder.can_borrow) or ( + is_final + and builder.expression_depth > 1 + and borrow_scope(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION + ) return builder.builder.get_attr( obj, expr.name, rtype, expr.line, borrow=borrow, is_final=is_final ) +def borrow_scope(builder: IRBuilder, v: Value) -> int: + if isinstance(v, GetAttr) and v.is_borrowed: + obj_scope = borrow_scope(builder, v.obj) + get_attr_scope = KEEP_ALIVE_SHORT_LIVED if not v.is_final else KEEP_ALIVE_WHOLE_EXPRESSION + return min(get_attr_scope, obj_scope) + elif isinstance(v, Cast) and v.is_borrowed: + return borrow_scope(builder, v.src) + elif isinstance(v, CallC) and v.function_name == "CPyList_GetItemBorrow": + return KEEP_ALIVE_SHORT_LIVED + return 999 + + def check_instance_attribute_access_through_class( builder: IRBuilder, expr: MemberExpr, typ: ProperType | None ) -> None: diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 9c5960314e77..3702ce450587 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -831,7 +831,7 @@ def get_attr( and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr) ): - op = GetAttr(obj, attr, line, borrow=borrow) + op = GetAttr(obj, attr, line, borrow=borrow, is_final=is_final) # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test index 2f4624495db4..febbd2475c08 100644 --- a/mypyc/test-data/irbuild-final.test +++ b/mypyc/test-data/irbuild-final.test @@ -195,3 +195,59 @@ L0: r3 = CPyStr_Equal(r1, r2) keep_alive r0, c return r3 + +[case testFinalAttrExpressionKinds2] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.f: Final = s + +def f(c: C) -> str: + return c.d.f + "a" + +def g(a: list[D], n: int) -> str: + return a[n].f + "a" +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + self.f = s + return 1 +def f(c): + c :: __main__.C + r0 :: __main__.D + r1, r2, r3 :: str +L0: + r0 = borrow c.d + r1 = borrow r0.f + r2 = 'a' + r3 = PyUnicode_Concat(r1, r2) + keep_alive c, r0 + return r3 +def g(a, n): + a :: list + n :: int + r0 :: object + r1 :: __main__.D + r2, r3, r4 :: str +L0: + r0 = CPyList_GetItemBorrow(a, n) + r1 = borrow cast(__main__.D, r0) + r2 = r1.f + keep_alive a, n, r0 + r3 = 'a' + r4 = PyUnicode_Concat(r2, r3) + return r4 + diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index ea667da7ae55..668e2129d1fa 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2414,3 +2414,157 @@ L0: r0 = borrow c.s r1 = C(r0) return r1 + +[case testBorrowedFinalAttribute2] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def g(c: C, n: int) -> str: + a = [c] + return a[n].s + "a" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def g(c, n): + c :: __main__.C + n :: int + r0 :: list + r1 :: ptr + a :: list + r2 :: object + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = PyList_New(1) + r1 = list_items r0 + inc_ref c + buf_init_item r1, 0, c + a = r0 + r2 = CPyList_GetItemBorrow(a, n) + r3 = borrow cast(__main__.C, r2) + r4 = r3.s + dec_ref a + r5 = 'a' + r6 = PyUnicode_Concat(r4, r5) + dec_ref r4 + return r6 + +[case testBorrowedFinalAttribute3] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d = d + +class CC: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.f: Final = s + +def f() -> str: + c = C(D("x")) + return c.d.f + "a" + +def g(d: D) -> str: + c = CC(d) + return c.d.f + "a" +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def CC.__init__(self, d): + self :: __main__.CC + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + inc_ref s + self.f = s + return 1 +def f(): + r0 :: str + r1 :: __main__.D + r2, c :: __main__.C + r3 :: __main__.D + r4, r5, r6 :: str +L0: + r0 = 'x' + r1 = D(r0) + r2 = C(r1) + dec_ref r1 + c = r2 + r3 = borrow c.d + r4 = r3.f + dec_ref c + r5 = 'a' + r6 = PyUnicode_Concat(r4, r5) + dec_ref r4 + return r6 +def g(d): + d :: __main__.D + r0, c :: __main__.CC + r1 :: __main__.D + r2, r3, r4 :: str +L0: + r0 = CC(d) + c = r0 + r1 = borrow c.d + r2 = borrow r1.f + r3 = 'a' + r4 = PyUnicode_Concat(r2, r3) + dec_ref c + return r4 + +[case testBorrowedFinalAttribute4] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + + def method(self) -> int: + return self.d.foo() + +class D: + def foo(self) -> int: + return 1 +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def C.method(self): + self :: __main__.C + r0 :: __main__.D + r1 :: int +L0: + r0 = borrow self.d + r1 = r0.foo() + return r1 +def D.foo(self): + self :: __main__.D +L0: + return 2 From 49f6d90e19a26c4a0d9cb6b9e65b8982af8f94f0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Jun 2026 15:48:17 +0100 Subject: [PATCH 10/33] WIP failing vec test case --- mypyc/test-data/refcount.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 668e2129d1fa..46f70b1bae9f 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2568,3 +2568,16 @@ def D.foo(self): self :: __main__.D L0: return 2 + +[case testBorrowedFinalAttribute5] +from typing import Final +from librt.vecs import vec + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def g(c: C, n: int) -> str: + a = vec[C]([c]) + return a[n].s + "a" +[out] From 8ee2588a2d116f52dc1838f1ce41479150eab826 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Jun 2026 13:10:00 +0100 Subject: [PATCH 11/33] Handle vec borrowing properly --- mypyc/irbuild/builder.py | 5 +-- mypyc/irbuild/expression.py | 3 ++ mypyc/test-data/refcount.test | 61 ++++++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index c37b03b2bc61..778095982e5f 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -310,10 +310,7 @@ def accept(self, node: Expression, *, can_borrow: bool = False) -> Value: ... def accept(self, node: Statement) -> None: ... def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> Value | None: - """Transform a sub-expression or a statement. - - NOTE: Use accept_top_level_expr() for top-level expressions so that keep-alives - will be flushed correctly. + """Transform an expression or a statement. If can_borrow is true, prefer to generate a borrowed reference. Borrowed references are faster since they don't require reference count diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index a2ad52de4a4b..dbc55c0a9767 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -78,6 +78,7 @@ LoadAddress, LoadLiteral, PrimitiveDescription, + PrimitiveOp, RaiseStandardError, Register, TupleGet, @@ -332,6 +333,8 @@ def borrow_scope(builder: IRBuilder, v: Value) -> int: return borrow_scope(builder, v.src) elif isinstance(v, CallC) and v.function_name == "CPyList_GetItemBorrow": return KEEP_ALIVE_SHORT_LIVED + elif isinstance(v, PrimitiveOp) and v.desc.name == "vec_get_item_unsafe_borrow": + return KEEP_ALIVE_SHORT_LIVED return 999 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 46f70b1bae9f..55e4f9dcbd69 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2572,12 +2572,71 @@ L0: [case testBorrowedFinalAttribute5] from typing import Final from librt.vecs import vec +from mypy_extensions import i64 class C: def __init__(self, s: str) -> None: self.s: Final = s -def g(c: C, n: int) -> str: +def g(c: C, n: i64) -> str: a = vec[C]([c]) return a[n].s + "a" [out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def g(c, n): + c :: __main__.C + n :: i64 + r0 :: object + r1 :: ptr + r2 :: vec[__main__.C] + r3, r4 :: ptr + a :: vec[__main__.C] + r5 :: native_int + r6 :: bit + r7 :: i64 + r8 :: bit + r9 :: bool + r10 :: i64 + r11 :: __main__.C + r12, r13, r14 :: str +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = VecTApi.alloc(1, 1, r1) + r3 = r2.items + inc_ref c + set_mem r3, c :: __main__.C* + r4 = r3 + 8 + a = r2 + r5 = a.len + r6 = n < r5 :: unsigned + if r6 goto L4 else goto L1 :: bool +L1: + r7 = n + r5 + r8 = r7 < r5 :: unsigned + if r8 goto L3 else goto L6 :: bool +L2: + r9 = raise IndexError + unreachable +L3: + r10 = r7 + goto L5 +L4: + r10 = n +L5: + r11 = vec_get_item_unsafe_borrow[__main__.C] a, r10 + r12 = r11.s + dec_ref a + r13 = 'a' + r14 = PyUnicode_Concat(r12, r13) + dec_ref r12 + return r14 +L6: + dec_ref a + goto L2 From b5dc263138cf2556e6bd63ccf668fd36f138d907 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Jun 2026 13:15:06 +0100 Subject: [PATCH 12/33] Fix typos --- mypyc/common.py | 2 +- mypyc/irbuild/ll_builder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index 0b5f0bddda72..918889ea936f 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -69,7 +69,7 @@ BITMAP_BITS: Final = 32 # Constant for keeping a (often borrowed) Value alive for a short time (e.g. a few subexpressions), -# until flush_kee_alives() is called with this value. +# until flush_keep_alives() is called with this value. KEEP_ALIVE_SHORT_LIVED: Final = 0 # Keep value alive until the current top-level expression has been fully evaluated. # This allows keeping alive values across function calls and arbitrary computation. diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 3702ce450587..40469069c82f 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -835,7 +835,7 @@ def get_attr( # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - # Final attributes can be borrwed over arbitrary compuation, since they + # Final attributes can be borrowed over arbitrary computation, since they # won't be rebound, as long as we keep the object alive scope = KEEP_ALIVE_SHORT_LIVED if not is_final else KEEP_ALIVE_WHOLE_EXPRESSION self.keep_alives.append((obj, scope)) From 7584998b2ea6e0a3ec41a56264bb571a3bb41c11 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Jun 2026 17:43:03 +0100 Subject: [PATCH 13/33] More general fix --- mypyc/common.py | 4 ++-- mypyc/irbuild/expression.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index 918889ea936f..e69c3c2bcb08 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -68,8 +68,8 @@ BITMAP_TYPE: Final = "uint32_t" BITMAP_BITS: Final = 32 -# Constant for keeping a (often borrowed) Value alive for a short time (e.g. a few subexpressions), -# until flush_keep_alives() is called with this value. +# Constant for keeping a (often borrowed) op alive for a short time (e.g. a few subexpressions), +# until flush_keep_alives() is called. KEEP_ALIVE_SHORT_LIVED: Final = 0 # Keep value alive until the current top-level expression has been fully evaluated. # This allows keeping alive values across function calls and arbitrary computation. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index dbc55c0a9767..0d6669727efe 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -331,9 +331,9 @@ def borrow_scope(builder: IRBuilder, v: Value) -> int: return min(get_attr_scope, obj_scope) elif isinstance(v, Cast) and v.is_borrowed: return borrow_scope(builder, v.src) - elif isinstance(v, CallC) and v.function_name == "CPyList_GetItemBorrow": - return KEEP_ALIVE_SHORT_LIVED - elif isinstance(v, PrimitiveOp) and v.desc.name == "vec_get_item_unsafe_borrow": + elif isinstance(v, (CallC, PrimitiveOp)) and v.is_borrowed: + # Values borrowed from a C function (e.g. a borrowed list/vec item) may be + # invalidated by arbitrary computation, so they are only short-lived. return KEEP_ALIVE_SHORT_LIVED return 999 From f2121cfde567b0f232211240c981609e4e3659eb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 1 Jul 2026 13:54:52 +0100 Subject: [PATCH 14/33] Don't borrow over walrus expressions --- mypyc/ir/ops.py | 7 ++-- mypyc/irbuild/builder.py | 59 ++++++++++++++++++++++++++++++ mypyc/irbuild/expression.py | 28 +++++++++----- mypyc/irbuild/ll_builder.py | 18 +++++---- mypyc/irbuild/statement.py | 9 ++++- mypyc/test-data/irbuild-final.test | 1 - mypyc/test-data/refcount.test | 30 +++++++++++++++ 7 files changed, 130 insertions(+), 22 deletions(-) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index d692b792f79d..14e12559be1e 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -31,7 +31,7 @@ class to enable the new behavior. Sometimes adding a new abstract from mypy_extensions import trait -from mypyc.common import PROPSET_PREFIX +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, PROPSET_PREFIX from mypyc.ir.deps import Dependency from mypyc.ir.rtypes import ( RArray, @@ -898,7 +898,7 @@ def __init__( *, borrow: bool = False, allow_error_value: bool = False, - is_final: bool = False, + borrow_scope: int = KEEP_ALIVE_SHORT_LIVED, ) -> None: super().__init__(line) self.obj = obj @@ -913,7 +913,8 @@ def __init__( elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted - self.is_final = is_final + # How long a borrowed result of this op stays valid (a KEEP_ALIVE_* constant). + self.borrow_scope = borrow_scope def sources(self) -> list[Value]: return [self.obj] diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 778095982e5f..d7afc716152d 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -20,6 +20,7 @@ TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, ArgKind, + AssignmentExpr, CallExpr, Decorator, Expression, @@ -42,6 +43,7 @@ TypeParam, Var, ) +from mypy.traverser import TraverserVisitor from mypy.types import ( AnyType, DeletedType, @@ -82,6 +84,7 @@ BasicBlock, Branch, Call, + Cast, ComparisonOp, GetAttr, InitStatic, @@ -287,6 +290,10 @@ def __init__( self.can_borrow = False self.expression_depth = 0 + # Symbols (local vars) reassigned via a walrus expression within the current + # top-level expression. Used to avoid borrowing an attribute over the whole + # expression when the borrow root could be rebound (and thus freed) partway. + self.reassigned_in_expr: set[SymbolNode] = set() # When set, load_globals_dict uses this module instead of self.module_name. # Used by generate_attr_defaults_init for cross-module inherited defaults. @@ -319,6 +326,8 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V with self.catch_errors(node.line): if isinstance(node, Expression): self.expression_depth += 1 + if self.expression_depth == 1: + self.reassigned_in_expr = find_walrus_targets(node) old_can_borrow = self.can_borrow self.can_borrow = can_borrow try: @@ -336,6 +345,7 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V self.expression_depth -= 1 if self.expression_depth == 0: self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) + self.reassigned_in_expr = set() return res else: try: @@ -1598,6 +1608,32 @@ def is_final_instance_attr_ref(self, expr: MemberExpr) -> bool: and any(expr.name in ir.final_attributes for ir in obj_rtype.class_ir.mro) ) + def root_is_reassigned(self, v: Value) -> bool: + """Is the root local variable a borrow chain 'v' reads from reassigned this expression? + + A whole-expression borrow of an attribute keeps the borrow root alive only + via the register holding it. If that register belongs to a local variable + that is rebound (via a walrus assignment) during the same top-level + expression, the old value may be freed while the borrow is still live. + """ + if not self.reassigned_in_expr: + return False + # Peel borrowed links back to the root value the chain reads from. + while True: + if isinstance(v, GetAttr) and v.is_borrowed: + v = v.obj + elif isinstance(v, Cast) and v.is_borrowed: + v = v.src + else: + break + if not isinstance(v, Register): + return False + for symbol in self.reassigned_in_expr: + target = self.symtables[-1].get(symbol) + if isinstance(target, AssignmentTargetRegister) and target.register is v: + return True + return False + def mark_block_unreachable(self) -> None: """Mark statements in the innermost block being processed as unreachable. @@ -1712,6 +1748,29 @@ def get_call_target_fullname(ref: RefExpr) -> str: return ref.fullname +class WalrusTargetCollector(TraverserVisitor): + """Collect the symbols assigned to by walrus expressions in a subtree.""" + + def __init__(self) -> None: + self.targets: set[SymbolNode] = set() + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + if o.target.node is not None: + self.targets.add(o.target.node) + super().visit_assignment_expr(o) + + +def find_walrus_targets(expr: Expression) -> set[SymbolNode]: + """Return the symbols reassigned via a walrus expression within 'expr'. + + Walrus (':=') is the only way to rebind a variable in the middle of evaluating + an expression, so this is the complete set of in-expression reassignments. + """ + collector = WalrusTargetCollector() + expr.accept(collector) + return collector.targets + + def create_type_params( builder: IRBuilder, typing_mod: Value, type_args: list[TypeParam], line: int ) -> list[Value]: diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 0d6669727efe..b0052cd51a76 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -314,23 +314,33 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) is_final = builder.is_final_instance_attr_ref(expr) - borrow = (can_borrow and builder.can_borrow) or ( + scope = KEEP_ALIVE_SHORT_LIVED + if ( is_final and builder.expression_depth > 1 - and borrow_scope(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION - ) + and borrow_scope_of(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION + # Don't borrow across the whole expression if the borrow root can be + # rebound (via a walrus assignment) while the borrow is still live, since + # rebinding could free the object we borrow from. + and not builder.root_is_reassigned(obj) + ): + scope = KEEP_ALIVE_WHOLE_EXPRESSION + borrow = (can_borrow and builder.can_borrow) or scope == KEEP_ALIVE_WHOLE_EXPRESSION return builder.builder.get_attr( - obj, expr.name, rtype, expr.line, borrow=borrow, is_final=is_final + obj, expr.name, rtype, expr.line, borrow=borrow, borrow_scope=scope ) -def borrow_scope(builder: IRBuilder, v: Value) -> int: +def borrow_scope_of(builder: IRBuilder, v: Value) -> int: + """Compute how long an existing borrowed value can safely be kept alive. + + Returns a KEEP_ALIVE_* constant (or a large value if 'v' is not borrowed and + thus has no borrowing constraint). + """ if isinstance(v, GetAttr) and v.is_borrowed: - obj_scope = borrow_scope(builder, v.obj) - get_attr_scope = KEEP_ALIVE_SHORT_LIVED if not v.is_final else KEEP_ALIVE_WHOLE_EXPRESSION - return min(get_attr_scope, obj_scope) + return min(v.borrow_scope, borrow_scope_of(builder, v.obj)) elif isinstance(v, Cast) and v.is_borrowed: - return borrow_scope(builder, v.src) + return borrow_scope_of(builder, v.src) elif isinstance(v, (CallC, PrimitiveOp)) and v.is_borrowed: # Values borrowed from a C function (e.g. a borrowed list/vec item) may be # invalidated by arbitrary computation, so they are only short-lived. diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 40469069c82f..104a28df6e4b 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -20,7 +20,6 @@ FAST_PREFIX, IS_FREE_THREADED, KEEP_ALIVE_SHORT_LIVED, - KEEP_ALIVE_WHOLE_EXPRESSION, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, @@ -823,22 +822,25 @@ def get_attr( line: int, *, borrow: bool = False, - is_final: bool = False, + borrow_scope: int = KEEP_ALIVE_SHORT_LIVED, ) -> Value: - """Get a native or Python attribute of an object.""" + """Get a native or Python attribute of an object. + + If the result is borrowed, borrow_scope controls how long it stays valid + (a KEEP_ALIVE_* constant). The caller is responsible for choosing a scope + that is actually safe (e.g. downgrading to short-lived if the borrow root + may be rebound during the borrow's live range). + """ if ( isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr) ): - op = GetAttr(obj, attr, line, borrow=borrow, is_final=is_final) + op = GetAttr(obj, attr, line, borrow=borrow, borrow_scope=borrow_scope) # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - # Final attributes can be borrowed over arbitrary computation, since they - # won't be rebound, as long as we keep the object alive - scope = KEEP_ALIVE_SHORT_LIVED if not is_final else KEEP_ALIVE_WHOLE_EXPRESSION - self.keep_alives.append((obj, scope)) + self.keep_alives.append((obj, borrow_scope)) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 39398c6bc27e..c28c0933ffe0 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -86,7 +86,12 @@ object_rprimitive, ) from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional -from mypyc.irbuild.builder import IRBuilder, create_type_params, int_borrow_friendly_op +from mypyc.irbuild.builder import ( + IRBuilder, + create_type_params, + find_walrus_targets, + int_borrow_friendly_op, +) from mypyc.irbuild.for_helpers import for_loop_helper from mypyc.irbuild.generator import add_raise_exception_blocks_to_generator_class from mypyc.irbuild.nonlocalcontrol import ( @@ -164,8 +169,10 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: # ExpressionStmts do not need to be coerced like other Expressions, so # we shouldn't call builder.accept here. builder.expression_depth += 1 + builder.reassigned_in_expr = find_walrus_targets(stmt.expr) stmt.expr.accept(builder.visitor) builder.expression_depth -= 1 + builder.reassigned_in_expr = set() builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test index febbd2475c08..90aeaf24bee4 100644 --- a/mypyc/test-data/irbuild-final.test +++ b/mypyc/test-data/irbuild-final.test @@ -250,4 +250,3 @@ L0: r3 = 'a' r4 = PyUnicode_Concat(r2, r3) return r4 - diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 55e4f9dcbd69..b5d7e2ed97eb 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2640,3 +2640,33 @@ L5: L6: dec_ref a goto L2 + +[case testBorrowedFinalAttributeReassignViaWalrus] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C, d: C) -> str: + return c.s + (c := d).s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(c, d): + c, d :: __main__.C + r0, r1, r2 :: str +L0: + r0 = c.s + inc_ref d + c = d + dec_ref c + r1 = borrow d.s + r2 = PyUnicode_Concat(r0, r1) + dec_ref r0 + return r2 From 32e1a46666fb69b48293c3706323028ba1a5ebc3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 1 Jul 2026 14:47:17 +0100 Subject: [PATCH 15/33] Fix lambdas --- mypyc/irbuild/builder.py | 17 ++++++++++++++ mypyc/test-data/run-classes.test | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index d7afc716152d..b1fa0d6e38aa 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -27,6 +27,7 @@ FuncDef, IndexExpr, IntExpr, + LambdaExpr, Lvalue, MemberExpr, MypyFile, @@ -294,6 +295,9 @@ def __init__( # top-level expression. Used to avoid borrowing an attribute over the whole # expression when the borrow root could be rebound (and thus freed) partway. self.reassigned_in_expr: set[SymbolNode] = set() + # Saved expression state for enclosing functions (see enter()/leave()). + self.expression_depth_stack: list[int] = [] + self.reassigned_in_expr_stack: list[set[SymbolNode]] = [] # When set, load_globals_dict uses this module instead of self.module_name. # Used by generate_attr_defaults_init for cross-module inherited defaults. @@ -1360,6 +1364,13 @@ def enter(self, fn_info: FuncInfo | str = "", *, ret_type: RType = none_rprimiti self.fn_info = fn_info self.fn_infos.append(self.fn_info) self.ret_types.append(ret_type) + # A function body is its own top-level expression context, even when the + # function (e.g. a lambda) is being generated in the middle of an outer + # expression. Save the outer expression state and start fresh. + self.expression_depth_stack.append(self.expression_depth) + self.reassigned_in_expr_stack.append(self.reassigned_in_expr) + self.expression_depth = 0 + self.reassigned_in_expr = set() if fn_info.is_generator: self.nonlocal_control.append(GeneratorNonlocalControl()) else: @@ -1373,6 +1384,8 @@ def leave(self) -> tuple[list[Register], list[RuntimeArg], list[BasicBlock], RTy ret_type = self.ret_types.pop() fn_info = self.fn_infos.pop() self.nonlocal_control.pop() + self.expression_depth = self.expression_depth_stack.pop() + self.reassigned_in_expr = self.reassigned_in_expr_stack.pop() self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] return builder.args, runtime_args, builder.blocks, ret_type, fn_info @@ -1759,6 +1772,10 @@ def visit_assignment_expr(self, o: AssignmentExpr) -> None: self.targets.add(o.target.node) super().visit_assignment_expr(o) + def visit_lambda_expr(self, o: LambdaExpr) -> None: + # A lambda body forms its own expression context, so don't descend into it. + pass + def find_walrus_targets(expr: Expression) -> set[SymbolNode]: """Return the symbols reassigned via a walrus expression within 'expr'. diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 73927bb037a7..1931ea568301 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -6104,3 +6104,43 @@ base = sys.getrefcount(BIG) o.v = BIG o.v = BIG assert sys.getrefcount(BIG) == base, "reassignment leaked refs" + +[case testBorrowedFinalAttributeInLambdaAndNestedFunction] +from typing import Final, Callable + +class Box: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, b: Box) -> None: + self.b: Final = b + +def use(x: Box, y: Box) -> int: + return x.n + y.n + +def spend(x: Box) -> Box: + # Allocate a lot so a freed Box slot is likely to be reused, turning any + # use-after-free of a borrowed attribute into an observable wrong result. + junk = [Box(i) for i in range(100)] + return Box(x.n + junk[50].n - 50) + +def make_lambda() -> Callable[[], int]: + # The lambda body creates a C, borrows its final attribute, then calls + # spend() (which allocates heavily) before reading the borrow. + return lambda: use(C(Box(111)).b, spend(Box(0))) + +def make_nested() -> Callable[[], int]: + def inner() -> int: + return use(C(Box(111)).b, spend(Box(0))) + return inner + +def test_borrowed_final_attribute_in_lambda() -> None: + f = make_lambda() + for _ in range(1000): + assert f() == 111 + +def test_borrowed_final_attribute_in_nested_function() -> None: + g = make_nested() + for _ in range(1000): + assert g() == 111 From 66745763befc66cce8e396e1e02c93d13e330a23 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 1 Jul 2026 14:58:16 +0100 Subject: [PATCH 16/33] Tweaks --- mypyc/common.py | 4 ++-- mypyc/irbuild/expression.py | 11 +++++------ mypyc/irbuild/ll_builder.py | 9 +-------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index e69c3c2bcb08..afe97e0a6d4c 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -68,8 +68,8 @@ BITMAP_TYPE: Final = "uint32_t" BITMAP_BITS: Final = 32 -# Constant for keeping a (often borrowed) op alive for a short time (e.g. a few subexpressions), -# until flush_keep_alives() is called. +# Constant for keeping a (often borrowed) op alive for a short time (a few subexpressions, +# no arbitrary computation or memory allocations), until flush_keep_alives() is called. KEEP_ALIVE_SHORT_LIVED: Final = 0 # Keep value alive until the current top-level expression has been fully evaluated. # This allows keeping alive values across function calls and arbitrary computation. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index b0052cd51a76..b1c324c43b7c 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -318,10 +318,9 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: if ( is_final and builder.expression_depth > 1 - and borrow_scope_of(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION + and value_borrow_scope(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION # Don't borrow across the whole expression if the borrow root can be - # rebound (via a walrus assignment) while the borrow is still live, since - # rebinding could free the object we borrow from. + # rebound via a walrus assignment and not builder.root_is_reassigned(obj) ): scope = KEEP_ALIVE_WHOLE_EXPRESSION @@ -331,16 +330,16 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: ) -def borrow_scope_of(builder: IRBuilder, v: Value) -> int: +def value_borrow_scope(builder: IRBuilder, v: Value) -> int: """Compute how long an existing borrowed value can safely be kept alive. Returns a KEEP_ALIVE_* constant (or a large value if 'v' is not borrowed and thus has no borrowing constraint). """ if isinstance(v, GetAttr) and v.is_borrowed: - return min(v.borrow_scope, borrow_scope_of(builder, v.obj)) + return min(v.borrow_scope, value_borrow_scope(builder, v.obj)) elif isinstance(v, Cast) and v.is_borrowed: - return borrow_scope_of(builder, v.src) + return value_borrow_scope(builder, v.src) elif isinstance(v, (CallC, PrimitiveOp)) and v.is_borrowed: # Values borrowed from a C function (e.g. a borrowed list/vec item) may be # invalidated by arbitrary computation, so they are only short-lived. diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 104a28df6e4b..b202c1ed6e3e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -317,14 +317,7 @@ def goto_and_activate(self, block: BasicBlock) -> None: self.goto(block) self.activate_block(block) - def keep_alive( - self, - values: list[Value], - line: int, - *, - scope: int = KEEP_ALIVE_SHORT_LIVED, - steal: bool = False, - ) -> None: + def keep_alive(self, values: list[Value], line: int, *, steal: bool = False) -> None: self.add(KeepAlive(values, line, steal=steal)) def load_mem(self, ptr: Value, value_type: RType, *, borrow: bool = False) -> Value: From 400eb53faeadddd7d5d586793c20c29ac662aefc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 1 Jul 2026 18:21:00 +0100 Subject: [PATCH 17/33] Fix conditional control flow in expressions --- mypyc/irbuild/expression.py | 4 + mypyc/irbuild/ll_builder.py | 48 ++++++++-- mypyc/irbuild/vec.py | 2 +- mypyc/test-data/refcount.test | 170 ++++++++++++++++++++++++++++++++++ 4 files changed, 215 insertions(+), 9 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index b1c324c43b7c..0a3063f08dff 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -913,15 +913,19 @@ def transform_conditional_expr(builder: IRBuilder, expr: ConditionalExpr) -> Val target = Register(expr_type) builder.activate_block(if_body) + true_checkpoint = builder.builder.keep_alive_checkpoint() true_value = builder.accept(expr.if_expr) true_value = builder.coerce(true_value, expr_type, expr.line) builder.add(Assign(target, true_value, expr.line)) + builder.builder.flush_keep_alives_since(expr.line, true_checkpoint) builder.goto(next_block) builder.activate_block(else_body) + false_checkpoint = builder.builder.keep_alive_checkpoint() false_value = builder.accept(expr.else_expr) false_value = builder.coerce(false_value, expr_type, expr.line) builder.add(Assign(target, false_value, expr.line)) + builder.builder.flush_keep_alives_since(expr.line, false_checkpoint) builder.goto(next_block) builder.activate_block(next_block) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index b202c1ed6e3e..41944766981b 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -251,6 +251,15 @@ BOOL_BINARY_OPS: Final = {"&", "&=", "|", "|=", "^", "^=", "==", "!=", "<", "<=", ">", ">="} +class PendingKeepAlive: + __slots__ = ("value", "scope", "seq") + + def __init__(self, value: Value, scope: int, seq: int) -> None: + self.value = value + self.scope = scope + self.seq = seq + + class LowLevelIRBuilder: """A "low-level" IR builder class. @@ -279,7 +288,8 @@ def __init__(self, errors: Errors | None, options: CompilerOptions) -> None: # temporaries. Use flush_keep_alives() to mark the end of the live range. # The second value is the scope/duration of keep alive (KEEP_ALIVE_* constant). # Different values must be kept alive for different durations. - self.keep_alives: list[tuple[Value, int]] = [] + self.keep_alives: list[PendingKeepAlive] = [] + self.next_keep_alive_seq = 0 def set_module(self, module_name: str, module_path: str) -> None: """Set the name and path of the current module.""" @@ -371,9 +381,27 @@ def self(self) -> Register: return self.args[0] def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: - if any(s == scope for _, s in self.keep_alives): - self.add(KeepAlive([v for v, s in self.keep_alives if s == scope], line)) - self.keep_alives = [(v, s) for v, s in self.keep_alives if s != scope] + if any(entry.scope == scope for entry in self.keep_alives): + self.add( + KeepAlive( + [entry.value for entry in self.keep_alives if entry.scope == scope], line + ) + ) + self.keep_alives = [entry for entry in self.keep_alives if entry.scope != scope] + + def keep_alive_checkpoint(self) -> int: + return self.next_keep_alive_seq + + def flush_keep_alives_since(self, line: int, checkpoint: int) -> None: + """Flush keep-alives added after 'checkpoint' without touching earlier ones.""" + new_keep_alives = [entry for entry in self.keep_alives if entry.seq >= checkpoint] + if new_keep_alives: + self.add(KeepAlive([entry.value for entry in new_keep_alives], line)) + self.keep_alives = [entry for entry in self.keep_alives if entry.seq < checkpoint] + + def add_keep_alive(self, value: Value, scope: int) -> None: + self.keep_alives.append(PendingKeepAlive(value, scope, self.next_keep_alive_seq)) + self.next_keep_alive_seq += 1 def debug_print(self, toprint: str | Value) -> None: if isinstance(toprint, str): @@ -403,7 +431,7 @@ def unbox_or_cast( return self.add(Unbox(src, target_type, line)) else: if can_borrow: - self.keep_alives.append((src, KEEP_ALIVE_SHORT_LIVED)) + self.add_keep_alive(src, KEEP_ALIVE_SHORT_LIVED) return self.add(Cast(src, target_type, line, borrow=can_borrow, unchecked=unchecked)) def coerce( @@ -833,7 +861,7 @@ def get_attr( # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - self.keep_alives.append((obj, borrow_scope)) + self.add_keep_alive(obj, borrow_scope) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) @@ -2128,18 +2156,22 @@ def shortcircuit_helper( # it is the right side if the left is false. true_body, false_body = (right_body, left_body) if op == "and" else (left_body, right_body) + left_checkpoint = self.keep_alive_checkpoint() left_value = left() self.add_bool_branch(left_value, true_body, false_body) self.activate_block(left_body) left_coerced = self.coerce(left_value, expr_type, line) self.add(Assign(target, left_coerced, line)) + self.flush_keep_alives_since(line, left_checkpoint) self.goto(next_block) self.activate_block(right_body) + right_checkpoint = self.keep_alive_checkpoint() right_value = right() right_coerced = self.coerce(right_value, expr_type, line) self.add(Assign(target, right_coerced, line)) + self.flush_keep_alives_since(line, right_checkpoint) self.goto(next_block) self.activate_block(next_block) @@ -2297,7 +2329,7 @@ def call_c( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append((arg, KEEP_ALIVE_SHORT_LIVED)) + self.add_keep_alive(arg, KEEP_ALIVE_SHORT_LIVED) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE @@ -2418,7 +2450,7 @@ def primitive_op( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append((arg, KEEP_ALIVE_SHORT_LIVED)) + self.add_keep_alive(arg, KEEP_ALIVE_SHORT_LIVED) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index e965ddb567e7..8dffc12a2213 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -351,7 +351,7 @@ def vec_get_item_unsafe_lower( vtype = base.type item_addr = vec_item_ptr(builder, base, index) result = vec_load_mem_item(builder, item_addr, vtype.item_type, can_borrow=can_borrow) - builder.keep_alives.append((base, KEEP_ALIVE_SHORT_LIVED)) + builder.add_keep_alive(base, KEEP_ALIVE_SHORT_LIVED) return result diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index b5d7e2ed97eb..30c054969adf 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2670,3 +2670,173 @@ L0: r2 = PyUnicode_Concat(r0, r1) dec_ref r0 return r2 + +[case testBorrowedFinalAttributeConditionalMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(b: bool) -> str: + return (C("x").s if b else "y") + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(b): + b :: bool + r0 :: str + r1 :: __main__.C + r2, r3, r4, r5, r6 :: str +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = 'x' + r1 = C(r0) + r2 = borrow r1.s + inc_ref r2 + r3 = r2 + dec_ref r1 + goto L3 +L2: + r4 = 'y' + inc_ref r4 + r3 = r4 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r3, r5) + dec_ref r3 + return r6 + +[case testBorrowedFinalAttributeAndMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> str: + return (s and C("x").s) + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s :: str + r0 :: bit + r1, r2 :: str + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = CPyStr_IsTrue(s) + if r0 goto L2 else goto L1 :: bool +L1: + inc_ref s + r1 = s + goto L3 +L2: + r2 = 'x' + r3 = C(r2) + r4 = borrow r3.s + inc_ref r4 + r1 = r4 + dec_ref r3 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r1, r5) + dec_ref r1 + return r6 + +[case testBorrowedFinalAttributeOrMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> str: + return (s or C("x").s) + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s :: str + r0 :: bit + r1, r2 :: str + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = CPyStr_IsTrue(s) + if r0 goto L1 else goto L2 :: bool +L1: + inc_ref s + r1 = s + goto L3 +L2: + r2 = 'x' + r3 = C(r2) + r4 = borrow r3.s + inc_ref r4 + r1 = r4 + dec_ref r3 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r1, r5) + dec_ref r1 + return r6 + +[case testBorrowedFinalAttributeChainedComparisonMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> bool: + return s == "x" == C("x").s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s, r0 :: str + r1, r2 :: bool + r3 :: str + r4 :: __main__.C + r5 :: str + r6 :: bool +L0: + r0 = 'x' + r1 = CPyStr_EqualLiteral(s, r0, 1) + if r1 goto L2 else goto L1 :: bool +L1: + r2 = r1 + goto L3 +L2: + r3 = 'x' + r4 = C(r3) + r5 = borrow r4.s + r6 = CPyStr_EqualLiteral(r5, r0, 1) + r2 = r6 + dec_ref r4 +L3: + return r2 + From f6d60d82f0d7fa53becebef312a4a0551e8557f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 2 Jul 2026 11:54:41 +0100 Subject: [PATCH 18/33] Fix comprehensions --- mypyc/irbuild/builder.py | 12 ++++++++++++ mypyc/irbuild/for_helpers.py | 11 +++++++---- mypyc/test-data/run-classes.test | 33 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index b1fa0d6e38aa..ef8cf51f664c 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1421,6 +1421,18 @@ def enter_scope(self, fn_info: FuncInfo) -> Iterator[None]: self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] + @contextmanager + def enter_borrow_scope(self, line: int) -> Iterator[None]: + """Enter new borrow scope from which borrows can't leak to outer expressions.""" + checkpoint = self.builder.keep_alive_checkpoint() + old_expression_depth = self.expression_depth + self.expression_depth = 0 + try: + yield + self.builder.flush_keep_alives_since(line, checkpoint) + finally: + self.expression_depth = old_expression_depth + @contextmanager def enter_method( self, diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index fc539a1fc090..5a36dcb80ff7 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -293,7 +293,8 @@ def sequence_from_generator_preallocate_helper( target_op = empty_op_llbuilder(length, line) def set_item(item_index: Value) -> None: - e = builder.accept(gen.left_expr) + with builder.enter_borrow_scope(line): + e = builder.accept(gen.left_expr) set_item_op(target_op, item_index, e, line) for_loop_helper_with_index( @@ -446,9 +447,10 @@ def loop_contents( remaining_loop_params: the parameters for any further nested loops; if it's empty we'll instead evaluate the "gen_inner_stmts" function """ - # Check conditions, in order, short circuiting them. + # Check conditions, in order, short-circuiting them. for cond in conds: - cond_val = builder.accept(cond) + with builder.enter_borrow_scope(line): + cond_val = builder.accept(cond) cont_block, rest_block = BasicBlock(), BasicBlock() # If the condition is true we'll skip the continue. builder.add_bool_branch(cond_val, rest_block, cont_block) @@ -462,7 +464,8 @@ def loop_contents( else: # We finally reached the actual body of the generator. # Generate the IR for the inner loop body. - gen_inner_stmts() + with builder.enter_borrow_scope(line): + gen_inner_stmts() handle_loop(loop_params) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 1931ea568301..4e8b67a0061a 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -6144,3 +6144,36 @@ def test_borrowed_final_attribute_in_nested_function() -> None: g = make_nested() for _ in range(1000): assert g() == 111 + +[case testBorrowedFinalAttributeInComprehension] +from typing import Final + +class Box: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, b: Box) -> None: + self.b: Final = b + +def use(x: Box, y: Box) -> int: + return x.n + y.n + +def spend(x: Box) -> Box: + # Allocate a lot so a freed Box slot is likely to be reused, turning any + # use-after-free of a borrowed attribute into an observable wrong result. + junk = [Box(i) for i in range(100)] + return Box(x.n + junk[50].n - 50) + +def comp(ns: list[int]) -> list[int]: + # Each iteration allocates a fresh C, borrows its final attribute .b, then + # calls spend() (which allocates heavily) before reading the borrow. The + # borrow container must be kept alive for the current iteration only; if it + # is scoped to the whole (comprehension) expression, the refcount pass + # dec_refs an uninitialized temporary on the loop back-edge and leaks/uses + # freed memory. + return [use(C(Box(111)).b, spend(Box(i))) for i in ns] + +def test_borrowed_final_attribute_in_comprehension() -> None: + for _ in range(1000): + assert comp([1, 2, 3, 4, 5]) == [112, 113, 114, 115, 116] From a71ba1430f2a5371580d46191fcc2bdf0ebdc7be Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 2 Jul 2026 13:53:42 +0100 Subject: [PATCH 19/33] WIP test case --- mypyc/common.py | 5 +++-- mypyc/test-data/refcount.test | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index afe97e0a6d4c..7cf5a3a01e43 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -71,8 +71,9 @@ # Constant for keeping a (often borrowed) op alive for a short time (a few subexpressions, # no arbitrary computation or memory allocations), until flush_keep_alives() is called. KEEP_ALIVE_SHORT_LIVED: Final = 0 -# Keep value alive until the current top-level expression has been fully evaluated. -# This allows keeping alive values across function calls and arbitrary computation. +# Keep value alive longer, up to until the current top-level expression has been fully +# evaluated. This allows keeping alive values across function calls and arbitrary +# computation. KEEP_ALIVE_WHOLE_EXPRESSION: Final = 1 # Runtime C library files that are always included (some ops may bring diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 30c054969adf..cd11f27caea0 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2840,3 +2840,13 @@ L2: L3: return r2 +[case testBorrowedFinalInListComprehension] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(a: list[C]) -> list[str]: + return [x.s for x in a] +[out] From aee68855c7de83a1b184c597de886db150291079 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 2 Jul 2026 14:18:42 +0100 Subject: [PATCH 20/33] Add comprehension test cases --- mypyc/test-data/refcount.test | 115 ++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index cd11f27caea0..a1a512880124 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2850,3 +2850,118 @@ class C: def f(a: list[C]) -> list[str]: return [x.s for x in a] [out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(a): + a :: list + r0 :: native_int + r1 :: list + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x :: __main__.C + r7 :: str + r8 :: native_int +L0: + r0 = var_object_size a + r1 = PyList_New(r0) + r2 = 0 +L1: + r3 = var_object_size a + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool +L2: + r5 = list_get_item_unsafe a, r2 + r6 = cast(__main__.C, r5) + x = r6 + r7 = x.s + dec_ref x + CPyList_SetItemUnsafe(r1, r2, r7) +L3: + r8 = r2 + 1 + r2 = r8 + goto L1 +L4: + return r1 + +[case testBorrowedFinalInSetComprehension] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.s: Final = s + +def use(s: str) -> bool: + return True + +def f(a: list[C]) -> set[C]: + return {x for x in a if use(x.d.s)} +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + inc_ref s + self.s = s + return 1 +def use(s): + s :: str +L0: + return 1 +def f(a): + a :: list + r0 :: set + r1, r2 :: native_int + r3 :: bit + r4 :: object + r5, x :: __main__.C + r6 :: __main__.D + r7 :: str + r8 :: bool + r9 :: i32 + r10 :: bit + r11 :: native_int +L0: + r0 = PySet_New(0) + r1 = 0 +L1: + r2 = var_object_size a + r3 = r1 < r2 :: signed + if r3 goto L2 else goto L5 :: bool +L2: + r4 = list_get_item_unsafe a, r1 + r5 = cast(__main__.C, r4) + x = r5 + r6 = borrow x.d + r7 = borrow r6.s + r8 = use(r7) + if r8 goto L3 else goto L6 :: bool +L3: + r9 = PySet_Add(r0, x) + dec_ref x + r10 = r9 >= 0 :: signed +L4: + r11 = r1 + 1 + r1 = r11 + goto L1 +L5: + return r0 +L6: + dec_ref x + goto L4 From a9304f25b78b1ce623e9a8a55102d0f65351dccf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 2 Jul 2026 18:14:12 +0100 Subject: [PATCH 21/33] Fix await, yield and yield from --- mypyc/irbuild/builder.py | 46 +++++++++++++++++++++++++++++ mypyc/irbuild/expression.py | 3 ++ mypyc/irbuild/statement.py | 3 ++ mypyc/test-data/run-async.test | 23 +++++++++++++++ mypyc/test-data/run-generators.test | 43 +++++++++++++++++++++++++++ 5 files changed, 118 insertions(+) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index ef8cf51f664c..e94ce191adc8 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -21,6 +21,7 @@ TYPE_VAR_TUPLE_KIND, ArgKind, AssignmentExpr, + AwaitExpr, CallExpr, Decorator, Expression, @@ -43,6 +44,8 @@ TypeInfo, TypeParam, Var, + YieldExpr, + YieldFromExpr, ) from mypy.traverser import TraverserVisitor from mypy.types import ( @@ -295,9 +298,15 @@ def __init__( # top-level expression. Used to avoid borrowing an attribute over the whole # expression when the borrow root could be rebound (and thus freed) partway. self.reassigned_in_expr: set[SymbolNode] = set() + # Whether the current top-level expression contains a suspension point + # (await, yield or yield from). A whole-expression borrow can't span such a + # point, since the borrowed value (and its root) live in registers that are + # not spilled into the generator environment across the suspend. + self.expr_has_suspend = False # Saved expression state for enclosing functions (see enter()/leave()). self.expression_depth_stack: list[int] = [] self.reassigned_in_expr_stack: list[set[SymbolNode]] = [] + self.expr_has_suspend_stack: list[bool] = [] # When set, load_globals_dict uses this module instead of self.module_name. # Used by generate_attr_defaults_init for cross-module inherited defaults. @@ -332,6 +341,7 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V self.expression_depth += 1 if self.expression_depth == 1: self.reassigned_in_expr = find_walrus_targets(node) + self.expr_has_suspend = expr_has_suspend(node) old_can_borrow = self.can_borrow self.can_borrow = can_borrow try: @@ -350,6 +360,7 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V if self.expression_depth == 0: self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) self.reassigned_in_expr = set() + self.expr_has_suspend = False return res else: try: @@ -1369,8 +1380,10 @@ def enter(self, fn_info: FuncInfo | str = "", *, ret_type: RType = none_rprimiti # expression. Save the outer expression state and start fresh. self.expression_depth_stack.append(self.expression_depth) self.reassigned_in_expr_stack.append(self.reassigned_in_expr) + self.expr_has_suspend_stack.append(self.expr_has_suspend) self.expression_depth = 0 self.reassigned_in_expr = set() + self.expr_has_suspend = False if fn_info.is_generator: self.nonlocal_control.append(GeneratorNonlocalControl()) else: @@ -1386,6 +1399,7 @@ def leave(self) -> tuple[list[Register], list[RuntimeArg], list[BasicBlock], RTy self.nonlocal_control.pop() self.expression_depth = self.expression_depth_stack.pop() self.reassigned_in_expr = self.reassigned_in_expr_stack.pop() + self.expr_has_suspend = self.expr_has_suspend_stack.pop() self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] return builder.args, runtime_args, builder.blocks, ret_type, fn_info @@ -1800,6 +1814,38 @@ def find_walrus_targets(expr: Expression) -> set[SymbolNode]: return collector.targets +class SuspendDetector(TraverserVisitor): + """Detect await/yield/yield from expressions in a subtree.""" + + def __init__(self) -> None: + self.found = False + + def visit_await_expr(self, o: AwaitExpr) -> None: + self.found = True + + def visit_yield_expr(self, o: YieldExpr) -> None: + self.found = True + + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: + self.found = True + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + # A lambda body forms its own function (and suspension) context. + pass + + +def expr_has_suspend(expr: Expression) -> bool: + """Does evaluating 'expr' involve a suspension point (await/yield/yield from)? + + A whole-expression borrow can't safely span a suspension point, since the + borrowed value and its borrow root are held in registers that aren't spilled + into the generator environment across the suspend. + """ + detector = SuspendDetector() + expr.accept(detector) + return detector.found + + def create_type_params( builder: IRBuilder, typing_mod: Value, type_args: list[TypeParam], line: int ) -> list[Value]: diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 0a3063f08dff..d1e6d025be81 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -322,6 +322,9 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: # Don't borrow across the whole expression if the borrow root can be # rebound via a walrus assignment and not builder.root_is_reassigned(obj) + # Don't borrow across a suspension point (await/yield/yield from), since + # the borrow would not survive the suspend (registers aren't spilled). + and not builder.expr_has_suspend ): scope = KEEP_ALIVE_WHOLE_EXPRESSION borrow = (can_borrow and builder.can_borrow) or scope == KEEP_ALIVE_WHOLE_EXPRESSION diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index c28c0933ffe0..21f47b190c30 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -89,6 +89,7 @@ from mypyc.irbuild.builder import ( IRBuilder, create_type_params, + expr_has_suspend, find_walrus_targets, int_borrow_friendly_op, ) @@ -170,9 +171,11 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: # we shouldn't call builder.accept here. builder.expression_depth += 1 builder.reassigned_in_expr = find_walrus_targets(stmt.expr) + builder.expr_has_suspend = expr_has_suspend(stmt.expr) stmt.expr.accept(builder.visitor) builder.expression_depth -= 1 builder.reassigned_in_expr = set() + builder.expr_has_suspend = False builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index cf9b54368c27..69779f9ff90a 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1974,3 +1974,26 @@ async def async_iter(vals: list[int]) -> AsyncIterator[int]: yield v [typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAwait] +from typing import Final +from testutil import async_val + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +async def f(c: C) -> str: + # c.s is a final attr; it must not be borrowed across the await, which is a + # suspension point in the middle of the expression. + return c.s + await async_val("hello") + +[file driver.py] +from native import f, C +from testutil import run_generator + +for i in range(50): + c = C("value" + str(i)) + yields, val = run_generator(f(c), inputs=["world"]) + assert yields == ("hello",), yields + assert val == "value" + str(i) + "world", repr(val) diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index cf1dac7c5733..a23af2ed6eea 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -944,3 +944,46 @@ from typing import Optional, Union def test_compiledGeneratorEmptyTuple() -> None: jobs: Generator[Optional[str], None, None] = (_ for _ in ()) assert list(jobs) == [] + +[case testBorrowedFinalAttrAcrossYield] +from typing import Final, Generator + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C) -> Generator[str, str, None]: + # c.s is a final attr; it must not be borrowed across the yield, which is + # a suspension point in the middle of the expression. + x = c.s + (yield "first") + yield x + +def test_borrow_across_yield() -> None: + for i in range(50): + c = C("value" + str(i)) + g = f(c) + assert next(g) == "first" + assert g.send("world") == "value" + str(i) + "world" + +def yield_from_gen(c: C) -> Generator[str, str, str]: + r = yield "inner" + return c.s + r + +def g2(c: C) -> Generator[str, str, str]: + # Borrow a final attr across a yield from expression. + x = c.s + (yield from yield_from_gen(c)) + return x + +def test_borrow_across_yield_from() -> None: + for i in range(50): + c = C("v" + str(i)) + gen = g2(c) + assert next(gen) == "inner" + try: + gen.send("!") + except StopIteration as e: + assert e.value == "v" + str(i) + "v" + str(i) + "!", e.value + else: + assert False + +[typing fixtures/typing-full.pyi] From e6df3f293df3fba61eaea732c1318fe0126a2371 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 7 Jul 2026 16:49:17 +0100 Subject: [PATCH 22/33] Fix async comprehensions --- mypyc/irbuild/builder.py | 15 +++++++++++++ mypyc/test-data/run-async.test | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index e94ce191adc8..81d6c7cdc429 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -24,8 +24,10 @@ AwaitExpr, CallExpr, Decorator, + DictionaryComprehension, Expression, FuncDef, + GeneratorExpr, IndexExpr, IntExpr, LambdaExpr, @@ -1829,6 +1831,19 @@ def visit_yield_expr(self, o: YieldExpr) -> None: def visit_yield_from_expr(self, o: YieldFromExpr) -> None: self.found = True + def visit_generator_expr(self, o: GeneratorExpr) -> None: + # An 'async for' clause suspends via an implicit await on __anext__ that + # isn't represented as an AwaitExpr node in the AST (list/set comprehensions + # delegate to a GeneratorExpr, so they are covered here too). + if any(o.is_async): + self.found = True + super().visit_generator_expr(o) + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + if any(o.is_async): + self.found = True + super().visit_dictionary_comprehension(o) + def visit_lambda_expr(self, o: LambdaExpr) -> None: # A lambda body forms its own function (and suspension) context. pass diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 69779f9ff90a..dd3e21a7bfc2 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1997,3 +1997,44 @@ for i in range(50): yields, val = run_generator(f(c), inputs=["world"]) assert yields == ("hello",), yields assert val == "value" + str(i) + "world", repr(val) + +[case testBorrowedFinalAttrAcrossAsyncComprehension] +import asyncio +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +class AsyncRange: + def __init__(self, n: int) -> None: + self.n = n + self.i = 0 + + def __aiter__(self) -> "AsyncRange": + return self + + async def __anext__(self) -> str: + if self.i >= self.n: + raise StopAsyncIteration + i = self.i + self.i += 1 + # Suspend in the middle of iteration. + await asyncio.sleep(0) + return "item" + str(i) + +async def f(c: C) -> str: + # c.s is a final attr; it must not be borrowed across the async comprehension, + # which suspends via an implicit await on __anext__ that isn't represented as + # an AwaitExpr node in the AST. + return c.s + "".join([x async for x in AsyncRange(3)]) + +async def test_async_comprehension_borrow() -> None: + for i in range(50): + c = C("value" + str(i)) + assert await f(c) == "value" + str(i) + "item0item1item2" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] From 1691a9f8ed8826a3c2b0a13c1ac165241977cd30 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 12:48:44 +0100 Subject: [PATCH 23/33] Refactor after rebase --- mypyc/irbuild/builder.py | 8 -------- mypyc/irbuild/expression.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 81d6c7cdc429..e94112b0e669 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1641,14 +1641,6 @@ def is_final_native_attr_ref(self, expr: MemberExpr) -> bool: return expr.name in ir.final_attributes return False - def is_final_instance_attr_ref(self, expr: MemberExpr) -> bool: - obj_rtype = self.node_type(expr.expr) - return ( - isinstance(obj_rtype, RInstance) - and obj_rtype.class_ir.is_ext_class - and any(expr.name in ir.final_attributes for ir in obj_rtype.class_ir.mro) - ) - def root_is_reassigned(self, v: Value) -> bool: """Is the root local variable a borrow chain 'v' reads from reassigned this expression? diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index d1e6d025be81..6d8de9758e38 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -313,7 +313,7 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) - is_final = builder.is_final_instance_attr_ref(expr) + is_final = builder.is_final_native_attr_ref(expr) scope = KEEP_ALIVE_SHORT_LIVED if ( is_final From 40527e64346d92146bc0cec84b7764076e9f204b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 14:16:03 +0100 Subject: [PATCH 24/33] Fix bug --- mypyc/irbuild/builder.py | 4 ++++ mypyc/test-data/run-async.test | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index e94112b0e669..f3db8c239506 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1442,12 +1442,16 @@ def enter_borrow_scope(self, line: int) -> Iterator[None]: """Enter new borrow scope from which borrows can't leak to outer expressions.""" checkpoint = self.builder.keep_alive_checkpoint() old_expression_depth = self.expression_depth + old_reassigned_in_expr = self.reassigned_in_expr + old_expr_has_suspend = self.expr_has_suspend self.expression_depth = 0 try: yield self.builder.flush_keep_alives_since(line, checkpoint) finally: self.expression_depth = old_expression_depth + self.reassigned_in_expr = old_reassigned_in_expr + self.expr_has_suspend = old_expr_has_suspend @contextmanager def enter_method( diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index dd3e21a7bfc2..a7a08e7ced5f 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2038,3 +2038,32 @@ async def test_async_comprehension_borrow() -> None: async def sleep(t: float) -> None: ... [typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAwaitAfterComprehension] +import asyncio +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +async def sink() -> str: + await asyncio.sleep(0) + return "z" + +async def f(c: C, xs: list[int]) -> str: + # A (sync) comprehension enters a nested borrow scope earlier in the same + # top-level expression. It must not clear the enclosing expression's suspend + # flag, or c.s (a final attr) gets borrowed across the await below, which + # produces invalid C (the borrow does not survive the suspension point). + return str([x for x in xs]) + (c.s + await sink()) + +async def test_borrow_final_attr_across_await_after_comprehension() -> None: + for i in range(50): + c = C("v" + str(i)) + assert await f(c, [1, 2, 3]) == "[1, 2, 3]" + "v" + str(i) + "z" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] From b32d34a4db0f6553321b88bf16ac5a0222524a1a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 14:50:45 +0100 Subject: [PATCH 25/33] Update comment --- mypyc/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/common.py b/mypyc/common.py index 7cf5a3a01e43..fa34647c5c72 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -73,7 +73,7 @@ KEEP_ALIVE_SHORT_LIVED: Final = 0 # Keep value alive longer, up to until the current top-level expression has been fully # evaluated. This allows keeping alive values across function calls and arbitrary -# computation. +# computation. Note that some expressions (e.g. lambdas), restrict the scope of borrowing. KEEP_ALIVE_WHOLE_EXPRESSION: Final = 1 # Runtime C library files that are always included (some ops may bring From de28c6b81aaa1731b7ce1e9f24ae222834973209 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 14:58:12 +0100 Subject: [PATCH 26/33] Update test --- mypyc/test-data/irbuild-final.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test index 90aeaf24bee4..0ede988a4320 100644 --- a/mypyc/test-data/irbuild-final.test +++ b/mypyc/test-data/irbuild-final.test @@ -196,7 +196,7 @@ L0: keep_alive r0, c return r3 -[case testFinalAttrExpressionKinds2] +[case testFinalAttrExpressionKinds2_withgil] from typing import Final class C: From 00e38505fe5135d9b225d39cba40e8c9f35a924f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 15:03:43 +0100 Subject: [PATCH 27/33] Fix more tests --- mypyc/test-data/refcount.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index a1a512880124..ded719c7f311 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2415,7 +2415,7 @@ L0: r1 = C(r0) return r1 -[case testBorrowedFinalAttribute2] +[case testBorrowedFinalAttribute2_withgil] from typing import Final class C: @@ -2457,7 +2457,7 @@ L0: dec_ref r4 return r6 -[case testBorrowedFinalAttribute3] +[case testBorrowedFinalAttribute3_withgil] from typing import Final class C: @@ -2840,7 +2840,7 @@ L2: L3: return r2 -[case testBorrowedFinalInListComprehension] +[case testBorrowedFinalInListComprehension_withgil] from typing import Final class C: @@ -2889,7 +2889,7 @@ L3: L4: return r1 -[case testBorrowedFinalInSetComprehension] +[case testBorrowedFinalInSetComprehension_withgil] from typing import Final class C: From 30de6e0b0f4faa809ade88861f2afe4474659ad5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 15:37:16 +0100 Subject: [PATCH 28/33] Refactor --- mypyc/irbuild/builder.py | 13 +++++++++---- mypyc/irbuild/expression.py | 18 ++++++++---------- mypyc/irbuild/ll_builder.py | 37 ++++++++++++++++++++++++------------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index f3db8c239506..d8ae71e33bf5 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1439,15 +1439,20 @@ def enter_scope(self, fn_info: FuncInfo) -> Iterator[None]: @contextmanager def enter_borrow_scope(self, line: int) -> Iterator[None]: - """Enter new borrow scope from which borrows can't leak to outer expressions.""" - checkpoint = self.builder.keep_alive_checkpoint() + """Enter new borrow scope from which borrows can't leak to outer expressions. + + This is a borrow region (see LowLevelIRBuilder.borrow_region) that also + resets the per-expression borrowing heuristic state, since the body forms + its own top-level expression context (e.g. a comprehension iteration or a + lambda body). + """ old_expression_depth = self.expression_depth old_reassigned_in_expr = self.reassigned_in_expr old_expr_has_suspend = self.expr_has_suspend self.expression_depth = 0 try: - yield - self.builder.flush_keep_alives_since(line, checkpoint) + with self.builder.borrow_region(line): + yield finally: self.expression_depth = old_expression_depth self.reassigned_in_expr = old_reassigned_in_expr diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 6d8de9758e38..bf2809b31497 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -916,19 +916,17 @@ def transform_conditional_expr(builder: IRBuilder, expr: ConditionalExpr) -> Val target = Register(expr_type) builder.activate_block(if_body) - true_checkpoint = builder.builder.keep_alive_checkpoint() - true_value = builder.accept(expr.if_expr) - true_value = builder.coerce(true_value, expr_type, expr.line) - builder.add(Assign(target, true_value, expr.line)) - builder.builder.flush_keep_alives_since(expr.line, true_checkpoint) + with builder.builder.borrow_region(expr.line): + true_value = builder.accept(expr.if_expr) + true_value = builder.coerce(true_value, expr_type, expr.line) + builder.add(Assign(target, true_value, expr.line)) builder.goto(next_block) builder.activate_block(else_body) - false_checkpoint = builder.builder.keep_alive_checkpoint() - false_value = builder.accept(expr.else_expr) - false_value = builder.coerce(false_value, expr_type, expr.line) - builder.add(Assign(target, false_value, expr.line)) - builder.builder.flush_keep_alives_since(expr.line, false_checkpoint) + with builder.builder.borrow_region(expr.line): + false_value = builder.accept(expr.else_expr) + false_value = builder.coerce(false_value, expr_type, expr.line) + builder.add(Assign(target, false_value, expr.line)) builder.goto(next_block) builder.activate_block(next_block) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 41944766981b..09400fd957b1 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -7,7 +7,8 @@ from __future__ import annotations import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from typing import Final, TypeGuard, cast from mypy.argmap import map_actuals_to_formals @@ -399,6 +400,18 @@ def flush_keep_alives_since(self, line: int, checkpoint: int) -> None: self.add(KeepAlive([entry.value for entry in new_keep_alives], line)) self.keep_alives = [entry for entry in self.keep_alives if entry.seq < checkpoint] + @contextmanager + def borrow_region(self, line: int) -> Iterator[None]: + """Confine borrows created in this region: flush them when leaving it. + + Keep-alives added inside the region (regardless of their duration scope) + are flushed on exit, so borrows can't leak past a lexical boundary such as + a conditional branch, an 'and'/'or' branch or a comprehension iteration. + """ + checkpoint = self.keep_alive_checkpoint() + yield + self.flush_keep_alives_since(line, checkpoint) + def add_keep_alive(self, value: Value, scope: int) -> None: self.keep_alives.append(PendingKeepAlive(value, scope, self.next_keep_alive_seq)) self.next_keep_alive_seq += 1 @@ -2156,22 +2169,20 @@ def shortcircuit_helper( # it is the right side if the left is false. true_body, false_body = (right_body, left_body) if op == "and" else (left_body, right_body) - left_checkpoint = self.keep_alive_checkpoint() - left_value = left() - self.add_bool_branch(left_value, true_body, false_body) + with self.borrow_region(line): + left_value = left() + self.add_bool_branch(left_value, true_body, false_body) - self.activate_block(left_body) - left_coerced = self.coerce(left_value, expr_type, line) - self.add(Assign(target, left_coerced, line)) - self.flush_keep_alives_since(line, left_checkpoint) + self.activate_block(left_body) + left_coerced = self.coerce(left_value, expr_type, line) + self.add(Assign(target, left_coerced, line)) self.goto(next_block) self.activate_block(right_body) - right_checkpoint = self.keep_alive_checkpoint() - right_value = right() - right_coerced = self.coerce(right_value, expr_type, line) - self.add(Assign(target, right_coerced, line)) - self.flush_keep_alives_since(line, right_checkpoint) + with self.borrow_region(line): + right_value = right() + right_coerced = self.coerce(right_value, expr_type, line) + self.add(Assign(target, right_coerced, line)) self.goto(next_block) self.activate_block(next_block) From 2be6c4bf19d35a1f82adfd2ae79fe2745a2647d1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 16:04:00 +0100 Subject: [PATCH 29/33] Add codespell ignore work --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 361290e55a1d..ec31c8c8ea16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: codespell args: - - --ignore-words-list=HAX,Nam,ccompiler,ot,statics,whet,zar + - --ignore-words-list=HAX,Nam,alives,ccompiler,ot,statics,whet,zar exclude: ^(mypy/test/|mypy/typeshed/|mypyc/test-data/|test-data/).+$ - repo: https://github.com/rhysd/actionlint rev: v1.7.7 From fb65dfaf32eee9dc84d4da08754c2a4cf8e36804 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 16:05:31 +0100 Subject: [PATCH 30/33] Properly fix codespell --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec31c8c8ea16..548b568621a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: codespell args: - - --ignore-words-list=HAX,Nam,alives,ccompiler,ot,statics,whet,zar + - --ignore-words-list=HAX,Nam,ccompiler,keep-alives,ot,statics,whet,zar exclude: ^(mypy/test/|mypy/typeshed/|mypyc/test-data/|test-data/).+$ - repo: https://github.com/rhysd/actionlint rev: v1.7.7 From 883c2509712b356057bd557e19c82b33053131c0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Jul 2026 13:37:43 +0100 Subject: [PATCH 31/33] Fix borrowing over casts --- mypyc/irbuild/expression.py | 2 + mypyc/irbuild/ll_builder.py | 17 ++++- mypyc/test-data/refcount.test | 116 ++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index bf2809b31497..91e1e55d8da2 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -342,6 +342,8 @@ def value_borrow_scope(builder: IRBuilder, v: Value) -> int: if isinstance(v, GetAttr) and v.is_borrowed: return min(v.borrow_scope, value_borrow_scope(builder, v.obj)) elif isinstance(v, Cast) and v.is_borrowed: + # A cast just propagates the value (when successful), so the scope is + # determined by the source. return value_borrow_scope(builder, v.src) elif isinstance(v, (CallC, PrimitiveOp)) and v.is_borrowed: # Values borrowed from a C function (e.g. a borrowed list/vec item) may be diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 09400fd957b1..aafdb73fed84 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -416,6 +416,21 @@ def add_keep_alive(self, value: Value, scope: int) -> None: self.keep_alives.append(PendingKeepAlive(value, scope, self.next_keep_alive_seq)) self.next_keep_alive_seq += 1 + def keep_borrow_source_alive(self, value: Value, borrow_scope: int) -> None: + """Ensure the source of borrowed value will be alive for given borrow scope. + + A borrow keeps reading from 'value', so 'value' must stay valid for the + borrow's whole scope. When 'value' is itself a borrowed cast it holds no + reference of its own (and coerce() only kept its source alive short-term), + so we follow the cast to the underlying owned value and keep *that* alive + for the requested scope too. Otherwise, a longer-lived borrow through a cast + (e.g. a Final attribute read via cast(T, make())) could read freed memory. + """ + self.add_keep_alive(value, borrow_scope) + while isinstance(value, Cast) and value.is_borrowed: + value = value.src + self.add_keep_alive(value, borrow_scope) + def debug_print(self, toprint: str | Value) -> None: if isinstance(toprint, str): toprint = self.load_str(toprint) @@ -874,7 +889,7 @@ def get_attr( # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - self.add_keep_alive(obj, borrow_scope) + self.keep_borrow_source_alive(obj, borrow_scope) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index ded719c7f311..6aba56e9ecfe 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2965,3 +2965,119 @@ L5: L6: dec_ref x goto L4 + +[case testBorrowedFinalAttributeThroughCast] +from typing import Final, cast + +class C: + def __init__(self, x: object) -> None: + self.x: Final = x + +def make() -> object: + return C(1) + +def g(a: object) -> object: + return a + +def h() -> object: + return g(cast(C, make()).x) +[out] +def C.__init__(self, x): + self :: __main__.C + x :: object +L0: + inc_ref x + self.x = x + return 1 +def make(): + r0 :: object + r1 :: __main__.C +L0: + r0 = object 1 + r1 = C(r0) + return r1 +def g(a): + a :: object +L0: + inc_ref a + return a +def h(): + r0 :: object + r1 :: __main__.C + r2, r3 :: object +L0: + r0 = make() + r1 = borrow cast(__main__.C, r0) + r2 = borrow r1.x + r3 = g(r2) + dec_ref r0 + return r3 + +[case testBorrowedFinalAttributeCastChain] +from typing import Final, cast + +class Inner: + def __init__(self, x: object) -> None: + self.x: Final = x + +class Outer: + def __init__(self, inner: object) -> None: + self.inner: Final = inner + +def g(a: object) -> object: + return a + +def make() -> object: + return Outer(Inner(1)) + +def h() -> object: + # cast( Inner, ).x + o = cast(Outer, make()) + return g(cast(Inner, o.inner).x) +[out] +def Inner.__init__(self, x): + self :: __main__.Inner + x :: object +L0: + inc_ref x + self.x = x + return 1 +def Outer.__init__(self, inner): + self :: __main__.Outer + inner :: object +L0: + inc_ref inner + self.inner = inner + return 1 +def g(a): + a :: object +L0: + inc_ref a + return a +def make(): + r0 :: object + r1 :: __main__.Inner + r2 :: __main__.Outer +L0: + r0 = object 1 + r1 = Inner(r0) + r2 = Outer(r1) + dec_ref r1 + return r2 +def h(): + r0 :: object + r1, o :: __main__.Outer + r2 :: object + r3 :: __main__.Inner + r4, r5 :: object +L0: + r0 = make() + r1 = cast(__main__.Outer, r0) + o = r1 + r2 = borrow o.inner + r3 = borrow cast(__main__.Inner, r2) + r4 = borrow r3.x + r5 = g(r4) + dec_ref o + return r5 + From 8dd7527be9ba6dcd0dd1e48d2f6501dcc1c4726e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:40:18 +0000 Subject: [PATCH 32/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/test-data/refcount.test | 1 - 1 file changed, 1 deletion(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 6aba56e9ecfe..d6f745cc8fb2 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -3080,4 +3080,3 @@ L0: r5 = g(r4) dec_ref o return r5 - From 082428cd16d4f1d25ed3bdde9ecf2dbb861cff46 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Jul 2026 13:59:55 +0100 Subject: [PATCH 33/33] Work around 32-bit platform test failure --- mypyc/test-data/refcount.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index d6f745cc8fb2..bedfabac791d 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2569,7 +2569,7 @@ def D.foo(self): L0: return 2 -[case testBorrowedFinalAttribute5] +[case testBorrowedFinalAttribute5_64bit] from typing import Final from librt.vecs import vec from mypy_extensions import i64