diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index a0d2dc81e1648..1645e9498f207 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -354,6 +354,117 @@ The ``closed`` keyword can also be used in class-based syntax: class HasOnlyName(TypedDict, closed=True): name: str +Extra items +----------- + +Instead of fully closing a TypedDict, you can use the ``extra_items`` +keyword, introduced to ``TypedDict`` in Python 3.15 (and available via +``typing_extensions.TypedDict`` in older versions), to allow keys that +are not known in advance, as long as their values have a specific type +(:pep:`728`): + +.. code-block:: python + + class Movie(TypedDict, extra_items=bool): + name: str + + a: Movie = {"name": "Blade Runner", "novel_adaptation": True} # OK + b: Movie = { + "name": "Blade Runner", + "year": 1982, # Error: "int" is not assignable to "bool" + } + +Extra items are treated as non-required items with the declared value +type, so they can be read, assigned, and deleted like any other +non-required key: + +.. code-block:: python + + def f(movie: Movie) -> None: + reveal_type(movie["name"]) # Revealed type is "str" + reveal_type(movie["novel_adaptation"]) # Revealed type is "bool" + movie["novel_adaptation"] = False # OK + del movie["novel_adaptation"] # OK + +The ``extra_items`` argument accepts the ``ReadOnly[]`` qualifier, in +which case the extra items cannot be mutated, and behave covariantly +during assignability checks; ``Required[]`` and ``NotRequired[]`` are +not allowed. ``extra_items`` cannot be combined with ``closed`` in the +same definition, because ``closed=True`` is equivalent to +``extra_items=Never``. + +Like regular items, ``extra_items`` is inherited by subclasses, and can +only be redeclared (with a narrower type) if it is read-only in the +parent. Any field a subclass adds must be compatible with the parent's +``extra_items``: assignable to it if it is read-only, non-required and +equivalent to it otherwise: + +.. code-block:: python + + class MovieBase(TypedDict, extra_items=int | None): + name: str + + class MovieA(MovieBase): # Error: "year" is required + year: int | None + + class MovieB(MovieBase): # OK + year: NotRequired[int | None] + +A TypedDict with ``extra_items`` also accepts arbitrary keyword +arguments of that type when used with ``Unpack`` to type ``**kwargs``: + +.. code-block:: python + + class MovieExtra(TypedDict, extra_items=int): + name: str + + def g(**kwargs: Unpack[MovieExtra]) -> None: ... + + # Equivalent to: + def g2(*, name: str, **kwargs: int) -> None: ... + + g(name="No Country for Old Men", year=2007) # OK + +Conversely, when calling a function with ``**movie``, the extra items +that ``movie`` may contain are checked against the function's +``**kwargs`` type: + +.. code-block:: python + + def plain(**kwargs: str) -> None: ... + + def h(movie: MovieExtra) -> None: + plain(**movie) # Error: incompatible type "**MovieExtra"; expected "str" + +Since all the value types of a TypedDict with ``extra_items`` (or +``closed=True``) are known, it is assignable to ``Mapping[str, VT]`` +whenever they are all assignable to ``VT``, its ``values()`` and +``items()`` methods return precisely typed views, and it can be indexed +with an arbitrary ``str`` key: + +.. code-block:: python + + def h(movie: MovieExtra, key: str) -> None: + m: Mapping[str, int | str] = movie # OK + reveal_type(movie.items()) # Revealed type is "dict_items[str, str | int]" + reveal_type(movie[key]) # Revealed type is "str | int" + +Moreover, if every item is non-required, mutable, and consistent with +``extra_items``, the TypedDict is assignable to ``dict[str, VT]`` and +supports operations that would otherwise be unsafe on a TypedDict, such +as ``clear()``, ``popitem()``, and assigning or deleting arbitrary +``str`` keys: + +.. code-block:: python + + class IntDict(TypedDict, extra_items=int): + pass + + def counters(d: IntDict, key: str) -> None: + v: dict[str, int] = d # OK + d[key] = 42 # OK + d.clear() # OK + Unions of TypedDicts -------------------- diff --git a/mypy/argmap.py b/mypy/argmap.py index c30f6bd43f13f..2128d6f623a87 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -91,6 +91,16 @@ def map_actuals_to_formals( formal_to_actual[formal_names.index(name)].append(ai) elif nodes.ARG_STAR2 in formal_kinds: formal_to_actual[formal_kinds.index(nodes.ARG_STAR2)].append(ai) + if ( + actualt.extra_items is not None + and not actualt.is_closed + and nodes.ARG_STAR2 in formal_kinds + ): + # The TypedDict may contain extra keys of the extra_items type + # (PEP 728); map the pseudo-item to the callee's **kwargs so it + # is checked as well. Without a **kwargs formal the extra keys + # are ignored, like the possible extra keys of an open TypedDict. + formal_to_actual[formal_kinds.index(nodes.ARG_STAR2)].append(ai) else: # We don't exactly know which **kwargs are provided by the # caller, so we'll defer until all the other unambiguous @@ -248,7 +258,12 @@ def expand_actual_type( assert formal_name is not None else: # Pick an arbitrary item if no specified keyword is expected. - formal_name = (set(actual_type.items.keys()) - self.kwargs_used).pop() + unused_names = set(actual_type.items.keys()) - self.kwargs_used + if not unused_names and actual_type.extra_items is not None: + # The named items are exhausted, so this mapping is for the + # extra_items pseudo-item (PEP 728). + return actual_type.extra_items + formal_name = unused_names.pop() self.kwargs_used.add(formal_name) return actual_type.items[formal_name] elif isinstance(actual_type, Instance) and is_subtype( diff --git a/mypy/cache.py b/mypy/cache.py index ebe36e8940b81..6bd5b21895321 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -69,7 +69,7 @@ from mypy_extensions import u8 # High-level cache layout format -CACHE_VERSION: Final = 10 +CACHE_VERSION: Final = 11 # Type used internally to represent errors: # (path, line, column, end_line, end_column, severity, message, code) diff --git a/mypy/checker.py b/mypy/checker.py index 212ddd65d9122..c7a51edb700f7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3229,7 +3229,7 @@ def check_typeddict_inheritance(self, defn: ClassDef) -> None: data = defn.info.typeddict_data if data is None or not data.ready: return - for base, base_items in data.bases: + for base, base_items, base_extra_items in data.bases: assert base.typeddict_type for field_name, base_type in base_items.items(): field_type = td.items[field_name] @@ -3259,6 +3259,47 @@ def check_typeddict_inheritance(self, defn: ClassDef) -> None: f"with a mutually compatible type", source.ctx, ) + if base_extra_items is None: + continue + base_extra_readonly = base.typeddict_type.extra_items_readonly + if not isinstance(get_proper_type(base_extra_items), UninhabitedType): + # Fields not known to the base must be compatible with its extra_items + # pseudo-item, like any other item redeclaration. (For a closed base, + # semantic analysis already rejected such fields.) + for field_name, field_type in td.items.items(): + if field_name in base_items: + continue + if base_extra_readonly: + is_compatible = is_subtype(field_type, base_extra_items) + else: + is_compatible = is_equivalent(field_type, base_extra_items) + if is_compatible: + continue + source = data.field_sources[field_name] + if source.base is None: + self.fail( + f'Type of field "{field_name}" incompatible with "extra_items" ' + f'of base class "{base.name}"', + source.ctx, + ) + else: + self.fail( + f'Type of field "{field_name}" from base class ' + f'"{source.base.name}" incompatible with "extra_items" of base ' + f'class "{base.name}"', + source.ctx, + ) + if td.extra_items is not None: + # The resolved pseudo-item must be compatible with each base pseudo-item. + if base_extra_readonly: + is_compatible = is_subtype(td.extra_items, base_extra_items) + else: + is_compatible = is_equivalent(td.extra_items, base_extra_items) + if not is_compatible: + self.fail( + f'Definition of "extra_items" incompatible with base class "{base.name}"', + defn, + ) def visit_import_from(self, node: ImportFrom) -> None: for name, _ in node.names: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 44855f49afaf9..94b201434d6ce 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -127,6 +127,7 @@ is_same_type, is_subtype, non_method_protocol_members, + typed_dict_dict_value_type, ) from mypy.traverser import ( all_name_and_member_expressions, @@ -857,6 +858,13 @@ def validate_typeddict_kwargs( last_open_star_found, code=codes.TYPEDDICT_ITEM, ) + elif callee.extra_items is not None: + self.chk.fail( + "Cannot unpack item that may contain incompatible extra keys into a " + 'TypedDict with "extra_items"', + last_open_star_found, + code=codes.TYPEDDICT_ITEM, + ) absent_keys = [] for key in callee.items: if key not in callee.required_keys and key not in result: @@ -931,8 +939,26 @@ def validate_star_typeddict_item( # If this key is not required at least in some item of a union # it may not shadow previous item, so we need to type check both. result[key].append(arg) - all_closed = all(t.is_closed for t in possible_tds) - return True, any_fallback or not all_closed + all_absorbed = all(self.star_source_absorbed(td, callee) for td in possible_tds) + return True, any_fallback or not all_absorbed + + def star_source_absorbed(self, source: TypedDictType, callee: TypedDictType) -> bool: + """Can ** unpacking `source` add no undeclared or incompatible keys to `callee`? + + Keys declared on `source` are checked individually elsewhere; this is about + the keys covered by its PEP 728 pseudo-item. + """ + src_extra = source.extra_item() + if src_extra.typ is None: + # A default-open source may contain arbitrary extra keys. + return False + if isinstance(get_proper_type(src_extra.typ), UninhabitedType): + # A closed source contributes no undeclared keys. + return True + callee_extra = callee.extra_item() + if callee_extra.typ is None or callee.is_closed: + return False + return is_subtype(src_extra.typ, callee_extra.typ) def valid_unpack_fallback_item(self, typ: ProperType) -> bool: if isinstance(typ, AnyType): @@ -952,7 +978,12 @@ def match_typeddict_call_with_dict( result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee) if result is not None: validated_kwargs, _ = result - return callee.required_keys <= set(validated_kwargs.keys()) <= set(callee.items.keys()) + keys = set(validated_kwargs.keys()) + if not callee.required_keys <= keys: + return False + return keys <= set(callee.items.keys()) or ( + callee.extra_items is not None and not callee.is_closed + ) else: return False @@ -993,13 +1024,22 @@ def typeddict_callable(self, info: TypeInfo) -> CallableType: def typeddict_callable_from_context( self, callee: TypedDictType, variables: Sequence[TypeVarLikeType] | None = None ) -> CallableType: + arg_types = list(callee.items.values()) + arg_kinds = [ + ArgKind.ARG_NAMED if name in callee.required_keys else ArgKind.ARG_NAMED_OPT + for name in callee.items + ] + arg_names: list[str | None] = list(callee.items.keys()) + if callee.extra_items is not None and not callee.is_closed: + # PEP 728: arbitrary extra keyword arguments of the extra_items type + # are accepted when constructing a TypedDict. + arg_types.append(callee.extra_items) + arg_kinds.append(ArgKind.ARG_STAR2) + arg_names.append(None) return CallableType( - list(callee.items.values()), - [ - ArgKind.ARG_NAMED if name in callee.required_keys else ArgKind.ARG_NAMED_OPT - for name in callee.items - ], - list(callee.items.keys()), + arg_types, + arg_kinds, + arg_names, callee, self.named_type("builtins.type"), variables=variables, @@ -1015,14 +1055,19 @@ def check_typeddict_call_with_kwargs( always_present_keys: set[str], ) -> Type: actual_keys = kwargs.keys() + extra_keys = actual_keys - callee.items.keys() + # PEP 728: a TypedDict with a non-Never extra_items pseudo-item accepts + # arbitrary extra keys, whose values are checked against it below. + allows_extra_keys = callee.extra_items is not None and not callee.is_closed if callee.to_be_mutated: assigned_readonly_keys = actual_keys & callee.readonly_keys + if allows_extra_keys and callee.extra_items_readonly: + assigned_readonly_keys |= extra_keys if assigned_readonly_keys: self.msg.readonly_keys_mutated(assigned_readonly_keys, context=context) - if not ( - callee.required_keys <= always_present_keys and actual_keys <= callee.items.keys() - ): - if not (actual_keys <= callee.items.keys()): + known_keys_only = actual_keys <= callee.items.keys() or allows_extra_keys + if not (callee.required_keys <= always_present_keys and known_keys_only): + if not known_keys_only: self.msg.unexpected_typeddict_keys( callee, expected_keys=[ @@ -1098,6 +1143,19 @@ def check_typeddict_call_with_kwargs( lvalue_name=f'TypedDict item "{item_name}"', rvalue_name="expression", ) + if allows_extra_keys and ret_type.extra_items is not None: + for item_name in extra_keys: + for item_value in kwargs[item_name]: + self.chk.check_simple_assignment( + lvalue_type=ret_type.extra_items, + rvalue=item_value, + context=item_value, + msg=ErrorMessage( + message_registry.INCOMPATIBLE_TYPES.value, code=codes.TYPEDDICT_ITEM + ), + lvalue_name=f'extra TypedDict item "{item_name}"', + rvalue_name="expression", + ) return orig_ret_type @@ -4822,19 +4880,42 @@ def visit_typeddict_index_expr( ): key_names.append(key_type.value) else: + result = self.typeddict_arbitrary_str_key_item(td_type, typ, setitem) + if result is not None: + return result, set() self.msg.typeddict_key_must_be_string_literal(td_type, index) return AnyType(TypeOfAny.from_error), set() value_types = [] for key_name in key_names: - value_type = td_type.items.get(key_name) - if value_type is None: + item = td_type.item(key_name) + if item.typ is None or isinstance(get_proper_type(item.typ), UninhabitedType): + # The key is unknown (default-open TypedDict) or provably absent + # (closed TypedDict). Extra keys of a TypedDict with extra_items + # have the pseudo-item type (PEP 728). self.msg.typeddict_key_not_found(td_type, key_name, index, setitem) return AnyType(TypeOfAny.from_error), set() - else: - value_types.append(value_type) + value_types.append(item.typ) return make_simplified_union(value_types), set(key_names) + def typeddict_arbitrary_str_key_item( + self, td_type: TypedDictType, key_type: Type, setitem: bool + ) -> Type | None: + """Handle indexing a PEP 728 TypedDict with an arbitrary str key. + + Reads are allowed on any TypedDict with closed=True or extra_items= set, + yielding the union of all value types. Writes are only allowed if the + TypedDict behaves like dict[str, VT]. Return None if the operation is + not allowed. + """ + if td_type.extra_items is None: + return None + if not is_subtype(key_type, self.named_type("builtins.str")): + return None + if setitem: + return typed_dict_dict_value_type(td_type) + return make_simplified_union(td_type.value_types_with_extra()) + def visit_enum_index_expr( self, enum_type: TypeInfo, index: Expression, context: Context ) -> Type: diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e75a8ed7a5b03..7eb0b208a1618 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -42,7 +42,7 @@ is_final_node, ) from mypy.plugin import AttributeContext -from mypy.subtypes import is_subtype +from mypy.subtypes import is_subtype, typed_dict_dict_value_type from mypy.typeops import ( bind_self, erase_to_bound, @@ -1382,6 +1382,9 @@ def analyze_typeddict_access( typ, mx.context.index, setitem=True ) assigned_readonly_keys = typ.readonly_keys & key_names + if typ.extra_items_readonly: + # Extra keys inherit the read-only qualifier of extra_items (PEP 728). + assigned_readonly_keys |= {k for k in key_names if k not in typ.items} if assigned_readonly_keys and not mx.suppress_errors: mx.msg.readonly_keys_mutated(assigned_readonly_keys, context=mx.context) else: @@ -1407,6 +1410,15 @@ def analyze_typeddict_access( fallback=mx.chk.named_type("builtins.function"), name=name, ) + if typ.fallback.type.get(name) is None: + # The member is not defined on the TypedDict fallback. A TypedDict that + # behaves like dict[str, VT] additionally supports dict methods, e.g. + # clear() and popitem() (PEP 728). + value_type = typed_dict_dict_value_type(typ) + if value_type is not None: + str_type = mx.chk.named_type("builtins.str") + dict_type = mx.chk.named_generic_type("builtins.dict", [str_type, value_type]) + return _analyze_member_access(name, dict_type, mx, None) return _analyze_member_access(name, typ.fallback, mx, override_info) diff --git a/mypy/constraints.py b/mypy/constraints.py index 48cc23f742227..d6a5037638947 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1373,13 +1373,25 @@ def visit_typeddict_type(self, template: TypedDictType) -> list[Constraint]: actual = self.actual if isinstance(actual, TypedDictType): res: list[Constraint] = [] - # NOTE: Non-matching keys are ignored. Compatibility is checked - # elsewhere so this shouldn't be unsafe. - for item_name, template_item_type, actual_item_type in template.zip(actual): - res.extend(infer_constraints(template_item_type, actual_item_type, self.direction)) + # NOTE: Compatibility is checked elsewhere so this shouldn't be unsafe. + # Keys named on only one side are matched against the other side's + # PEP 728 pseudo-item; a None item type means the key is untyped + # (missing in a default-open TypedDict), so nothing can be inferred. + for item_name, template_item, actual_item in template.zipall(actual): + if template_item.typ is None or actual_item.typ is None: + continue + res.extend(infer_constraints(template_item.typ, actual_item.typ, self.direction)) + if template.extra_items is not None and actual.extra_items is not None: + # The pseudo-items themselves cover the keys named on neither side. + res.extend( + infer_constraints(template.extra_items, actual.extra_items, self.direction) + ) return res elif isinstance(actual, AnyType): - return self.infer_against_any(template.items.values(), actual) + values: list[Type] = list(template.items.values()) + if template.extra_items is not None: + values.append(template.extra_items) + return self.infer_against_any(values, actual) else: return [] diff --git a/mypy/copytype.py b/mypy/copytype.py index 12dda9d263975..f7ea7a905f709 100644 --- a/mypy/copytype.py +++ b/mypy/copytype.py @@ -109,7 +109,12 @@ def visit_typeddict_type(self, t: TypedDictType) -> ProperType: return self.copy_common( t, TypedDictType( - t.items, t.required_keys, t.readonly_keys, t.fallback, is_closed=t.is_closed + t.items, + t.required_keys, + t.readonly_keys, + t.fallback, + extra_items=t.extra_items, + extra_items_readonly=t.extra_items_readonly, ), ) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index fd507216a6be9..62314fa0e039a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -327,8 +327,8 @@ def _possible_callable_kwargs(cls, repl: Parameters, dict_type: Instance) -> Pro """Given a callable, extract all parameters that can be passed as `**kwargs`. If the function only accepts **kwargs, this will be a `dict[str, KwargsValueType]`. - Otherwise, this will be a `TypedDict` containing all explicit args and ignoring - `**kwargs` (until PEP 728 `extra_items` is supported). TypedDict entries will + Otherwise, this will be a `TypedDict` containing all explicit args, with any + `**kwargs` type mapped to `extra_items` (PEP 728). TypedDict entries will be required iff the corresponding argument is kw-only and has no default. """ if repl.variables: @@ -348,9 +348,15 @@ def _possible_callable_kwargs(cls, repl: Parameters, dict_type: Instance) -> Pro kwargs[name] = type if not kwargs and extra_items is not None: return Instance(dict_type.type, [dict_type.args[0], extra_items]) - # TODO: when PEP 728 `extra_items` is implemented, pass extra_items below. - is_closed = extra_items is None - return TypedDictType(kwargs, required_names, set(), dict_type, is_closed=is_closed) + # A callable without **kwargs accepts no extra keyword arguments, i.e. the + # TypedDict is closed; **kwargs of type T maps to extra_items=T (PEP 728). + return TypedDictType( + kwargs, + required_names, + set(), + dict_type, + extra_items=extra_items if extra_items is not None else UninhabitedType(), + ) def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: # Sometimes solver may need to expand a type variable with (a copy of) itself @@ -558,7 +564,12 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: return cached fallback = t.fallback.accept(self) assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) - result = t.copy_modified(item_types=self.expand_types(t.items.values()), fallback=fallback) + extra_items = t.extra_items.accept(self) if t.extra_items is not None else None + result = t.copy_modified( + item_types=self.expand_types(t.items.values()), + fallback=fallback, + extra_items=extra_items, + ) self.set_cached(t, result) return result diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 1c9323be056dd..29c56e55f63d3 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -258,11 +258,11 @@ def expr_to_unanalyzed_type( if not expr.items: raise TypeTranslationError() items: dict[str, Type] = {} - extra_items_from = [] + merged_from = [] for item_name, value in expr.items: if not isinstance(item_name, StrExpr): if item_name is None: - extra_items_from.append( + merged_from.append( expr_to_unanalyzed_type( value, options, @@ -279,7 +279,7 @@ def expr_to_unanalyzed_type( result = TypedDictType( items, set(), set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column ) - result.extra_items_from = extra_items_from + result.merged_from = merged_from return result else: raise TypeTranslationError() diff --git a/mypy/fastparse.py b/mypy/fastparse.py index d9e2d5df8f4c1..87a95cbf693d3 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2135,16 +2135,16 @@ def visit_Dict(self, n: ast3.Dict) -> Type: if not n.keys: return self.invalid_type(n) items: dict[str, Type] = {} - extra_items_from = [] + merged_from = [] for item_name, value in zip(n.keys, n.values): if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str): if item_name is None: - extra_items_from.append(self.visit(value)) + merged_from.append(self.visit(value)) continue return self.invalid_type(n) items[item_name.value] = self.visit(value) result = TypedDictType(items, set(), set(), _dummy_fallback, n.lineno, n.col_offset) - result.extra_items_from = extra_items_from + result.merged_from = merged_from return result # Attribute(expr value, identifier attr, expr_context ctx) diff --git a/mypy/fixup.py b/mypy/fixup.py index 48ed7c26d57ba..860f91c11d114 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -313,6 +313,8 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: if tdt.items: for it in tdt.items.values(): it.accept(self) + if tdt.extra_items is not None: + tdt.extra_items.accept(self) if tdt.fallback is not None: if tdt.fallback.type_ref is not None: if ( diff --git a/mypy/indirection.py b/mypy/indirection.py index 6bbda859de8f9..98391731142ba 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -153,6 +153,8 @@ def visit_tuple_type(self, t: types.TupleType) -> None: def visit_typeddict_type(self, t: types.TypedDictType) -> None: self._visit_type_list(list(t.items.values())) + if t.extra_items is not None: + self._visit(t.extra_items) self._visit(t.fallback) def visit_literal_type(self, t: types.LiteralType) -> None: diff --git a/mypy/join.py b/mypy/join.py index c609b303bb087..8e45a4e2b910f 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -662,9 +662,18 @@ def visit_typeddict_type(self, t: TypedDictType) -> ProperType: readonly_keys.add(item_name) fallback = self.s.create_anonymous_fallback() - is_closed = self.s.is_closed and t.is_closed + # Join the PEP 728 pseudo-items like any other item. A None result + # leaves the join implicitly open. + extra_items, _, extra_items_readonly = self.resolve_typeddict_item( + "extra_items", self.s.extra_item(), t.extra_item() + ) return TypedDictType( - items, required_keys, readonly_keys, fallback, is_closed=is_closed + items, + required_keys, + readonly_keys, + fallback, + extra_items=extra_items, + extra_items_readonly=extra_items_readonly if extra_items is not None else False, ) elif isinstance(self.s, Instance): return join_types(self.s, t.fallback) diff --git a/mypy/meet.py b/mypy/meet.py index bfc6d88a1e209..c2ae32e4b83d4 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1162,7 +1162,25 @@ def resolve_typeddict_item_type( def visit_typeddict_type(self, t: TypedDictType) -> ProperType: if isinstance(self.s, TypedDictType): - is_closed = self.s.is_closed or t.is_closed + # Meet the PEP 728 pseudo-items first, like any other item. As with + # named items, incompatible mutable pseudo-items make the meet + # uninhabited. + s_extra = self.s.extra_item() + t_extra = t.extra_item() + extra_items: Type | None + extra_items_readonly = False + if s_extra.typ is None and t_extra.typ is None: + extra_items = None + else: + extra_meet, extra_items_readonly = self.resolve_typeddict_item_type( + "extra_items", s_extra, t_extra + ) + if extra_meet is None: + return self.default(self.s) + extra_items = extra_meet + is_closed = extra_items is not None and isinstance( + get_proper_type(extra_items), UninhabitedType + ) items: dict[str, Type] = {} readonly_keys: set[str] = set() for name, s_item, t_item in self.s.zipall(t): @@ -1187,7 +1205,12 @@ def visit_typeddict_type(self, t: TypedDictType) -> ProperType: fallback = self.s.create_anonymous_fallback() required_keys = self.s.required_keys | t.required_keys return TypedDictType( - items, required_keys, readonly_keys, fallback, is_closed=is_closed + items, + required_keys, + readonly_keys, + fallback, + extra_items=extra_items, + extra_items_readonly=extra_items_readonly, ) elif isinstance(self.s, Instance) and is_subtype(t, self.s): return t @@ -1345,7 +1368,9 @@ def typed_dict_mapping_overlap( mutable_mapping = next( (base for base in other.type.mro if base.fullname == "typing.MutableMapping"), None ) - if mutable_mapping is not None and typed.readonly_keys: + if mutable_mapping is not None and ( + typed.readonly_keys or (typed.extra_items is not None and typed.extra_items_readonly) + ): return False mapping = next(base for base in other.type.mro if base.fullname == "typing.Mapping") @@ -1367,5 +1392,11 @@ def typed_dict_mapping_overlap( else: if not overlapping(key_type, str_type): return False - non_required = set(typed.items.keys()) - typed.required_keys - return any(overlapping(typed.items[k], value_type) for k in non_required) + candidate_values = [typed.items[k] for k in typed.items.keys() - typed.required_keys] + extra_item = typed.extra_item() + if extra_item.typ is not None and not isinstance( + get_proper_type(extra_item.typ), UninhabitedType + ): + # The PEP 728 pseudo-item may also provide overlapping values. + candidate_values.append(extra_item.typ) + return any(overlapping(t, value_type) for t in candidate_values) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index d5af1c8c6ec8d..7ff29dc1e39d6 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -966,15 +966,15 @@ def read_type(state: State, data: ReadBuffer) -> Type: n = read_int_bare(data) values = [read_type(state, data) for i in range(n)] td_items = {} - extra_items_from = [] + merged_from = [] for key, val in zip(keys, values): if key is None: assert isinstance(val, ProperType) - extra_items_from.append(val) + merged_from.append(val) else: td_items[key] = val typeddict_type = TypedDictType(td_items, set(), set(), _dummy_fallback) - typeddict_type.extra_items_from = extra_items_from + typeddict_type.merged_from = merged_from read_loc(data, typeddict_type) expect_end_tag(data) return typeddict_type diff --git a/mypy/nodes.py b/mypy/nodes.py index e2ea348d2df11..a966a2c4da5e8 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5272,13 +5272,16 @@ class TypedDictData: # If False, the type definition referenced a placeholder ready: bool - bases: list[tuple[TypeInfo, dict[str, mypy.types.Type]]] + # Each entry is a base TypeInfo together with its field types and PEP 728 + # extra_items pseudo-item type (or None), both mapped through the base's + # type arguments. + bases: list[tuple[TypeInfo, dict[str, mypy.types.Type], mypy.types.Type | None]] field_sources: dict[str, TypedDictFieldSource] def __init__( self, ready: bool, - bases: list[tuple[TypeInfo, dict[str, mypy.types.Type]]], + bases: list[tuple[TypeInfo, dict[str, mypy.types.Type], mypy.types.Type | None]], field_sources: dict[str, TypedDictFieldSource], ) -> None: self.ready = ready diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index a7d1c9e4426ab..dd89795371f2c 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -61,7 +61,7 @@ create_singledispatch_function_callback, singledispatch_register_callback, ) -from mypy.subtypes import is_subtype +from mypy.subtypes import is_subtype, typed_dict_dict_value_type from mypy.typeops import is_literal_type_like, make_simplified_union from mypy.types import ( TPDICT_FB_NAMES, @@ -76,6 +76,7 @@ TypedDictType, TypeOfAny, TypeVarType, + UninhabitedType, UnionType, get_proper_type, get_proper_types, @@ -84,6 +85,9 @@ TD_SETDEFAULT_NAMES: Final = {n + ".setdefault" for n in TPDICT_FB_NAMES} TD_POP_NAMES: Final = {n + ".pop" for n in TPDICT_FB_NAMES} TD_DELITEM_NAMES: Final = {n + ".__delitem__" for n in TPDICT_FB_NAMES} +# In some test fixtures values() is inherited from Mapping, like get(). +TD_VALUES_NAMES: Final = {n + ".values" for n in TPDICT_FB_NAMES} | {"typing.Mapping.values"} +TD_ITEMS_NAMES: Final = {n + ".items" for n in TPDICT_FB_NAMES} TD_UPDATE_METHOD_NAMES: Final = ( {n + ".update" for n in TPDICT_FB_NAMES} @@ -154,6 +158,10 @@ def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | No return typed_dict_pop_callback elif fullname in TD_DELITEM_NAMES: return typed_dict_delitem_callback + elif fullname in TD_VALUES_NAMES: + return typed_dict_values_callback + elif fullname in TD_ITEMS_NAMES: + return typed_dict_items_callback elif fullname == "_ctypes.Array.__getitem__": return array_getitem_callback elif fullname == "_ctypes.Array.__iter__": @@ -291,10 +299,16 @@ def typed_dict_get_callback(ctx: MethodContext) -> Type: output_types: list[Type] = [] for key in keys: - value_type: Type | None = ctx.type.items.get(key) + item = ctx.type.item(key) + value_type: Type | None = item.typ if value_type is None: - if not ctx.type.is_closed: - return ctx.default_return_type + # Nothing is known about the key in a default-open TypedDict. + return ctx.default_return_type + if ( + isinstance(get_proper_type(value_type), UninhabitedType) + and key not in ctx.type.items + ): + # The key is provably absent (closed TypedDict). output_types.append(default_type) elif key in ctx.type.required_keys: output_types.append(value_type) @@ -366,15 +380,19 @@ def typed_dict_pop_callback(ctx: MethodContext) -> Type: value_types = [] for key in keys: - if key in ctx.type.required_keys or key in ctx.type.readonly_keys: + item = ctx.type.item(key) + if key in ctx.type.required_keys or (item.typ is not None and item.readonly): ctx.api.msg.typeddict_key_cannot_be_deleted(ctx.type, key, key_expr) - value_type = ctx.type.items.get(key) - if value_type: - value_types.append(value_type) - else: + if item.typ is None or ( + key not in ctx.type.items + and isinstance(get_proper_type(item.typ), UninhabitedType) + ): + # The key is unknown (default-open TypedDict) or provably + # absent (closed TypedDict). ctx.api.msg.typeddict_key_not_found(ctx.type, key, key_expr) return AnyType(TypeOfAny.from_error) + value_types.append(item.typ) if len(ctx.args[1]) == 0: return make_simplified_union(value_types) @@ -425,6 +443,9 @@ def typed_dict_setdefault_callback(ctx: MethodContext) -> Type: return AnyType(TypeOfAny.from_error) assigned_readonly_keys = ctx.type.readonly_keys & set(keys) + if ctx.type.extra_items_readonly: + # Extra keys inherit the read-only qualifier of extra_items (PEP 728). + assigned_readonly_keys |= {k for k in keys if k not in ctx.type.items} if assigned_readonly_keys: ctx.api.msg.readonly_keys_mutated(assigned_readonly_keys, context=key_expr) @@ -433,9 +454,13 @@ def typed_dict_setdefault_callback(ctx: MethodContext) -> Type: value_types = [] for key in keys: - value_type = ctx.type.items.get(key) + item = ctx.type.item(key) + value_type = item.typ - if value_type is None: + if value_type is None or ( + key not in ctx.type.items + and isinstance(get_proper_type(value_type), UninhabitedType) + ): ctx.api.msg.typeddict_key_not_found(ctx.type, key, key_expr) return AnyType(TypeOfAny.from_error) @@ -465,6 +490,10 @@ def typed_dict_delitem_callback(ctx: MethodContext) -> Type: key_expr = ctx.args[0][0] keys = try_getting_str_literals(key_expr, ctx.arg_types[0][0]) if keys is None: + if typed_dict_dict_value_type(ctx.type) is not None: + # Arbitrary str keys can be deleted from a TypedDict that + # behaves like dict[str, VT] (PEP 728). + return ctx.default_return_type ctx.api.fail( message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, key_expr, @@ -473,13 +502,52 @@ def typed_dict_delitem_callback(ctx: MethodContext) -> Type: return AnyType(TypeOfAny.from_error) for key in keys: - if key in ctx.type.required_keys or key in ctx.type.readonly_keys: + item = ctx.type.item(key) + if key in ctx.type.required_keys or (item.typ is not None and item.readonly): ctx.api.msg.typeddict_key_cannot_be_deleted(ctx.type, key, key_expr) - elif key not in ctx.type.items: + elif item.typ is None or ( + key not in ctx.type.items + and isinstance(get_proper_type(item.typ), UninhabitedType) + ): + # The key is unknown (default-open TypedDict) or provably + # absent (closed TypedDict). ctx.api.msg.typeddict_key_not_found(ctx.type, key, key_expr) return ctx.default_return_type +def typed_dict_values_callback(ctx: MethodContext) -> Type: + """Infer a precise return type for TypedDict.values (PEP 728). + + The values of a TypedDict with closed=True or extra_items= set are known + exactly, so the imprecise object value type can be replaced with their union. + """ + if isinstance(ctx.type, TypedDictType) and ctx.type.extra_items is not None: + default = get_proper_type(ctx.default_return_type) + if isinstance(default, Instance) and default.args: + args = list(default.args) + args[-1] = make_simplified_union(ctx.type.value_types_with_extra()) + return default.copy_modified(args=args) + return ctx.default_return_type + + +def typed_dict_items_callback(ctx: MethodContext) -> Type: + """Infer a precise return type for TypedDict.items (PEP 728).""" + if isinstance(ctx.type, TypedDictType) and ctx.type.extra_items is not None: + default = get_proper_type(ctx.default_return_type) + if isinstance(default, Instance) and default.args: + value_type = make_simplified_union(ctx.type.value_types_with_extra()) + args = list(default.args) + last = get_proper_type(args[-1]) + if isinstance(last, TupleType) and len(last.items) == 2: + # e.g. Iterable[tuple[str, object]] + args[-1] = last.copy_modified(items=[last.items[0], value_type]) + else: + # e.g. dict_items[str, object] + args[-1] = value_type + return default.copy_modified(args=args) + return ctx.default_return_type + + _TP_DICT_MUTATING_METHODS: Final = frozenset({"update of TypedDict", "__ior__ of TypedDict"}) diff --git a/mypy/semanal.py b/mypy/semanal.py index e010273b0781f..328e907ffc877 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2248,8 +2248,15 @@ def get_fullname_for_hook(self, expr: Expression) -> str | None: return None def analyze_class_keywords(self, defn: ClassDef) -> None: - for value in defn.keywords.values(): - value.accept(self) + for key, value in defn.keywords.items(): + if key == "extra_items": + # The value of a PEP 728 TypedDict "extra_items" argument is a type + # expression, and may validly reference type variables; it is analyzed + # as a type in the TypedDict analyzer. + with self.allow_unbound_tvars_set(): + value.accept(self) + else: + value.accept(self) def enter_class(self, info: TypeInfo) -> None: # Remember previous active class diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index b1ba9f6c3abec..7cdab2dc19ef2 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -53,6 +53,7 @@ TypedDictType, TypeOfAny, TypeVarLikeType, + UninhabitedType, get_proper_type, ) @@ -69,6 +70,21 @@ class FieldSource(NamedTuple): ctx: Context +class TypedDictArgs(NamedTuple): + """Parsed arguments of a functional-syntax TypedDict() call.""" + + typename: str + items: list[str] + types: list[Type] + total: bool + # The PEP 728 pseudo-item (closed=True is represented as an uninhabited type). + extra_items: Type | None + extra_items_readonly: bool + tvar_defs: list[TypeVarLikeType] + # False if there was an error during parsing. + ok: bool + + class TypedDictAnalyzer: def __init__( self, options: Options, api: SemanticAnalyzerInterface, msg: MessageBuilder @@ -116,6 +132,18 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N is_closed = require_bool_literal_argument( self.api, defn.keywords["closed"], "closed", False ) + extra_items: Type | None = None + extra_items_readonly = False + if "extra_items" in defn.keywords: + if "closed" in defn.keywords: + self.fail( + 'Cannot use "closed" and "extra_items" in the same TypedDict definition', defn + ) + is_closed = None + res = self.analyze_extra_items_argument(defn.keywords["extra_items"]) + if res is None: + return True, None # Defer + extra_items, extra_items_readonly = res if ( len(defn.base_type_exprs) == 1 @@ -134,7 +162,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N field_types, required_keys, readonly_keys, - is_closed or False, + UninhabitedType() if is_closed else extra_items, + extra_items_readonly, defn.line, existing_info, ) @@ -174,7 +203,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N else: self.fail("All bases of a new TypedDict must be TypedDict types", defn) - bases_info: list[tuple[TypeInfo, dict[str, Type]]] = [] + bases_info: list[tuple[TypeInfo, dict[str, Type], Type | None]] = [] for base in typeddict_bases: base_info = self.fetch_keys_and_types_from_base(base, defn) if base_info is not None: @@ -182,8 +211,15 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N new_field_sources, new_statements = self.analyze_typeddict_classdef_fields(defn) if new_field_sources is None: return True, None # Defer - field_types, required_keys, readonly_keys, is_closed, field_sources = ( - self.resolve_field_inheritance(bases_info, new_field_sources, is_closed, defn) + ( + field_types, + required_keys, + readonly_keys, + extra_items, + extra_items_readonly, + field_sources, + ) = self.resolve_field_inheritance( + bases_info, new_field_sources, is_closed, extra_items, extra_items_readonly, defn ) typeddict_data = TypedDictData(True, bases_info, field_sources) info = self.build_typeddict_typeinfo( @@ -191,7 +227,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N field_types, required_keys, readonly_keys, - is_closed, + extra_items, + extra_items_readonly, defn.line, existing_info, typeddict_data=typeddict_data, @@ -204,7 +241,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N def fetch_keys_and_types_from_base( self, base: Expression, ctx: Context - ) -> tuple[TypeInfo, dict[str, Type]] | None: + ) -> tuple[TypeInfo, dict[str, Type], Type | None] | None: info = self._parse_typeddict_base(base, ctx) base_args: list[Type] = [] if isinstance(base, IndexExpr): @@ -227,14 +264,19 @@ def fetch_keys_and_types_from_base( any_kind = TypeOfAny.from_error base_args = [AnyType(any_kind) for _ in tvars] + extra_items = base_typed_dict.extra_items with state.strict_optional_set(self.options.strict_optional): valid_items = self.map_items_to_base(valid_items, tvars, base_args) + if extra_items is not None and tvars: + extra_items = expand_type( + extra_items, {t.id: a for (t, a) in zip(tvars, base_args)} + ) - return info, valid_items + return info, valid_items, extra_items def field_sources_in_reverse_order( self, - bases: list[tuple[TypeInfo, dict[str, Type]]], + bases: list[tuple[TypeInfo, dict[str, Type], Type | None]], child_field_sources: dict[str, FieldSource], ctx: Context, ) -> dict[str, list[FieldSource]]: @@ -243,7 +285,7 @@ def field_sources_in_reverse_order( Iterate bases in reverse order to preserve key ordering for display. """ result: dict[str, list[FieldSource]] = {} - for base_info, base_fields in reversed(bases): + for base_info, base_fields, _ in reversed(bases): assert base_info.typeddict_type is not None for field_name, field_type in base_fields.items(): source = FieldSource( @@ -274,6 +316,7 @@ def verify_requiredness_compatibility( field_name: str, source: FieldSource, is_required: bool, + is_readonly: bool, primary_source_base: TypeInfo | None, ctx: Context, ) -> None: @@ -284,6 +327,12 @@ def verify_requiredness_compatibility( self.fail( f'Field "{field_name}" is required in base class "{source.base.name}"', ctx ) + elif is_readonly: + self.fail( + f'Field "{field_name}" is required in base class "{source.base.name}" but ' + f'not in base class "{primary_source_base.name}"', + ctx, + ) else: self.fail( f'Field "{field_name}" is required in base class "{source.base.name}" but can ' @@ -302,56 +351,131 @@ def verify_requiredness_compatibility( ctx, ) - def verify_field_against_closed_bases( + def verify_field_against_extra_items( self, field_name: str, - closed_bases: Collection[tuple[TypeInfo, Collection[str]]], + is_required: bool, + extra_bases: Collection[tuple[TypeInfo, Collection[str], Type, bool]], primary_source_base: TypeInfo | None, ctx: Context, ) -> None: - for closed_base_type, closed_base_fields in closed_bases: - if field_name in closed_base_fields: + """Verify a field against the extra_items pseudo-items of base classes. + + Only requiredness and closedness are checked here; the compatibility of the + field type with the extra_items type requires subtype checks, so it is + deferred to checker.check_typeddict_inheritance(). + """ + for base_type, base_fields, base_extra_items, base_readonly in extra_bases: + if field_name in base_fields: continue - if primary_source_base: - self.fail( - f'Cannot extend closed base class "{closed_base_type.name}" with field ' - f'"{field_name}" from base class "{primary_source_base.name}"', - ctx, - ) - else: - self.fail( - f'Cannot extend closed base class "{closed_base_type.name}" with new field ' - f'"{field_name}"', - ctx, - ) + if isinstance(get_proper_type(base_extra_items), UninhabitedType): + if primary_source_base: + self.fail( + f'Cannot extend closed base class "{base_type.name}" with field ' + f'"{field_name}" from base class "{primary_source_base.name}"', + ctx, + ) + else: + self.fail( + f'Cannot extend closed base class "{base_type.name}" with new field ' + f'"{field_name}"', + ctx, + ) + elif not base_readonly and is_required: + # Extra items are non-required, so a field not known to the base can + # only be required if the base's extra_items is read-only. + if primary_source_base: + self.fail( + f'Field "{field_name}" from base class "{primary_source_base.name}" ' + f'must be non-required, since "extra_items" of base class ' + f'"{base_type.name}" is not read-only', + ctx, + ) + else: + self.fail( + f'Field "{field_name}" must be non-required, since "extra_items" of ' + f'base class "{base_type.name}" is not read-only', + ctx, + ) def resolve_field_inheritance( self, - bases: list[tuple[TypeInfo, dict[str, Type]]], + bases: list[tuple[TypeInfo, dict[str, Type], Type | None]], child_field_sources: dict[str, FieldSource], child_is_closed: bool | None, + child_extra_items: Type | None, + child_extra_items_readonly: bool, ctx: Context, - ) -> tuple[dict[str, Type], set[str], set[str], bool, dict[str, TypedDictFieldSource]]: - """Determine field types, requiredness, readonlyness, and closedness.""" + ) -> tuple[ + dict[str, Type], set[str], set[str], Type | None, bool, dict[str, TypedDictFieldSource] + ]: + """Determine field types, requiredness, readonlyness, and the extra_items pseudo-item.""" field_sources = self.field_sources_in_reverse_order(bases, child_field_sources, ctx) field_types: dict[str, Type] = {} chosen_sources: dict[str, TypedDictFieldSource] = {} required_keys: set[str] = set() readonly_keys: set[str] = set() - closed_bases = [ - (base_info, base_fields.keys()) - for (base_info, base_fields) in bases - if base_info.typeddict_type and base_info.typeddict_type.is_closed + # Bases that restrict extra items (closed or with extra_items= set). + extra_bases = [ + ( + base_info, + base_fields.keys(), + base_extra_items, + base_info.typeddict_type.extra_items_readonly, + ) + for (base_info, base_fields, base_extra_items) in bases + if base_info.typeddict_type and base_extra_items is not None ] - if child_is_closed is False and closed_bases: - for base_info, _ in closed_bases: - self.fail( - f"Open TypedDict class cannot subclass closed TypedDict class " - f'"{base_info.name}"', - ctx, - ) + if child_is_closed is False and extra_bases: + for base_info, _, base_extra_items, _ in extra_bases: + if isinstance(get_proper_type(base_extra_items), UninhabitedType): + self.fail( + f"Open TypedDict class cannot subclass closed TypedDict class " + f'"{base_info.name}"', + ctx, + ) + else: + self.fail( + f"Open TypedDict class cannot subclass TypedDict class " + f'"{base_info.name}" with "extra_items"', + ctx, + ) + + # Resolve the extra_items pseudo-item. Compatibility of its type with the + # base pseudo-items requires subtype checks, so it is deferred to + # checker.check_typeddict_inheritance(). + extra_items: Type | None + extra_items_readonly = False + if child_extra_items is not None: + extra_items = child_extra_items + extra_items_readonly = child_extra_items_readonly + if extra_items_readonly: + # Like fields, a mutable pseudo-item cannot be redeclared as read-only. + for base_info, _, base_extra_items, base_readonly in extra_bases: + if not base_readonly and not isinstance( + get_proper_type(base_extra_items), UninhabitedType + ): + self.fail( + f'Cannot redeclare mutable "extra_items" of base class ' + f'"{base_info.name}" as read-only', + ctx, + ) + elif child_is_closed: + extra_items = UninhabitedType() + elif child_is_closed is None: + # Inherit from the bases. Prefer the first mutable pseudo-item; mutual + # compatibility with the other bases is verified in the checker. + extra_items = None + for _, _, base_extra_items, base_readonly in extra_bases: + if extra_items is None or (extra_items_readonly and not base_readonly): + extra_items = base_extra_items + extra_items_readonly = base_readonly + else: + # Explicitly open with closed=False (errors were reported above if the + # bases restrict extra items). + extra_items = None for field_name, sources in field_sources.items(): primary_source = self.primary_source(sources) @@ -366,13 +490,8 @@ def resolve_field_inheritance( base=primary_source.base, ctx=primary_source.ctx ) - if primary_source.is_readonly: - # If the primary source is readonly, all sources are readonly - is_readonly = True - is_required = any(source.is_required for source in sources) - else: - is_readonly = False - is_required = primary_source.is_required + is_readonly = primary_source.is_readonly + is_required = primary_source.is_required if is_required: required_keys.add(field_name) @@ -381,15 +500,36 @@ def resolve_field_inheritance( for source in sources: if source is not primary_source: + if is_readonly and not source.is_readonly: + # Only reachable when the child redeclares a mutable base + # field as read-only: a mutable base source would otherwise + # have been selected as the primary source. + assert source.base is not None + self.fail( + f'Cannot redeclare mutable field "{field_name}" of base class ' + f'"{source.base.name}" as read-only', + primary_source.ctx, + ) self.verify_requiredness_compatibility( - field_name, source, is_required, primary_source.base, primary_source.ctx + field_name, + source, + is_required, + is_readonly, + primary_source.base, + primary_source.ctx, ) - self.verify_field_against_closed_bases( - field_name, closed_bases, primary_source.base, primary_source.ctx + self.verify_field_against_extra_items( + field_name, is_required, extra_bases, primary_source.base, primary_source.ctx ) - is_closed = bool(closed_bases) if child_is_closed is None else child_is_closed - return field_types, required_keys, readonly_keys, is_closed, chosen_sources + return ( + field_types, + required_keys, + readonly_keys, + extra_items, + extra_items_readonly, + chosen_sources, + ) def _parse_typeddict_base(self, base: Expression, ctx: Context) -> TypeInfo: if isinstance(base, RefExpr): @@ -482,7 +622,7 @@ def analyze_typeddict_classdef_fields( self.api, defn.keywords["total"], "total", True ) continue - elif key == "closed": + elif key in ("closed", "extra_items"): continue for_function = ' for "__init_subclass__" of "TypedDict"' self.msg.unexpected_keyword_argument_for_function(for_function, key, defn) @@ -575,6 +715,36 @@ def extract_meta_info( typ = typ.item return typ, is_required, readonly + def analyze_extra_items_argument(self, expr: Expression) -> tuple[Type, bool] | None: + """Analyze the type given to an extra_items= argument (PEP 728). + + Return a (type, readonly) pair with any ReadOnly[] qualifier unwrapped, + or None if the type is not ready yet (the caller should defer). + """ + try: + unanalyzed = expr_to_unanalyzed_type(expr, self.options, self.api.is_stub_file) + except TypeTranslationError: + self.fail('Invalid "extra_items" argument', expr, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error), False + analyzed = self.api.anal_type( + unanalyzed, + allow_typed_dict_special_forms=True, + allow_placeholder=not self.api.is_func_scope(), + prohibit_self_type='"extra_items" argument', + ) + if analyzed is None: + return None # Need to defer + typ, is_required, readonly = self.extract_meta_info(analyzed, expr) + if is_required is not None: + self.fail( + '"{}[]" is not allowed in "extra_items"'.format( + "Required" if is_required else "NotRequired" + ), + expr, + code=codes.VALID_TYPE, + ) + return typ, readonly + def check_typeddict( self, node: Expression, name: str ) -> tuple[bool, TypeInfo | None, list[TypeVarLikeType]]: @@ -604,10 +774,14 @@ def check_typeddict( # This is a valid typed dict, but some type is not ready. # The caller should defer this until next iteration. return True, None, [] - typename, items, wrapped_types, total, closed, tvar_defs, ok = res + typename, items, wrapped_types, total, extra_items, extra_items_readonly, tvar_defs, ok = ( + res + ) if not ok: # Error. Construct dummy return value. - info = self.build_typeddict_typeinfo(name, {}, set(), set(), False, call.line, None) + info = self.build_typeddict_typeinfo( + name, {}, set(), set(), None, False, call.line, None + ) else: if "@" not in name and name != typename: self.fail( @@ -648,7 +822,8 @@ def check_typeddict( dict(zip(items, types)), required_keys, readonly_keys, - closed, + extra_items, + extra_items_readonly, call.line, existing_info, ) @@ -661,19 +836,17 @@ def check_typeddict( call.analyzed.set_line(call) return True, info, tvar_defs - def parse_typeddict_args( - self, call: CallExpr - ) -> tuple[str, list[str], list[Type], bool, bool, list[TypeVarLikeType], bool] | None: + def parse_typeddict_args(self, call: CallExpr) -> TypedDictArgs | None: """Parse typed dict call expression. - Return names, types, totality, open/closed, was there an error during parsing. - If some type is not ready, return None. + Return the parsed arguments (with ok=False if there was an error during + parsing). If some type is not ready, return None. """ # TODO: Share code with check_argument_count in checkexpr.py? args = call.args if len(args) < 2: return self.fail_typeddict_arg("Too few arguments for TypedDict()", call) - if len(args) > 4: + if len(args) > 5: return self.fail_typeddict_arg("Too many arguments for TypedDict()", call) if call.arg_kinds[:2] != [ARG_POS, ARG_POS] or any( arg_kind != ARG_NAMED for arg_kind in call.arg_kinds[2:] @@ -681,7 +854,7 @@ def parse_typeddict_args( return self.fail_typeddict_arg("Unexpected arguments to TypedDict()", call) seen_arg_names = set() for arg_name in call.arg_names[2:]: - if arg_name not in ("total", "closed"): + if arg_name not in ("total", "closed", "extra_items"): return self.fail_typeddict_arg( f'Unexpected keyword argument "{arg_name}" for "TypedDict"', call ) @@ -699,25 +872,49 @@ def parse_typeddict_args( "TypedDict() expects a dictionary literal as the second argument", call ) total: bool | None = True - closed: bool = False + closed: bool | None = None + extra_items_expr: Expression | None = None for arg_name, arg in zip(call.arg_names[2:], call.args[2:]): assert arg_name + if arg_name == "extra_items": + extra_items_expr = arg + continue value = require_bool_literal_argument(self.api, arg, arg_name) if value is None: - return "", [], [], True, False, [], False + return TypedDictArgs("", [], [], True, None, False, [], False) if arg_name == "closed": closed = value else: total = value + if closed is not None and extra_items_expr is not None: + self.fail( + 'Cannot use "closed" and "extra_items" in the same TypedDict definition', call + ) + closed = None dictexpr = args[1] - tvar_defs = self.api.get_and_bind_all_tvars([t for k, t in dictexpr.items]) + type_exprs = [t for k, t in dictexpr.items] + if extra_items_expr is not None: + type_exprs.append(extra_items_expr) + tvar_defs = self.api.get_and_bind_all_tvars(type_exprs) res = self.parse_typeddict_fields_with_types(dictexpr.items) if res is None: # One of the types is not ready, defer. return None items, types, ok = res + extra_items: Type | None = None + extra_items_readonly = False + if extra_items_expr is not None: + extra_res = self.analyze_extra_items_argument(extra_items_expr) + if extra_res is None: + # The extra_items type is not ready, defer. + return None + extra_items, extra_items_readonly = extra_res + elif closed: + extra_items = UninhabitedType() assert total is not None - return args[0].value, items, types, total, closed, tvar_defs, ok + return TypedDictArgs( + args[0].value, items, types, total, extra_items, extra_items_readonly, tvar_defs, ok + ) def parse_typeddict_fields_with_types( self, dict_items: list[tuple[Expression | None, Expression]] @@ -759,11 +956,9 @@ def parse_typeddict_fields_with_types( types.append(analyzed) return items, types, True - def fail_typeddict_arg( - self, message: str, context: Context - ) -> tuple[str, list[str], list[Type], bool, bool, list[TypeVarLikeType], bool]: + def fail_typeddict_arg(self, message: str, context: Context) -> TypedDictArgs: self.fail(message, context) - return "", [], [], True, False, [], False + return TypedDictArgs("", [], [], True, None, False, [], False) def build_typeddict_typeinfo( self, @@ -771,7 +966,8 @@ def build_typeddict_typeinfo( item_types: dict[str, Type], required_keys: set[str], readonly_keys: set[str], - is_closed: bool, + extra_items: Type | None, + extra_items_readonly: bool, line: int, existing_info: TypeInfo | None, typeddict_data: TypedDictData | None = None, @@ -785,14 +981,21 @@ def build_typeddict_typeinfo( assert fallback is not None info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) typeddict_type = TypedDictType( - item_types, required_keys, readonly_keys, fallback, is_closed=is_closed + item_types, + required_keys, + readonly_keys, + fallback, + extra_items=extra_items, + extra_items_readonly=extra_items_readonly, ) any_placeholder = has_placeholder(typeddict_type) if typeddict_data: - for _, base_fields in typeddict_data.bases: + for _, base_fields, base_extra_items in typeddict_data.bases: for field_type in base_fields.values(): if has_placeholder(field_type): any_placeholder = True + if base_extra_items is not None and has_placeholder(base_extra_items): + any_placeholder = True else: typeddict_data = TypedDictData(True, [], {}) if any_placeholder: diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 7b7a3d6c84564..e60bcdb8fe4e5 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -500,7 +500,8 @@ def visit_typeddict_type(self, typ: TypedDictType) -> SnapshotItem: items = tuple((key, snapshot_type(item_type)) for key, item_type in typ.items.items()) required = tuple(sorted(typ.required_keys)) readonly = tuple(sorted(typ.readonly_keys)) - return ("TypedDictType", items, required, readonly, typ.is_closed) + extra_items = snapshot_optional_type(typ.extra_items) + return ("TypedDictType", items, required, readonly, extra_items, typ.extra_items_readonly) def visit_literal_type(self, typ: LiteralType) -> SnapshotItem: return ("LiteralType", snapshot_type(typ.fallback), typ.value) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 5b723711405a1..ea71bb3d17240 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -518,6 +518,8 @@ def visit_parameters(self, typ: Parameters) -> None: def visit_typeddict_type(self, typ: TypedDictType) -> None: for value_type in typ.items.values(): value_type.accept(self) + if typ.extra_items is not None: + typ.extra_items.accept(self) typ.fallback.accept(self) def visit_raw_expression_type(self, t: RawExpressionType) -> None: diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 29cd8ac8b4635..11d7f8c38c781 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -1092,6 +1092,8 @@ def visit_typeddict_type(self, typ: TypedDictType) -> list[str]: triggers = [] for item in typ.items.values(): triggers.extend(self.get_type_triggers(item)) + if typ.extra_items is not None: + triggers.extend(self.get_type_triggers(typ.extra_items)) triggers.extend(self.get_type_triggers(typ.fallback)) return triggers diff --git a/mypy/stubgen.py b/mypy/stubgen.py index fe90c3d256b7a..71527ccfa0ac6 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1101,6 +1101,7 @@ def process_typeddict(self, lvalue: NameExpr, rvalue: CallExpr) -> None: items: list[tuple[str, Expression]] = [] total: Expression | None = None closed: Expression | None = None + extra_items: Expression | None = None if len(rvalue.args) > 1 and rvalue.arg_kinds[1] == ARG_POS: if not isinstance(rvalue.args[1], DictExpr): self.annotate_as_incomplete(lvalue) @@ -1115,6 +1116,8 @@ def process_typeddict(self, lvalue: NameExpr, rvalue: CallExpr) -> None: total = arg elif arg_name == "closed": closed = arg + elif arg_name == "extra_items": + extra_items = arg else: self.annotate_as_incomplete(lvalue) return @@ -1139,6 +1142,8 @@ def process_typeddict(self, lvalue: NameExpr, rvalue: CallExpr) -> None: bases += f", total={total.accept(p)}" if closed is not None: bases += f", closed={closed.accept(p)}" + if extra_items is not None: + bases += f", extra_items={extra_items.accept(p)}" class_def = f"{self._indent}class {lvalue.name}({bases}):" if len(items) == 0: self.add(f"{class_def} ...\n") diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 2d97dd534d9f5..b50ea1e87c158 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -290,6 +290,26 @@ def is_same_type( ) +def typed_dict_dict_value_type(typ: TypedDictType) -> Type | None: + """Return VT if a TypedDict behaves like dict[str, VT] (PEP 728), else None. + + This holds if every item, including the extra_items pseudo-item, is + non-required, mutable, and consistent with the extra_items type. Such a + TypedDict additionally supports dict operations that are normally unsafe, + like clear() or __setitem__ with an arbitrary str key. + """ + extra_item = typ.extra_item() + if extra_item.typ is None or extra_item.readonly: + return None + if isinstance(get_proper_type(extra_item.typ), UninhabitedType): + return None + if typ.required_keys or typ.readonly_keys: + return None + if not all(is_equivalent(item, extra_item.typ) for item in typ.items.values()): + return None + return extra_item.typ + + # This is a common entry point for subtyping checks (both proper and non-proper). # Never call this private function directly, use the public versions. def _is_subtype( @@ -934,14 +954,95 @@ def is_top_type(self, typ: Type) -> bool: def visit_typeddict_type(self, left: TypedDictType) -> bool: right = self.right if isinstance(right, Instance): - return self._is_subtype(left.fallback, right) + if self._is_subtype(left.fallback, right): + return True + if left.extra_items is None: + # A default-open TypedDict implicitly allows extra items of type + # ReadOnly[object], so the fallback (which derives from + # Mapping[str, object]) is already its most precise Mapping view. + return False + mapping_info = next( + (base for base in left.fallback.type.mro if base.fullname == "typing.Mapping"), + None, + ) + if mapping_info is None: + return False + str_type = map_instance_to_supertype(left.fallback, mapping_info).args[0] + extra_item = left.extra_item() + assert extra_item.typ is not None + # PEP 728: a TypedDict with closed=True or extra_items= set is + # assignable to Mapping[str, VT] when all of its value types are + # assignable to VT. + precise_mapping = Instance( + mapping_info, [str_type, UnionType.make_union(left.value_types_with_extra())] + ) + if self._is_subtype(precise_mapping, right): + return True + # PEP 728: it is additionally assignable to dict[str, VT] (or + # MutableMapping[str, VT]) if every item, including the extra_items + # pseudo-item, is non-required, mutable, and consistent with VT. + if ( + right.type.fullname not in ("builtins.dict", "typing.MutableMapping") + or len(right.args) != 2 + ): + return False + if extra_item.readonly or isinstance(get_proper_type(extra_item.typ), UninhabitedType): + return False + if left.required_keys or left.readonly_keys: + return False + + def consistent(l: Type, r: Type) -> bool: + if self.proper_subtype: + return is_same_type(l, r) + return is_equivalent( + l, + r, + ignore_type_params=self.subtype_context.ignore_type_params, + options=self.options, + ) + + right_key, right_value = right.args + if not consistent(str_type, right_key): + return False + if not consistent(extra_item.typ, right_value): + return False + return all(consistent(t, right_value) for t in left.items.values()) elif isinstance(right, TypedDictType): if left == right: return True # Fast path - # A closed type must remain closed - if right.is_closed and not left.is_closed: - return False + # Compare the PEP 728 pseudo-items, covering the keys named on + # neither side. (Keys named on only one side are covered by the + # zipall() loops below, which fall back to the pseudo-item.) + r_extra = right.extra_item() + if r_extra.typ is not None: + l_extra = left.extra_item() + if r_extra.readonly: + # Read-only pseudo-items behave covariantly. A missing + # pseudo-item is implicitly ReadOnly[object]. + l_extra_type = ( + l_extra.typ + if l_extra.typ is not None + else Instance(left.fallback.type.mro[-1], []) + ) + if not self._is_subtype(l_extra_type, r_extra.typ): + return False + else: + # A mutable pseudo-item must remain mutable and equivalent, + # like any other mutable item. This subsumes "a closed type + # must remain closed" (closed == extra_items=Never). + if l_extra.typ is None or l_extra.readonly: + return False + if self.proper_subtype: + if not is_same_type(l_extra.typ, r_extra.typ): + return False + elif not is_equivalent( + l_extra.typ, + r_extra.typ, + ignore_type_params=self.subtype_context.ignore_type_params, + options=self.options, + ): + return False # Perform fast key-based checks before recursing into value types for name, l, r in left.zipall(right): @@ -978,9 +1079,11 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: if r.typ is None: check = True elif l.typ is None: - # Keys cannot be dropped in an open subtype - # as this would implicitly widen the type constraint - check = False + # The key is missing in a default-open left, so its implicit + # item is ReadOnly[NotRequired[object]] (PEP 728). Only a + # top-typed item on the right can accommodate that. + obj = Instance(left.fallback.type.mro[-1], []) + check = self._is_subtype(obj, r.typ) else: check = self._is_subtype(l.typ, r.typ) if not check: diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 6de922cb7f234..86665401b0c52 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -230,14 +230,21 @@ def test_typeddict_type_constructor_signature(self) -> None: assert typ.fallback is self.fx.a assert_equal(typ.line, 10) assert_equal(typ.column, 20) + assert typ.extra_items is None assert not typ.is_closed - closed = TypedDictType({"x": self.fx.o}, {"x"}, set(), self.fx.a, is_closed=True) + closed = TypedDictType( + {"x": self.fx.o}, {"x"}, set(), self.fx.a, extra_items=UninhabitedType() + ) assert closed.is_closed + extra = TypedDictType({"x": self.fx.o}, {"x"}, set(), self.fx.a, extra_items=self.fx.b) + assert not extra.is_closed + assert extra.extra_items is self.fx.b + with self.assertRaises(TypeError): TypedDictType( # type: ignore[call-arg] - {"x": self.fx.o}, {"x"}, set(), self.fx.a, 10, 20, True + {"x": self.fx.o}, {"x"}, set(), self.fx.a, 10, 20, UninhabitedType() ) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 7052c80118710..58702c03aadeb 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -288,7 +288,8 @@ def visit_typeddict_type(self, t: TypedDictType, /) -> Type: cast(Any, t.fallback.accept(self)), t.line, t.column, - is_closed=t.is_closed, + extra_items=t.extra_items.accept(self) if t.extra_items is not None else None, + extra_items_readonly=t.extra_items_readonly, ) self.set_cached(t, result) return result @@ -432,7 +433,8 @@ def visit_tuple_type(self, t: TupleType, /) -> T: return self.query_types([t.partial_fallback] + t.items) def visit_typeddict_type(self, t: TypedDictType, /) -> T: - return self.query_types(t.items.values()) + extra = [t.extra_items] if t.extra_items is not None else [] + return self.query_types(list(t.items.values()) + extra) def visit_raw_expression_type(self, t: RawExpressionType, /) -> T: return self.strategy([]) @@ -572,7 +574,8 @@ def visit_tuple_type(self, t: TupleType, /) -> bool: return self.query_types([t.partial_fallback] + t.items) def visit_typeddict_type(self, t: TypedDictType, /) -> bool: - return self.query_types(list(t.items.values())) + extra = [t.extra_items] if t.extra_items is not None else [] + return self.query_types(list(t.items.values()) + extra) def visit_raw_expression_type(self, t: RawExpressionType, /) -> bool: return self.default diff --git a/mypy/typeanal.py b/mypy/typeanal.py index ff3c8bd2816e1..c7799f6e635fe 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1391,10 +1391,11 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: " must be enabled with --enable-incomplete-feature=InlineTypedDict", t, ) - is_closed = False + extra_items: Type | None = None + extra_items_readonly = False required_keys = req_keys fallback = self.named_type("typing._TypedDict") - for typ in t.extra_items_from: + for typ in t.merged_from: analyzed = self.analyze_type(typ) p_analyzed = get_proper_type(analyzed) if not isinstance(p_analyzed, TypedDictType): @@ -1414,9 +1415,17 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: readonly_keys = t.readonly_keys required_keys = t.required_keys fallback = t.fallback - is_closed = t.is_closed + extra_items = t.extra_items + extra_items_readonly = t.extra_items_readonly return TypedDictType( - items, required_keys, readonly_keys, fallback, t.line, t.column, is_closed=is_closed + items, + required_keys, + readonly_keys, + fallback, + t.line, + t.column, + extra_items=extra_items, + extra_items_readonly=extra_items_readonly, ) def visit_raw_expression_type(self, t: RawExpressionType) -> Type: diff --git a/mypy/types.py b/mypy/types.py index da420d1c012d2..7a084aea974b8 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2496,9 +2496,17 @@ def with_unpacked_kwargs(self) -> NormalizedCallableType: ArgKind.ARG_NAMED if name in last_type.required_keys else ArgKind.ARG_NAMED_OPT for name in last_type.items ] + extra_names: list[str | None] = list(last_type.items) + extra_types: list[Type] = list(last_type.items.values()) + if last_type.extra_items is not None and not last_type.is_closed: + # PEP 728: **kwargs: Unpack[TD] with extra_items also accepts arbitrary + # keyword arguments of the extra_items type. + extra_kinds.append(ArgKind.ARG_STAR2) + extra_names.append(None) + extra_types.append(last_type.extra_items) new_arg_kinds = self.arg_kinds[:-1] + extra_kinds - new_arg_names = self.arg_names[:-1] + list(last_type.items) - new_arg_types = self.arg_types[:-1] + list(last_type.items.values()) + new_arg_names = self.arg_names[:-1] + extra_names + new_arg_types = self.arg_types[:-1] + extra_types return NormalizedCallableType( self.copy_modified( arg_kinds=new_arg_kinds, @@ -2988,12 +2996,11 @@ def slice( class TypedDictItem(NamedTuple): """Type, mutability and requiredness of an item in a TypedDict. - If typ is `None`, the item comes from a missing item in an open TypedDict, and - the type should be treated as if it were a `builtins.object`. (Missing items in - closed TypedDicts will have an uninhabited type.) - - TODO: pass a `builtins.object` instead of None when TypedDictType gains a - proper extra_items field. + If typ is `None`, the item comes from a missing item in a default-open + TypedDict (one with neither `closed=True` nor `extra_items=` given), and + the type should be treated as if it were a `builtins.object`. (Missing + items in TypedDicts with `extra_items=` set will have that type; in + closed TypedDicts it is an uninhabited type.) """ typ: Type | None @@ -3022,6 +3029,13 @@ class TypedDictType(ProperType): (ex: "Point") whose TypeInfo has a typeddict_type that is anonymous. This is similar to how named tuples work. + The PEP 728 pseudo-item is stored in `extra_items` (with any ReadOnly[] + qualifier stripped into `extra_items_readonly`, mirroring how named items + are stored). `extra_items` is None for a default-open TypedDict; a closed + TypedDict (`closed=True`) stores an uninhabited type, per the PEP's + `closed=True` == `extra_items=Never` equivalence. `extra_items_readonly` + is only meaningful when `extra_items` is not None. + TODO: The fallback structure is perhaps overly complicated. """ @@ -3029,18 +3043,23 @@ class TypedDictType(ProperType): "items", "required_keys", "readonly_keys", - "is_closed", + "extra_items", + "extra_items_readonly", "fallback", - "extra_items_from", + "merged_from", "to_be_mutated", ) items: dict[str, Type] # item_name -> item_type required_keys: set[str] readonly_keys: set[str] + extra_items: Type | None + extra_items_readonly: bool fallback: Instance - extra_items_from: list[ProperType] # only used during semantic analysis + # TypedDicts merged in with ** in an inline TypedDict; only used during + # semantic analysis. Unrelated to the PEP 728 extra_items field above. + merged_from: list[ProperType] to_be_mutated: bool # only used in a plugin for `.update`, `|=`, etc def __init__( @@ -3052,19 +3071,28 @@ def __init__( line: int = -1, column: int = -1, *, - is_closed: bool = False, + extra_items: Type | None = None, + extra_items_readonly: bool = False, ) -> None: super().__init__(line, column) self.items = items self.required_keys = required_keys self.readonly_keys = readonly_keys - self.is_closed = is_closed + self.extra_items = extra_items + self.extra_items_readonly = extra_items_readonly self.fallback = fallback self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 - self.extra_items_from = [] + self.merged_from = [] self.to_be_mutated = False + @property + def is_closed(self) -> bool: + """Whether no extra items are allowed (PEP 728 closed=True or extra_items=Never).""" + return self.extra_items is not None and isinstance( + get_proper_type(self.extra_items), UninhabitedType + ) + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_typeddict_type(self) @@ -3075,7 +3103,8 @@ def __hash__(self) -> int: self.fallback, frozenset(self.required_keys), frozenset(self.readonly_keys), - self.is_closed, + self.extra_items, + self.extra_items_readonly, ) ) @@ -3093,7 +3122,8 @@ def __eq__(self, other: object) -> bool: and self.fallback == other.fallback and self.required_keys == other.required_keys and self.readonly_keys == other.readonly_keys - and self.is_closed == other.is_closed + and self.extra_items == other.extra_items + and self.extra_items_readonly == other.extra_items_readonly ) def serialize(self) -> JsonDict: @@ -3103,7 +3133,8 @@ def serialize(self) -> JsonDict: "required_keys": sorted(self.required_keys), "readonly_keys": sorted(self.readonly_keys), "fallback": self.fallback.serialize(), - "is_closed": self.is_closed, + "extra_items": self.extra_items.serialize() if self.extra_items is not None else None, + "extra_items_readonly": self.extra_items_readonly, } @classmethod @@ -3114,7 +3145,10 @@ def deserialize(cls, data: JsonDict) -> TypedDictType: set(data["required_keys"]), set(data["readonly_keys"]), Instance.deserialize(data["fallback"]), - is_closed=bool(data["is_closed"]), + extra_items=( + deserialize_type(data["extra_items"]) if data["extra_items"] is not None else None + ), + extra_items_readonly=bool(data["extra_items_readonly"]), ) def write(self, data: WriteBuffer) -> None: @@ -3123,7 +3157,8 @@ def write(self, data: WriteBuffer) -> None: write_type_map(data, self.items) write_str_list(data, sorted(self.required_keys)) write_str_list(data, sorted(self.readonly_keys)) - write_bool(data, self.is_closed) + write_type_opt(data, self.extra_items) + write_bool(data, self.extra_items_readonly) write_tag(data, END_TAG) @classmethod @@ -3135,7 +3170,8 @@ def read(cls, data: ReadBuffer) -> TypedDictType: set(read_str_list(data)), set(read_str_list(data)), fallback, - is_closed=read_bool(data), + extra_items=read_type_opt(data), + extra_items_readonly=read_bool(data), ) assert read_tag(data) == END_TAG return ret @@ -3161,7 +3197,8 @@ def copy_modified( item_names: list[str] | None = None, required_keys: set[str] | None = None, readonly_keys: set[str] | None = None, - is_closed: bool | None = None, + extra_items: Bogus[Type | None] = _dummy, + extra_items_readonly: Bogus[bool] = _dummy, ) -> TypedDictType: if fallback is None: fallback = self.fallback @@ -3173,8 +3210,10 @@ def copy_modified( required_keys = self.required_keys if readonly_keys is None: readonly_keys = self.readonly_keys - if is_closed is None: - is_closed = self.is_closed + if extra_items is _dummy: + extra_items = self.extra_items + if extra_items_readonly is _dummy: + extra_items_readonly = self.extra_items_readonly if item_names is not None: items = {k: v for (k, v) in items.items() if k in item_names} required_keys &= set(item_names) @@ -3185,7 +3224,8 @@ def copy_modified( fallback, self.line, self.column, - is_closed=is_closed, + extra_items=extra_items, + extra_items_readonly=extra_items_readonly, ) def zip(self, right: TypedDictType) -> Iterable[tuple[str, Type, Type]]: @@ -3198,16 +3238,30 @@ def zip(self, right: TypedDictType) -> Iterable[tuple[str, Type, Type]]: def item(self, item_name: str) -> TypedDictItem: item_type = self.items.get(item_name) if item_type is not None: - is_required = item_name in self.required_keys - is_readonly = item_name in self.readonly_keys - elif self.is_closed: - item_type = UninhabitedType() - is_required = False - is_readonly = False - else: - is_required = False - is_readonly = True - return TypedDictItem(item_type, is_required, is_readonly) + return TypedDictItem( + item_type, item_name in self.required_keys, item_name in self.readonly_keys + ) + return self.extra_item() + + def extra_item(self) -> TypedDictItem: + """The PEP 728 pseudo-item describing the keys not in `items`.""" + if self.extra_items is not None: + # Extra items are always non-required. + return TypedDictItem(self.extra_items, False, self.extra_items_readonly) + # Default-open TypedDicts implicitly allow NotRequired[ReadOnly[object]] items. + return TypedDictItem(None, False, True) + + def value_types_with_extra(self) -> list[Type]: + """All value types, including the non-Never extra_items pseudo-item (PEP 728). + + Only meaningful for a TypedDict with closed=True or extra_items= set, whose + complete set of value types is statically known. + """ + assert self.extra_items is not None + value_types: list[Type] = list(self.items.values()) + if not self.is_closed: + value_types.append(self.extra_items) + return value_types def zipall(self, right: TypedDictType) -> Iterable[tuple[str, TypedDictItem, TypedDictItem]]: left = self @@ -4058,7 +4112,13 @@ def item_str(name: str, typ: str) -> str: + "}" ) if t.is_closed: + # closed=True is preferred over the equivalent extra_items=Never (PEP 728). s += ", closed=True" + elif t.extra_items is not None: + extra = t.extra_items.accept(self) + if t.extra_items_readonly: + extra = f"ReadOnly[{extra}]" + s += f", extra_items={extra}" prefix = "" if t.fallback and t.fallback.type: if t.fallback.type.fullname not in TPDICT_FB_NAMES: diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index 9cbf9d95067a9..15d7e8c60ee3c 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -106,6 +106,8 @@ def visit_tuple_type(self, t: TupleType, /) -> None: def visit_typeddict_type(self, t: TypedDictType, /) -> None: self.traverse_types(t.items.values()) + if t.extra_items is not None: + t.extra_items.accept(self) t.fallback.accept(self) def visit_union_type(self, t: UnionType, /) -> None: diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index d1e928441a9ec..51afb4bdb89c7 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -2750,7 +2750,7 @@ reveal_type(Sneaky(f5, 1).kwargs) # N: Revealed type is "TypedDict('builtins.di reveal_type(Sneaky(f5, 1, 2).kwargs) # N: Revealed type is "TypedDict('builtins.dict', {'x'?: builtins.int, 'y'?: builtins.int}, closed=True)" reveal_type(Sneaky(f6, x=1).kwargs) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" reveal_type(Sneaky(f6, x=1, y=2).kwargs) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" -reveal_type(Sneaky(f7, 1, y='').kwargs) # N: Revealed type is "TypedDict('builtins.dict', {'x'?: builtins.int})" +reveal_type(Sneaky(f7, 1, y='').kwargs) # N: Revealed type is "TypedDict('builtins.dict', {'x'?: builtins.int}, extra_items=builtins.str)" reveal_type(Sneaky(f8, 1, y='').kwargs) # N: Revealed type is "builtins.dict[builtins.str, builtins.str]" reveal_type(Sneaky(f9, 1, y=0).kwargs) # N: Revealed type is "TypedDict('builtins.dict', {'x'?: builtins.int, 'y': builtins.int, 'z'?: builtins.str}, closed=True)" reveal_type(Sneaky(f9, 1, y=0, z='').kwargs) # N: Revealed type is "TypedDict('builtins.dict', {'x'?: builtins.int, 'y': builtins.int, 'z'?: builtins.str}, closed=True)" diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index f504e00688cff..913183c11e584 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -1106,6 +1106,26 @@ main:2: note: Revealed type is "TypedDict('m.D', {'x': builtins.int, 'y': builti -- Modules -- + +[case testSerializeExtraItemsTypedDict] +from m import d, r +reveal_type(d) +reveal_type(r) +[file m.py] +from typing import ReadOnly, TypedDict +D = TypedDict('D', {'x': int}, extra_items=int) +R = TypedDict('R', {'x': int}, extra_items=ReadOnly[str]) +d: D +r: R +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out1] +main:2: note: Revealed type is "TypedDict('m.D', {'x': builtins.int}, extra_items=builtins.int)" +main:3: note: Revealed type is "TypedDict('m.R', {'x': builtins.int}, extra_items=ReadOnly[builtins.str])" +[out2] +main:2: note: Revealed type is "TypedDict('m.D', {'x': builtins.int}, extra_items=builtins.int)" +main:3: note: Revealed type is "TypedDict('m.R', {'x': builtins.int}, extra_items=ReadOnly[builtins.str])" + [case testSerializeImport] import b b.c.f() diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 01414ee2f457a..75da359681055 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -4800,13 +4800,13 @@ class A(TypedDict, total=True): class B(TypedDict, total=False): key: ReadOnly[int] -class C(B, A): +class C(B, A): # E: Field "key" is required in base class "A" but not in base class "B" pass class D(A, B): pass c: C -reveal_type(c) # N: Revealed type is "TypedDict('__main__.C', {'key'=: builtins.int})" +reveal_type(c) # N: Revealed type is "TypedDict('__main__.C', {'key'?=: builtins.int})" d: D reveal_type(d) # N: Revealed type is "TypedDict('__main__.D', {'key'=: builtins.int})" [builtins fixtures/dict.pyi] @@ -6126,3 +6126,694 @@ class Callback(Model): def __init__(self, **kwargs: Unpack[_CallbackInit]) -> None: ... [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +# Extra items +# See https://peps.python.org/pep-0728/ + +[case testTypedDictExtraItemsClassAndFunctionalSyntax] +from typing import ReadOnly, TypedDict +class Movie(TypedDict, extra_items=bool): + name: str +m: Movie +reveal_type(m) # N: Revealed type is "TypedDict('__main__.Movie', {'name': builtins.str}, extra_items=builtins.bool)" +MovieF = TypedDict('MovieF', {'name': str}, extra_items=bool) +mf: MovieF +reveal_type(mf) # N: Revealed type is "TypedDict('__main__.MovieF', {'name': builtins.str}, extra_items=builtins.bool)" +class MovieRO(TypedDict, extra_items=ReadOnly[int]): + name: str +mr: MovieRO +reveal_type(mr) # N: Revealed type is "TypedDict('__main__.MovieRO', {'name': builtins.str}, extra_items=ReadOnly[builtins.int])" +class MovieNone(TypedDict, extra_items=None): + name: str +mn: MovieNone +reveal_type(mn) # N: Revealed type is "TypedDict('__main__.MovieNone', {'name': builtins.str}, extra_items=None)" +class MovieTotal(TypedDict, total=False, extra_items=bool): + name: str +mt: MovieTotal +reveal_type(mt) # N: Revealed type is "TypedDict('__main__.MovieTotal', {'name'?: builtins.str}, extra_items=builtins.bool)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsCannotBeUsedWithClosed] +from typing import TypedDict +class Movie(TypedDict, closed=True, extra_items=int): # E: Cannot use "closed" and "extra_items" in the same TypedDict definition + name: str +MovieF = TypedDict('MovieF', {'name': str}, closed=True, extra_items=int) # E: Cannot use "closed" and "extra_items" in the same TypedDict definition +m: Movie +mf: MovieF +# extra_items wins for error recovery: +reveal_type(m) # N: Revealed type is "TypedDict('__main__.Movie', {'name': builtins.str}, extra_items=builtins.int)" +reveal_type(mf) # N: Revealed type is "TypedDict('__main__.MovieF', {'name': builtins.str}, extra_items=builtins.int)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsInvalidArgument] +from typing import NotRequired, Required, TypedDict +class A(TypedDict, extra_items=Required[int]): # E: "Required[]" is not allowed in "extra_items" + x: int +class B(TypedDict, extra_items=NotRequired[int]): # E: "NotRequired[]" is not allowed in "extra_items" + x: int +C = TypedDict('C', {'x': int}, extra_items=Required[str]) # E: "Required[]" is not allowed in "extra_items" +D = TypedDict('D', {'x': int}, extra_items=[str]) # E: Bracketed expression "[...]" is not valid as a type # N: Did you mean "List[...]"? +F = TypedDict('F', {'x': int}, extra_items=lambda: 0) # E: Invalid "extra_items" argument +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsRepeatedKeyword_no_parallel] +# The native parser treats a repeated keyword argument as a blocking syntax +# error, so this case is kept out of the parallel suite (which forces it on). +from typing import TypedDict +E = TypedDict('E', {'x': int}, extra_items=int, extra_items=str) # E: Repeated keyword argument "extra_items" for "TypedDict" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubclassingInherited] +from typing import ReadOnly, TypedDict +class MovieBase(TypedDict, extra_items=ReadOnly[int]): + name: str +class Movie(MovieBase): + year: int +m: Movie +reveal_type(m) # N: Revealed type is "TypedDict('__main__.Movie', {'name': builtins.str, 'year': builtins.int}, extra_items=ReadOnly[builtins.int])" +class MovieChild(Movie): + pass +mc: MovieChild +reveal_type(mc) # N: Revealed type is "TypedDict('__main__.MovieChild', {'name': builtins.str, 'year': builtins.int}, extra_items=ReadOnly[builtins.int])" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubclassingMutable] +from typing import NotRequired, TypedDict +class MovieBase(TypedDict, extra_items=int): + name: str +class MovieRequiredYear(MovieBase): + year: int # E: Field "year" must be non-required, since "extra_items" of base class "MovieBase" is not read-only +class MovieWrongType(MovieBase): + year: NotRequired[str] # E: Type of field "year" incompatible with "extra_items" of base class "MovieBase" +class MovieWithYear(MovieBase): + year: NotRequired[int] +m: MovieWithYear +reveal_type(m) # N: Revealed type is "TypedDict('__main__.MovieWithYear', {'name': builtins.str, 'year'?: builtins.int}, extra_items=builtins.int)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubclassingReadOnly] +from typing import ReadOnly, TypedDict +from typing_extensions import Never +class BookBase(TypedDict, extra_items=ReadOnly[str]): + title: str +class Book(BookBase, extra_items=str): # OK, narrowing a read-only pseudo-item + year: str +class BookBadField(BookBase): + year: int # E: Type of field "year" incompatible with "extra_items" of base class "BookBase" +class BookBadExtra(BookBase, extra_items=int): # E: Definition of "extra_items" incompatible with base class "BookBase" + pass +class BookClosed(BookBase, closed=True): + pass +bc: BookClosed +reveal_type(bc) # N: Revealed type is "TypedDict('__main__.BookClosed', {'title': builtins.str}, closed=True)" +class BookNever(BookBase, extra_items=Never): + pass +bn: BookNever +reveal_type(bn) # N: Revealed type is "TypedDict('__main__.BookNever', {'title': builtins.str}, closed=True)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsRedeclare] +from typing import ReadOnly, TypedDict +class Parent(TypedDict, extra_items=int): + pass +class ChildSame(Parent, extra_items=int): # OK, evolution-compatible redeclaration + pass +class ChildOther(Parent, extra_items=str): # E: Definition of "extra_items" incompatible with base class "Parent" + pass +class ChildReadOnly(Parent, extra_items=ReadOnly[int]): # E: Cannot redeclare mutable "extra_items" of base class "Parent" as read-only + pass +class ChildClosed(Parent, closed=True): # E: Definition of "extra_items" incompatible with base class "Parent" + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubclassingClosedFalse] +from typing import TypedDict +class MovieBase(TypedDict, extra_items=int): + name: str +class MovieOpen(MovieBase, closed=False): # E: Open TypedDict class cannot subclass TypedDict class "MovieBase" with "extra_items" + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsMultipleInheritance] +from typing import NotRequired, ReadOnly, TypedDict +class A(TypedDict, extra_items=int): + a: NotRequired[int] +class B(TypedDict, extra_items=int): + b: NotRequired[int] +class AB(A, B): + pass +ab: AB +reveal_type(ab) # N: Revealed type is "TypedDict('__main__.AB', {'b'?: builtins.int, 'a'?: builtins.int}, extra_items=builtins.int)" +class C(TypedDict, extra_items=str): + c: NotRequired[str] +class AC(A, C): # E: Type of field "c" from base class "C" incompatible with "extra_items" of base class "A" \ + # E: Type of field "a" from base class "A" incompatible with "extra_items" of base class "C" \ + # E: Definition of "extra_items" incompatible with base class "C" + pass +class RO(TypedDict, extra_items=ReadOnly[object]): + r: NotRequired[int] +class ARO(A, RO): # OK, mutable int pseudo-item satisfies ReadOnly[object] + pass +aro: ARO +reveal_type(aro) # N: Revealed type is "TypedDict('__main__.ARO', {'r'?: builtins.int, 'a'?: builtins.int}, extra_items=builtins.int)" +# With mutable extra_items, fields of one base must satisfy the pseudo-item of the other: +class D(TypedDict, extra_items=int): + d: str +class AD(A, D): # E: Field "d" from base class "D" must be non-required, since "extra_items" of base class "A" is not read-only \ + # E: Type of field "d" from base class "D" incompatible with "extra_items" of base class "A" + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubtyping] +from typing import TypedDict +class MovieExtraInt(TypedDict, extra_items=int): + name: str +class MovieExtraIntAlso(TypedDict, extra_items=int): + name: str +class MovieExtraStr(TypedDict, extra_items=str): + name: str +class MovieNotClosed(TypedDict): + name: str +class MovieClosed(TypedDict, closed=True): + name: str +def f( + extra_int: MovieExtraInt, + extra_int2: MovieExtraIntAlso, + extra_str: MovieExtraStr, + not_closed: MovieNotClosed, + closed: MovieClosed, +) -> None: + a: MovieExtraInt = extra_int2 + b: MovieExtraInt = extra_str # E: Incompatible types in assignment (expression has type "MovieExtraStr", variable has type "MovieExtraInt") + c: MovieExtraStr = extra_int # E: Incompatible types in assignment (expression has type "MovieExtraInt", variable has type "MovieExtraStr") + d: MovieExtraInt = not_closed # E: Incompatible types in assignment (expression has type "MovieNotClosed", variable has type "MovieExtraInt") + e: MovieNotClosed = extra_int + g: MovieExtraInt = closed # E: Incompatible types in assignment (expression has type "MovieClosed", variable has type "MovieExtraInt") + h: MovieClosed = extra_int # E: Incompatible types in assignment (expression has type "MovieExtraInt", variable has type "MovieClosed") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubtypingReadOnly] +from typing import NotRequired, ReadOnly, TypedDict +class Movie(TypedDict, extra_items=ReadOnly[object]): + name: str +class MovieDetails(TypedDict, extra_items=int): + name: str + year: NotRequired[int] +class MovieNotClosed(TypedDict): + name: str +class MovieClosed(TypedDict, closed=True): + name: str +def f(details: MovieDetails, not_closed: MovieNotClosed, closed: MovieClosed) -> None: + a: Movie = details # OK, 'int' is assignable to ReadOnly[object] + b: Movie = not_closed # OK, implicit pseudo-item is ReadOnly[object] + c: Movie = closed +class MovieStrInt(TypedDict, extra_items=ReadOnly[str]): + name: str +def g(details: MovieDetails) -> None: + m: MovieStrInt = details # E: Incompatible types in assignment (expression has type "MovieDetails", variable has type "MovieStrInt") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsSubtypingRequiredness] +from typing import NotRequired, TypedDict +class Movie(TypedDict, extra_items=int): + name: str +class MovieWithYear(TypedDict, extra_items=int): + name: str + year: int +def f(with_year: MovieWithYear) -> None: + # 'year' is required in MovieWithYear but must be non-required to match + # the mutable pseudo-item of Movie + movie: Movie = with_year # E: Incompatible types in assignment (expression has type "MovieWithYear", variable has type "Movie") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsGeneric] +from typing import Generic, ReadOnly, TypeVar, TypedDict +T = TypeVar('T') +class TD(TypedDict, Generic[T], extra_items=ReadOnly[T]): + name: str +def f(x: TD[int]) -> None: + reveal_type(x) # N: Revealed type is "TypedDict('__main__.TD', {'name': builtins.str}, extra_items=ReadOnly[builtins.int])" +TDF = TypedDict('TDF', {'name': str}, extra_items=T) +def g(y: TDF[str]) -> None: + reveal_type(y) # N: Revealed type is "TypedDict('__main__.TDF', {'name': builtins.str}, extra_items=builtins.str)" +class Child(TD[int]): + year: int +def h(c: Child) -> None: + reveal_type(c) # N: Revealed type is "TypedDict('__main__.Child', {'name': builtins.str, 'year': builtins.int}, extra_items=ReadOnly[builtins.int])" +class BadChild(TD[str]): + year: int # E: Type of field "year" incompatible with "extra_items" of base class "TD" +def i(x: TD[int], y: TD[object]) -> None: + a: TD[object] = x # OK, read-only pseudo-items are covariant + b: TD[int] = y # E: Incompatible types in assignment (expression has type "TD[object]", variable has type "TD[int]") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testJoinOfTypedDictsWithExtraItems] +from typing import TypedDict +class ExtraInt(TypedDict, extra_items=int): + name: str +class ExtraIntAlso(TypedDict, extra_items=int): + name: str +class ExtraStr(TypedDict, extra_items=str): + name: str +class Closed(TypedDict, closed=True): + name: str +class Open(TypedDict): + name: str +def f(a: ExtraInt, b: ExtraIntAlso, c: ExtraStr, d: Closed, e: Open, flag: bool) -> None: + reveal_type(a if flag else b) # N: Revealed type is "TypedDict('__main__.ExtraInt', {'name': builtins.str}, extra_items=builtins.int)" + reveal_type(a if flag else d) # N: Revealed type is "TypedDict({'name': builtins.str}, extra_items=ReadOnly[builtins.int])" + reveal_type(a if flag else e) # N: Revealed type is "TypedDict('__main__.Open', {'name': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testMeetOfTypedDictsWithExtraItems] +from typing import Callable, TypedDict, TypeVar +T = TypeVar('T') +def meet(x: Callable[[T, T], None]) -> T: pass +class ExtraInt(TypedDict, extra_items=int): + name: str +class ExtraIntAlso(TypedDict, extra_items=int): + name: str +class ExtraStr(TypedDict, extra_items=str): + name: str +class Open(TypedDict): + name: str +def fII(x: ExtraInt, y: ExtraIntAlso) -> None: pass +def fIS(x: ExtraInt, y: ExtraStr) -> None: pass +def fIO(x: ExtraInt, y: Open) -> None: pass +if int(): + reveal_type(meet(fII)) # N: Revealed type is "TypedDict('__main__.ExtraInt', {'name': builtins.str}, extra_items=builtins.int)" +if int(): + # Incompatible mutable pseudo-items make the meet uninhabited + reveal_type(meet(fIS)) # N: Revealed type is "Never" +if int(): + reveal_type(meet(fIO)) # N: Revealed type is "TypedDict('__main__.ExtraInt', {'name': builtins.str}, extra_items=builtins.int)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictSubtypingTopTypeReadOnlyKeyCanBeMissing] +from typing import NotRequired, ReadOnly, TypedDict +class A(TypedDict): + name: str + x: ReadOnly[NotRequired[object]] +class ANonTop(TypedDict): + name: str + x: ReadOnly[NotRequired[int]] +class AReq(TypedDict): + name: str + x: ReadOnly[object] +class B(TypedDict): + name: str +class BExtra(TypedDict, extra_items=int): + name: str +def f(b: B, be: BExtra) -> None: + # PEP 728: an item that is read-only, not required, and of top value type + # does not require the corresponding key to be present in the subtype. + a: A = b + a2: A = be + bad: ANonTop = b # E: Incompatible types in assignment (expression has type "B", variable has type "ANonTop") + bad2: AReq = b # E: Incompatible types in assignment (expression has type "B", variable has type "AReq") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsConstruction] +from typing import TypedDict, ReadOnly + +class Movie(TypedDict, extra_items=bool): + name: str +a: Movie = {"name": "Blade Runner", "novel_adaptation": True} +b: Movie = {"name": "Blade Runner", "year": 1982} # E: Incompatible types (expression has type "int", extra TypedDict item "year" has type "bool") + +class ExtraMovie(TypedDict, extra_items=int): + name: str +ExtraMovie(name="No Country for Old Men") +ExtraMovie(name="No Country for Old Men", year=2007) +ExtraMovie(name="No Country for Old Men", language="English") # E: Incompatible types (expression has type "str", extra TypedDict item "language" has type "int") + +class ClosedMovie(TypedDict, closed=True): + name: str +ClosedMovie(name="No Country for Old Men") +ClosedMovie(name="No Country for Old Men", year=2007) # E: Extra key "year" for TypedDict "ClosedMovie" + +class ROMovie(TypedDict, extra_items=ReadOnly[int]): + name: str +# Constructing read-only extra items is allowed +r: ROMovie = {"name": "x", "year": 1982} +r2 = ROMovie(name="x", year=1982) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsConstructionGenericInference] +from typing import Generic, TypedDict, TypeVar +T = TypeVar('T') +class TD(TypedDict, Generic[T], extra_items=T): + name: str +reveal_type(TD(name="x", extra=1)) # N: Revealed type is "TypedDict('__main__.TD', {'name': builtins.str}, extra_items=builtins.int)" +reveal_type(TD(name="x", e1=1, e2="s")) # N: Revealed type is "TypedDict('__main__.TD', {'name': builtins.str}, extra_items=builtins.object)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsGetItemSetItemDelItem] +from typing import TypedDict, ReadOnly + +class Movie(TypedDict, extra_items=int): + name: str +class ROMovie(TypedDict, extra_items=ReadOnly[int]): + name: str + +def f(m: Movie, r: ROMovie) -> None: + reveal_type(m["name"]) # N: Revealed type is "builtins.str" + reveal_type(m["year"]) # N: Revealed type is "builtins.int" + m["year"] = 1982 + m["year"] = "x" # E: Value of "year" has incompatible type "str"; expected "int" + del m["year"] + del m["name"] # E: Key "name" of TypedDict "Movie" cannot be deleted + reveal_type(r["year"]) # N: Revealed type is "builtins.int" + r["year"] = 1982 # E: ReadOnly TypedDict key "year" TypedDict is mutated + del r["year"] # E: Key "year" of TypedDict "ROMovie" cannot be deleted +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsMethods] +from typing import TypedDict, ReadOnly + +class Movie(TypedDict, extra_items=int): + name: str +class ROMovie(TypedDict, extra_items=ReadOnly[int]): + name: str +class Closed(TypedDict, closed=True): + name: str + +def f(m: Movie, r: ROMovie, c: Closed) -> None: + reveal_type(m.get("year")) # N: Revealed type is "builtins.int | None" + reveal_type(m.get("year", 0)) # N: Revealed type is "builtins.int" + reveal_type(c.get("year", 0)) # N: Revealed type is "Literal[0]?" + reveal_type(m.pop("year")) # N: Revealed type is "builtins.int" + reveal_type(m.pop("year", 0)) # N: Revealed type is "builtins.int" + r.pop("year") # E: Key "year" of TypedDict "ROMovie" cannot be deleted + c.pop("year") # E: TypedDict "Closed" has no key "year" + reveal_type(m.setdefault("year", 0)) # N: Revealed type is "builtins.int" + m.setdefault("year", "x") # E: Argument 2 to "setdefault" of "TypedDict" has incompatible type "str"; expected "int" + r.setdefault("year", 0) # E: ReadOnly TypedDict key "year" TypedDict is mutated + m.update({"year": 1982}) + m.update({"year": "x"}) # E: Incompatible types (expression has type "str", extra TypedDict item "year" has type "int") + r.update({"year": 1982}) # E: ReadOnly TypedDict key "year" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsUnpackKwargs] +from typing import TypedDict +from typing_extensions import Unpack + +class MovieNoExtra(TypedDict): + name: str +class MovieExtra(TypedDict, extra_items=int): + name: str + +def f(**kwargs: Unpack[MovieNoExtra]) -> None: ... +def g(**kwargs: Unpack[MovieExtra]) -> None: ... + +f(name="No Country for Old Men", year=2007) # E: Unexpected keyword argument "year" for "f" +g(name="No Country for Old Men", year=2007) +g(name="No Country for Old Men", year="") # E: Argument "year" to "g" has incompatible type "str"; expected "int" + +# Equivalent to an explicit signature with typed **kwargs: +def g2(*, name: str, **kwargs: int) -> None: ... +gv = g +gv = g2 +g2v = g2 +g2v = g +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsUnpackStar] +# flags: --extra-checks +from typing import TypedDict + +class Dest(TypedDict, extra_items=int): + name: str +class SrcCompat(TypedDict, extra_items=int): + name: str +class SrcBool(TypedDict, extra_items=bool): + name: str +class SrcStr(TypedDict, extra_items=str): + name: str +class SrcOpen(TypedDict): + name: str +class SrcClosed(TypedDict, closed=True): + name: str + +def f(compat: SrcCompat, srcb: SrcBool, srcs: SrcStr, op: SrcOpen, cl: SrcClosed) -> None: + d: Dest = {**compat} + d2: Dest = {**srcb} + d3: Dest = {**cl} + d4: Dest = {**srcs} # E: Cannot unpack item that may contain incompatible extra keys into a TypedDict with "extra_items" + d5: Dest = {**op} # E: Cannot unpack item that may contain incompatible extra keys into a TypedDict with "extra_items" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsMappingAssignability] +from typing import Mapping, TypedDict, Union + +class MovieExtraStr(TypedDict, extra_items=str): + name: str +class MovieExtraInt(TypedDict, extra_items=int): + name: str +class Closed(TypedDict, closed=True): + name: str + year: int +class Open(TypedDict): + name: str + +def f(extra_str: MovieExtraStr, extra_int: MovieExtraInt, closed: Closed, open_td: Open) -> None: + a: Mapping[str, str] = extra_str + b: Mapping[str, int] = extra_int # E: Incompatible types in assignment (expression has type "MovieExtraInt", variable has type "Mapping[str, int]") + c: Mapping[str, Union[int, str]] = extra_int + d: Mapping[str, Union[str, int]] = closed + e: Mapping[str, str] = open_td # E: Incompatible types in assignment (expression has type "Open", variable has type "Mapping[str, str]") + g: Mapping[str, object] = open_td +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsDictAssignability] +from typing import MutableMapping, NotRequired, ReadOnly, TypedDict + +class IntDict(TypedDict, extra_items=int): + pass +class IntDictWithNum(IntDict): + num: NotRequired[int] +class ReqDict(TypedDict, extra_items=int): + num: int +class RODict(TypedDict, extra_items=ReadOnly[int]): + pass +class Closed(TypedDict, closed=True): + pass + +def f(x: IntDict, n: IntDictWithNum, r: ReqDict, ro: RODict, c: Closed) -> None: + a: dict[str, int] = x + b: dict[str, int] = n + m: MutableMapping[str, int] = n + bad1: dict[str, object] = x # E: Incompatible types in assignment (expression has type "IntDict", variable has type "dict[str, object]") + bad2: dict[str, int] = r # E: Incompatible types in assignment (expression has type "ReqDict", variable has type "dict[str, int]") + bad3: dict[str, int] = ro # E: Incompatible types in assignment (expression has type "RODict", variable has type "dict[str, int]") + bad4: dict[str, int] = c # E: Incompatible types in assignment (expression has type "Closed", variable has type "dict[str, int]") + +def g(might_not_be_a_builtin_dict: dict[str, int]) -> None: + int_dict: IntDict = might_not_be_a_builtin_dict # E: Incompatible types in assignment (expression has type "dict[str, int]", variable has type "IntDict") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsValuesItems] +from typing import TypedDict + +class MovieExtraInt(TypedDict, extra_items=int): + name: str +class Closed(TypedDict, closed=True): + name: str + year: int +class Open(TypedDict): + name: str + +def f(movie: MovieExtraInt, closed: Closed, open_td: Open) -> None: + reveal_type(movie.values()) # N: Revealed type is "typing.Iterable[builtins.str | builtins.int]" + reveal_type(movie.items()) # N: Revealed type is "typing._ItemsView[builtins.str, builtins.str | builtins.int]" + reveal_type(closed.values()) # N: Revealed type is "typing.Iterable[builtins.str | builtins.int]" + reveal_type(closed.items()) # N: Revealed type is "typing._ItemsView[builtins.str, builtins.str | builtins.int]" + reveal_type(open_td.values()) # N: Revealed type is "typing.Iterable[builtins.object]" + reveal_type(open_td.items()) # N: Revealed type is "typing._ItemsView[builtins.str, builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsArbitraryStrKey] +from typing import NotRequired, TypedDict + +class MovieExtraInt(TypedDict, extra_items=int): + name: str +class Closed(TypedDict, closed=True): + name: str + year: int +class Open(TypedDict): + name: str +class IntDictWithNum(TypedDict, extra_items=int): + num: NotRequired[int] + +def f(movie: MovieExtraInt, closed: Closed, open_td: Open, d: IntDictWithNum, key: str) -> None: + reveal_type(movie[key]) # N: Revealed type is "builtins.str | builtins.int" + reveal_type(closed[key]) # N: Revealed type is "builtins.str | builtins.int" + open_td[key] # E: TypedDict key must be a string literal; expected one of ("name") + # Writes with arbitrary keys need the TypedDict to behave like dict[str, VT]: + movie[key] = 1 # E: TypedDict key must be a string literal; expected one of ("name") + d[key] = 42 + del d[key] + d.clear() + reveal_type(d.popitem()) # N: Revealed type is "tuple[builtins.str, builtins.int]" + movie.clear() # E: "MovieExtraInt" has no attribute "clear" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsRecursive] +from typing import TypedDict + +class Tree(TypedDict, extra_items="Tree"): + name: str + +t: Tree = {"name": "root", "child": {"name": "leaf"}} +t2: Tree = {"name": "root", "child": {"name": "leaf", "bad": 1}} # E: Incompatible types (expression has type "int", extra TypedDict item "bad" has type "Tree") + +def f(t: Tree) -> None: + reveal_type(t["child"]) # N: Revealed type is "TypedDict('__main__.Tree', {'name': builtins.str}, extra_items=TypedDict('__main__.Tree', {'name': builtins.str}, extra_items=...))" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsUnpackProtocolEquivalence] +from typing import Protocol, TypedDict +from typing_extensions import Unpack + +class MovieExtra(TypedDict, extra_items=int): + name: str + +class UnpackSig(Protocol): + def __call__(self, **kwargs: Unpack[MovieExtra]) -> None: ... +class ExplicitSig(Protocol): + def __call__(self, *, name: str, **kwargs: int) -> None: ... +class WrongSig(Protocol): + def __call__(self, *, name: str, **kwargs: str) -> None: ... + +def g(**kwargs: Unpack[MovieExtra]) -> None: ... +def g_explicit(*, name: str, **kwargs: int) -> None: ... + +p1: ExplicitSig = g +p2: UnpackSig = g_explicit +p3: UnpackSig = g +p4: ExplicitSig = g_explicit +p5: WrongSig = g # E: Incompatible types in assignment (expression has type "def g(**kwargs: Unpack[MovieExtra]) -> None", variable has type "WrongSig") \ + # N: "WrongSig.__call__" has type "def __call__(self, *, name: str, **kwargs: str) -> None" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsCallSiteStarStar] +from typing import ReadOnly, TypedDict +from typing_extensions import Unpack + +class TDInt(TypedDict, extra_items=int): + name: str +class TDStr(TypedDict, extra_items=str): + name: str +class TDRO(TypedDict, extra_items=ReadOnly[int]): + name: str +class TDClosed(TypedDict, closed=True): + name: str +class TDOpen(TypedDict): + name: str + +def f(*, name: str, **kwargs: int) -> None: ... +def g(*, name: str) -> None: ... +def u(**kwargs: Unpack[TDInt]) -> None: ... + +def test(ti: TDInt, ts: TDStr, tro: TDRO, tc: TDClosed, to: TDOpen) -> None: + f(**ti) + f(**ts) # E: Argument 1 to "f" has incompatible type "**TDStr"; expected "int" + f(**tro) # Reading read-only extra items is fine. + f(**tc) # A closed TypedDict contributes no extra keys. + f(**to) # The possible extra keys of a default-open TypedDict are not typed. + g(**ti) # No **kwargs in the callee: the extra keys may all be absent. + u(**ti) + u(**ts) # E: Argument 1 to "u" has incompatible type "**TDStr"; expected "int" + u(**tc) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyRedeclarationRequiredness] +from typing import NotRequired, ReadOnly, Required, TypedDict + +class Base(TypedDict): + a: Required[int] + b: ReadOnly[NotRequired[int]] + c: ReadOnly[Required[int]] + +class MutableAsReadOnly(Base): + a: ReadOnly[int] # E: Cannot redeclare mutable field "a" of base class "Base" as read-only + +class StrengthenReadOnly(Base): # OK: a read-only item may be made required + b: ReadOnly[Required[int]] + +class WeakenReadOnly(Base): + c: ReadOnly[NotRequired[int]] # E: Field "c" is required in base class "Base" + +class Left(TypedDict): + x: ReadOnly[NotRequired[int]] + +class Right(TypedDict): + x: ReadOnly[Required[int]] + +class Conflict(Left, Right): # E: Field "x" is required in base class "Right" but not in base class "Left" + pass + +class Resolvable(Right, Left): # OK: "Right" strengthens the item declared in "Left" + pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictExtraItemsConstraintInference] +from typing import Generic, NotRequired, TypedDict, TypeVar + +T = TypeVar("T") + +class G(TypedDict, Generic[T]): + x: NotRequired[T] + +class GE(TypedDict, Generic[T], extra_items=T): + pass + +class TDInt(TypedDict, extra_items=int): + pass + +class TDNamed(TypedDict, extra_items=int): + y: NotRequired[int] + +def f(g: G[T]) -> T: + return g["x"] + +def h(e: GE[T]) -> T: + return e["y"] + +def test(ti: TDInt, tn: TDNamed) -> None: + # The named key on one side is matched against the pseudo-item of the other. + reveal_type(f(ti)) # N: Revealed type is "builtins.int" + reveal_type(h(tn)) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test index c4bbf7d3f2283..5417ab2477f20 100644 --- a/test-data/unit/diff.test +++ b/test-data/unit/diff.test @@ -694,6 +694,29 @@ p = Point(dict(x=42, y=1337)) __main__.Point __main__.p + +[case testTypedDictExtraItems] +from typing import ReadOnly, TypedDict +class Point(TypedDict, extra_items=int): + x: int +class Point2(TypedDict, extra_items=int): + x: int +class Point3(TypedDict, extra_items=int): + x: int +[file next.py] +from typing import ReadOnly, TypedDict +class Point(TypedDict, extra_items=str): + x: int +class Point2(TypedDict, extra_items=ReadOnly[int]): + x: int +class Point3(TypedDict, extra_items=int): + x: int +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +__main__.Point +__main__.Point2 + [case testTypeAliasSimple] A = int B = int diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index ef2e8c0b343bf..22777d6108c35 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3822,6 +3822,30 @@ def bar(x: C) -> int: == b.py:4: error: TypedDict "A" has no key "b" + +[case testTypedDictUpdateExtraItems] +import b +[file a.py] +from typing import TypedDict +class A(TypedDict, extra_items=int): + a: int +[file a.py.2] +from typing import TypedDict +class A(TypedDict, extra_items=str): + a: int +[file b.py] +from typing import TypedDict +from a import A +class B(TypedDict, extra_items=int): + a: int +def foo(x: A) -> None: + y: B = x +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +== +b.py:6: error: Incompatible types in assignment (expression has type "A", variable has type "B") + [case testBasicAliasUpdate] import b [file a.py] diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 4dc9aa3e7713e..926fdf98d5567 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -38,6 +38,8 @@ class dict(Mapping[KT, VT]): @overload def get(self, key: KT, default: T, /) -> Union[VT, T]: pass def __len__(self) -> int: ... + def clear(self) -> None: ... + def popitem(self) -> Tuple[KT, VT]: ... class int: # for convenience def __add__(self, x: Union[int, complex]) -> int: pass diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index 0bc5637b32708..0078433296f1d 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -79,5 +79,13 @@ class _TypedDict(Mapping[str, object]): def pop(self, k: NoReturn, default: T = ...) -> object: ... def update(self: T, __m: T) -> None: ... def __delitem__(self, k: NoReturn) -> None: ... + # Approximate return type (the real one is dict_items); a plugin hook + # makes the value type precise for TypedDicts with closed=True or + # extra_items= set (PEP 728). + def items(self) -> _ItemsView[str, object]: ... + +# Approximation of dict_items, to give the return type of _TypedDict.items +# the same shape as in typeshed. +class _ItemsView(Iterable[T_co], Generic[T, T_co]): ... class _SpecialForm: pass diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 0c8b74ecf29a1..91d5c359fea29 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -3822,11 +3822,15 @@ def f(x: str | None) -> None: ... [case testTypeddict] import typing, x +from typing import ReadOnly D1 = typing.TypedDict('D1', {'a': int, 'b': str}) D2 = typing.TypedDict('D2', {'a': int, 'b': str}, total=False) D3 = typing.TypedDict('D3', {'a': int, 'b': str}, closed=True) D4 = typing.TypedDict('D4', {'a': int, 'b': str}, closed=True, total=False) +D5 = typing.TypedDict('D5', {'a': int, 'b': str}, extra_items=int) +D6 = typing.TypedDict('D6', {'a': int, 'b': str}, extra_items=ReadOnly[str], total=False) [out] +from typing import ReadOnly from typing_extensions import TypedDict class D1(TypedDict): @@ -3845,6 +3849,14 @@ class D4(TypedDict, total=False, closed=True): a: int b: str +class D5(TypedDict, extra_items=int): + a: int + b: str + +class D6(TypedDict, total=False, extra_items=ReadOnly[str]): + a: int + b: str + [case testTypeddictClassWithKeyword] from typing import TypedDict class MyDict(TypedDict, total=False):