Implement TypedDict extra_items= (PEP 728)

docs/source/typed_dict.rst

@@ -354,6 +354,117 @@ The ``closed`` keyword can also be used in class-based syntax:
    class HasOnlyName(TypedDict, closed=True):
        name: str
 
+Extra items
+-----------
+
+Instead of fully closing a TypedDict, you can use the ``extra_items``
+keyword, introduced to ``TypedDict`` in Python 3.15 (and available via
+``typing_extensions.TypedDict`` in older versions), to allow keys that
+are not known in advance, as long as their values have a specific type
+(:pep:`728`):
+
+.. code-block:: python
+
+   class Movie(TypedDict, extra_items=bool):
+       name: str
+
+   a: Movie = {"name": "Blade Runner", "novel_adaptation": True}  # OK
+   b: Movie = {
+       "name": "Blade Runner",
+       "year": 1982,  # Error: "int" is not assignable to "bool"
+   }
+
+Extra items are treated as non-required items with the declared value
+type, so they can be read, assigned, and deleted like any other
+non-required key:
+
+.. code-block:: python
+
+   def f(movie: Movie) -> None:
+       reveal_type(movie["name"])              # Revealed type is "str"
+       reveal_type(movie["novel_adaptation"])  # Revealed type is "bool"
+       movie["novel_adaptation"] = False  # OK
+       del movie["novel_adaptation"]  # OK
+
+The ``extra_items`` argument accepts the ``ReadOnly[]`` qualifier, in
+which case the extra items cannot be mutated, and behave covariantly
+during assignability checks; ``Required[]`` and ``NotRequired[]`` are
+not allowed. ``extra_items`` cannot be combined with ``closed`` in the
+same definition, because ``closed=True`` is equivalent to
+``extra_items=Never``.
+
+Like regular items, ``extra_items`` is inherited by subclasses, and can
+only be redeclared (with a narrower type) if it is read-only in the
+parent. Any field a subclass adds must be compatible with the parent's
+``extra_items``: assignable to it if it is read-only, non-required and
+equivalent to it otherwise:
+
+.. code-block:: python
+
+   class MovieBase(TypedDict, extra_items=int | None):
+       name: str
+
+   class MovieA(MovieBase):  # Error: "year" is required
+       year: int | None
+
+   class MovieB(MovieBase):  # OK
+       year: NotRequired[int | None]
+
+A TypedDict with ``extra_items`` also accepts arbitrary keyword
+arguments of that type when used with ``Unpack`` to type ``**kwargs``:
+
+.. code-block:: python
+
+   class MovieExtra(TypedDict, extra_items=int):
+       name: str
+
+   def g(**kwargs: Unpack[MovieExtra]) -> None: ...
+
+   # Equivalent to:
+   def g2(*, name: str, **kwargs: int) -> None: ...
+
+   g(name="No Country for Old Men", year=2007)  # OK
+
+Conversely, when calling a function with ``**movie``, the extra items
+that ``movie`` may contain are checked against the function's
+``**kwargs`` type:
+
+.. code-block:: python
+
+   def plain(**kwargs: str) -> None: ...
+
+   def h(movie: MovieExtra) -> None:
+       plain(**movie)  # Error: incompatible type "**MovieExtra"; expected "str"
+
+Since all the value types of a TypedDict with ``extra_items`` (or
+``closed=True``) are known, it is assignable to ``Mapping[str, VT]``
+whenever they are all assignable to ``VT``, its ``values()`` and
+``items()`` methods return precisely typed views, and it can be indexed
+with an arbitrary ``str`` key:
+
+.. code-block:: python
+
+   def h(movie: MovieExtra, key: str) -> None:
+       m: Mapping[str, int | str] = movie  # OK
+       reveal_type(movie.items())  # Revealed type is "dict_items[str, str | int]"
+       reveal_type(movie[key])     # Revealed type is "str | int"
+
+Moreover, if every item is non-required, mutable, and consistent with
+``extra_items``, the TypedDict is assignable to ``dict[str, VT]`` and
+supports operations that would otherwise be unsafe on a TypedDict, such
+as ``clear()``, ``popitem()``, and assigning or deleting arbitrary
+``str`` keys:
+
+.. code-block:: python
+
+   class IntDict(TypedDict, extra_items=int):
+       pass
+
+   def counters(d: IntDict, key: str) -> None:
+       v: dict[str, int] = d  # OK
+       d[key] = 42            # OK
+       d.clear()              # OK
+
 Unions of TypedDicts
 --------------------
 

mypy/argmap.py

@@ -91,6 +91,16 @@ def map_actuals_to_formals(
                         formal_to_actual[formal_names.index(name)].append(ai)
                     elif nodes.ARG_STAR2 in formal_kinds:
                         formal_to_actual[formal_kinds.index(nodes.ARG_STAR2)].append(ai)
+                if (
+                    actualt.extra_items is not None
+                    and not actualt.is_closed
+                    and nodes.ARG_STAR2 in formal_kinds
+                ):
+                    # The TypedDict may contain extra keys of the extra_items type
+                    # (PEP 728); map the pseudo-item to the callee's **kwargs so it
+                    # is checked as well. Without a **kwargs formal the extra keys
+                    # are ignored, like the possible extra keys of an open TypedDict.
+                    formal_to_actual[formal_kinds.index(nodes.ARG_STAR2)].append(ai)
             else:
                 # We don't exactly know which **kwargs are provided by the
                 # caller, so we'll defer until all the other unambiguous
@@ -248,7 +258,12 @@ def expand_actual_type(
                     assert formal_name is not None
                 else:
                     # Pick an arbitrary item if no specified keyword is expected.
-                    formal_name = (set(actual_type.items.keys()) - self.kwargs_used).pop()
+                    unused_names = set(actual_type.items.keys()) - self.kwargs_used
+                    if not unused_names and actual_type.extra_items is not None:
+                        # The named items are exhausted, so this mapping is for the
+                        # extra_items pseudo-item (PEP 728).
+                        return actual_type.extra_items
+                    formal_name = unused_names.pop()
                 self.kwargs_used.add(formal_name)
                 return actual_type.items[formal_name]
             elif isinstance(actual_type, Instance) and is_subtype(

mypy/cache.py

@@ -69,7 +69,7 @@
 from mypy_extensions import u8
 
 # High-level cache layout format
-CACHE_VERSION: Final = 10
+CACHE_VERSION: Final = 11
 
 # Type used internally to represent errors:
 #   (path, line, column, end_line, end_column, severity, message, code)

mypy/checker.py

@@ -3229,7 +3229,7 @@ def check_typeddict_inheritance(self, defn: ClassDef) -> None:
         data = defn.info.typeddict_data
         if data is None or not data.ready:
             return
-        for base, base_items in data.bases:
+        for base, base_items, base_extra_items in data.bases:
             assert base.typeddict_type
             for field_name, base_type in base_items.items():
                 field_type = td.items[field_name]
@@ -3259,6 +3259,47 @@ def check_typeddict_inheritance(self, defn: ClassDef) -> None:
                                 f"with a mutually compatible type",
                                 source.ctx,
                             )
+            if base_extra_items is None:
+                continue
+            base_extra_readonly = base.typeddict_type.extra_items_readonly
+            if not isinstance(get_proper_type(base_extra_items), UninhabitedType):
+                # Fields not known to the base must be compatible with its extra_items
+                # pseudo-item, like any other item redeclaration. (For a closed base,
+                # semantic analysis already rejected such fields.)
+                for field_name, field_type in td.items.items():
+                    if field_name in base_items:
+                        continue
+                    if base_extra_readonly:
+                        is_compatible = is_subtype(field_type, base_extra_items)
+                    else:
+                        is_compatible = is_equivalent(field_type, base_extra_items)
+                    if is_compatible:
+                        continue
+                    source = data.field_sources[field_name]
+                    if source.base is None:
+                        self.fail(
+                            f'Type of field "{field_name}" incompatible with "extra_items" '
+                            f'of base class "{base.name}"',
+                            source.ctx,
+                        )
+                    else:
+                        self.fail(
+                            f'Type of field "{field_name}" from base class '
+                            f'"{source.base.name}" incompatible with "extra_items" of base '
+                            f'class "{base.name}"',
+                            source.ctx,
+                        )
+            if td.extra_items is not None:
+                # The resolved pseudo-item must be compatible with each base pseudo-item.
+                if base_extra_readonly:
+                    is_compatible = is_subtype(td.extra_items, base_extra_items)
+                else:
+                    is_compatible = is_equivalent(td.extra_items, base_extra_items)
+                if not is_compatible:
+                    self.fail(
+                        f'Definition of "extra_items" incompatible with base class "{base.name}"',
+                        defn,
+                    )
 
     def visit_import_from(self, node: ImportFrom) -> None:
         for name, _ in node.names:

mypy/checkexpr.py

@@ -127,6 +127,7 @@
     is_same_type,
     is_subtype,
     non_method_protocol_members,
+    typed_dict_dict_value_type,
 )
 from mypy.traverser import (
     all_name_and_member_expressions,
@@ -857,6 +858,13 @@ def validate_typeddict_kwargs(
                     last_open_star_found,
                     code=codes.TYPEDDICT_ITEM,
                 )
+            elif callee.extra_items is not None:
+                self.chk.fail(
+                    "Cannot unpack item that may contain incompatible extra keys into a "
+                    'TypedDict with "extra_items"',
+                    last_open_star_found,
+                    code=codes.TYPEDDICT_ITEM,
+                )
             absent_keys = []
             for key in callee.items:
                 if key not in callee.required_keys and key not in result:
@@ -931,8 +939,26 @@ def validate_star_typeddict_item(
                 # If this key is not required at least in some item of a union
                 # it may not shadow previous item, so we need to type check both.
                 result[key].append(arg)
-        all_closed = all(t.is_closed for t in possible_tds)
-        return True, any_fallback or not all_closed
+        all_absorbed = all(self.star_source_absorbed(td, callee) for td in possible_tds)
+        return True, any_fallback or not all_absorbed
+
+    def star_source_absorbed(self, source: TypedDictType, callee: TypedDictType) -> bool:
+        """Can ** unpacking `source` add no undeclared or incompatible keys to `callee`?
+
+        Keys declared on `source` are checked individually elsewhere; this is about
+        the keys covered by its PEP 728 pseudo-item.
+        """
+        src_extra = source.extra_item()
+        if src_extra.typ is None:
+            # A default-open source may contain arbitrary extra keys.
+            return False
+        if isinstance(get_proper_type(src_extra.typ), UninhabitedType):
+            # A closed source contributes no undeclared keys.
+            return True
+        callee_extra = callee.extra_item()
+        if callee_extra.typ is None or callee.is_closed:
+            return False
+        return is_subtype(src_extra.typ, callee_extra.typ)
 
     def valid_unpack_fallback_item(self, typ: ProperType) -> bool:
         if isinstance(typ, AnyType):
@@ -952,7 +978,12 @@ def match_typeddict_call_with_dict(
             result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee)
         if result is not None:
             validated_kwargs, _ = result
-            return callee.required_keys <= set(validated_kwargs.keys()) <= set(callee.items.keys())
+            keys = set(validated_kwargs.keys())
+            if not callee.required_keys <= keys:
+                return False
+            return keys <= set(callee.items.keys()) or (
+                callee.extra_items is not None and not callee.is_closed
+            )
         else:
             return False
 
@@ -993,13 +1024,22 @@ def typeddict_callable(self, info: TypeInfo) -> CallableType:
     def typeddict_callable_from_context(
         self, callee: TypedDictType, variables: Sequence[TypeVarLikeType] | None = None
     ) -> CallableType:
+        arg_types = list(callee.items.values())
+        arg_kinds = [
+            ArgKind.ARG_NAMED if name in callee.required_keys else ArgKind.ARG_NAMED_OPT
+            for name in callee.items
+        ]
+        arg_names: list[str | None] = list(callee.items.keys())
+        if callee.extra_items is not None and not callee.is_closed:
+            # PEP 728: arbitrary extra keyword arguments of the extra_items type
+            # are accepted when constructing a TypedDict.
+            arg_types.append(callee.extra_items)
+            arg_kinds.append(ArgKind.ARG_STAR2)
+            arg_names.append(None)
         return CallableType(
-            list(callee.items.values()),
-            [
-                ArgKind.ARG_NAMED if name in callee.required_keys else ArgKind.ARG_NAMED_OPT
-                for name in callee.items
-            ],
-            list(callee.items.keys()),
+            arg_types,
+            arg_kinds,
+            arg_names,
             callee,
             self.named_type("builtins.type"),
             variables=variables,
@@ -1015,14 +1055,19 @@ def check_typeddict_call_with_kwargs(
         always_present_keys: set[str],
     ) -> Type:
         actual_keys = kwargs.keys()
+        extra_keys = actual_keys - callee.items.keys()
+        # PEP 728: a TypedDict with a non-Never extra_items pseudo-item accepts
+        # arbitrary extra keys, whose values are checked against it below.
+        allows_extra_keys = callee.extra_items is not None and not callee.is_closed
         if callee.to_be_mutated:
             assigned_readonly_keys = actual_keys & callee.readonly_keys
+            if allows_extra_keys and callee.extra_items_readonly:
+                assigned_readonly_keys |= extra_keys
             if assigned_readonly_keys:
                 self.msg.readonly_keys_mutated(assigned_readonly_keys, context=context)
-        if not (
-            callee.required_keys <= always_present_keys and actual_keys <= callee.items.keys()
-        ):
-            if not (actual_keys <= callee.items.keys()):
+        known_keys_only = actual_keys <= callee.items.keys() or allows_extra_keys
+        if not (callee.required_keys <= always_present_keys and known_keys_only):
+            if not known_keys_only:
                 self.msg.unexpected_typeddict_keys(
                     callee,
                     expected_keys=[
@@ -1098,6 +1143,19 @@ def check_typeddict_call_with_kwargs(
                         lvalue_name=f'TypedDict item "{item_name}"',
                         rvalue_name="expression",
                     )
+        if allows_extra_keys and ret_type.extra_items is not None:
+            for item_name in extra_keys:
+                for item_value in kwargs[item_name]:
+                    self.chk.check_simple_assignment(
+                        lvalue_type=ret_type.extra_items,
+                        rvalue=item_value,
+                        context=item_value,
+                        msg=ErrorMessage(
+                            message_registry.INCOMPATIBLE_TYPES.value, code=codes.TYPEDDICT_ITEM
+                        ),
+                        lvalue_name=f'extra TypedDict item "{item_name}"',
+                        rvalue_name="expression",
+                    )
 
         return orig_ret_type
 
@@ -4822,19 +4880,42 @@ def visit_typeddict_index_expr(
                 ):
                     key_names.append(key_type.value)
                 else:
+                    result = self.typeddict_arbitrary_str_key_item(td_type, typ, setitem)
+                    if result is not None:
+                        return result, set()
                     self.msg.typeddict_key_must_be_string_literal(td_type, index)
                     return AnyType(TypeOfAny.from_error), set()
 
         value_types = []
         for key_name in key_names:
-            value_type = td_type.items.get(key_name)
-            if value_type is None:
+            item = td_type.item(key_name)
+            if item.typ is None or isinstance(get_proper_type(item.typ), UninhabitedType):
+                # The key is unknown (default-open TypedDict) or provably absent
+                # (closed TypedDict). Extra keys of a TypedDict with extra_items
+                # have the pseudo-item type (PEP 728).
                 self.msg.typeddict_key_not_found(td_type, key_name, index, setitem)
                 return AnyType(TypeOfAny.from_error), set()
-            else:
-                value_types.append(value_type)
+            value_types.append(item.typ)
         return make_simplified_union(value_types), set(key_names)
 
+    def typeddict_arbitrary_str_key_item(
+        self, td_type: TypedDictType, key_type: Type, setitem: bool
+    ) -> Type | None:
+        """Handle indexing a PEP 728 TypedDict with an arbitrary str key.
+
+        Reads are allowed on any TypedDict with closed=True or extra_items= set,
+        yielding the union of all value types. Writes are only allowed if the
+        TypedDict behaves like dict[str, VT]. Return None if the operation is
+        not allowed.
+        """
+        if td_type.extra_items is None:
+            return None
+        if not is_subtype(key_type, self.named_type("builtins.str")):
+            return None
+        if setitem:
+            return typed_dict_dict_value_type(td_type)
+        return make_simplified_union(td_type.value_types_with_extra())
+
     def visit_enum_index_expr(
         self, enum_type: TypeInfo, index: Expression, context: Context
     ) -> Type: