Add support for PEP 661 (sentinels)

.github/workflows/test.yml

@@ -226,6 +226,10 @@ jobs:
     - name: Setup tox environment
       run: |
         tox run -e ${{ matrix.toxenv }} --notest
+    - name: Install local librt in tox environment
+      if: ${{ !matrix.test_mypyc && matrix.toxenv == 'py' }}
+      # Use local version of librt so that fixes not yet in a PyPI release are available.
+      run: tox exec -e ${{ matrix.toxenv }} -- pip install -U mypyc/lib-rt
     - name: Test
       run: tox run -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }}
       continue-on-error: ${{ matrix.allow_failure == 'true' }}

mypy/cache.py

@@ -48,7 +48,10 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import Any, Final, TypeAlias as _TypeAlias
+from typing import TYPE_CHECKING, Any, Final, TypeAlias as _TypeAlias
+
+if TYPE_CHECKING:
+    from mypy.types import SentinelValue
 
 from librt.internal import (
     ReadBuffer as ReadBuffer,
@@ -69,7 +72,7 @@
 from mypy_extensions import u8
 
 # High-level cache layout format
-CACHE_VERSION: Final = 10
+CACHE_VERSION: Final = 11
 
 # Type used internally to represent errors:
 #   (path, line, column, end_line, end_column, severity, message, code)
@@ -308,6 +311,7 @@ def read(cls, data: ReadBuffer) -> CacheMetaEx | None:
 LITERAL_BYTES: Final[Tag] = 5
 LITERAL_FLOAT: Final[Tag] = 6
 LITERAL_COMPLEX: Final[Tag] = 7
+LITERAL_SENTINEL: Final[Tag] = 8
 
 # Collections.
 LIST_GEN: Final[Tag] = 20
@@ -328,7 +332,7 @@ def read(cls, data: ReadBuffer) -> CacheMetaEx | None:
 END_TAG: Final[Tag] = 255
 
 
-def read_literal(data: ReadBuffer, tag: Tag) -> int | str | bool | float:
+def read_literal(data: ReadBuffer, tag: Tag) -> int | str | bool | float | SentinelValue:
     if tag == LITERAL_INT:
         return read_int_bare(data)
     elif tag == LITERAL_STR:
@@ -339,12 +343,18 @@ def read_literal(data: ReadBuffer, tag: Tag) -> int | str | bool | float:
         return True
     elif tag == LITERAL_FLOAT:
         return read_float_bare(data)
+    elif tag == LITERAL_SENTINEL:
+        from mypy.types import SentinelValue as _SentinelValue
+
+        return _SentinelValue(read_str_bare(data), read_str_bare(data))
     assert False, f"Unknown literal tag {tag}"
 
 
 # There is an intentional asymmetry between read and write for literals because
 # None and/or complex values are only allowed in some contexts but not in others.
-def write_literal(data: WriteBuffer, value: int | str | bool | float | complex | None) -> None:
+def write_literal(
+    data: WriteBuffer, value: int | str | bool | float | complex | SentinelValue | None
+) -> None:
     if isinstance(value, bool):
         write_bool(data, value)
     elif isinstance(value, int):
@@ -360,8 +370,12 @@ def write_literal(data: WriteBuffer, value: int | str | bool | float | complex |
         write_tag(data, LITERAL_COMPLEX)
         write_float_bare(data, value.real)
         write_float_bare(data, value.imag)
-    else:
+    elif value is None:
         write_tag(data, LITERAL_NONE)
+    else:
+        write_tag(data, LITERAL_SENTINEL)
+        write_str_bare(data, value.fullname)
+        write_str_bare(data, value.name)
 
 
 def read_int(data: ReadBuffer) -> int:

mypy/messages.py

@@ -2783,13 +2783,18 @@ def format_literal_value(typ: LiteralType) -> str:
                 modifier += "="
             items.append(f"{item_name!r}{modifier}: {format(item_type)}")
         return f"TypedDict({{{', '.join(items)}}})"
+    elif isinstance(typ, LiteralType) and typ.is_sentinel_literal():
+        return format_literal_value(typ)
     elif isinstance(typ, LiteralType):
         return f"Literal[{format_literal_value(typ)}]"
     elif isinstance(typ, UnionType):
         typ = get_proper_type(ignore_last_known_values(typ))
         if not isinstance(typ, UnionType):
             return format(typ)
         literal_items, union_items = separate_union_literals(typ)
+        sentinel_items = [item for item in literal_items if item.is_sentinel_literal()]
+        literal_items = [item for item in literal_items if not item.is_sentinel_literal()]
+        union_items = [*sentinel_items, *union_items]
 
         # Coalesce multiple Literal[] members. This also changes output order.
         # If there's just one Literal item, retain the original ordering.

mypy/nodes.py

@@ -1416,6 +1416,7 @@ def is_dynamic(self) -> bool:
     "from_module_getattr",
     "has_explicit_value",
     "allow_incompatible_override",
+    "is_sentinel",
 ]
 
 
@@ -1454,6 +1455,7 @@ class Var(SymbolNode):
         "allow_incompatible_override",
         "invalid_partial_type",
         "is_argument",
+        "is_sentinel",
     )
 
     __match_args__ = ("name", "type", "final_value")
@@ -1516,6 +1518,8 @@ def __init__(self, name: str, type: mypy.types.Type | None = None) -> None:
         self.invalid_partial_type = False
         # Is it a variable symbol for a function argument?
         self.is_argument = False
+        # Was this variable created by PEP 661 sentinel()/Sentinel() syntax?
+        self.is_sentinel = False
 
     @property
     def name(self) -> str:
@@ -1598,6 +1602,7 @@ def write(self, data: WriteBuffer) -> None:
                 self.from_module_getattr,
                 self.has_explicit_value,
                 self.allow_incompatible_override,
+                self.is_sentinel,
             ],
         )
         write_literal(data, self.final_value)
@@ -1635,12 +1640,15 @@ def read(cls, data: ReadBuffer) -> Var:
             v.from_module_getattr,
             v.has_explicit_value,
             v.allow_incompatible_override,
-        ) = read_flags(data, num_flags=19)
+            v.is_sentinel,
+        ) = read_flags(data, num_flags=20)
         tag = read_tag(data)
         if tag == LITERAL_COMPLEX:
             v.final_value = complex(read_float_bare(data), read_float_bare(data))
         elif tag != LITERAL_NONE:
-            v.final_value = read_literal(data, tag)
+            val = read_literal(data, tag)
+            assert not isinstance(val, mypy.types.SentinelValue)
+            v.final_value = val
         assert read_tag(data) == END_TAG
         return v
 

mypy/semanal.py

@@ -271,6 +271,7 @@
     OVERRIDE_DECORATOR_NAMES,
     PROTOCOL_NAMES,
     REVEAL_TYPE_NAMES,
+    SENTINEL_TYPE_NAMES,
     TPDICT_NAMES,
     TYPE_ALIAS_NAMES,
     TYPE_CHECK_ONLY_NAMES,
@@ -289,6 +290,7 @@
     ParamSpecType,
     PlaceholderType,
     ProperType,
+    SentinelValue,
     TrivialSyntheticTypeTranslator,
     TupleType,
     Type,
@@ -3377,9 +3379,15 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         # may be set to True while there were still placeholders due to forward refs.
         s.is_alias_def = False
 
+        sentinel_definition = self.is_sentinel_declaration(s)
+
         # OK, this is a regular assignment, perform the necessary analysis steps.
         s.is_final_def = self.unwrap_final(s)
+        if sentinel_definition:
+            s.is_final_def = True
         self.analyze_lvalues(s)
+        if sentinel_definition:
+            self.setup_sentinel_var(s)
         self.check_final_implicit_def(s)
         self.store_final_status(s)
         self.check_classvar(s)
@@ -3392,6 +3400,51 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         self.process__deletable__(s)
         self.process__slots__(s)
 
+    def is_sentinel_declaration(self, s: AssignmentStmt) -> bool:
+        """Does this assignment define a PEP 661 sentinel singleton?"""
+        if self.is_func_scope() or s.unanalyzed_type is not None:
+            return False
+        if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
+            return False
+        if not isinstance(s.rvalue, CallExpr):
+            return False
+        call = s.rvalue
+        if not isinstance(call.callee, RefExpr):
+            return False
+        if call.callee.fullname not in SENTINEL_TYPE_NAMES:
+            return False
+        if not call.args or call.arg_kinds[0] != ARG_POS or not isinstance(call.args[0], StrExpr):
+            return False
+        return True
+
+    def setup_sentinel_var(self, s: AssignmentStmt) -> None:
+        lvalue = s.lvalues[0]
+        assert isinstance(lvalue, NameExpr)
+        if not isinstance(lvalue.node, Var):
+            return
+        var = lvalue.node
+        var.is_sentinel = True
+        typ = self.sentinel_type_for_var(var, s.rvalue)
+        if typ is not None:
+            s.type = typ
+
+    def sentinel_type_for_var(self, var: Var, rvalue: Expression) -> Instance | None:
+        assert isinstance(rvalue, CallExpr)
+        callee = rvalue.callee
+        assert isinstance(callee, RefExpr)
+        typ = self.named_type_or_none(callee.fullname)
+        if typ is None:
+            return None
+        name = f"{self.type.name}.{var.name}" if self.type is not None else var.name
+        return typ.copy_modified(
+            last_known_value=LiteralType(
+                SentinelValue(var.fullname, name),
+                fallback=typ,
+                line=rvalue.line,
+                column=rvalue.column,
+            )
+        )
+
     def analyze_identity_global_assignment(self, s: AssignmentStmt) -> bool:
         """Special case 'X = X' in global scope.
 
@@ -3556,6 +3609,8 @@ def is_type_ref(self, rv: Expression, bare: bool = False) -> bool:
             # Assignment color = Color['RED'] defines a variable, not an alias.
             return not rv.node.is_enum
         if isinstance(rv.node, Var):
+            if rv.node.is_sentinel:
+                return True
             return rv.node.fullname in NEVER_NAMES
 
         if isinstance(rv, NameExpr):
@@ -4763,6 +4818,7 @@ def store_declared_types(self, lvalue: Lvalue, typ: Type) -> None:
                     var.is_final
                     and isinstance(typ, Instance)
                     and typ.last_known_value
+                    and not isinstance(typ.last_known_value.value, SentinelValue)
                     and (not self.type or not self.type.is_enum)
                 ):
                     var.final_value = typ.last_known_value.value

mypy/test/testtypes.py

@@ -5,6 +5,9 @@
 import re
 from unittest import TestCase, skipUnless
 
+from librt.internal import ReadBuffer, WriteBuffer
+
+from mypy.cache import read_tag
 from mypy.erasetype import erase_type, remove_instance_last_known_values
 from mypy.indirection import TypeIndirectionVisitor
 from mypy.join import join_types
@@ -30,13 +33,15 @@
 from mypy.test.typefixture import InterfaceTypeFixture, TypeFixture
 from mypy.typeops import false_only, make_simplified_union, true_only
 from mypy.types import (
+    LITERAL_TYPE,
     AnyType,
     CallableType,
     Instance,
     LiteralType,
     NoneType,
     Overloaded,
     ProperType,
+    SentinelValue,
     TupleType,
     Type,
     TypedDictType,
@@ -66,6 +71,25 @@ def setUp(self) -> None:
     def test_any(self) -> None:
         assert_equal(str(AnyType(TypeOfAny.special_form)), "Any")
 
+    def test_sentinel_literal_json_roundtrip(self) -> None:
+        literal = LiteralType(SentinelValue("__main__.MISSING", "MISSING"), self.fx.a)
+        assert_equal(str(literal), "MISSING")
+        data = literal.serialize()
+        assert isinstance(data, dict)
+        roundtrip = LiteralType.deserialize(data)
+        self.assertEqual(roundtrip.value, literal.value)
+        self.assertEqual(roundtrip.fallback.type_ref, self.fx.a.type.fullname)
+
+    def test_sentinel_literal_ff_roundtrip(self) -> None:
+        literal = LiteralType(SentinelValue("__main__.MISSING", "MISSING"), self.fx.a)
+        data = WriteBuffer()
+        literal.write(data)
+        buffer = ReadBuffer(data.getvalue())
+        assert read_tag(buffer) == LITERAL_TYPE
+        roundtrip = LiteralType.read(buffer)
+        self.assertEqual(roundtrip.value, literal.value)
+        self.assertEqual(roundtrip.fallback.type_ref, self.fx.a.type.fullname)
+
     def test_simple_unbound_type(self) -> None:
         u = UnboundType("Foo")
         assert_equal(str(u), "Foo?")

mypy/typeanal.py

@@ -1050,6 +1050,16 @@ def analyze_unbound_type_without_type_info(
                 column=t.column,
             )
 
+        if isinstance(sym.node, Var) and sym.node.is_sentinel:
+            typ = get_proper_type(sym.node.type)
+            if isinstance(typ, Instance) and typ.last_known_value is not None:
+                return LiteralType(
+                    value=typ.last_known_value.value,
+                    fallback=typ.last_known_value.fallback,
+                    line=t.line,
+                    column=t.column,
+                )
+
         # None of the above options worked. We parse the args (if there are any)
         # to make sure there are no remaining semanal-only types, then give up.
         t = t.copy_modified(args=self.anal_array(t.args))

mypy/typeops.py

@@ -1058,7 +1058,7 @@ def is_singleton_identity_type(typ: ProperType) -> bool:
             or (typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES)
         )
     if isinstance(typ, LiteralType):
-        return typ.is_enum_literal() or isinstance(typ.value, bool)
+        return typ.is_enum_literal() or typ.is_sentinel_literal() or isinstance(typ.value, bool)
     if isinstance(typ, TypeType) and isinstance(typ.item, Instance) and typ.item.type.is_final:
         return True
     if isinstance(typ, FunctionLike) and typ.is_type_obj() and typ.type_object().is_final: