From c56f1038dbc775331bbbcd9336ab4b9565cc492b Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Thu, 24 Sep 2026 18:07:15 +0200 Subject: [PATCH 01/24] Accept inferred explicit-Any variable as base class Fixes #21998: mypy rejected subclassing a variable whose type is inferred as Any (e.g. `x = args[1]` where `args: *tuple[int, Any]`), even though an explicitly `Any`-annotated variable is accepted as a base class. Semantic analysis runs before type inference, so a base class that is a variable with an uninferred type cannot be validated there. Such bases are now provisionally treated as Any and validated by the type checker once the type is known (TypeChecker.check_deferred_base_classes). An inferred `Any` is accepted only if it comes from an explicit annotation (TypeOfAny.explicit); `Any` leaking in from unannotated code is still rejected, preserving existing behavior (e.g. dynamic class plugin negatives). `--disallow-subclassing-any` is honored for inferred-Any bases as well. --- mypy/checker.py | 111 ++++++++++++++++++++++++++++++ mypy/nodes.py | 7 ++ mypy/semanal.py | 105 +++++++++++++++++----------- mypy/typeanal.py | 25 +++++++ test-data/unit/check-classes.test | 33 +++++++++ 5 files changed, 240 insertions(+), 41 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d4ba6c54fb35..6a40b3ee6124 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -207,6 +207,7 @@ def __init__(self) -> None: WithStmt, YieldExpr, get_func_def, + get_member_expr_fullname, is_final_node, ) from mypy.operators import flip_ops, int_op_to_method, neg_ops @@ -291,10 +292,12 @@ def __init__(self) -> None: UninhabitedType, UnionType, UnpackType, + extend_args_for_prefix_and_suffix, find_unpack_in_list, flatten_nested_unions, get_proper_type, get_proper_types, + get_variadic_item, instance_cache, is_literal_type, is_named_instance, @@ -2805,9 +2808,78 @@ def check__exit__return_type(self, defn: FuncItem) -> None: ): self.msg.incorrect__exit__return(defn) + def is_explicit_any(self, typ: ProperType) -> bool: + """Is this an Any type that comes from an explicit annotation? + + Unlike ``Any`` that leaks in from unannotated (dynamically typed) code, + an explicitly written ``Any`` means the user opted into dynamic typing, + so it is treated like an explicitly ``Any``-typed variable. + """ + while ( + isinstance(typ, AnyType) + and typ.type_of_any == TypeOfAny.from_another_any + and typ.source_any is not None + ): + typ = get_proper_type(typ.source_any) + return isinstance(typ, AnyType) and typ.type_of_any != TypeOfAny.unannotated + + def check_deferred_base_classes(self, defn: ClassDef) -> None: + """Validate base classes deferred from semantic analysis. + + A base class that is a variable with an inferred (not declared) type -- + e.g. ``x = args[1]`` where ``args[1]`` is ``Any`` -- could not be + validated during semantic analysis, since the type is only inferred by + the type checker. Such bases were provisionally treated as ``Any``; + now that the type is known, accept it if it is ``Any`` (like an + explicitly ``Any``-typed variable) and otherwise report the errors that + semantic analysis deferred. + """ + for var, base_expr in defn.info.deferred_base_classes: + typ = get_proper_type(var.type) + # Mirror the cases semantic analysis accepts for an explicitly + # typed variable (see TypeAnalyser.analyze_unbound_type_without_type_info). + # An inferred plain ``Any`` is only accepted if it comes from an + # explicit annotation; ``Any`` leaking in from unannotated code is + # still rejected, as before. + is_any = ( + self.is_explicit_any(typ) + or (isinstance(typ, Instance) and typ.type.fullname == "builtins.type") + or (isinstance(typ, TypeType) and self.is_explicit_any(get_proper_type(typ.item))) + ) + if is_any: + if self.options.disallow_subclassing_any: + if isinstance(base_expr, (NameExpr, MemberExpr)): + msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' + else: + msg = 'Class cannot subclass value of type "Any"' + self.fail(msg, base_expr) + continue + if isinstance(base_expr, NameExpr): + name_repr: str | None = base_expr.name + elif isinstance(base_expr, MemberExpr): + name_repr = get_member_expr_fullname(base_expr) + else: + name_repr = None + self.fail( + f'Variable "{var.fullname}" is not valid as a type', + base_expr, + code=codes.VALID_TYPE, + ) + self.note( + "See https://mypy.readthedocs.io/en/stable/common_issues.html" + "#variables-vs-type-aliases", + base_expr, + code=codes.VALID_TYPE, + ) + msg = "Invalid base class" + if name_repr: + msg += f' "{name_repr}"' + self.fail(msg, base_expr) + def visit_class_def(self, defn: ClassDef) -> None: """Type check a class definition.""" typ = defn.info + self.check_deferred_base_classes(defn) for base in typ.mro[1:]: if base.is_final: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) @@ -4389,6 +4461,36 @@ def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: res.append(lv) return res + def adjust_rvalue_type_if_possible( + self, rvalue_type: TupleType, lvalues: list[Lvalue] + ) -> TupleType: + """Adjust type of rvalue to match the shape/structure of lvalues. + + Currently, we only allow this if the rvalue type has contains *tuple[Any, ...]. + """ + right_variadic = get_variadic_item(rvalue_type) + if right_variadic is None: + return rvalue_type + right_unpack_index, right_item = right_variadic + if not isinstance(get_proper_type(right_item), AnyType): + return rvalue_type + left_star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), None + ) + if left_star_index is None: + extra = len(lvalues) - len(rvalue_type.items) + 1 + if extra < 0: + return rvalue_type + return rvalue_type.copy_modified( + items=rvalue_type.items[:right_unpack_index] + + [right_item] * extra + + rvalue_type.items[right_unpack_index + 1 :] + ) + new_items = extend_args_for_prefix_and_suffix( + tuple(rvalue_type.items), left_star_index, len(lvalues) - left_star_index - 1 + ) + return rvalue_type.copy_modified(items=list(new_items)) + def check_multi_assignment_from_tuple( self, lvalues: list[Lvalue], @@ -4399,6 +4501,9 @@ def check_multi_assignment_from_tuple( infer_lvalue_type: bool = True, ) -> None: rvalue_unpack = find_unpack_in_list(rvalue_type.items) + if rvalue_unpack is not None: + rvalue_type = self.adjust_rvalue_type_if_possible(rvalue_type, lvalues) + rvalue_unpack = find_unpack_in_list(rvalue_type.items) if self.check_rvalue_count_in_assignment( lvalues, len(rvalue_type.items), context, rvalue_unpack=rvalue_unpack ): @@ -4440,8 +4545,14 @@ def check_multi_assignment_from_tuple( if isinstance(reinferred_rvalue_type, TupleType): # This branch will usually be taken, but in some cases context can # e.g. select a different overload + # TODO: reinferred tuple may be of a different (invalid) shape. rvalue_type = reinferred_rvalue_type + # Reinferring the type can undo the shape adjustment, so do it again. + rvalue_unpack = find_unpack_in_list(rvalue_type.items) + if rvalue_unpack is not None: + rvalue_type = self.adjust_rvalue_type_if_possible(rvalue_type, lvalues) + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( rvalue_type.items, star_index, len(lvalues) ) diff --git a/mypy/nodes.py b/mypy/nodes.py index dc86550cebe4..57e1ea18772b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3686,6 +3686,7 @@ class is generic then it will be a type constructor of higher kind. "type_object_type", "default_depends", "typeddict_data", + "deferred_base_classes", ) _fullname: str # Fully qualified name @@ -3920,6 +3921,12 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.dataclass_transform_spec = None self.is_type_check_only = False self.deprecated = None + # Base classes that are variables whose types had not been inferred yet + # during semantic analysis. Each entry is a (variable, expression) tuple. + # The type checker validates them once the types are known (see + # TypeChecker.check_deferred_base_classes). This is not serialized; it + # is only meaningful within a single build. + self.deferred_base_classes: list[tuple[Var, Expression]] = [] self.type_object_type = None self.default_depends = {} self.typeddict_data = None diff --git a/mypy/semanal.py b/mypy/semanal.py index cab56021c55e..d0d76aa6be21 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -243,6 +243,7 @@ from mypy.tvar_scope import TypeVarLikeScope from mypy.typeanal import ( SELF_TYPE_NAMES, + DeferredBaseClassVar, FindTypeVarVisitor, TypeAnalyser, TypeVarDefaultTranslator, @@ -719,24 +720,26 @@ def refresh_partial( self.accept(node) del self.patches + def ad_hoc_error(self, msg: str) -> None: + n = TempNode(AnyType(TypeOfAny.special_form)) + n.line = 1 + n.column = 0 + n.end_line = 1 + n.end_column = 0 + self.fail(msg, n) + def refresh_top_level(self, file_node: MypyFile) -> None: """Reanalyze a stale module top-level in fine-grained incremental mode.""" if self.options.allow_redefinition and not self.options.local_partial_types: - n = TempNode(AnyType(TypeOfAny.special_form)) - n.line = 1 - n.column = 0 - n.end_line = 1 - n.end_column = 0 - self.fail("--local-partial-types must be enabled if using --allow-redefinition", n) + self.ad_hoc_error( + "--local-partial-types must be enabled if using --allow-redefinition" + ) if self.options.allow_redefinition and self.options.allow_redefinition_old: - n = TempNode(AnyType(TypeOfAny.special_form)) - n.line = 1 - n.column = 0 - n.end_line = 1 - n.end_column = 0 - self.fail( - "--allow-redefinition-old and --allow-redefinition should not be used together", n + self.ad_hoc_error( + "--allow-redefinition-old and --allow-redefinition should not be used together" ) + if not self.options.local_partial_types and self.options.num_workers > 0: + self.ad_hoc_error("--local-partial-types must be enabled in parallel mode") self.recurse_into_functions = False self.add_implicit_module_attrs(file_node) for d in file_node.defs: @@ -2032,7 +2035,7 @@ def analyze_class(self, defn: ClassDef) -> None: self.mark_incomplete(defn.name, defn) return - base_types, base_error = bases_result + base_types, base_error, deferred_bases = bases_result if any(isinstance(base, PlaceholderType) for base, _ in base_types): # We need to know the TypeInfo of each base to construct the MRO. Placeholder types # are okay in nested positions, since they can't affect the MRO. @@ -2069,6 +2072,7 @@ def analyze_class(self, defn: ClassDef) -> None: defn.info.default_depends = default_depends if base_error: defn.info.fallback_to_any = True + defn.info.deferred_base_classes = deferred_bases if any_meta: defn.info.meta_fallback_to_any = True @@ -2620,7 +2624,7 @@ def get_name_repr_of_expr(self, expr: Expression) -> str | None: def analyze_base_classes( self, cls_name: str, base_type_exprs: list[Expression] - ) -> tuple[list[tuple[ProperType, Expression]], bool] | None: + ) -> tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression]]] | None: """Analyze base class types. Return None if some definition was incomplete. Otherwise, return a tuple @@ -2628,9 +2632,15 @@ def analyze_base_classes( * List of (analyzed type, original expression) tuples * Boolean indicating whether one of the bases had a semantic analysis error + * List of (variable, original expression) tuples for bases whose validity + could not be checked yet, because the variable's type has not been + inferred (inference happens in the type checker). These are + provisionally treated as having an Any base; the type checker + validates them once the type is known. """ is_error = False bases = [] + deferred: list[tuple[Var, Expression]] = [] for i, base_expr in enumerate(base_type_exprs): if ( isinstance(base_expr, RefExpr) @@ -2654,6 +2664,13 @@ def analyze_base_classes( allow_type_any=True, unique_name=inline_base(cls_name, i), ) + except DeferredBaseClassVar as e: + # The base is a variable whose type will only be known after + # type checking (e.g. `x = args[1]` where `args[1]` is `Any`). + # Provisionally treat it as Any; the type checker verifies the + # inferred type (see TypeChecker.check_deferred_base_classes). + deferred.append((e.var, base_expr)) + base = AnyType(TypeOfAny.special_form) except TypeTranslationError: name = self.get_name_repr_of_expr(base_expr) if isinstance(base_expr, CallExpr): @@ -2669,7 +2686,7 @@ def analyze_base_classes( return None base = get_proper_type(base) bases.append((base, base_expr)) - return bases, is_error + return bases, is_error, deferred def configure_base_classes( self, defn: ClassDef, bases: list[tuple[ProperType, Expression]] @@ -2693,11 +2710,18 @@ def configure_base_classes( base_types.append(base) elif isinstance(base, AnyType): if self.options.disallow_subclassing_any: - if isinstance(base_expr, (NameExpr, MemberExpr)): - msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' - else: - msg = 'Class cannot subclass value of type "Any"' - self.fail(msg, base_expr) + # A deferred base class is provisionally treated as Any; + # the type checker reports this error once the actual + # inferred type is known, if it is Any. + is_deferred = any( + expr is base_expr for _, expr in defn.info.deferred_base_classes + ) + if not is_deferred: + if isinstance(base_expr, (NameExpr, MemberExpr)): + msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' + else: + msg = 'Class cannot subclass value of type "Any"' + self.fail(msg, base_expr) info.fallback_to_any = True elif isinstance(base, TypedDictType): base_types.append(base.fallback) @@ -4334,9 +4358,8 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: elif isinstance(s.rvalue, RefExpr): s.rvalue.is_alias_rvalue = True + updated = False if existing: - # An alias gets updated. - updated = False if isinstance(existing.node, TypeAlias): # Invalidate recursive status cache in case it was previously set. existing.node._is_recursive = None @@ -4352,15 +4375,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # Otherwise just replace existing placeholder with type alias *in place*. existing._node = alias_node updated = True - # TODO: switch type aliases to if has_placeholder(): process_placeholder() pattern. - # Type aliases are last notable exception from this logic. - if updated: - if self.final_iteration: - self.cannot_resolve_name(lvalue.name, "name", s) - return True - else: - # We need to defer so that this change can get propagated to base classes. - self.defer(s, force_progress=True) else: self.add_symbol(lvalue.name, alias_node, s) if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias): @@ -4368,6 +4382,10 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: current_node = existing.node if existing else alias_node assert isinstance(current_node, TypeAlias) self.disable_invalid_recursive_aliases(s, current_node, s.rvalue) + # Check for placeholders only after we disable invalid recursive aliases. + # Otherwise, we may get an infinite recursion while visiting the target. + if updated or has_placeholder(res): + self.process_placeholder(lvalue.name, "name", s, force_progress=updated) if self.is_class_scope(): assert self.type is not None if self.type.is_protocol: @@ -4803,6 +4821,10 @@ def analyze_member_lvalue( self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True) for func in self.scope.functions: func.def_or_infer_vars = True + + if self.is_self_member_ref(lval) or self.is_cls_member_ref(lval): + assert self.type, "Self or cls member outside a class" + cur_node = self.type.names.get(lval.name) if ( cur_node and isinstance(cur_node.node, Var) @@ -4820,6 +4842,13 @@ def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: node = memberexpr.expr.node return isinstance(node, Var) and node.is_self + def is_cls_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of cls?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_cls + def check_lvalue_validity(self, node: Expression | SymbolNode | None, ctx: Context) -> None: if isinstance(node, TypeVarExpr): self.fail("Invalid assignment target", ctx) @@ -5899,12 +5928,12 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: alias_node.default_depends = default_depends s.alias_node = alias_node + updated = False if ( existing and isinstance(existing.node, (PlaceholderNode, TypeAlias)) and existing.node.line == s.line ): - updated = False if isinstance(existing.node, TypeAlias): # Invalidate recursive status cache in case it was previously set. existing.node._is_recursive = None @@ -5922,20 +5951,14 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: # Otherwise just replace existing placeholder with type alias *in place*. existing._node = alias_node updated = True - - if updated: - if self.final_iteration: - self.cannot_resolve_name(s.name.name, "name", s) - return - else: - # We need to defer so that this change can get propagated to base classes. - self.defer(s, force_progress=True) else: self.add_symbol(s.name.name, alias_node, s) current_node = existing.node if existing else alias_node assert isinstance(current_node, TypeAlias) self.disable_invalid_recursive_aliases(s, current_node, s.value) + if updated or has_placeholder(res): + self.process_placeholder(s.name.name, "name", s, force_progress=updated) s.name.accept(self) finally: self.pop_type_args(s.type_args) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6717f4f18c90..9e638f1c30bb 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -186,6 +186,25 @@ def analyze_type_alias( return res, analyzer.aliases_used +class DeferredBaseClassVar(Exception): + """A base class is a variable whose type hasn't been inferred yet. + + Semantic analysis runs before type checking, so when a class uses a variable + with an inferred (not declared) type as a base class -- e.g. ``x = args[1]`` + where ``args[1]`` is ``Any`` -- the variable's type is not yet known. The + class is provisionally treated as having an ``Any`` base; the type checker + validates the inferred type once it is known (see + ``TypeChecker.check_deferred_base_classes``). + + The variable is carried on the exception so the checker can inspect its + inferred type without re-resolving the base class expression. + """ + + def __init__(self, var: nodes.Var) -> None: + super().__init__() + self.var = var + + class TypeAnalyser(SyntheticTypeVisitor[Type], TypeAnalyzerPluginInterface): """Semantic analyzer for types. @@ -1018,6 +1037,12 @@ def analyze_unbound_type_without_type_info( return AnyType(TypeOfAny.special_form) if isinstance(typ, TypeType) and isinstance(typ.item, AnyType): return AnyType(TypeOfAny.from_another_any, source_any=typ.item) + if typ is None: + # The variable's type has not been inferred yet (inference + # happens in the type checker, after semantic analysis). + # Defer the validity check; the type checker will verify + # the inferred type. Provisionally treat the base as Any. + raise DeferredBaseClassVar(sym.node) # Option 2: # Unbound type variable. Currently these may be still valid, # for example when defining a generic type alias. diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index dc5d74abc07d..16141bc132f6 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9727,3 +9727,36 @@ def f() -> None: class X: ... undefined # E: Name "undefined" is not defined + +[case testPartialNoneTypeClassMethod] +# flags: --local-partial-types + +class C: + x = None + + @classmethod + def foo(cls) -> None: + if not cls.x: + cls.x = 1 + +reveal_type(C.x) # N: Revealed type is "builtins.int | None" +[builtins fixtures/classmethod.pyi] + +[case testSubclassInferredAnyVariable] +# flags: --python-version=3.11 +from typing import Any +def f(*args: *tuple[int, Any]) -> Any: + x = args[1] + class Sub(x): pass + return Sub() +[builtins fixtures/tuple.pyi] + +[case testSubclassInferredNonAnyVariable] +from typing import Any +def f(x: Any) -> None: + y = x + class Sub1(y): pass + z = 1 + class Sub2(z): pass # E: Variable "z" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "z" From 033523792fd0764db6bca696f310f4f72fb74dda Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:35:30 +0000 Subject: [PATCH 02/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/annotate.py | 3 ++- mypyc/codegen/emitclass.py | 1 + mypyc/codegen/emitmodule.py | 3 ++- mypyc/codegen/emitwrapper.py | 1 + mypyc/ir/func_ir.py | 1 + mypyc/irbuild/ast_helpers.py | 1 + mypyc/irbuild/builder.py | 5 +++-- mypyc/irbuild/classdef.py | 1 + mypyc/irbuild/constant_fold.py | 3 ++- mypyc/irbuild/context.py | 1 + mypyc/irbuild/env_class.py | 1 + mypyc/irbuild/expression.py | 1 + mypyc/irbuild/for_helpers.py | 1 + mypyc/irbuild/format_str_tokenizer.py | 3 ++- mypyc/irbuild/function.py | 1 + mypyc/irbuild/generator.py | 1 + mypyc/irbuild/ll_builder.py | 3 ++- mypyc/irbuild/main.py | 3 ++- mypyc/irbuild/mapper.py | 1 + mypyc/irbuild/match.py | 1 + mypyc/irbuild/missingtypevisitor.py | 1 + mypyc/irbuild/prebuildvisitor.py | 1 + mypyc/irbuild/prepare.py | 3 ++- mypyc/irbuild/specialize.py | 1 + mypyc/irbuild/statement.py | 1 + mypyc/irbuild/util.py | 1 + mypyc/irbuild/visitor.py | 1 + mypyc/test/test_statement.py | 3 ++- mypyc/test/testutil.py | 3 ++- 29 files changed, 40 insertions(+), 11 deletions(-) diff --git a/mypyc/annotate.py b/mypyc/annotate.py index bc282fc3ea6c..cba886f1acaa 100644 --- a/mypyc/annotate.py +++ b/mypyc/annotate.py @@ -11,7 +11,6 @@ from html import escape from typing import Final -from mypy.build import BuildResult from mypy.nodes import ( AssignmentStmt, CallExpr, @@ -39,6 +38,8 @@ Var, WithStmt, ) + +from mypy.build import BuildResult from mypy.traverser import TraverserVisitor from mypy.types import AnyType, Instance, ProperType, Type, TypeOfAny, get_proper_type from mypy.util import FancyFormatter diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 953fd8a9e435..51970849f82b 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Mapping from mypy.nodes import ARG_STAR, ARG_STAR2 + from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import ( Emitter, diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 1b21b51709db..f2895c164c4f 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -11,6 +11,8 @@ from collections.abc import Iterable from typing import TypeVar +from mypy.nodes import MypyFile, TypeInfo + from mypy.build import ( BuildResult, BuildSource, @@ -23,7 +25,6 @@ ) from mypy.errors import CompileError from mypy.fscache import FileSystemCache -from mypy.nodes import MypyFile, TypeInfo from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 5213de19011f..9918e8c5969a 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -15,6 +15,7 @@ from collections.abc import Sequence from mypy.nodes import ARG_NAMED, ARG_NAMED_OPT, ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, ArgKind + from mypy.operators import op_methods_to_symbols, reverse_op_method_names, reverse_op_methods from mypyc.codegen.emit import AssignHandler, Emitter, ErrorHandler, GotoHandler, ReturnHandler from mypyc.common import ( diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py index d11fef42feb5..ca72e393a5c7 100644 --- a/mypyc/ir/func_ir.py +++ b/mypyc/ir/func_ir.py @@ -7,6 +7,7 @@ from typing import Final from mypy.nodes import ARG_POS, ArgKind, Block, FuncDef + from mypyc.common import BITMAP_BITS, JsonDict, bitmap_name, get_id_from_name, short_id_from_name from mypyc.ir.ops import ( Assign, diff --git a/mypyc/irbuild/ast_helpers.py b/mypyc/irbuild/ast_helpers.py index 8fff4b19b33d..74b516152145 100644 --- a/mypyc/irbuild/ast_helpers.py +++ b/mypyc/irbuild/ast_helpers.py @@ -20,6 +20,7 @@ UnaryExpr, Var, ) + from mypyc.ir.ops import BasicBlock from mypyc.ir.rtypes import is_fixed_width_rtype, is_tagged from mypyc.irbuild.builder import IRBuilder diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 957cc2704441..6413373fdbc2 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -9,8 +9,6 @@ from contextlib import contextmanager from typing import Any, Final, overload -from mypy.build import Graph -from mypy.maptype import map_instance_to_supertype from mypy.nodes import ( ARG_NAMED, ARG_POS, @@ -50,6 +48,9 @@ YieldExpr, YieldFromExpr, ) + +from mypy.build import Graph +from mypy.maptype import map_instance_to_supertype from mypy.traverser import TraverserVisitor from mypy.types import ( AnyType, diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 3eb617cf8655..8490c872963d 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -29,6 +29,7 @@ Var, is_class_var, ) + from mypy.types import Instance, UnboundType, get_proper_type from mypyc.common import GENERATOR_HELPER_NAME, MYPYC_DEFAULTS_SETUP, PROPSET_PREFIX from mypyc.ir.class_ir import ClassIR, NonExtClassInfo diff --git a/mypyc/irbuild/constant_fold.py b/mypyc/irbuild/constant_fold.py index 53274dd3f971..96b2d441533e 100644 --- a/mypyc/irbuild/constant_fold.py +++ b/mypyc/irbuild/constant_fold.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Final -from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op from mypy.nodes import ( BytesExpr, ComplexExpr, @@ -26,6 +25,8 @@ UnaryExpr, Var, ) + +from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op from mypyc.irbuild.util import bytes_from_str if TYPE_CHECKING: diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index c1997a921333..dbc4a3664048 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -3,6 +3,7 @@ from __future__ import annotations from mypy.nodes import FuncItem + from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import INVALID_FUNC_DEF from mypyc.ir.ops import BasicBlock, Value diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 4e5c58158cae..92927b8fcedb 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -18,6 +18,7 @@ def g() -> int: from __future__ import annotations from mypy.nodes import Argument, FuncDef, SymbolNode, Var + from mypyc.common import ( BITMAP_BITS, ENV_ATTR_NAME, diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 8bc4568e0d47..6092225cde49 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -49,6 +49,7 @@ UnaryExpr, Var, ) + from mypy.types import ( AnyType, Instance, diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index f1536fb3111a..9d08fd19c369 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -27,6 +27,7 @@ TypeAlias, Var, ) + from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types from mypyc.common import GENERATOR_HELPER_NAME, IS_FREE_THREADED from mypyc.ir.ops import ( diff --git a/mypyc/irbuild/format_str_tokenizer.py b/mypyc/irbuild/format_str_tokenizer.py index f4c6167023bf..e2716c6cfed6 100644 --- a/mypyc/irbuild/format_str_tokenizer.py +++ b/mypyc/irbuild/format_str_tokenizer.py @@ -5,6 +5,8 @@ from enum import Enum, unique from typing import Final +from mypy.nodes import Context, Expression + from mypy.checkstrformat import ( ConversionSpecifier, parse_conversion_specifiers, @@ -12,7 +14,6 @@ ) from mypy.errors import Errors from mypy.messages import MessageBuilder -from mypy.nodes import Context, Expression from mypy.options import Options from mypyc.ir.ops import Integer, Value from mypyc.ir.rtypes import ( diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 588e1cd51dbb..0f7c457f0895 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -28,6 +28,7 @@ TypeInfo, Var, ) + from mypy.types import CallableType, Type, UnboundType, get_proper_type from mypyc.common import FAST_PREFIX, LAMBDA_NAME, PROPSET_PREFIX, SELF_NAME from mypyc.ir.class_ir import ClassIR, NonExtClassInfo diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 7cd4f7de2846..31e9f0449a22 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -13,6 +13,7 @@ from collections.abc import Callable from mypy.nodes import ARG_OPT, FuncDef, Var + from mypyc.common import ( ENV_ATTR_NAME, GENERATOR_ATTRIBUTE_PREFIX, diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index aafdb73fed84..da0d863252e1 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -11,8 +11,9 @@ from contextlib import contextmanager from typing import Final, TypeGuard, cast -from mypy.argmap import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind + +from mypy.argmap import map_actuals_to_formals from mypy.operators import op_methods, unary_op_methods from mypy.types import AnyType, TypeOfAny from mypyc.common import ( diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index bc5354c34203..4c7d5632ce0e 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -25,8 +25,9 @@ def f(x: int) -> int: from collections.abc import Callable from typing import Any, TypeVar, cast -from mypy.build import Graph from mypy.nodes import ClassDef, Expression, FuncDef, MypyFile + +from mypy.build import Graph from mypy.state import state from mypy.types import Type from mypyc.analysis.attrdefined import analyze_always_defined_attrs diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 523c718c9b86..62a96020e67c 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -3,6 +3,7 @@ from __future__ import annotations from mypy.nodes import ARG_STAR, ARG_STAR2, GDEF, ArgKind, FuncDef, RefExpr, SymbolNode, TypeInfo + from mypy.types import ( AnyType, CallableType, diff --git a/mypyc/irbuild/match.py b/mypyc/irbuild/match.py index 7b08cf1437ca..fcd2c6046c89 100644 --- a/mypyc/irbuild/match.py +++ b/mypyc/irbuild/match.py @@ -4,6 +4,7 @@ from contextlib import contextmanager from mypy.nodes import MatchStmt, NameExpr, TypeInfo + from mypy.patterns import ( AsPattern, ClassPattern, diff --git a/mypyc/irbuild/missingtypevisitor.py b/mypyc/irbuild/missingtypevisitor.py index e655d270a4a4..c251f38d1899 100644 --- a/mypyc/irbuild/missingtypevisitor.py +++ b/mypyc/irbuild/missingtypevisitor.py @@ -1,6 +1,7 @@ from __future__ import annotations from mypy.nodes import Expression, Node + from mypy.traverser import ExtendedTraverserVisitor from mypy.types import AnyType, Type, TypeOfAny diff --git a/mypyc/irbuild/prebuildvisitor.py b/mypyc/irbuild/prebuildvisitor.py index 2d9535f06a52..39f822dce5e1 100644 --- a/mypyc/irbuild/prebuildvisitor.py +++ b/mypyc/irbuild/prebuildvisitor.py @@ -22,6 +22,7 @@ TupleExpr, Var, ) + from mypy.traverser import ExtendedTraverserVisitor, TraverserVisitor from mypy.types import Type from mypyc.errors import Errors diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 99ce7de8df1f..312237c7d4fc 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -17,7 +17,6 @@ from collections.abc import Iterable from typing import NamedTuple -from mypy.build import Graph from mypy.nodes import ( ARG_STAR, ARG_STAR2, @@ -38,6 +37,8 @@ Var, ) from mypy.semanal import refers_to_fullname + +from mypy.build import Graph from mypy.traverser import TraverserVisitor from mypy.types import Instance, Type, get_proper_type from mypyc.common import ( diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index b2ff57847d6a..8dd220787fa3 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -36,6 +36,7 @@ TupleExpr, Var, ) + from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( BasicBlock, diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index a797b8f5ab34..cbb0c0333a5c 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,6 +49,7 @@ YieldExpr, YieldFromExpr, ) + from mypyc.common import ( GENERATOR_HELPER_NAME, KEEP_ALIVE_SHORT_LIVED, diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index a2378fa55bcc..c28b5f373660 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -32,6 +32,7 @@ is_class_var, ) from mypy.semanal import refers_to_fullname + from mypy.types import FINAL_DECORATOR_NAMES from mypyc.errors import Errors from mypyc.ir.class_ir import ClassIR diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index bb8aa4932e39..a5caac41be3b 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -84,6 +84,7 @@ YieldExpr, YieldFromExpr, ) + from mypyc.ir.ops import Value from mypyc.irbuild.builder import IRBuilder, IRVisitor, UnsupportedException from mypyc.irbuild.classdef import transform_class_def diff --git a/mypyc/test/test_statement.py b/mypyc/test/test_statement.py index 35773f934122..a571ddbd6039 100644 --- a/mypyc/test/test_statement.py +++ b/mypyc/test/test_statement.py @@ -3,8 +3,9 @@ import unittest from typing import cast -from mypy.build import Graph from mypy.nodes import Import, MypyFile + +from mypy.build import Graph from mypy.options import Options from mypyc.errors import Errors from mypyc.irbuild.builder import IRBuilder diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 2d139ace7c8e..6f51e54fbe5a 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -9,10 +9,11 @@ import shutil from collections.abc import Callable, Iterator +from mypy.nodes import Expression, MypyFile + from mypy import build from mypy.errors import CompileError from mypy.main import process_options -from mypy.nodes import Expression, MypyFile from mypy.options import Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite From 868b5419f649205d024b12c4caf77be6e4122849 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Thu, 24 Sep 2026 19:01:28 +0200 Subject: [PATCH 03/24] Restore mypy core modules deleted by bad push The previous push accidentally deleted mypy/checker.py, mypy/nodes.py, mypy/semanal.py and mypy/typeanal.py, breaking every CI job. Restore them byte-identical from upstream master (8ddae783). The regression tests for #21998 are kept; the underlying fix still needs to be re-implemented against the current layout. --- mypy/checker.py | 10110 +++++++++++++++++++++++++++++++++++++++++++++ mypy/nodes.py | 5608 +++++++++++++++++++++++++ mypy/semanal.py | 8662 ++++++++++++++++++++++++++++++++++++++ mypy/typeanal.py | 2961 +++++++++++++ 4 files changed, 27341 insertions(+) create mode 100644 mypy/checker.py create mode 100644 mypy/nodes.py create mode 100644 mypy/semanal.py create mode 100644 mypy/typeanal.py diff --git a/mypy/checker.py b/mypy/checker.py new file mode 100644 index 000000000000..f03d06d965c3 --- /dev/null +++ b/mypy/checker.py @@ -0,0 +1,10110 @@ +"""Mypy type checker. + +Infer types of expressions and type-check the AST. We use multi-pass architecture, +this means we re-visit the AST until either all types are inferred, or we hit the +upper limit on number of passes (currently 3). The number of passes is intentionally +low, mostly to keep performance predictable. An example of code where more than one +pass is needed: + + def foo() -> None: + C().x += 1 # we don't know the type of x before we visit C + + class C: + def __init__(self) -> None: + self.x = + +In this example, encountering attribute C.x will trigger handle_cannot_determine_type() +which in turn will schedule foo() to be re-visited in the next pass. This process is +called deferring, some notes on it: +* Only a top-level function or method can be deferred. For historical reasons we don't + defer module top-levels. +* There are two reasons that can trigger deferral: a type of name/attribute used in + a function body is not inferred yet, or type of superclass node is not known when + checking LSP. +* In first case, the actual deferred node will be always FuncDef, even if the actual + top-level function is either a Decorator or an OverloadedFuncDef. +* In the second case, the deferred node will be actual top-level node, unless FuncDef + is sufficient (in case when deferring a method of a class nested in function). +* When a function was deferred we suppress most errors, and infer all expressions as + Any until the end of deferred function. This is done to avoid false positives. We may + change this in future by adding a special kind of Any that signifies not ready type. + +Second important aspect of architecture is two-phase checking. Currently, it is only +used in parallel type-checking mode, but at some point it will be the default in +sequential mode as well. + +In first phase (called interface phase) we type-check only module top-levels, function +signatures, and bodies of functions/methods that were found during semantic analysis as +(possibly) affecting external module interface (most notably __init__ methods). This is +done by setting recurse_into_functions flag to False. We handle all deferrals triggered +during this phase (still with recurse_into_functions being False) before moving to +the next one. + +In second phase (called implementation phase) we type-check only the bodies of functions +and methods not type-checked in interface phase. This is done by setting +recurse_into_functions to True, and gathering all functions/methods with def_or_infer_vars +flag being False. Note that we execute only parts of the relevant visit methods that +were not executed before (i.e. we don't handle function signatures twice), see +check_partial() method for more details. + +The boundary between function signature logic and function body logic is somewhat arbitrary, +and currently mostly sits where it was historically. Some rules about this are: +* Any logic that can *change* function signature *must* be executed in phase one. +* In general, we want to put as much of logic into phase two for better performance. +* Try keeping things clear/consistent, right now the boundary always sits at the point when + we push the function on the scope stack and call check_func_item(). + +Note: some parts of the type-checker live in other `check*.py` files, like `checkexpr.py`, +the orchestration between various type-checking phases and passes is done in `build.py`. +""" + +from __future__ import annotations + +import itertools +from collections import defaultdict +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set as AbstractSet +from contextlib import ExitStack, contextmanager +from typing import ( + Final, + Generic, + Literal, + NamedTuple, + TypeAlias as _TypeAlias, + TypeGuard, + TypeVar, + cast, + overload, +) + +import mypy.checkexpr +from mypy import errorcodes as codes, join, message_registry, nodes, operators +from mypy.binder import ConditionalTypeBinder, Frame, get_declaration +from mypy.checker_shared import CheckerScope, TypeCheckerSharedApi, TypeRange +from mypy.checker_state import checker_state +from mypy.checkmember import ( + MemberContext, + analyze_class_attribute_access, + analyze_instance_member_access, + analyze_member_access, + is_instance_var, +) +from mypy.checkpattern import PatternChecker +from mypy.constraints import SUPERTYPE_OF +from mypy.erasetype import ( + erase_type, + erase_typevars, + remove_instance_last_known_values, + shallow_erase_type_for_equality, +) +from mypy.errorcodes import TYPE_VAR, UNUSED_AWAITABLE, UNUSED_COROUTINE, ErrorCode +from mypy.errors import ( + ErrorInfo, + Errors, + ErrorWatcher, + IterationDependentErrors, + IterationErrorWatcher, + report_internal_error, +) +from mypy.expandtype import expand_type +from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash +from mypy.maptype import map_instance_to_supertype +from mypy.meet import is_overlapping_types, meet_types +from mypy.message_registry import ErrorMessage +from mypy.messages import ( + SUGGESTED_TEST_FIXTURES, + MessageBuilder, + append_invariance_notes, + append_union_note, + format_type, + format_type_bare, + format_type_distinctly, + make_inferred_type_note, + pretty_seq, +) +from mypy.mro import MroError, calculate_mro +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + ARG_STAR, + CONTRAVARIANT, + COVARIANT, + FUNC_NO_INFO, + GDEF, + IMPLICITLY_ABSTRACT, + INVARIANT, + IS_ABSTRACT, + LDEF, + LITERAL_TYPE, + MDEF, + NOT_ABSTRACT, + SYMBOL_FUNCBASE_TYPES, + AssertStmt, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + BytesExpr, + CallExpr, + ClassDef, + ComparisonExpr, + Context, + ContinueStmt, + Decorator, + DelStmt, + DictExpr, + EllipsisExpr, + Expression, + ExpressionStmt, + FloatExpr, + ForStmt, + FuncBase, + FuncDef, + FuncItem, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportBase, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListExpr, + Lvalue, + MatchStmt, + MemberExpr, + MypyFile, + NameExpr, + Node, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + OverloadPart, + PassStmt, + PromoteExpr, + RaiseStmt, + RefExpr, + ReturnStmt, + SetExpr, + SplittingVisitor, + StarExpr, + Statement, + StrExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TempNode, + TryStmt, + TupleExpr, + TypeAlias, + TypeAliasStmt, + TypeInfo, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + get_func_def, + is_final_node, +) +from mypy.operators import flip_ops, int_op_to_method, neg_ops +from mypy.options import PRECISE_TUPLE_TYPES, Options +from mypy.patterns import AsPattern, StarredPattern +from mypy.plugin import Plugin +from mypy.plugins import dataclasses as dataclasses_plugin +from mypy.scope import Scope +from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name +from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS +from mypy.semanal_shared import SemanticAnalyzerCoreInterface +from mypy.sharedparse import BINARY_MAGIC_METHODS +from mypy.state import state +from mypy.subtypes import ( + find_member, + infer_class_variances, + is_callable_compatible, + is_equivalent, + is_more_precise, + is_proper_subtype, + is_same_type, + is_subtype, + restrict_subtype_away, + unify_generic_callable, +) +from mypy.traverser import TraverserVisitor, all_return_statements, has_return_statement +from mypy.treetransform import TransformVisitor +from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type, make_optional_type +from mypy.typeops import ( + bind_self, + can_have_shared_disjoint_base, + coerce_to_literal, + custom_special_method, + erase_def_to_union_or_bound, + erase_to_bound, + erase_to_union_or_bound, + false_only, + fixup_partial_type, + function_type, + is_literal_type_like, + is_singleton_equality_type, + is_singleton_identity_type, + make_simplified_union, + true_only, + try_expanding_sum_type_to_union, + try_getting_int_literals_from_type, + try_getting_str_literals, + try_getting_str_literals_from_type, + tuple_fallback, + type_object_type, +) +from mypy.types import ( + ANY_STRATEGY, + MYPYC_NATIVE_INT_NAMES, + NOT_IMPLEMENTED_TYPE_NAMES, + OVERLOAD_NAMES, + AnyType, + BoolTypeQuery, + CallableType, + DeletedType, + ErasedType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + PartialType, + ProperType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeGuardedType, + TypeOfAny, + TypeTranslator, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + extend_args_for_prefix_and_suffix, + find_unpack_in_list, + flatten_nested_unions, + get_proper_type, + get_proper_types, + get_variadic_item, + instance_cache, + is_literal_type, + is_named_instance, + is_unannotated_any, +) +from mypy.types_utils import is_overlapping_none, remove_optional, store_argument_type, strip_type +from mypy.typetraverser import TypeTraverserVisitor +from mypy.typevars import fill_typevars, fill_typevars_with_any, has_no_typevars +from mypy.util import is_dunder, is_sunder +from mypy.visitor import NodeVisitor + +T = TypeVar("T") + +DEFAULT_LAST_PASS: Final = 2 # Pass numbers start at 0 + +# Maximum length of fixed tuple types inferred when narrowing from variadic tuples. +MAX_PRECISE_TUPLE_SIZE: Final = 8 + +DeferredNodeType: _TypeAlias = FuncDef | OverloadedFuncDef | Decorator +FineGrainedDeferredNodeType: _TypeAlias = FuncDef | MypyFile | OverloadedFuncDef + + +# A node which is postponed to be processed during the next pass. +# In normal mode one can defer functions and methods (also decorated and/or overloaded) +# but not lambda expressions. Nested functions can't be deferred -- only top-level functions +# and methods of classes not defined within a function can be deferred. +class DeferredNode(NamedTuple): + node: DeferredNodeType + # And its TypeInfo (for semantic analysis self type handling) + active_typeinfo: TypeInfo | None + + +# Same as above, but for fine-grained mode targets. Only top-level functions/methods +# and module top levels are allowed as such. +class FineGrainedDeferredNode(NamedTuple): + node: FineGrainedDeferredNodeType + active_typeinfo: TypeInfo | None + + +# Data structure returned by find_isinstance_check representing +# information learned from the truth or falsehood of a condition. The +# dict maps nodes representing expressions like 'a[0].x' to their +# refined types under the assumption that the condition has a +# particular truth value. A value of None means that the condition can +# never have that truth value. + +# NB: The keys of this dict are nodes in the original source program, +# which are compared by reference equality--effectively, being *the +# same* expression of the program, not just two identical expressions +# (such as two references to the same variable). TODO: it would +# probably be better to have the dict keyed by the nodes' literal_hash +# field instead. +TypeMap: _TypeAlias = dict[Expression, Type] + + +# Keeps track of partial types in a single scope. In fine-grained incremental +# mode partial types initially defined at the top level cannot be completed in +# a function, and we use the 'is_function' attribute to enforce this. +class PartialTypeScope: + def __init__(self, map: dict[Var, Context], is_function: bool, is_local: bool) -> None: + self.map: Final = map + self.is_function: Final = is_function + self.is_local: Final = is_local + + +class LocalTypeMap: + """Store inferred types into a temporary type map (returned). + + This can be used to perform type checking "experiments" without + affecting exported types (which are used by mypyc). + """ + + def __init__(self, chk: TypeChecker) -> None: + self.chk = chk + + def __enter__(self) -> dict[Expression, Type]: + temp_type_map: dict[Expression, Type] = {} + self.chk._type_maps.append(temp_type_map) + return temp_type_map + + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> Literal[False]: + self.chk._type_maps.pop() + return False + + +class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi, SplittingVisitor): + """Mypy type checker. + + Type check mypy source files that have been semantically analyzed. + + You must create a separate instance for each source file. + """ + + # Are we type checking a stub? + is_stub = False + # Error message reporter + errors: Errors + # Utility for generating messages + msg: MessageBuilder + # Types of type checked nodes. The first item is the "master" type + # map that will store the final, exported types. Additional items + # are temporary type maps used during type inference, and these + # will be eventually popped and either discarded or merged into + # the master type map. + # + # Avoid accessing this directly, but prefer the lookup_type(), + # has_type() etc. helpers instead. + _type_maps: list[dict[Expression, Type]] + + # Helper for managing conditional types + binder: ConditionalTypeBinder + # Helper for type checking expressions + _expr_checker: mypy.checkexpr.ExpressionChecker + + pattern_checker: PatternChecker + + tscope: Scope + scope: CheckerScope + # Stack of function return types + return_types: list[Type] + # Flags; true for dynamically typed functions + dynamic_funcs: list[bool] + # Stack of collections of variables with partial types + partial_types: list[PartialTypeScope] + # Vars for which partial type errors are already reported + # (to avoid logically duplicate errors with different error context). + partial_reported: set[Var] + # Short names of Var nodes whose previous inferred type has been widened via assignment. + # NOTE: The names might not be unique, they are only for debugging purposes. + widened_vars: list[str] + # Global variables widened inside a function body, to be propagated to + # the module-level binder after the function is type checked (with --allow-redefinition-new). + _globals_widened_in_func: list[tuple[NameExpr, Type]] + globals: SymbolTable + modules: dict[str, MypyFile] + # Nodes that couldn't be checked because some types weren't available. We'll run + # another pass and try these again. + deferred_nodes: list[DeferredNode] + # Type checking pass number (0 = first pass) + pass_num = 0 + # Last pass number to take + last_pass = DEFAULT_LAST_PASS + # Have we deferred the current function? If yes, don't infer additional + # types during this pass within the function. + current_node_deferred = False + # Is this file a typeshed stub? + is_typeshed_stub = False + options: Options + # Used for collecting inferred attribute types so that they can be checked + # for consistency. + inferred_attribute_types: dict[Var, Type] | None = None + # Don't infer partial None types if we are processing assignment from Union + no_partial_types: bool = False + # Extra module references not detected during semantic analysis (these are rare cases + # e.g. access to class-level import via instance). + module_refs: set[str] + # A map from variable nodes to a snapshot of the frame ids of the + # frames that were active when the variable was declared. This can + # be used to determine nearest common ancestor frame of a variable's + # declaration and the current frame, which lets us determine if it + # was declared in a different branch of the same `if` statement + # (if that frame is a conditional_frame). + var_decl_frames: dict[Var, set[int]] + + # Plugin that provides special type checking rules for specific library + # functions such as open(), etc. + plugin: Plugin + + # A helper state to produce unique temporary names on demand. + _unique_id: int + # Fake concrete type used when checking variance + _variance_dummy_type: Instance | None + + def __init__( + self, + errors: Errors, + modules: dict[str, MypyFile], + options: Options, + tree: MypyFile, + path: str, + plugin: Plugin, + per_line_checking_time_ns: dict[int, int], + ) -> None: + """Construct a type checker. + + Use errors to report type check errors. + """ + self.errors = errors + self.modules = modules + self.options = options + self.tree = tree + self.path = path + self.msg = MessageBuilder(errors, modules) + self.plugin = plugin + self.tscope = Scope() + self.scope = CheckerScope(tree) + self.binder = ConditionalTypeBinder(options) + self.globals_binder = self.binder + self.globals = tree.names + self.globals_unreachable: set[int] = set() + self.return_types = [] + self.dynamic_funcs = [] + self.partial_types = [] + self.partial_reported = set() + self.var_decl_frames = {} + self.deferred_nodes = [] + self.widened_vars = [] + self._globals_widened_in_func = [] + self._type_maps = [{}] + self.module_refs = set() + self.pass_num = 0 + self.current_node_deferred = False + self.is_stub = tree.is_stub + self.is_typeshed_stub = tree.is_typeshed_file(options) + self.inferred_attribute_types = None + self.allow_constructor_cache = True + self.local_type_map = LocalTypeMap(self) + + self.can_skip_diagnostics: Final = ( + self.options.ignore_errors + or self.path in self.msg.errors.ignored_files + or (self.options.test_env and self.is_typeshed_stub) + ) + self.recurse_into_functions = True + # This internal flag is used to track whether we a currently type-checking + # a final declaration (assignment), so that some errors should be suppressed. + # Should not be set manually, use get_final_context/enter_final_context instead. + # NOTE: we use the context manager to avoid "threading" an additional `is_final_def` + # argument through various `checker` and `checkmember` functions. + self._is_final_def = False + + # Track when we enter an overload implementation. Some checks should not be applied + # to the implementation signature when specific overloads are available. + # Use `enter_overload_impl` to modify. + self.overload_impl_stack: list[OverloadPart] = [] + + # This flag is set when we run type-check or attribute access check for the purpose + # of giving a note on possibly missing "await". It is used to avoid infinite recursion. + self.checking_missing_await = False + + # While this is True, allow passing an abstract class where Type[T] is expected. + # although this is technically unsafe, this is desirable in some context, for + # example when type-checking class decorators. + self.allow_abstract_call = False + + # Child checker objects for specific AST node types + self._expr_checker = mypy.checkexpr.ExpressionChecker( + self, self.msg, self.plugin, per_line_checking_time_ns + ) + + self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) + self._unique_id = 0 + self._variance_dummy_type = None + + # Assignment expression is the only *expression* that can have externally + # visible effects via binder. We bump this whenever this happens to handle + # situations when some expressions are accepted multiple times. + self.assignment_expression_effect = 0 + + @property + def expr_checker(self) -> mypy.checkexpr.ExpressionChecker: + return self._expr_checker + + @property + def type_context(self) -> list[Type | None]: + return self._expr_checker.type_context + + def reset(self) -> None: + """Cleanup stale state that might be left over from a typechecking run. + + This allows us to reuse TypeChecker objects in fine-grained + incremental mode. + """ + # TODO: verify this is still actually worth it over creating new checkers + self.partial_reported.clear() + self.module_refs.clear() + self.binder = ConditionalTypeBinder(self.options) + self._type_maps[1:] = [] + self._type_maps[0].clear() + self.expr_checker.reset() + self.deferred_nodes = [] + self.partial_types = [] + self.inferred_attribute_types = None + self.scope = CheckerScope(self.tree) + self.globals_unreachable.clear() + + def mark_unreachable(self, block: list[Statement], after: Statement) -> None: + """Marks all statements in block after a given one (inclusive) as unreachable.""" + last_line = (last := block[-1]).end_line or last.line + self.globals_unreachable.update(range(after.line, last_line + 1)) + + def check_first_pass(self, recurse_into_functions: bool = True) -> None: + """Type check the entire file, but defer functions with unresolved references. + + Unresolved references are forward references to variables + whose types haven't been inferred yet. They may occur later + in the same file or in a different file that's being processed + later (usually due to an import cycle). + + Deferred functions will be processed by check_second_pass(). + """ + self.recurse_into_functions = recurse_into_functions + with state.strict_optional_set(self.options.strict_optional), checker_state.set(self): + self.errors.set_file( + self.path, self.tree.fullname, scope=self.tscope, options=self.options + ) + with self.tscope.module_scope(self.tree.fullname): + with self.enter_partial_types(), self.binder.top_frame_context(): + marked_unreachable = False + reported_unreachable = False + for d in self.tree.defs: + if self.binder.is_unreachable(): + finish = False + if not marked_unreachable: + self.mark_unreachable(self.tree.defs, after=d) + marked_unreachable = True + if not self.should_report_unreachable_issues(): + finish = True + if ( + not finish + and not reported_unreachable + and not self.is_noop_for_reachability(d) + ): + self.msg.unreachable_statement(d) + self.binder.suppress_unreachable_warnings() + finish = True + reported_unreachable = True + + if finish and not self.options.check_unreachable: + break + + if not self.options.check_unreachable: + continue + + self.accept(d) + + assert not self.current_node_deferred + + all_ = self.globals.get("__all__") + if all_ is not None and all_.type is not None: + all_node = all_.node + assert all_node is not None + seq_str = self.named_generic_type( + "typing.Sequence", [self.named_type("builtins.str")] + ) + if not is_subtype(all_.type, seq_str): + str_seq_s, all_s = format_type_distinctly( + seq_str, all_.type, options=self.options + ) + self.fail( + message_registry.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), all_node + ) + + def check_second_pass( + self, + todo: Sequence[DeferredNode | FineGrainedDeferredNode] | None = None, + *, + allow_constructor_cache: bool = True, + recurse_into_functions: bool = True, + impl_only: bool = False, + ) -> bool: + """Run second or following pass of type checking. + + This goes through deferred nodes, returning True if there were any. + """ + self.allow_constructor_cache = allow_constructor_cache + self.recurse_into_functions = recurse_into_functions + with state.strict_optional_set(self.options.strict_optional), checker_state.set(self): + if not todo and not self.deferred_nodes: + return False + self.errors.set_file( + self.path, self.tree.fullname, scope=self.tscope, options=self.options + ) + with self.tscope.module_scope(self.tree.fullname): + self.pass_num += 1 + if not todo: + todo = self.deferred_nodes + else: + assert not self.deferred_nodes + self.deferred_nodes = [] + done: set[DeferredNodeType | FineGrainedDeferredNodeType] = set() + for node, active_typeinfo in todo: + if node in done: + continue + # This is useful for debugging: + # print("XXX in pass %d, class %s, function %s" % + # (self.pass_num, type_name, node.fullname or node.name)) + done.add(node) + with ExitStack() as stack: + if active_typeinfo: + stack.enter_context(self.tscope.class_scope(active_typeinfo)) + stack.enter_context(self.scope.push_class(active_typeinfo)) + self.check_partial(node, impl_only=impl_only) + return True + + def check_partial( + self, node: DeferredNodeType | FineGrainedDeferredNodeType, impl_only: bool + ) -> None: + self.widened_vars = [] + if isinstance(node, MypyFile): + self.check_top_level(node) + else: + with self.binder.top_frame_context(): + # TODO: use impl_only in the daemon as well. + if not impl_only: + self.accept(node) + return + if node.line in self.globals_unreachable: + return + if isinstance(node, (FuncDef, Decorator)): + self.check_partial_impl(node) + else: + # Overloads need some special logic, since settable + # properties are stored as overloads. + for i, item in enumerate(node.items): + # Setter and/or deleter can technically be empty. + allow_empty = not node.is_property or i > 0 + assert isinstance(item, Decorator) + # Although the actual bodies of overload items are empty, we + # still need to execute some logic that doesn't affect signature. + with self.tscope.function_scope(item.func): + self.check_func_item( + item.func, name=node.name, allow_empty=allow_empty + ) + if node.impl is not None: + # Handle the implementation as a regular function. + with self.enter_overload_impl(node.impl): + self.check_partial_impl(node.impl) + + def check_partial_impl(self, impl: FuncDef | Decorator) -> None: + """Check only the body (not the signature) of a non-overloaded function.""" + if isinstance(impl, FuncDef): + self.visit_func_def_impl(impl) + else: + if any(refers_to_fullname(d, "typing.no_type_check") for d in impl.decorators): + return + with self.tscope.function_scope(impl.func): + self.check_func_item(impl.func, name=impl.func.name) + + def check_top_level(self, node: MypyFile) -> None: + """Check only the top-level of a module, skipping function definitions.""" + with self.enter_partial_types(): + with self.binder.top_frame_context(): + for d in node.defs: + d.accept(self) + + assert not self.current_node_deferred + # TODO: Handle __all__ + + def defer_node(self, node: DeferredNodeType, enclosing_class: TypeInfo | None) -> None: + """Defer a node for processing during next type-checking pass. + + Args: + node: function/method being deferred + enclosing_class: for methods, the class where the method is defined + NOTE: this can't handle nested functions/methods. + """ + # We don't freeze the entire scope since only top-level functions and methods + # can be deferred. Only module/class level scope information is needed. + # Module-level scope information is preserved in the TypeChecker instance. + self.deferred_nodes.append(DeferredNode(node, enclosing_class)) + + def handle_cannot_determine_type(self, name: str, context: Context) -> None: + node = self.scope.top_level_function() + if self.pass_num < self.last_pass and isinstance(node, FuncDef): + # Don't report an error yet. Just defer. Note that we don't defer + # lambdas because they are coupled to the surrounding function + # through the binder and the inferred type of the lambda, so it + # would get messy. + enclosing_class = self.scope.enclosing_class(node) + self.defer_node(node, enclosing_class) + # Set a marker so that we won't infer additional types in this + # function. Any inferred types could be bogus, because there's at + # least one type that we don't know. + self.current_node_deferred = True + else: + self.msg.cannot_determine_type(name, context) + + def accept(self, stmt: Statement) -> None: + """Type check a node in the given type context.""" + try: + stmt.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, stmt.line, self.errors, self.options) + + def accept_loop( + self, + body: Statement, + else_body: Statement | None = None, + *, + exit_condition: Expression | None = None, + on_enter_body: Callable[[], None] | None = None, + ) -> None: + """Repeatedly type check a loop body until the frame doesn't change.""" + + # The outer frame accumulates the results of all iterations: + with self.binder.frame_context(can_skip=False, conditional_frame=True): + # Check for potential decreases in the number of partial types so as not to stop the + # iteration too early: + partials_old = sum(len(pts.map) for pts in self.partial_types) + # Check if assignment widened the inferred type of a variable; in this case we + # need to iterate again (we only do one extra iteration, since this could go + # on without bound otherwise) + widened_old = len(self.widened_vars) + + iter_errors = IterationDependentErrors() + iter = 1 + while True: + with self.binder.frame_context(can_skip=True, break_frame=2, continue_frame=1): + if on_enter_body is not None: + on_enter_body() + + with IterationErrorWatcher(self.msg.errors, iter_errors): + self.accept(body) + + partials_new = sum(len(pts.map) for pts in self.partial_types) + widened_new = len(self.widened_vars) + # Perform multiple iterations if something changed that might affect + # inferred types. Also limit the number of iterations. The limits are + # somewhat arbitrary, but they were chosen to 1) avoid slowdown from + # multiple iterations in common cases and 2) support common, valid use + # cases. Limits are needed since otherwise we could infer infinitely + # complex types. + if ( + (partials_new == partials_old) + and (not self.binder.last_pop_changed or iter > 3) + and (widened_new == widened_old or iter > 1) + ): + break + partials_old = partials_new + widened_old = widened_new + iter += 1 + if iter == 20: + raise RuntimeError("Too many iterations when checking a loop") + + self.msg.iteration_dependent_errors(iter_errors) + + # If exit_condition is set, assume it must be False on exit from the loop: + if exit_condition: + _, else_map = self.find_isinstance_check(exit_condition) + self.push_type_map(else_map) + + # Check the else body: + if else_body: + self.accept(else_body) + + # + # Definitions + # + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + # We always process overload as part of the top-level to infer various + # externally visible properties like its type, similar to visit_decorator(). + # Only the body of the implementation is checked as a function-level target. + # TODO: clean-up deferral logic and the daemon to avoid unnecessary work. + with self.tscope.function_scope(defn): + self._visit_overloaded_func_def(defn) + + def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + num_abstract = 0 + if not defn.items: + # In this case we have already complained about none of these being + # valid overloads. + return + if len(defn.items) == 1: + self.fail(message_registry.MULTIPLE_OVERLOADS_REQUIRED, defn) + + if defn.is_property: + # HACK: Infer the type of the property. + assert isinstance(defn.items[0], Decorator) + self.visit_decorator(defn.items[0]) + if defn.items[0].var.is_settable_property: + # Perform a reduced visit just to infer the actual setter type. + self.visit_decorator_inner(defn.setter, skip_first_item=True) + setter_type = defn.setter.var.type + # Check if the setter can accept two positional arguments. + any_type = AnyType(TypeOfAny.special_form) + fallback_setter_type = CallableType( + arg_types=[any_type, any_type], + arg_kinds=[ARG_POS, ARG_POS], + arg_names=[None, None], + ret_type=any_type, + fallback=self.named_type("builtins.function"), + ) + if setter_type and not is_subtype(setter_type, fallback_setter_type): + self.fail("Invalid property setter signature", defn.setter.func) + setter_type = self.extract_callable_type(setter_type, defn) + if not isinstance(setter_type, CallableType) or len(setter_type.arg_types) != 2: + # TODO: keep precise type for callables with tricky but valid signatures. + setter_type = fallback_setter_type + defn.items[0].var.setter_type = setter_type + if isinstance(defn.type, Overloaded): + # Update legacy property type for decorated properties. + getter_type = self.extract_callable_type(defn.items[0].var.type, defn) + if getter_type is not None: + getter_type.definition = defn.items[0] + defn.type.items[0] = getter_type + for i, fdef in enumerate(defn.items): + assert isinstance(fdef, Decorator) + if defn.is_property: + assert isinstance(defn.items[0], Decorator) + settable = defn.items[0].var.is_settable_property + # Do not visit the second time the items we checked above. + if (settable and i > 1) or (not settable and i > 0): + # Type check initialization expressions. + # TODO: initializers can infer types if they contain a walrus, + # it may be not safe to optimize them away completely. + if not self.can_skip_diagnostics: + self.check_default_params(fdef.func) + if self.recurse_into_functions or fdef.func.def_or_infer_vars: + with ( + self.tscope.function_scope(fdef.func), + self.set_recurse_into_functions(), + ): + self.check_func_item(fdef.func, name=fdef.func.name, allow_empty=True) + else: + # Perform full check for real overloads to infer type of all decorated + # overload variants. + self.visit_decorator_inner(fdef, allow_empty=True) + if fdef.func.abstract_status in (IS_ABSTRACT, IMPLICITLY_ABSTRACT): + num_abstract += 1 + if num_abstract not in (0, len(defn.items)): + self.fail(message_registry.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + if defn.impl: + with self.enter_overload_impl(defn.impl): + defn.impl.accept(self) + if not defn.is_property: + self.check_overlapping_overloads(defn) + if defn.type is None: + item_types = [] + for item in defn.items: + assert isinstance(item, Decorator) + item_type = self.extract_callable_type(item.var.type, item) + if item_type is not None: + item_type.definition = item + item_types.append(item_type) + if item_types: + defn.type = Overloaded(item_types) + elif defn.type is None: + # We store the getter type as an overall overload type, as some + # code paths are getting property type this way. + assert isinstance(defn.items[0], Decorator) + var_type = self.extract_callable_type(defn.items[0].var.type, defn) + if not isinstance(var_type, CallableType): + # Construct a fallback type, invalid types should be already reported. + any_type = AnyType(TypeOfAny.special_form) + var_type = CallableType( + arg_types=[any_type], + arg_kinds=[ARG_POS], + arg_names=[None], + ret_type=any_type, + fallback=self.named_type("builtins.function"), + ) + defn.type = Overloaded([var_type]) + # Check override validity after we analyzed current definition. + if not self.can_skip_diagnostics and defn.info: + found_method_base_classes = self.check_method_override(defn) + if ( + defn.is_explicit_override + and not found_method_base_classes + and found_method_base_classes is not None + # If the class has Any fallback, we can't be certain that a method + # is really missing - it might come from unfollowed import. + and not defn.info.fallback_to_any + ): + self.msg.no_overridable_method(defn.name, defn) + self.check_explicit_override_decorator(defn, found_method_base_classes, defn.impl) + self.check_inplace_operator_method(defn) + + @contextmanager + def enter_overload_impl(self, impl: OverloadPart) -> Iterator[None]: + self.overload_impl_stack.append(impl) + try: + yield + finally: + assert self.overload_impl_stack.pop() == impl + + def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> CallableType | None: + """Get type as seen by an overload item caller.""" + inner_type = get_proper_type(inner_type) + outer_type: FunctionLike | None = None + if inner_type is None or isinstance(inner_type, AnyType): + return None + if isinstance(inner_type, TypeVarLikeType): + inner_type = get_proper_type(inner_type.upper_bound) + if isinstance(inner_type, TypeType): + inner_type = get_proper_type( + self.expr_checker.analyze_type_type_callee(inner_type.item, ctx) + ) + + if isinstance(inner_type, FunctionLike): + outer_type = inner_type + elif isinstance(inner_type, Instance): + inner_call = get_proper_type( + analyze_member_access( + name="__call__", + typ=inner_type, + context=ctx, + is_lvalue=False, + is_super=False, + is_operator=True, + original_type=inner_type, + chk=self, + ) + ) + if isinstance(inner_call, FunctionLike): + outer_type = inner_call + elif isinstance(inner_type, UnionType): + union_type = make_simplified_union(inner_type.items) + if isinstance(union_type, UnionType): + items = [] + for item in union_type.items: + callable_item = self.extract_callable_type(item, ctx) + if callable_item is None: + break + items.append(callable_item) + else: + joined_type = get_proper_type(join.join_type_list(items)) + if isinstance(joined_type, FunctionLike): + outer_type = joined_type + else: + return self.extract_callable_type(union_type, ctx) + + if outer_type is None: + self.msg.not_callable(inner_type, ctx) + return None + if isinstance(outer_type, Overloaded): + return None + + assert isinstance(outer_type, CallableType) + return outer_type + + def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: + # At this point we should have set the impl already, and all remaining + # items are decorators + + if self.can_skip_diagnostics: + return + + # Compute some info about the implementation (if it exists) for use below + impl_type: CallableType | None = None + if defn.impl: + if isinstance(defn.impl, FuncDef): + inner_type: Type | None = defn.impl.type + elif isinstance(defn.impl, Decorator): + inner_type = defn.impl.var.type + else: + assert False, "Impl isn't the right type" + + # This can happen if we've got an overload with a different + # decorator or if the implementation is untyped -- we gave up on the types. + impl_type = self.extract_callable_type(inner_type, defn.impl) + + is_descriptor_get = defn.info and defn.name == "__get__" + for i, item in enumerate(defn.items): + assert isinstance(item, Decorator) + sig1 = self.extract_callable_type(item.var.type, item) + if sig1 is None: + continue + + for j, item2 in enumerate(defn.items[i + 1 :]): + assert isinstance(item2, Decorator) + sig2 = self.extract_callable_type(item2.var.type, item2) + if sig2 is None: + continue + + if not are_argument_counts_overlapping(sig1, sig2): + continue + + if overload_can_never_match(sig1, sig2): + self.msg.overloaded_signature_will_never_match(i + 1, i + j + 2, item2.func) + elif not is_descriptor_get: + # Note: we force mypy to check overload signatures in strict-optional mode + # so we don't incorrectly report errors when a user tries typing an overload + # that happens to have a 'if the argument is None' fallback. + # + # For example, the following is fine in strict-optional mode but would throw + # the unsafe overlap error when strict-optional is disabled: + # + # @overload + # def foo(x: None) -> int: ... + # @overload + # def foo(x: str) -> str: ... + # + # See Python 2's map function for a concrete example of this kind of overload. + current_class = self.scope.active_class() + type_vars = current_class.defn.type_vars if current_class else [] + with state.strict_optional_set(True): + if is_unsafe_overlapping_overload_signatures(sig1, sig2, type_vars): + flip_note = ( + j == 0 + and not is_unsafe_overlapping_overload_signatures( + sig2, sig1, type_vars + ) + and not overload_can_never_match(sig2, sig1) + ) + self.msg.overloaded_signatures_overlap( + i + 1, i + j + 2, flip_note, item.func + ) + + if impl_type is not None: + assert defn.impl is not None + + # This is what we want from implementation, it should accept all arguments + # of an overload, but the return types should go the opposite way. + if is_callable_compatible( + impl_type, + sig1, + is_compat=is_subtype, + is_proper_subtype=False, + is_compat_return=lambda l, r: is_subtype(r, l), + ): + continue + # If the above check didn't work, we repeat some key steps in + # is_callable_compatible() to give a better error message. + + # We perform a unification step that's very similar to what + # 'is_callable_compatible' does -- the only difference is that + # we check and see if the impl_type's return value is a + # *supertype* of the overload alternative, not a *subtype*. + # + # This is to match the direction the implementation's return + # needs to be compatible in. + if impl_type.variables: + impl: CallableType | None = unify_generic_callable( + # Normalize both before unifying + impl_type.with_unpacked_kwargs(), + sig1.with_unpacked_kwargs(), + ignore_return=False, + return_constraint_direction=SUPERTYPE_OF, + ) + if impl is None: + self.msg.overloaded_signatures_typevar_specific(i + 1, defn.impl) + continue + else: + impl = impl_type + + # Prevent extra noise from inconsistent use of @classmethod by copying + # the first arg from the method being checked against. + if sig1.arg_types and defn.info: + impl = impl.copy_modified(arg_types=[sig1.arg_types[0]] + impl.arg_types[1:]) + + # Is the overload alternative's arguments subtypes of the implementation's? + if not is_callable_compatible( + impl, sig1, is_compat=is_subtype, is_proper_subtype=False, ignore_return=True + ): + self.msg.overloaded_signatures_param_specific(i + 1, defn.impl) + + # Is the overload alternative's return type a subtype of the implementation's? + if not ( + is_subtype(sig1.ret_type, impl.ret_type) + or is_subtype(impl.ret_type, sig1.ret_type) + ): + self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl) + + # Here's the scoop about generators and coroutines. + # + # There are two kinds of generators: classic generators (functions + # with `yield` or `yield from` in the body) and coroutines + # (functions declared with `async def`). The latter are specified + # in PEP 492 and only available in Python >= 3.5. + # + # Classic generators can be parameterized with three types: + # - ty is the Yield type (the type of y in `yield y`) + # - tc is the type reCeived by yield (the type of c in `c = yield`). + # - tr is the Return type (the type of r in `return r`) + # + # A classic generator must define a return type that's either + # `Generator[ty, tc, tr]`, Iterator[ty], or Iterable[ty] (or + # object or Any). If tc/tr are not given, both are None. + # + # A coroutine must define a return type corresponding to tr; the + # other two are unconstrained. The "external" return type (seen + # by the caller) is Awaitable[tr]. + # + # In addition, there's the synthetic type AwaitableGenerator: it + # inherits from both Awaitable and Generator and can be used both + # in `yield from` and in `await`. This type is set automatically + # for functions decorated with `@types.coroutine` or + # `@asyncio.coroutine`. Its single parameter corresponds to tr. + # + # PEP 525 adds a new type, the asynchronous generator, which was + # first released in Python 3.6. Async generators are `async def` + # functions that can also `yield` values. They can be parameterized + # with two types, ty and tc, because they cannot return a value. + # + # There are several useful methods, each taking a type t and a + # flag c indicating whether it's for a generator or coroutine: + # + # - is_generator_return_type(t, c) returns whether t is a Generator, + # Iterator, Iterable (if not c), or Awaitable (if c), or + # AwaitableGenerator (regardless of c). + # - is_async_generator_return_type(t) returns whether t is an + # AsyncGenerator. + # - get_generator_yield_type(t, c) returns ty. + # - get_generator_receive_type(t, c) returns tc. + # - get_generator_return_type(t, c) returns tr. + + def is_generator_return_type(self, typ: Type, is_coroutine: bool) -> bool: + """Is `typ` a valid type for a generator/coroutine? + + True if `typ` is a *supertype* of Generator or Awaitable. + Also true it it's *exactly* AwaitableGenerator (modulo type parameters). + """ + typ = get_proper_type(typ) + if is_coroutine: + # This means we're in Python 3.5 or later. + at = self.named_generic_type("typing.Awaitable", [AnyType(TypeOfAny.special_form)]) + if is_subtype(at, typ): + return True + else: + any_type = AnyType(TypeOfAny.special_form) + gt = self.named_generic_type("typing.Generator", [any_type, any_type, any_type]) + if is_subtype(gt, typ): + return True + return isinstance(typ, Instance) and typ.type.fullname == "typing.AwaitableGenerator" + + def is_async_generator_return_type(self, typ: Type) -> bool: + """Is `typ` a valid type for an async generator? + + True if `typ` is a supertype of AsyncGenerator. + """ + try: + any_type = AnyType(TypeOfAny.special_form) + agt = self.named_generic_type("typing.AsyncGenerator", [any_type, any_type]) + except KeyError: + # we're running on a version of typing that doesn't have AsyncGenerator yet + return False + return is_subtype(agt, typ) + + def get_generator_yield_type(self, return_type: Type, is_coroutine: bool) -> Type: + """Given the declared return type of a generator (t), return the type it yields (ty).""" + return_type = get_proper_type(return_type) + + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + elif isinstance(return_type, UnionType): + return make_simplified_union( + [self.get_generator_yield_type(item, is_coroutine) for item in return_type.items] + ) + elif not self.is_generator_return_type( + return_type, is_coroutine + ) and not self.is_async_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or + # Awaitable) return type, anything is permissible. + return AnyType(TypeOfAny.from_error) + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType(TypeOfAny.from_error) + elif return_type.type.fullname == "typing.Awaitable": + # Awaitable: ty is Any. + return AnyType(TypeOfAny.special_form) + elif return_type.args: + # AwaitableGenerator, Generator, AsyncGenerator, Iterator, or Iterable; ty is args[0]. + ret_type = return_type.args[0] + # TODO not best fix, better have dedicated yield token + return ret_type + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. IOW, ty is Any. + # (However, see https://github.com/python/mypy/issues/1933) + return AnyType(TypeOfAny.special_form) + + def get_generator_receive_type(self, return_type: Type, is_coroutine: bool) -> Type: + """Given a declared generator return type (t), return the type its yield receives (tc).""" + return_type = get_proper_type(return_type) + + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + elif isinstance(return_type, UnionType): + return make_simplified_union( + [self.get_generator_receive_type(item, is_coroutine) for item in return_type.items] + ) + elif not self.is_generator_return_type( + return_type, is_coroutine + ) and not self.is_async_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or + # Awaitable) return type, anything is permissible. + return AnyType(TypeOfAny.from_error) + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType(TypeOfAny.from_error) + elif return_type.type.fullname == "typing.Awaitable": + # Awaitable, AwaitableGenerator: tc is Any. + return AnyType(TypeOfAny.special_form) + elif ( + return_type.type.fullname in ("typing.Generator", "typing.AwaitableGenerator") + and len(return_type.args) >= 3 + ): + # Generator: tc is args[1]. + return return_type.args[1] + elif return_type.type.fullname == "typing.AsyncGenerator" and len(return_type.args) >= 2: + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. IOW, tc is None. + return NoneType() + + def get_coroutine_return_type(self, return_type: Type) -> Type: + return_type = get_proper_type(return_type) + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + assert isinstance(return_type, Instance), "Should only be called on coroutine functions." + # Note: return type is the 3rd type parameter of Coroutine. + return return_type.args[2] + + def get_generator_return_type(self, return_type: Type, is_coroutine: bool) -> Type: + """Given the declared return type of a generator (t), return the type it returns (tr).""" + return_type = get_proper_type(return_type) + + if isinstance(return_type, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=return_type) + elif isinstance(return_type, UnionType): + return make_simplified_union( + [self.get_generator_return_type(item, is_coroutine) for item in return_type.items] + ) + elif not self.is_generator_return_type(return_type, is_coroutine): + # If the function doesn't have a proper Generator (or + # Awaitable) return type, anything is permissible. + return AnyType(TypeOfAny.from_error) + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType(TypeOfAny.from_error) + elif return_type.type.fullname == "typing.Awaitable" and len(return_type.args) == 1: + # Awaitable: tr is args[0]. + return return_type.args[0] + elif ( + return_type.type.fullname in ("typing.Generator", "typing.AwaitableGenerator") + and len(return_type.args) >= 3 + ): + # AwaitableGenerator, Generator: tr is args[2]. + return return_type.args[2] + else: + # We have a supertype of Generator (Iterator, Iterable, object) + # Treat `Iterator[X]` as a shorthand for `Generator[X, Any, None]`. + return NoneType() + + def visit_func_def(self, defn: FuncDef) -> None: + # Type check initialization expressions as part of top-level. + if not self.can_skip_diagnostics: + self.check_default_params(defn) + if defn.original_def: + # Override previous definition (may affect externally visible types). + new_type = self.function_type(defn) + self.check_func_def_override(defn, new_type) + if not self.recurse_into_functions and not defn.def_or_infer_vars: + return + self.visit_func_def_impl(defn) + + def visit_func_def_impl(self, defn: FuncDef) -> None: + with self.tscope.function_scope(defn), self.set_recurse_into_functions(): + self.check_func_item(defn, name=defn.name) + if not self.can_skip_diagnostics: + if defn.info: + if not defn.is_overload and not defn.is_decorated: + # If the definition is the implementation for an + # overload, the legality of the override has already + # been typechecked, and decorated methods will be + # checked when the decorator is. + found_method_base_classes = self.check_method_override(defn) + self.check_explicit_override_decorator(defn, found_method_base_classes) + # TODO: we should use decorated signature for this check. + self.check_inplace_operator_method(defn) + + def check_func_item( + self, + defn: FuncItem, + type_override: CallableType | None = None, + name: str | None = None, + allow_empty: bool = False, + ) -> None: + """Type check a function. + + If type_override is provided, use it as the function type. + """ + self.dynamic_funcs.append(defn.is_dynamic() and not type_override) + + enclosing_node_deferred = self.current_node_deferred + with self.enter_partial_types(is_function=True): + typ = self.function_type(defn) + if type_override: + typ = type_override.copy_modified(line=typ.line, column=typ.column) + if isinstance(typ, CallableType): + with self.enter_attribute_inference_context(): + self.check_func_def(defn, typ, name, allow_empty) + else: + raise RuntimeError("Not supported") + + self.dynamic_funcs.pop() + self.current_node_deferred = enclosing_node_deferred + + if name == "__exit__": + self.check__exit__return_type(defn) + # TODO: the following logic should move to the dataclasses plugin + # https://github.com/python/mypy/issues/15515 + if name == "__post_init__": + if dataclasses_plugin.is_processed_dataclass(defn.info): + dataclasses_plugin.check_post_init(self, defn, defn.info) + + def check_func_def_override(self, defn: FuncDef, new_type: FunctionLike) -> None: + assert defn.original_def is not None + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + old_type = self.function_type(defn.original_def) + if not is_same_type(new_type, old_type): + self.msg.incompatible_conditional_function_def(defn, old_type, new_type) + else: + # Function definition overrides a variable initialized via assignment or a + # decorated function. + orig_type = defn.original_def.type + if orig_type is None: + # If other branch is unreachable, we don't type check it, and so we might + # not have a type for the original definition + return + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah, this is a partial type. Give it the type of the function. + orig_def = defn.original_def + if isinstance(orig_def, Decorator): + var = orig_def.var + else: + var = orig_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + defn.is_invalid_redefinition = True + self.fail(message_registry.INCOMPATIBLE_REDEFINITION, defn) + elif not defn.is_invalid_redefinition: + name_expr = NameExpr(defn.name) + name_expr.node = defn.original_def + self.binder.assign_type(name_expr, new_type, orig_type) + self.check_subtype( + new_type, + orig_type, + defn, + message_registry.INCOMPATIBLE_REDEFINITION, + "redefinition with type", + "original type", + ) + + @contextmanager + def enter_attribute_inference_context(self) -> Iterator[None]: + old_types = self.inferred_attribute_types + self.inferred_attribute_types = {} + yield None + self.inferred_attribute_types = old_types + + def check_func_def( + self, defn: FuncItem, typ: CallableType, name: str | None, allow_empty: bool = False + ) -> None: + """Type check a function definition.""" + # Expand type variables with value restrictions to ordinary types. + self.check_typevar_defaults(typ.variables) + expanded = self.expand_typevars(defn, typ) + original_typ = typ + for item, typ in expanded: + old_binder = self.binder + self.binder = ConditionalTypeBinder(self.options) + with self.binder.top_frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if not self.can_skip_diagnostics: + self.check_funcdef_item(item, typ, name, defn=defn, original_typ=original_typ) + + # Push return type. + self.return_types.append(typ.ret_type) + + with self.scope.push_function(defn): + # We temporarily push the definition to get the self type as + # visible from *inside* of this function/method. + ref_type: Type | None = self.scope.active_self_type() + + if typ.type_is: + arg_index = 0 + # For methods and classmethods, we want the second parameter + if ref_type is not None and defn.has_self_or_cls_argument: + arg_index = 1 + if arg_index < len(typ.arg_types) and not is_subtype( + typ.type_is, typ.arg_types[arg_index] + ): + self.fail( + message_registry.NARROWED_TYPE_NOT_SUBTYPE.format( + format_type(typ.type_is, self.options), + format_type(typ.arg_types[arg_index], self.options), + ), + item, + ) + + # Store argument types. + found_self = False + if isinstance(defn, FuncDef) and not defn.is_decorated: + found_self = self.require_correct_self_argument(typ, defn) + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + if isinstance(arg_type, TypeVarType): + # Refuse covariant parameter type variables + # TODO: check recursively for inner type variables + if ( + arg_type.variance == COVARIANT + and defn.name + not in {"__init__", "__new__", "__post_init__", "__replace__"} + and not is_private(defn.name) # private methods are not inherited + and (i != 0 or not found_self) + ): + ctx: Context = arg_type + if ctx.line < 0: + ctx = typ + self.fail(message_registry.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, ctx) + # Need to store arguments again for the expanded item. + store_argument_type(item, i, typ, self.named_generic_type) + + # Type check body in a new scope. + with self.binder.top_frame_context(): + # Copy some type narrowings from an outer function when it seems safe enough + # (i.e. we can't find an assignment that might change the type of the + # variable afterwards). + new_frame: Frame | None = None + for frame in old_binder.frames: + for key, narrowed_type in frame.types.items(): + key_var = extract_var_from_literal_hash(key) + if key_var is not None and not self.is_var_redefined_in_outer_context( + key_var, defn.line + ): + # It seems safe to propagate the type narrowing to a nested scope. + if new_frame is None: + new_frame = self.binder.push_frame() + new_frame.types[key] = narrowed_type + self.binder.declarations[key] = old_binder.declarations[key] + + if self.options.allow_redefinition and not self.is_stub: + # Add formal argument types to the binder. + for arg in defn.arguments: + # TODO: Add these directly using a fast path (possibly "put") + v = arg.variable + if v.type is not None: + n = NameExpr(v.name) + n.node = v + self.binder.assign_type(n, v.type, v.type) + + with self.scope.push_function(defn): + # We suppress reachability warnings for empty generator functions + # (return; yield) which have a "yield" that's unreachable by definition + # since it's only there to promote the function into a generator function. + # + # We also suppress reachability warnings when we use TypeVars with value + # restrictions: we only want to report a warning if a certain statement is + # marked as being suppressed in *all* of the expansions, but we currently + # have no good way of doing this. + # + # TODO: Find a way of working around this limitation + if _is_empty_generator_function(item) or len(expanded) >= 2: + self.binder.suppress_unreachable_warnings() + # When checking a third-party library, we can skip function body, + # if during semantic analysis we found that there are no attributes + # defined via self here. + if ( + not self.can_skip_diagnostics + or self.options.preserve_asts + or not isinstance(defn, FuncDef) + or defn.def_or_infer_vars + ): + self.accept(item.body) + unreachable = self.binder.is_unreachable() + if new_frame is not None: + self.binder.pop_frame(True, 0) + + if not unreachable: + body_is_trivial = is_trivial_body(defn.body) + if defn.is_generator or is_named_instance( + self.return_types[-1], "typing.AwaitableGenerator" + ): + return_type = self.get_generator_return_type( + self.return_types[-1], defn.is_coroutine + ) + elif defn.is_coroutine: + return_type = self.get_coroutine_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + return_type = get_proper_type(return_type) + + allow_empty = allow_empty or self.options.allow_empty_bodies + + show_error = ( + not body_is_trivial + or + # Allow empty bodies for abstract methods, overloads, in tests and stubs. + ( + not allow_empty + and not ( + isinstance(defn, FuncDef) and defn.abstract_status != NOT_ABSTRACT + ) + and not self.is_stub + ) + ) + + if ( + self.can_skip_diagnostics + # Ignore plugin generated methods, these usually don't need any bodies. + or ( + defn.info is not FUNC_NO_INFO + and ( + defn.name not in defn.info.names + or defn.info.names[defn.name].plugin_generated + ) + ) + # Ignore also definitions that appear in `if TYPE_CHECKING: ...` blocks. + # These can't be called at runtime anyway (similar to plugin-generated). + or (isinstance(defn, FuncDef) and defn.is_mypy_only) + # We want to minimize the fallout from checking empty bodies + # that was absent in many mypy versions. + or (body_is_trivial and is_subtype(NoneType(), return_type)) + ): + show_error = False + + if show_error and not self.current_node_deferred: + may_be_abstract = ( + body_is_trivial + and defn.info is not FUNC_NO_INFO + and defn.info.metaclass_type is not None + and defn.info.metaclass_type.type.has_base("abc.ABCMeta") + ) + if self.options.warn_no_return: + if not isinstance(return_type, (NoneType, AnyType)): + # Control flow fell off the end of a function that was + # declared to return a non-None type. + if isinstance(return_type, UninhabitedType): + # This is a NoReturn function + msg = message_registry.INVALID_IMPLICIT_RETURN + else: + msg = message_registry.MISSING_RETURN_STATEMENT + if body_is_trivial: + msg = ErrorMessage(msg.value, code=codes.EMPTY_BODY) + self.fail(msg, defn) + if may_be_abstract: + self.note(message_registry.EMPTY_BODY_ABSTRACT, defn) + else: + msg = message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE + if body_is_trivial: + msg = ErrorMessage(msg.value, code=codes.EMPTY_BODY) + # similar to code in check_return_stmt + if ( + not self.check_subtype( + subtype_label="implicitly returns", + subtype=NoneType(), + supertype_label="expected", + supertype=return_type, + context=defn, + msg=msg, + ) + and may_be_abstract + ): + self.note(message_registry.EMPTY_BODY_ABSTRACT, defn) + + self.return_types.pop() + + # Propagate any global variable widenings directly to the + # module-level binder (skipping any intermediate class binders). + if self._globals_widened_in_func: + for lvalue, widened_type in self._globals_widened_in_func: + self.globals_binder.put(lvalue, widened_type) + lit = literal_hash(lvalue) + if lit is not None: + self.globals_binder.declarations[lit] = widened_type + self._globals_widened_in_func = [] + + self.binder = old_binder + + def check_funcdef_item( + self, + item: FuncItem, + typ: CallableType, + name: str | None, + *, + defn: FuncItem, + original_typ: CallableType, + ) -> None: + if isinstance(item, FuncDef): + fdef = item + # Check if __init__ has an invalid return type. + if ( + fdef.info + and fdef.name in ("__init__", "__init_subclass__") + and not isinstance(get_proper_type(typ.ret_type), (NoneType, UninhabitedType)) + and not self.dynamic_funcs[-1] + ): + self.fail(message_registry.MUST_HAVE_NONE_RETURN_TYPE.format(fdef.name), item) + + # Check validity of __new__ signature + if fdef.info and fdef.name == "__new__": + self.check___new___signature(fdef, typ) + + self.check_for_missing_annotations(fdef) + if self.options.disallow_any_unimported: + if fdef.type and isinstance(fdef.type, CallableType): + ret_type = fdef.type.ret_type + if has_any_from_unimported_type(ret_type): + self.msg.unimported_type_becomes_any("Return type", ret_type, fdef) + for idx, arg_type in enumerate(fdef.type.arg_types): + if has_any_from_unimported_type(arg_type): + prefix = f'Argument {idx + 1} to "{fdef.name}"' + self.msg.unimported_type_becomes_any(prefix, arg_type, fdef) + check_for_explicit_any( + fdef.type, self.options, self.is_typeshed_stub, self.msg, context=fdef + ) + + if name: # Special method names + if ( + defn.info + and self.is_reverse_op_method(name) + and defn not in self.overload_impl_stack + ): + # TODO: we should use decorated signature for this check. + self.check_reverse_op_method(item, typ, name, defn) + elif name in ("__getattr__", "__getattribute__"): + self.check_getattr_method(typ, defn, name) + elif name == "__setattr__": + self.check_setattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail(message_registry.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, typ.ret_type) + self.check_unbound_return_typevar(typ) + elif isinstance(original_typ.ret_type, TypeVarType) and original_typ.ret_type.values: + # Since type vars with values are expanded, the return type is changed + # to a raw value. This is a hack to get it back. + self.check_unbound_return_typevar(original_typ) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if defn.is_async_generator: + if not self.is_async_generator_return_type(typ.ret_type): + self.fail(message_registry.INVALID_RETURN_TYPE_FOR_ASYNC_GENERATOR, typ) + else: + if not self.is_generator_return_type(typ.ret_type, defn.is_coroutine): + self.fail(message_registry.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + def require_correct_self_argument(self, func: Type, defn: FuncDef) -> bool: + func = get_proper_type(func) + if not isinstance(func, CallableType): + return False + + # Do not report errors for untyped methods in classes nested in untyped funcs. + if not ( + self.options.check_untyped_defs + or len(self.dynamic_funcs) < 2 + or not self.dynamic_funcs[-2] + or not defn.is_dynamic() + ): + return bool(func.arg_types) + + with self.scope.push_function(defn): + # We temporarily push the definition to get the self type as + # visible from *inside* of this function/method. + ref_type: Type | None = self.scope.active_self_type() + if ref_type is None: + return False + + if not defn.has_self_or_cls_argument or ( + func.arg_kinds and func.arg_kinds[0] in [nodes.ARG_STAR, nodes.ARG_STAR2] + ): + return False + + if not func.arg_types: + self.fail( + 'Method must have at least one argument. Did you forget the "self" argument?', defn + ) + return False + + if self.can_skip_diagnostics: + return True + + arg_type = func.arg_types[0] + if defn.is_class or defn.name == "__new__": + ref_type = mypy.types.TypeType.make_normalized(ref_type) + if is_same_type(arg_type, ref_type): + return True + + # This level of erasure matches the one in checkmember.check_self_arg(), + # better keep these two checks consistent. + erased = get_proper_type(erase_typevars(erase_to_bound(arg_type))) + if not is_subtype(ref_type, erased, ignore_type_params=True): + if ( + isinstance(erased, Instance) + and erased.type.is_protocol + or isinstance(erased, TypeType) + and isinstance(erased.item, Instance) + and erased.item.type.is_protocol + ): + # We allow the explicit self-type to be not a supertype of + # the current class if it is a protocol. For such cases + # the consistency check will be performed at call sites. + msg = None + elif func.arg_names[0] in {"self", "cls"}: + msg = message_registry.ERASED_SELF_TYPE_NOT_SUPERTYPE.format( + erased.str_with_options(self.options), ref_type.str_with_options(self.options) + ) + else: + msg = message_registry.MISSING_OR_INVALID_SELF_TYPE + if msg: + self.fail(msg, defn) + return True + + def is_var_redefined_in_outer_context(self, v: Var, after_line: int) -> bool: + """Can the variable be assigned to at module top level or outer function? + + Note that this doesn't do a full CFG analysis but uses a line number based + heuristic that isn't correct in some (rare) cases. + """ + outers = self.tscope.outer_functions() + if not outers: + # Top-level function -- outer context is top level, and we can't reason about + # globals + return True + for outer in outers: + if isinstance(outer, FuncDef): + if find_last_var_assignment_line(outer.body, v) >= after_line: + return True + return False + + def check_unbound_return_typevar(self, typ: CallableType) -> None: + """Fails when the return typevar is not defined in arguments.""" + if isinstance(typ.ret_type, TypeVarType) and typ.ret_type in typ.variables: + arg_type_visitor = CollectArgTypeVarTypes() + for argtype in typ.arg_types: + argtype.accept(arg_type_visitor) + + if typ.ret_type not in arg_type_visitor.arg_types: + self.fail(message_registry.UNBOUND_TYPEVAR, typ.ret_type, code=TYPE_VAR) + upper_bound = get_proper_type(typ.ret_type.upper_bound) + if not ( + isinstance(upper_bound, Instance) + and upper_bound.type.fullname == "builtins.object" + ): + self.note( + "Consider using the upper bound " + f"{format_type(typ.ret_type.upper_bound, self.options)} instead", + context=typ.ret_type, + code=TYPE_VAR, + ) + + def check_default_params(self, item: FuncItem, body_is_trivial: bool | None = None) -> None: + if body_is_trivial is None: + body_is_trivial = is_trivial_body(item.body) + # Although initializers are checked as part of the top-level, we do not + # show errors in them if they appear in an unannotated function. + self.dynamic_funcs.append(item.is_dynamic()) + for param in item.arguments: + if param.initializer is None: + continue + if body_is_trivial and isinstance(param.initializer, EllipsisExpr): + continue + name = param.variable.name + msg = "Incompatible default for " + if name.startswith("__tuple_param_"): + msg += f"tuple parameter {name[12:]}" + else: + msg += f'parameter "{name}"' + if ( + not self.options.implicit_optional + and isinstance(param.initializer, NameExpr) + and param.initializer.fullname == "builtins.None" + ): + notes = [ + "PEP 484 prohibits implicit Optional. " + "Accordingly, mypy has changed its default to no_implicit_optional=True", + "Use https://github.com/hauntsaninja/no_implicit_optional to automatically " + "upgrade your codebase", + ] + else: + notes = None + self.check_simple_assignment( + param.variable.type, + param.initializer, + context=param.initializer, + msg=ErrorMessage(msg, code=codes.ASSIGNMENT), + lvalue_name="parameter", + rvalue_name="default", + notes=notes, + ) + self.dynamic_funcs.pop() + + def is_forward_op_method(self, method_name: str) -> bool: + return method_name in operators.reverse_op_methods + + def is_reverse_op_method(self, method_name: str) -> bool: + return method_name in operators.reverse_op_method_set + + def check_for_missing_annotations(self, fdef: FuncItem) -> None: + show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub + if not show_untyped: + return + + has_explicit_annotation = isinstance(fdef.type, CallableType) and any( + not is_unannotated_any(t) for t in fdef.type.arg_types + [fdef.type.ret_type] + ) + + check_incomplete_defs = self.options.disallow_incomplete_defs and has_explicit_annotation + if self.options.disallow_untyped_defs or check_incomplete_defs: + if fdef.type is None and self.options.disallow_untyped_defs: + if not fdef.arguments or ( + len(fdef.arguments) == 1 + and (fdef.arg_names[0] == "self" or fdef.arg_names[0] == "cls") + ): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + if not has_return_statement(fdef) and not fdef.is_generator: + self.note( + 'Use "-> None" if function does not return a value', + fdef, + code=codes.NO_UNTYPED_DEF, + ) + else: + self.fail(message_registry.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + ret_type = get_proper_type(fdef.type.ret_type) + if is_unannotated_any(ret_type): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + elif fdef.is_generator: + if is_unannotated_any( + self.get_generator_return_type(ret_type, fdef.is_coroutine) + ): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + elif fdef.is_coroutine and isinstance(ret_type, Instance): + if is_unannotated_any(self.get_coroutine_return_type(ret_type)): + self.fail(message_registry.RETURN_TYPE_EXPECTED, fdef) + if any(is_unannotated_any(t) for t in fdef.type.arg_types): + self.fail(message_registry.PARAM_TYPE_EXPECTED, fdef) + + def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None: + self_type = fill_typevars_with_any(fdef.info) + bound_type = bind_self(typ, self_type, is_classmethod=True) + # Check that __new__ (after binding cls) returns an instance + # type (or any). + if fdef.info.is_metaclass(): + # This is a metaclass, so it must return a new unrelated type. + self.check_subtype( + bound_type.ret_type, + self.type_type(), + fdef, + message_registry.INVALID_NEW_TYPE, + "returns", + "but must return a subtype of", + ) + elif not isinstance( + get_proper_type(bound_type.ret_type), + (AnyType, Instance, TupleType, UninhabitedType, LiteralType), + ): + self.fail( + message_registry.NON_INSTANCE_NEW_TYPE.format( + format_type(bound_type.ret_type, self.options) + ), + fdef, + ) + else: + # And that it returns a subtype of the class + self.check_subtype( + bound_type.ret_type, + self_type, + fdef, + message_registry.INVALID_NEW_TYPE, + "returns", + "but must return a subtype of", + ) + + def check_reverse_op_method( + self, defn: FuncItem, reverse_type: CallableType, reverse_name: str, context: Context + ) -> None: + """Check a reverse operator method such as __radd__.""" + # Decides whether it's worth calling check_overlapping_op_methods(). + + # This used to check for some very obscure scenario. It now + # just decides whether it's worth calling + # check_overlapping_op_methods(). + + assert defn.info + + # First check for a valid signature + method_type = CallableType( + [AnyType(TypeOfAny.special_form), AnyType(TypeOfAny.special_form)], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + AnyType(TypeOfAny.special_form), + self.named_type("builtins.function"), + ) + if not is_subtype(reverse_type, method_type): + self.msg.invalid_signature(reverse_type, context) + return + + if reverse_name in ("__eq__", "__ne__"): + # These are defined for all objects => can't cause trouble. + return + + # With 'Any' or 'object' return type we are happy, since any possible + # return value is valid. + ret_type = get_proper_type(reverse_type.ret_type) + if isinstance(ret_type, AnyType): + return + if isinstance(ret_type, Instance): + if ret_type.type.fullname == "builtins.object": + return + if reverse_type.arg_kinds[0] == ARG_STAR: + reverse_type = reverse_type.copy_modified( + arg_types=[reverse_type.arg_types[0]] * 2, + arg_kinds=[ARG_POS] * 2, + arg_names=[reverse_type.arg_names[0], "_"], + ) + assert len(reverse_type.arg_types) >= 2 + + forward_name = operators.normal_from_reverse_op[reverse_name] + forward_inst = get_proper_type(reverse_type.arg_types[1]) + if isinstance(forward_inst, TypeVarType): + forward_inst = get_proper_type(forward_inst.upper_bound) + elif isinstance(forward_inst, TupleType): + forward_inst = tuple_fallback(forward_inst) + elif isinstance(forward_inst, (FunctionLike, TypedDictType, LiteralType)): + forward_inst = forward_inst.fallback + if isinstance(forward_inst, TypeType): + item = forward_inst.item + if isinstance(item, Instance): + opt_meta = item.type.metaclass_type + if opt_meta is not None: + forward_inst = opt_meta + + def has_readable_member(typ: UnionType | Instance, name: str) -> bool: + # TODO: Deal with attributes of TupleType etc. + if isinstance(typ, Instance): + return typ.type.has_readable_member(name) + return all( + (isinstance(x, UnionType) and has_readable_member(x, name)) + or (isinstance(x, Instance) and x.type.has_readable_member(name)) + for x in get_proper_types(typ.relevant_items()) + ) + + if not ( + isinstance(forward_inst, (Instance, UnionType)) + and has_readable_member(forward_inst, forward_name) + ): + return + forward_base = reverse_type.arg_types[1] + forward_type = self.expr_checker.analyze_external_member_access( + forward_name, forward_base, context=defn + ) + self.check_overlapping_op_methods( + reverse_type, + reverse_name, + defn.info, + forward_type, + forward_name, + forward_base, + context=defn, + ) + + def check_overlapping_op_methods( + self, + reverse_type: CallableType, + reverse_name: str, + reverse_class: TypeInfo, + forward_type: Type, + forward_name: str, + forward_base: Type, + context: Context, + ) -> None: + """Check for overlapping method and reverse method signatures. + + This function assumes that: + + - The reverse method has valid argument count and kinds. + - If the reverse operator method accepts some argument of type + X, the forward operator method also belong to class X. + + For example, if we have the reverse operator `A.__radd__(B)`, then the + corresponding forward operator must have the type `B.__add__(...)`. + """ + + # Note: Suppose we have two operator methods "A.__rOP__(B) -> R1" and + # "B.__OP__(C) -> R2". We check if these two methods are unsafely overlapping + # by using the following algorithm: + # + # 1. Rewrite "B.__OP__(C) -> R1" to "temp1(B, C) -> R1" + # + # 2. Rewrite "A.__rOP__(B) -> R2" to "temp2(B, A) -> R2" + # + # 3. Treat temp1 and temp2 as if they were both variants in the same + # overloaded function. (This mirrors how the Python runtime calls + # operator methods: we first try __OP__, then __rOP__.) + # + # If the first signature is unsafely overlapping with the second, + # report an error. + # + # 4. However, if temp1 shadows temp2 (e.g. the __rOP__ method can never + # be called), do NOT report an error. + # + # This behavior deviates from how we handle overloads -- many of the + # modules in typeshed seem to define __OP__ methods that shadow the + # corresponding __rOP__ method. + # + # Note: we do not attempt to handle unsafe overlaps related to multiple + # inheritance. (This is consistent with how we handle overloads: we also + # do not try checking unsafe overlaps due to multiple inheritance there.) + + for forward_item in flatten_nested_unions([forward_type]): + forward_item = get_proper_type(forward_item) + if isinstance(forward_item, CallableType): + if self.is_unsafe_overlapping_op(forward_item, forward_base, reverse_type): + self.msg.operator_method_signatures_overlap( + reverse_class, reverse_name, forward_base, forward_name, context + ) + elif isinstance(forward_item, Overloaded): + for item in forward_item.items: + if self.is_unsafe_overlapping_op(item, forward_base, reverse_type): + self.msg.operator_method_signatures_overlap( + reverse_class, reverse_name, forward_base, forward_name, context + ) + elif not isinstance(forward_item, AnyType): + self.msg.forward_operator_not_callable(forward_name, context) + + def is_unsafe_overlapping_op( + self, forward_item: CallableType, forward_base: Type, reverse_type: CallableType + ) -> bool: + # TODO: check argument kinds? + if len(forward_item.arg_types) < 1: + # Not a valid operator method -- can't succeed anyway. + return False + + # Erase the type if necessary to make sure we don't have a single + # TypeVar in forward_tweaked. (Having a function signature containing + # just a single TypeVar can lead to unpredictable behavior.) + forward_base_erased = forward_base + if isinstance(forward_base, TypeVarType): + forward_base_erased = erase_to_bound(forward_base) + + # Construct normalized function signatures corresponding to the + # operator methods. The first argument is the left operand and the + # second operand is the right argument -- we switch the order of + # the arguments of the reverse method. + + # TODO: this manipulation is dangerous if callables are generic. + # Shuffling arguments between callables can create meaningless types. + forward_tweaked = forward_item.copy_modified( + arg_types=[forward_base_erased, forward_item.arg_types[0]], + arg_kinds=[nodes.ARG_POS] * 2, + arg_names=[None] * 2, + ) + reverse_tweaked = reverse_type.copy_modified( + arg_types=[reverse_type.arg_types[1], reverse_type.arg_types[0]], + arg_kinds=[nodes.ARG_POS] * 2, + arg_names=[None] * 2, + ) + + reverse_base_erased = reverse_type.arg_types[0] + if isinstance(reverse_base_erased, TypeVarType): + reverse_base_erased = erase_to_bound(reverse_base_erased) + + if is_same_type(reverse_base_erased, forward_base_erased): + return False + elif is_subtype(reverse_base_erased, forward_base_erased): + first = reverse_tweaked + second = forward_tweaked + else: + first = forward_tweaked + second = reverse_tweaked + + current_class = self.scope.active_class() + type_vars = current_class.defn.type_vars if current_class else [] + return is_unsafe_overlapping_overload_signatures( + first, second, type_vars, partial_only=False + ) + + def check_inplace_operator_method(self, defn: FuncBase) -> None: + """Check an inplace operator method such as __iadd__. + + They cannot arbitrarily overlap with __add__. + """ + method = defn.name + if method not in operators.inplace_operator_methods: + return + typ = bind_self(self.function_type(defn)) + cls = defn.info + other_method = "__" + method[3:] + if cls.has_readable_member(other_method): + instance = fill_typevars(cls) + typ2 = get_proper_type( + self.expr_checker.analyze_external_member_access(other_method, instance, defn) + ) + fail = False + if isinstance(typ2, FunctionLike): + if not is_more_general_arg_prefix(typ, typ2): + fail = True + else: + # TODO overloads + fail = True + if fail: + self.msg.signatures_incompatible(method, other_method, defn) + + def check_getattr_method(self, typ: Type, context: Context, name: str) -> None: + if len(self.scope.stack) == 1: + # module scope + if name == "__getattribute__": + self.fail(message_registry.MODULE_LEVEL_GETATTRIBUTE, context) + return + # __getattr__ is fine at the module level as of Python 3.7 (PEP 562). We could + # show an error for Python < 3.7, but that would be annoying in code that supports + # both 3.7 and older versions. + method_type = CallableType( + [self.named_type("builtins.str")], + [nodes.ARG_POS], + [None], + AnyType(TypeOfAny.special_form), + self.named_type("builtins.function"), + ) + elif self.scope.active_class(): + method_type = CallableType( + [AnyType(TypeOfAny.special_form), self.named_type("builtins.str")], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + AnyType(TypeOfAny.special_form), + self.named_type("builtins.function"), + ) + else: + return + if not is_subtype(typ, method_type): + self.msg.invalid_signature_for_special_method(typ, context, name) + + def check_setattr_method(self, typ: Type, context: Context) -> None: + if not self.scope.active_class(): + return + method_type = CallableType( + [ + AnyType(TypeOfAny.special_form), + self.named_type("builtins.str"), + AnyType(TypeOfAny.special_form), + ], + [nodes.ARG_POS, nodes.ARG_POS, nodes.ARG_POS], + [None, None, None], + NoneType(), + self.named_type("builtins.function"), + ) + if not is_subtype(typ, method_type): + self.msg.invalid_signature_for_special_method(typ, context, "__setattr__") + + def check_slots_definition(self, typ: Type, context: Context) -> None: + """Check the type of __slots__.""" + str_type = self.named_type("builtins.str") + expected_type = UnionType( + [str_type, self.named_generic_type("typing.Iterable", [str_type])] + ) + self.check_subtype( + typ, + expected_type, + context, + message_registry.INVALID_TYPE_FOR_SLOTS, + "actual type", + "expected type", + code=codes.ASSIGNMENT, + ) + + def check_match_args(self, var: Var, typ: Type, context: Context) -> None: + """Check that __match_args__ contains literal strings""" + if not self.scope.active_class(): + return + typ = get_proper_type(typ) + if not isinstance(typ, TupleType) or not all( + is_string_literal(item) for item in typ.items + ): + self.msg.note( + "__match_args__ must be a tuple containing string literals for checking " + "of match statements to work", + context, + code=codes.LITERAL_REQ, + ) + + def expand_typevars( + self, defn: FuncItem, typ: CallableType + ) -> list[tuple[FuncItem, CallableType]]: + # TODO use generator + subst: list[list[tuple[TypeVarId, Type]]] = [] + tvars = list(typ.variables) or [] + if defn.info: + # Class type variables + tvars += defn.info.defn.type_vars or [] + for tvar in tvars: + if isinstance(tvar, TypeVarType) and tvar.values: + subst.append([(tvar.id, value) for value in tvar.values]) + # Make a copy of the function to check for each combination of + # value restricted type variables. (Except when running mypyc, + # where we need one canonical version of the function.) + if subst and not (self.options.mypyc or self.options.inspections): + result: list[tuple[FuncItem, CallableType]] = [] + for substitutions in itertools.product(*subst): + mapping = dict(substitutions) + result.append((expand_func(defn, mapping), expand_type(typ, mapping))) + return result + else: + return [(defn, typ)] + + def check_explicit_override_decorator( + self, + defn: FuncDef | OverloadedFuncDef, + found_method_base_classes: list[TypeInfo] | None, + context: Context | None = None, + ) -> None: + plugin_generated = False + if defn.info and (node := defn.info.get(defn.name)) and node.plugin_generated: + # Do not report issues for plugin generated nodes, + # they can't realistically use `@override` for their methods. + plugin_generated = True + + if ( + not plugin_generated + and found_method_base_classes + and not defn.is_explicit_override + and defn.name not in ("__init__", "__new__") + and not is_private(defn.name) + ): + self.msg.explicit_override_decorator_missing( + defn.name, found_method_base_classes[0].fullname, context or defn + ) + + def check_method_override( + self, defn: FuncDef | OverloadedFuncDef | Decorator + ) -> list[TypeInfo] | None: + """Check if function definition is compatible with base classes. + + This may defer the method if a signature is not available in at least one base class. + Return ``None`` if that happens. + + Return a list of base classes which contain an attribute with the method name. + """ + if self.can_skip_diagnostics: + return None + # Check against definitions in base classes. + check_override_compatibility = ( + defn.name not in ("__init__", "__new__", "__init_subclass__", "__post_init__") + and (self.options.check_untyped_defs or not defn.is_dynamic()) + and ( + # don't check override for synthesized __replace__ methods from dataclasses + defn.name != "__replace__" + or defn.info.metadata.get("dataclass_tag") is None + ) + ) + found_method_base_classes: list[TypeInfo] = [] + for base in defn.info.mro[1:]: + result = self.check_method_or_accessor_override_for_base( + defn, base, check_override_compatibility + ) + if result is None: + # Node was deferred, we will have another attempt later. + return None + if result: + found_method_base_classes.append(base) + return found_method_base_classes + + def check_method_or_accessor_override_for_base( + self, + defn: FuncDef | OverloadedFuncDef | Decorator, + base: TypeInfo, + check_override_compatibility: bool, + ) -> bool | None: + """Check if method definition is compatible with a base class. + + Return ``None`` if the node was deferred because one of the corresponding + superclass nodes is not ready. + + Return ``True`` if an attribute with the method name was found in the base class. + """ + found_base_method = False + if base: + name = defn.name + base_attr = base.names.get(name) + if base_attr: + # First, check if we override a final (always an error, even with Any types). + if is_final_node(base_attr.node) and not is_private(name): + self.msg.cant_override_final(name, base.name, defn) + # Second, final can't override anything writeable independently of types. + if defn.is_final: + self.check_if_final_var_override_writable(name, base_attr.node, defn) + found_base_method = True + if check_override_compatibility: + # Check compatibility of the override signature + # (__init__, __new__, __init_subclass__ are special). + if self.check_method_override_for_base_with_name(defn, name, base): + return None + if name in operators.inplace_operator_methods: + # Figure out the name of the corresponding operator method. + method = "__" + name[3:] + # An inplace operator method such as __iadd__ might not be + # always introduced safely if a base class defined __add__. + # TODO can't come up with an example where this is + # necessary; now it's "just in case" + if self.check_method_override_for_base_with_name(defn, method, base): + return None + return found_base_method + + def check_setter_type_override(self, defn: OverloadedFuncDef, base: TypeInfo) -> None: + """Check override of a setter type of a mutable attribute. + + Currently, this should be only called when either base node or the current node + is a custom settable property (i.e. where setter type is different from getter type). + Note that this check is contravariant. + """ + typ, _ = self.node_type_from_base(defn.name, defn.info, defn, setter_type=True) + original_type, _ = self.node_type_from_base(defn.name, base, defn, setter_type=True) + # The caller should handle deferrals. + assert typ is not None and original_type is not None + + if not is_subtype(original_type, typ): + self.msg.incompatible_setter_override(defn.setter, typ, original_type, base) + + def check_method_override_for_base_with_name( + self, defn: FuncDef | OverloadedFuncDef | Decorator, name: str, base: TypeInfo + ) -> bool: + """Check if overriding an attribute `name` of `base` with `defn` is valid. + + Return True if the supertype node was not analysed yet, and `defn` was deferred. + """ + base_attr = base.names.get(name) + if not base_attr: + return False + # The name of the method is defined in the base class. + + # Point errors at the 'def' line (important for backward compatibility + # of type ignores). + if not isinstance(defn, Decorator): + context = defn + else: + context = defn.func + + # Construct the type of the overriding method. + if isinstance(defn, (FuncDef, OverloadedFuncDef)): + override_class_or_static = defn.is_class or defn.is_static + else: + override_class_or_static = defn.func.is_class or defn.func.is_static + typ, _ = self.node_type_from_base(defn.name, defn.info, defn) + if typ is None: + # This may only happen if we're checking `x-redefinition` member + # and `x` itself is for some reason gone. Normally the node should + # be reachable from the containing class by its name. + # The redefinition is never removed, use this as a sanity check to verify + # the reasoning above. + assert f"{defn.name}-redefinition" in defn.info.names + return False + + original_node = base_attr.node + # `original_type` can be partial if (e.g.) it is originally an + # instance variable from an `__init__` block that becomes deferred. + supertype_ready = True + original_type, _ = self.node_type_from_base(name, base, defn) + if original_type is None: + supertype_ready = False + if self.pass_num < self.last_pass: + # If there are passes left, defer this node until next pass, + # otherwise try reconstructing the method type from available information. + # For consistency, defer an enclosing top-level function (if any). + top_level = self.scope.top_level_function() + if isinstance(top_level, FuncDef): + self.defer_node(top_level, self.scope.enclosing_class(top_level)) + else: + # Specify enclosing class explicitly, as we check type override before + # entering e.g. decorators or overloads. + self.defer_node(defn, defn.info) + return True + elif isinstance(original_node, (FuncDef, OverloadedFuncDef)): + original_type = self.function_type(original_node) + elif isinstance(original_node, Decorator): + original_type = self.function_type(original_node.func) + elif isinstance(original_node, Var): + # Super type can define method as an attribute. + # See https://github.com/python/mypy/issues/10134 + + # We also check that sometimes `original_node.type` is None. + # This is the case when we use something like `__hash__ = None`. + if original_node.type is not None: + original_type = get_proper_type(original_node.type) + else: + original_type = NoneType() + else: + # Will always fail to typecheck below, since we know the node is a method + original_type = NoneType() + + always_allow_covariant = False + if is_settable_property(defn) and ( + is_settable_property(original_node) or isinstance(original_node, Var) + ): + if is_custom_settable_property(defn) or (is_custom_settable_property(original_node)): + # Unlike with getter, where we try to construct some fallback type in case of + # deferral during last_pass, we can't make meaningful setter checks if the + # supertype is not known precisely. + if supertype_ready: + always_allow_covariant = True + self.check_setter_type_override(defn, base) + + if isinstance(original_node, (FuncDef, OverloadedFuncDef)): + original_class_or_static = original_node.is_class or original_node.is_static + elif isinstance(original_node, Decorator): + fdef = original_node.func + original_class_or_static = fdef.is_class or fdef.is_static + else: + original_class_or_static = False # a variable can't be class or static + + typ = get_proper_type(typ) + original_type = get_proper_type(original_type) + + if ( + is_property(defn) + and isinstance(original_node, Var) + and not original_node.is_final + and (not original_node.is_property or original_node.is_settable_property) + and isinstance(defn, Decorator) + ): + # We only give an error where no other similar errors will be given. + if not isinstance(original_type, AnyType): + self.msg.fail( + "Cannot override writeable attribute with read-only property", + # Give an error on function line to match old behaviour. + defn.func, + code=codes.OVERRIDE, + ) + + if isinstance(original_type, AnyType) or isinstance(typ, AnyType): + pass + elif isinstance(original_type, FunctionLike) and isinstance(typ, FunctionLike): + # Check that the types are compatible. + ok = self.check_override( + typ, + original_type, + defn.name, + name, + base.name if base.module_name == self.tree.fullname else base.fullname, + original_class_or_static, + override_class_or_static, + context, + ) + # Check if this override is covariant. + if ( + ok + and original_node + and codes.MUTABLE_OVERRIDE in self.options.enabled_error_codes + and self.is_writable_attribute(original_node) + and not always_allow_covariant + and not is_subtype(original_type, typ, ignore_pos_arg_names=True) + ): + base_str, override_str = format_type_distinctly( + original_type, typ, options=self.options + ) + msg = message_registry.COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE.with_additional_msg( + f' (base class "{base.name}" defined the type as {base_str},' + f" override has type {override_str})" + ) + self.fail(msg, context) + elif isinstance(original_type, UnionType) and any( + is_subtype(typ, orig_typ, ignore_pos_arg_names=True) + for orig_typ in original_type.items + ): + # This method is a subtype of at least one union variant. + if ( + original_node + and codes.MUTABLE_OVERRIDE in self.options.enabled_error_codes + and self.is_writable_attribute(original_node) + and not always_allow_covariant + ): + # Covariant override of mutable attribute. + base_str, override_str = format_type_distinctly( + original_type, typ, options=self.options + ) + msg = message_registry.COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE.with_additional_msg( + f' (base class "{base.name}" defined the type as {base_str},' + f" override has type {override_str})" + ) + self.fail(msg, context) + elif is_equivalent(original_type, typ): + # Assume invariance for a non-callable attribute here. Note + # that this doesn't affect read-only properties which can have + # covariant overrides. + pass + elif ( + original_node + and (not self.is_writable_attribute(original_node) or always_allow_covariant) + and is_subtype(typ, original_type) + ): + # If the attribute is read-only, allow covariance + pass + else: + self.msg.signature_incompatible_with_supertype( + defn.name, name, base.name, context, original=original_type, override=typ + ) + return False + + def get_op_other_domain(self, tp: FunctionLike) -> Type | None: + if isinstance(tp, CallableType): + if tp.arg_kinds and tp.arg_kinds[0] == ARG_POS: + # For generic methods, domain comparison is tricky, as a first + # approximation erase all remaining type variables. + return erase_typevars(tp.arg_types[0], {v.id for v in tp.variables}) + return None + elif isinstance(tp, Overloaded): + raw_items = [self.get_op_other_domain(it) for it in tp.items] + items = [it for it in raw_items if it] + if items: + return make_simplified_union(items) + return None + else: + assert False, "Need to check all FunctionLike subtypes here" + + def check_override( + self, + override: FunctionLike, + original: FunctionLike, + name: str, + name_in_super: str, + supertype: str, + original_class_or_static: bool, + override_class_or_static: bool, + node: Context, + ) -> bool: + """Check a method override with given signatures. + + Arguments: + override: The signature of the overriding method. + original: The signature of the original supertype method. + name: The name of the overriding method. + Used primarily for generating error messages. + name_in_super: The name of the overridden in the superclass. + Used for generating error messages only. + supertype: The name of the supertype. + original_class_or_static: Indicates whether the original method (from the superclass) + is either a class method or a static method. + override_class_or_static: Indicates whether the overriding method (from the subclass) + is either a class method or a static method. + node: Context node. + """ + # Use boolean variable to clarify code. + fail = False + op_method_wider_note = False + if not is_subtype(override, original, ignore_pos_arg_names=True): + fail = True + elif isinstance(override, Overloaded) and self.is_forward_op_method(name): + # Operator method overrides cannot extend the domain, as + # this could be unsafe with reverse operator methods. + original_domain = self.get_op_other_domain(original) + override_domain = self.get_op_other_domain(override) + if ( + original_domain + and override_domain + and not is_subtype(override_domain, original_domain) + ): + fail = True + op_method_wider_note = True + if isinstance(override, FunctionLike): + if original_class_or_static and not override_class_or_static: + fail = True + elif isinstance(original, CallableType) and isinstance(override, CallableType): + if original.type_guard is not None and override.type_guard is None: + fail = True + if original.type_is is not None and override.type_is is None: + fail = True + + if is_private(name): + fail = False + + if fail: + emitted_msg = False + + offset_arguments = isinstance(override, CallableType) and override.unpack_kwargs + # Normalize signatures, so we get better diagnostics. + if isinstance(override, (CallableType, Overloaded)): + override = override.with_unpacked_kwargs() + if isinstance(original, (CallableType, Overloaded)): + original = original.with_unpacked_kwargs() + + if ( + isinstance(override, CallableType) + and isinstance(original, CallableType) + and len(override.arg_types) == len(original.arg_types) + and override.min_args == original.min_args + ): + # Give more detailed messages for the common case of both + # signatures having the same number of arguments and no + # overloads. + + # override might have its own generic function type + # variables. If an argument or return type of override + # does not have the correct subtyping relationship + # with the original type even after these variables + # are erased, then it is definitely an incompatibility. + + override_ids = override.type_var_ids() + type_name = None + definition = get_func_def(override) + if isinstance(definition, FuncDef): + type_name = definition.info.name + + def erase_override(t: Type) -> Type: + return erase_typevars(t, ids_to_erase=override_ids) + + for i, (sub_kind, super_kind) in enumerate( + zip(override.arg_kinds, original.arg_kinds) + ): + if sub_kind.is_positional() and super_kind.is_positional(): + override_arg_type = override.arg_types[i] + original_arg_type = original.arg_types[i] + elif sub_kind.is_named() and super_kind.is_named() and not offset_arguments: + arg_name = override.arg_names[i] + if arg_name in original.arg_names: + override_arg_type = override.arg_types[i] + original_i = original.arg_names.index(arg_name) + original_arg_type = original.arg_types[original_i] + else: + continue + else: + continue + if not is_subtype(original_arg_type, erase_override(override_arg_type)): + context: Context = node + if ( + isinstance(node, FuncDef) + and not node.is_property + and ( + not node.is_decorated # fast path + # allow trivial decorators like @classmethod and @override + or not (sym := node.info.get(node.name)) + or not isinstance(sym.node, Decorator) + or not sym.node.decorators + ) + ): + # If there's any decorator, we can no longer map arguments 1:1 reliably. + arg_node = node.arguments[i + override.bound()] + if arg_node.line != -1: + context = arg_node + self.msg.argument_incompatible_with_supertype( + i + 1, + name, + type_name, + name_in_super, + original_arg_type, + supertype, + context, + origin_context=node, + ) + emitted_msg = True + + if not is_subtype(erase_override(override.ret_type), original.ret_type): + self.msg.return_type_incompatible_with_supertype( + name, name_in_super, supertype, original.ret_type, override.ret_type, node + ) + emitted_msg = True + elif isinstance(override, Overloaded) and isinstance(original, Overloaded): + # Give a more detailed message in the case where the user is trying to + # override an overload, and the subclass's overload is plausible, except + # that the order of the variants are wrong. + # + # For example, if the parent defines the overload f(int) -> int and f(str) -> str + # (in that order), and if the child swaps the two and does f(str) -> str and + # f(int) -> int + order = [] + for child_variant in override.items: + for i, parent_variant in enumerate(original.items): + if is_subtype(child_variant, parent_variant): + order.append(i) + break + + if len(order) == len(original.items) and order != sorted(order): + self.msg.overload_signature_incompatible_with_supertype( + name, name_in_super, supertype, node + ) + emitted_msg = True + + if not emitted_msg: + # Fall back to generic incompatibility message. + self.msg.signature_incompatible_with_supertype( + name, name_in_super, supertype, node, original=original, override=override + ) + if op_method_wider_note: + self.note( + "Overloaded operator methods can't have wider argument types in overrides", + node, + code=codes.OVERRIDE, + ) + return not fail + + def check__exit__return_type(self, defn: FuncItem) -> None: + """Generate error if the return type of __exit__ is problematic. + + If __exit__ always returns False but the return type is declared + as bool, mypy thinks that a with statement may "swallow" + exceptions even though this is not the case, resulting in + invalid reachability inference. + """ + if not defn.type or not isinstance(defn.type, CallableType): + return + + ret_type = get_proper_type(defn.type.ret_type) + if not has_bool_item(ret_type): + return + + returns = all_return_statements(defn) + if not returns: + return + + if all( + isinstance(ret.expr, NameExpr) and ret.expr.fullname == "builtins.False" + for ret in returns + ): + self.msg.incorrect__exit__return(defn) + + def visit_class_def(self, defn: ClassDef) -> None: + """Type check a class definition.""" + typ = defn.info + for base in typ.mro[1:]: + if base.is_final: + self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) + if not can_have_shared_disjoint_base(typ.bases): + self.fail(message_registry.INCOMPATIBLE_DISJOINT_BASES.format(typ.name), defn) + with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): + old_binder = self.binder + self.binder = ConditionalTypeBinder(self.options) + with self.binder.top_frame_context(): + with self.scope.push_class(defn.info): + self.accept(defn.defs) + self.binder = old_binder + if not self.can_skip_diagnostics: + if not (defn.info.typeddict_type or defn.info.tuple_type or defn.info.is_enum): + # If it is not a normal class (not a special form) check class keywords. + self.check_init_subclass(defn) + if not defn.has_incompatible_baseclass: + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) + self.check_metaclass_compatibility(typ) + self.check_final_deletable(typ) + if typ.typeddict_type: + self.check_typeddict_inheritance(defn) + + if defn.decorators: + sig: Type = type_object_type(defn.info) + # Decorators are applied in reverse order. + for decorator in reversed(defn.decorators): + if isinstance(decorator, CallExpr) and isinstance( + decorator.analyzed, PromoteExpr + ): + # _promote is a special type checking related construct. + continue + + dec = self.expr_checker.accept(decorator) + temp = self.temp_node(sig, context=decorator) + fullname = None + if isinstance(decorator, RefExpr): + fullname = decorator.fullname or None + + # TODO: Figure out how to have clearer error messages. + # (e.g. "class decorator must be a function that accepts a type." + old_allow_abstract_call = self.allow_abstract_call + self.allow_abstract_call = True + sig, _ = self.expr_checker.check_call( + dec, [temp], [nodes.ARG_POS], defn, callable_name=fullname + ) + self.allow_abstract_call = old_allow_abstract_call + # TODO: Apply the sig to the actual TypeInfo so we can handle decorators + # that completely swap out the type. (e.g. Callable[[Type[A]], Type[B]]) + if typ.defn.type_vars and typ.defn.type_args is None: + for base_inst in typ.bases: + for base_tvar, base_decl_tvar in zip( + base_inst.args, base_inst.type.defn.type_vars + ): + if ( + isinstance(base_tvar, TypeVarType) + and base_tvar.variance != INVARIANT + and isinstance(base_decl_tvar, TypeVarType) + and base_decl_tvar.variance != base_tvar.variance + ): + self.fail( + f'Variance of TypeVar "{base_tvar.name}" incompatible ' + "with variance in parent type", + context=defn, + code=codes.TYPE_VAR, + ) + if typ.defn.type_vars: + self.check_typevar_defaults(typ.defn.type_vars) + + if typ.is_protocol and typ.defn.type_vars: + self.check_protocol_variance(defn) + if not defn.has_incompatible_baseclass and defn.info.is_enum: + self.check_enum(defn) + infer_class_variances(defn.info) + + def check_final_deletable(self, typ: TypeInfo) -> None: + # These checks are only for mypyc. Only perform some checks that are easier + # to implement here than in mypyc. + for attr in typ.deletable_attributes: + node = typ.names.get(attr) + if node and isinstance(node.node, Var) and node.node.is_final: + self.fail(message_registry.CANNOT_MAKE_DELETABLE_FINAL, node.node) + + def check_init_subclass(self, defn: ClassDef) -> None: + """Check that keywords in a class definition are valid arguments for __init_subclass__(). + + In this example: + 1 class Base: + 2 def __init_subclass__(cls, thing: int): + 3 pass + 4 class Child(Base, thing=5): + 5 def __init_subclass__(cls): + 6 pass + 7 Child() + + Base.__init_subclass__(thing=5) is called at line 4. This is what we simulate here. + Child.__init_subclass__ is never called. + """ + if defn.info.metaclass_type and defn.info.metaclass_type.type.fullname not in ( + "builtins.type", + "abc.ABCMeta", + ): + # We can't safely check situations when both __init_subclass__ and a custom + # metaclass are present. + return + # At runtime, only Base.__init_subclass__ will be called, so + # we skip the current class itself. + for base in defn.info.mro[1:]: + if "__init_subclass__" not in base.names: + continue + name_expr = NameExpr(defn.name) + name_expr.node = base + callee = MemberExpr(name_expr, "__init_subclass__") + args = list(defn.keywords.values()) + arg_names: list[str | None] = list(defn.keywords.keys()) + # 'metaclass' keyword is consumed by the rest of the type machinery, + # and is never passed to __init_subclass__ implementations + if "metaclass" in arg_names: + idx = arg_names.index("metaclass") + arg_names.pop(idx) + args.pop(idx) + arg_kinds = [ARG_NAMED] * len(args) + call_expr = CallExpr(callee, args, arg_kinds, arg_names) + call_expr.line = defn.line + call_expr.column = defn.column + call_expr.end_line = defn.end_line + self.expr_checker.accept(call_expr, allow_none_return=True, always_allow_any=True) + # We are only interested in the first Base having __init_subclass__, + # all other bases have already been checked. + break + + def check_typevar_defaults(self, tvars: Sequence[TypeVarLikeType]) -> None: + for tv in tvars: + if not (isinstance(tv, TypeVarType) and tv.has_default()): + continue + if not is_subtype(tv.default, tv.upper_bound): + self.fail("TypeVar default must be a subtype of the bound type", tv) + if tv.values and not any(is_same_type(tv.default, value) for value in tv.values): + self.fail("TypeVar default must be one of the constraint types", tv) + + def check_enum(self, defn: ClassDef) -> None: + assert defn.info.is_enum + if defn.info.fullname not in ENUM_BASES and "__members__" in defn.info.names: + sym = defn.info.names["__members__"] + if isinstance(sym.node, Var) and sym.node.has_explicit_value: + # `__members__` will always be overwritten by `Enum` and is considered + # read-only, so we disallow assigning a value to it + self.fail(message_registry.ENUM_MEMBERS_ATTR_WILL_BE_OVERRIDDEN, sym.node) + for base in defn.info.mro[1:-1]: # we don't need self and `object` + if base.is_enum and base.fullname not in ENUM_BASES: + self.check_final_enum(defn, base) + + if self.is_stub and self.tree.fullname not in {"enum", "_typeshed"}: + if not defn.info.enum_members: + self.fail( + f'Detected enum "{defn.info.fullname}" in a type stub with zero members. ' + "There is a chance this is due to a recent change in the semantics of " + "enum membership. If so, use `member = value` to mark an enum member, " + "instead of `member: type`", + defn, + ) + self.note( + "See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members", + defn, + ) + + self.check_enum_bases(defn) + self.check_enum_new(defn) + + def check_final_enum(self, defn: ClassDef, base: TypeInfo) -> None: + if base.enum_members: + self.fail(f'Cannot extend enum with existing members: "{base.name}"', defn) + + def is_final_enum_value(self, sym: SymbolTableNode) -> bool: + if isinstance(sym.node, (FuncBase, Decorator)): + return False # A method is fine + if not isinstance(sym.node, Var): + return True # Can be a class or anything else + + # Now, only `Var` is left, we need to check: + # 1. Private name like in `__prop = 1` + # 2. Dunder name like `__hash__ = some_hasher` + # 3. Sunder name like `_order_ = 'a, b, c'` + # 4. If it is a method / descriptor like in `method = classmethod(func)` + if ( + is_private(sym.node.name) + or is_dunder(sym.node.name) + or is_sunder(sym.node.name) + # TODO: make sure that `x = @class/staticmethod(func)` + # and `x = property(prop)` both work correctly. + # Now they are incorrectly counted as enum members. + or isinstance(get_proper_type(sym.node.type), FunctionLike) + ): + return False + + return self.is_stub or sym.node.has_explicit_value + + def check_enum_bases(self, defn: ClassDef) -> None: + """ + Non-enum mixins cannot appear after enum bases; this is disallowed at runtime: + + class Foo: ... + class Bar(enum.Enum, Foo): ... + + But any number of enum mixins can appear in a class definition + (even if multiple enum bases define __new__). So this is fine: + + class Foo(enum.Enum): + def __new__(cls, val): ... + class Bar(enum.Enum): + def __new__(cls, val): ... + class Baz(int, Foo, Bar, enum.Flag): ... + """ + enum_base: Instance | None = None + for base in defn.info.bases: + if enum_base is None and base.type.is_enum: + enum_base = base + continue + elif enum_base is not None and not base.type.is_enum: + self.fail( + f'No non-enum mixin classes are allowed after "{enum_base.str_with_options(self.options)}"', + defn, + ) + break + + def check_enum_new(self, defn: ClassDef) -> None: + def has_new_method(info: TypeInfo) -> bool: + new_method = info.get("__new__") + return bool( + new_method + and new_method.node + and new_method.node.fullname != "builtins.object.__new__" + ) + + has_new = False + for base in defn.info.bases: + candidate = False + + if base.type.is_enum: + # If we have an `Enum`, then we need to check all its bases. + candidate = any(not b.is_enum and has_new_method(b) for b in base.type.mro[1:-1]) + else: + candidate = has_new_method(base.type) + + if candidate and has_new: + self.fail( + "Only a single data type mixin is allowed for Enum subtypes, " + 'found extra "{}"'.format(base.str_with_options(self.options)), + defn, + ) + elif candidate: + has_new = True + + def check_protocol_variance(self, defn: ClassDef) -> None: + """Check that protocol definition is compatible with declared + variances of type variables. + + Note that we also prohibit declaring protocol classes as invariant + if they are actually covariant/contravariant, since this may break + transitivity of subtyping, see PEP 544. + """ + if self.can_skip_diagnostics: + return + if defn.type_args is not None: + # Using new-style syntax (PEP 695), so variance will be inferred + return + info = defn.info + object_type = Instance(info.mro[-1], []) + tvars = info.defn.type_vars + if self._variance_dummy_type is None: + _, dummy_info = self.make_fake_typeinfo("", "Dummy", "Dummy", []) + self._variance_dummy_type = Instance(dummy_info, []) + dummy = self._variance_dummy_type + for i, tvar in enumerate(tvars): + if not isinstance(tvar, TypeVarType): + # Variance of TypeVarTuple and ParamSpec is underspecified by PEPs. + continue + up_args: list[Type] = [ + object_type if i == j else dummy.copy_modified() for j, _ in enumerate(tvars) + ] + down_args: list[Type] = [ + UninhabitedType() if i == j else dummy.copy_modified() for j, _ in enumerate(tvars) + ] + up, down = Instance(info, up_args), Instance(info, down_args) + # TODO: add advanced variance checks for recursive protocols + if is_subtype(down, up, ignore_declared_variance=True): + expected = COVARIANT + elif is_subtype(up, down, ignore_declared_variance=True): + expected = CONTRAVARIANT + else: + expected = INVARIANT + if expected != tvar.variance: + self.msg.bad_proto_variance(tvar.variance, tvar.name, expected, defn) + + def check_multiple_inheritance(self, typ: TypeInfo) -> None: + """Check for multiple inheritance related errors.""" + if len(typ.bases) <= 1: + # No multiple inheritance. + return + # Verify that inherited attributes are compatible. + mro = typ.mro[1:] + all_names = {name for base in mro for name in base.names} + for name in sorted(all_names - typ.names.keys()): + # Sort for reproducible message order. + # Attributes defined in both the type and base are skipped. + # Normal checks for attribute compatibility should catch any problems elsewhere. + if is_private(name): + continue + # Compare the first base defining a name with the rest. + # Remaining bases may not be pairwise compatible as the first base provides + # the used definition. + i, base = next((i, base) for i, base in enumerate(mro) if name in base.names) + for base2 in mro[i + 1 :]: + if name in base2.names and base2 not in base.mro: + self.check_compatibility(name, base, base2, typ) + + def check_compatibility( + self, name: str, base1: TypeInfo, base2: TypeInfo, ctx: TypeInfo + ) -> None: + """Check if attribute name in base1 is compatible with base2 in multiple inheritance. + + Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have + a direct subclass relationship (i.e., the compatibility requirement only derives from + multiple inheritance). + + This check verifies that a definition taken from base1 (and mapped to the current + class ctx), is type compatible with the definition taken from base2 (also mapped), so + that unsafe subclassing like this can be detected: + class A(Generic[T]): + def foo(self, x: T) -> None: ... + + class B: + def foo(self, x: str) -> None: ... + + class C(B, A[int]): ... # this is unsafe because... + + x: A[int] = C() + x.foo # ...runtime type is (str) -> None, while static type is (int) -> None + """ + if name in {"__init__", "__new__", "__init_subclass__", "__replace__"}: + # __init__ and friends can be incompatible -- it's a special case. + return + first = base1.names[name] + second = base2.names[name] + # Specify current_class explicitly as this function is called after leaving the class. + first_type, _ = self.node_type_from_base(name, base1, ctx, current_class=ctx) + second_type, _ = self.node_type_from_base(name, base2, ctx, current_class=ctx) + + # TODO: use more principled logic to decide is_subtype() vs is_equivalent(). + # We should rely on mutability of superclass node, not on types being Callable. + # (in particular handle settable properties with setter type different from getter). + + p_first_type = get_proper_type(first_type) + p_second_type = get_proper_type(second_type) + if isinstance(p_first_type, FunctionLike) and isinstance(p_second_type, FunctionLike): + if p_first_type.is_type_obj() and p_second_type.is_type_obj(): + # For class objects only check the subtype relationship of the classes, + # since we allow incompatible overrides of '__init__'/'__new__' + ok = is_subtype( + left=fill_typevars_with_any(p_first_type.type_object()), + right=fill_typevars_with_any(p_second_type.type_object()), + ) + else: + assert first_type and second_type + ok = is_subtype(first_type, second_type, ignore_pos_arg_names=True) + elif first_type and second_type: + if second.node is not None and not self.is_writable_attribute(second.node): + ok = is_subtype(first_type, second_type) + else: + ok = is_equivalent(first_type, second_type) + if ok: + if ( + first.node + and second.node + and self.is_writable_attribute(second.node) + and is_property(first.node) + and isinstance(first.node, Decorator) + and not isinstance(p_second_type, AnyType) + ): + self.msg.fail( + f'Cannot override writeable attribute "{name}" in base "{base2.name}"' + f' with read-only property in base "{base1.name}"', + ctx, + code=codes.OVERRIDE, + ) + else: + if first_type is None: + self.msg.cannot_determine_type_in_base(name, base1.name, ctx) + if second_type is None: + self.msg.cannot_determine_type_in_base(name, base2.name, ctx) + ok = True + # Final attributes can never be overridden, but can override + # non-final read-only attributes. + if is_final_node(second.node) and not is_private(name): + self.msg.cant_override_final(name, base2.name, ctx) + if is_final_node(first.node): + self.check_if_final_var_override_writable(name, second.node, ctx) + # Some attributes like __slots__ and __deletable__ are special, and the type can + # vary across class hierarchy. + if isinstance(second.node, Var) and second.node.allow_incompatible_override: + ok = True + if not ok: + self.msg.base_class_definitions_incompatible(name, base1, base2, ctx) + + def check_metaclass_compatibility(self, typ: TypeInfo) -> None: + """Ensures that metaclasses of all parent types are compatible.""" + if ( + typ.is_metaclass() + or typ.is_protocol + or typ.is_named_tuple + or typ.is_enum + or typ.typeddict_type is not None + ): + return # Reasonable exceptions from this check + + if typ.metaclass_type is None and any( + base.type.metaclass_type is not None for base in typ.bases + ): + self.fail( + "Metaclass conflict: the metaclass of a derived class must be " + "a (non-strict) subclass of the metaclasses of all its bases", + typ, + code=codes.METACLASS, + ) + explanation = typ.explain_metaclass_conflict() + if explanation: + self.note(explanation, typ, code=codes.METACLASS) + + def check_typeddict_inheritance(self, defn: ClassDef) -> None: + """Ensure that the final definition of a TypedDict is compatible with its base classes.""" + assert defn.info.typeddict_type + td = defn.info.typeddict_type + data = defn.info.typeddict_data + if data is None or not data.ready: + return + for base, base_items in data.bases: + assert base.typeddict_type + for field_name, base_type in base_items.items(): + field_type = td.items[field_name] + assert field_type + is_readonly = field_name in base.typeddict_type.readonly_keys + if is_readonly: + is_compatible = is_subtype(field_type, base_type) + else: + is_compatible = is_equivalent(field_type, base_type) + if not is_compatible: + source = data.field_sources[field_name] + if source.base is None: + self.fail( + f'Definition of field "{field_name}" incompatible with base class ' + f'"{base.name}"', + source.ctx, + ) + else: + self.fail( + f'Incompatible definitions of field "{field_name}" in base classes ' + f'"{base.name}" and "{source.base.name}"', + source.ctx, + ) + if field_name in td.readonly_keys: + self.note( + f'This can be resolved by redeclaring the field "{field_name}" ' + f"with a mutually compatible type", + source.ctx, + ) + + def visit_import_from(self, node: ImportFrom) -> None: + for name, _ in node.names: + if (sym := self.globals.get(name)) is not None: + self.warn_deprecated(sym.node, node) + self.check_import(node) + + def visit_import_all(self, node: ImportAll) -> None: + self.check_import(node) + + def visit_import(self, node: Import) -> None: + self.check_import(node) + + def check_import(self, node: ImportBase) -> None: + for assign in node.assignments: + lvalue = assign.lvalues[0] + lvalue_type, _, __ = self.check_lvalue(lvalue) + if lvalue_type is None: + # TODO: This is broken. + lvalue_type = AnyType(TypeOfAny.special_form) + assert isinstance(assign.rvalue, NameExpr) + message = message_registry.INCOMPATIBLE_IMPORT_OF.format(assign.rvalue.name) + self.check_simple_assignment( + lvalue_type, + assign.rvalue, + node, + msg=message, + lvalue_name="local name", + rvalue_name="imported name", + ) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + # This block was marked as being unreachable during semantic analysis. + # It turns out any blocks marked in this way are *intentionally* marked + # as unreachable -- so we don't display an error. + self.binder.unreachable() + return + marked_unreachable = False + reported_unreachable = False + for s in b.body: + if self.binder.is_unreachable(): + finish = False + if self.scope.top_level_function() is None and not marked_unreachable: + self.mark_unreachable(b.body, after=s) + marked_unreachable = True + if not self.should_report_unreachable_issues(): + finish = True + if ( + not finish + and not reported_unreachable + and not self.is_noop_for_reachability(s) + ): + self.msg.unreachable_statement(s) + self.binder.suppress_unreachable_warnings() + finish = True + reported_unreachable = True + + if finish and not self.options.check_unreachable: + break + + if not self.options.check_unreachable: + continue + + self.accept(s) + # Clear expression cache after each statement to avoid unlimited growth. + self.expr_checker.expr_cache.clear() + + def should_report_unreachable_issues(self) -> bool: + return ( + self.in_checked_function() + and self.options.warn_unreachable + and not self.current_node_deferred + and not self.binder.is_unreachable_warning_suppressed() + ) + + def is_noop_for_reachability(self, s: Statement) -> bool: + """Returns 'true' if the given statement either throws an error of some kind + or is a no-op. + + We use this function while handling the '--warn-unreachable' flag. When + that flag is present, we normally report an error on any unreachable statement. + But if that statement is just something like a 'pass' or a just-in-case 'assert False', + reporting an error would be annoying. + """ + if isinstance(s, AssertStmt) and is_false_literal(s.expr): + return True + elif isinstance(s, ReturnStmt) and is_literal_not_implemented(s.expr): + return True + elif isinstance(s, RaiseStmt): + return True + elif isinstance(s, ExpressionStmt): + if isinstance(s.expr, CallExpr): + with self.expr_checker.msg.filter_errors(filter_revealed_type=True): + typ = get_proper_type( + self.expr_checker.accept( + s.expr, allow_none_return=True, always_allow_any=True + ) + ) + + if isinstance(typ, UninhabitedType): + return True + return False + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + """Type check an assignment statement. + + Handle all kinds of assignment statements (simple, indexed, multiple). + """ + + # Avoid type checking type aliases in stubs to avoid false + # positives about modern type syntax available in stubs such + # as X | Y. + if not (s.is_alias_def and self.is_stub): + with self.enter_final_context(s.is_final_def): + self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) + + if s.is_alias_def: + self.check_type_alias_rvalue(s) + + if ( + s.type is not None + and self.options.disallow_any_unimported + and has_any_from_unimported_type(s.type) + ): + if isinstance(s.lvalues[-1], TupleExpr): + # This is a multiple assignment. Instead of figuring out which type is problematic, + # give a generic error message. + self.msg.unimported_type_becomes_any( + "A type on this line", AnyType(TypeOfAny.special_form), s + ) + else: + self.msg.unimported_type_becomes_any("Type of variable", s.type, s) + check_for_explicit_any(s.type, self.options, self.is_typeshed_stub, self.msg, context=s) + + if len(s.lvalues) > 1: + # Chained assignment (e.g. x = y = ...). + # Make sure that rvalue type will not be reinferred. + if not self.has_type(s.rvalue): + self.expr_checker.accept(s.rvalue) + rvalue = self.temp_node(self.lookup_type(s.rvalue), s) + for lv in s.lvalues[:-1]: + with self.enter_final_context(s.is_final_def): + self.check_assignment(lv, rvalue, s.type is None) + + self.check_final(s) + if ( + s.is_final_def + and s.type + and not has_no_typevars(s.type) + and self.scope.active_class() is not None + ): + self.fail(message_registry.DEPENDENT_FINAL_IN_CLASS_BODY, s) + + if s.unanalyzed_type and not self.in_checked_function(): + self.msg.annotation_in_unchecked_function(context=s) + + def check_type_alias_rvalue(self, s: AssignmentStmt) -> None: + with self.msg.filter_errors(): + alias_type = self.expr_checker.accept(s.rvalue) + self.store_type(s.lvalues[-1], alias_type) + + def check_assignment( + self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type: bool = True + ) -> None: + """Type check a single assignment: lvalue = rvalue.""" + if isinstance(lvalue, (TupleExpr, ListExpr)): + self.check_assignment_to_multiple_lvalues( + lvalue.items, rvalue, rvalue, infer_lvalue_type + ) + else: + self.try_infer_partial_generic_type_from_assignment(lvalue, rvalue, "=") + lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue, rvalue) + # If we're assigning to __getattr__ or similar methods, check that the signature is + # valid. + if isinstance(lvalue, NameExpr) and lvalue.node: + name = lvalue.node.name + if name in ("__setattr__", "__getattribute__", "__getattr__"): + # If an explicit type is given, use that. + if lvalue_type: + signature = lvalue_type + else: + signature = self.expr_checker.accept(rvalue) + if signature: + if name == "__setattr__": + self.check_setattr_method(signature, lvalue) + else: + self.check_getattr_method(signature, lvalue, name) + + if name == "__slots__" and self.scope.active_class() is not None: + typ = lvalue_type or self.expr_checker.accept(rvalue) + self.check_slots_definition(typ, lvalue) + if name == "__match_args__" and inferred is not None: + typ = self.expr_checker.accept(rvalue) + self.check_match_args(inferred, typ, lvalue) + if name == "__post_init__": + active_class = self.scope.active_class() + if active_class and dataclasses_plugin.is_processed_dataclass(active_class): + self.fail(message_registry.DATACLASS_POST_INIT_MUST_BE_A_FUNCTION, rvalue) + + if isinstance(lvalue, MemberExpr) and lvalue.name == "__match_args__": + self.fail(message_registry.CANNOT_MODIFY_MATCH_ARGS, lvalue) + + if lvalue_type: + if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: + # Try to infer a proper type for a variable with a partial None type. + rvalue_type = self.expr_checker.accept(rvalue) + if isinstance(get_proper_type(rvalue_type), NoneType): + # This doesn't actually provide any additional information -- multiple + # None initializers preserve the partial None type. + return + + var = lvalue_type.var + if is_valid_inferred_type( + rvalue_type, self.options, is_lvalue_final=var.is_final + ): + partial_types = self.find_partial_types(var) + if partial_types is not None: + if not self.current_node_deferred: + # Partial type can't be final, so strip any literal values. + rvalue_type = remove_instance_last_known_values(rvalue_type) + inferred_type = make_simplified_union([rvalue_type, NoneType()]) + self.set_inferred_type(var, lvalue, inferred_type) + else: + var.type = None + del partial_types[var] + lvalue_type = var.type + else: + # Try to infer a partial type. + if not self.infer_partial_type(var, lvalue, rvalue_type): + # If that also failed, give up and let the caller know that we + # cannot read their mind. The definition site will be reported later. + # Calling .put() directly because the newly inferred type is + # not a subtype of None - we are not looking for narrowing + fallback = self.inference_error_fallback_type(rvalue_type) + self.binder.put(lvalue, fallback) + # Same as self.set_inference_error_fallback_type but inlined + # to avoid computing fallback twice. + # We are replacing partial now, so the variable type + # should remain optional. + self.set_inferred_type(var, lvalue, make_optional_type(fallback)) + elif ( + isinstance(lvalue, MemberExpr) and lvalue.kind is None + ): # Ignore member access to modules + instance_type = self.expr_checker.accept(lvalue.expr) + rvalue_type, lvalue_type, infer_lvalue_type = self.check_member_assignment( + lvalue, instance_type, lvalue_type, rvalue, context=rvalue + ) + else: + # Hacky special case for assigning a literal None + # to a variable defined in a previous if + # branch. When we detect this, we'll go back and + # make the type optional. This is somewhat + # unpleasant, and a generalization of this would + # be an improvement! + if ( + not self.options.allow_redefinition + and is_literal_none(rvalue) + and isinstance(lvalue, NameExpr) + and lvalue.kind == LDEF + and isinstance(lvalue.node, Var) + and lvalue.node.type + and lvalue.node in self.var_decl_frames + and not isinstance(get_proper_type(lvalue_type), AnyType) + ): + decl_frame_map = self.var_decl_frames[lvalue.node] + # Check if the nearest common ancestor frame for the definition site + # and the current site is the enclosing frame of an if/elif/else block. + has_if_ancestor = False + for frame in reversed(self.binder.frames): + if frame.id in decl_frame_map: + has_if_ancestor = frame.conditional_frame + break + if has_if_ancestor: + lvalue_type = make_optional_type(lvalue_type) + self.set_inferred_type(lvalue.node, lvalue, lvalue_type) + + rvalue_type, lvalue_type = self.check_simple_assignment( + lvalue_type, rvalue, context=rvalue, inferred=inferred, lvalue=lvalue + ) + # The above call may update inferred variable type. Prevent further + # inference. + inferred = None + + # Special case: only non-abstract non-protocol classes can be assigned to + # variables with explicit type `Type[A]`, where A is protocol or abstract. + p_rvalue_type = get_proper_type(rvalue_type) + p_lvalue_type = get_proper_type(lvalue_type) + if ( + isinstance(p_rvalue_type, FunctionLike) + and p_rvalue_type.is_type_obj() + and ( + p_rvalue_type.type_object().is_abstract + or p_rvalue_type.type_object().is_protocol + ) + and isinstance(p_lvalue_type, TypeType) + and isinstance(p_lvalue_type.item, Instance) + and ( + p_lvalue_type.item.type.is_abstract or p_lvalue_type.item.type.is_protocol + ) + ): + self.msg.concrete_only_assign(p_lvalue_type, rvalue) + return + if rvalue_type and infer_lvalue_type and not isinstance(lvalue_type, PartialType): + # Don't use type binder for definitions of special forms, like named tuples. + if not (isinstance(lvalue, NameExpr) and lvalue.is_special_form): + self.binder.assign_type(lvalue, rvalue_type, lvalue_type) + if ( + isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and lvalue.node.is_inferred + and lvalue.node.is_index_var + and lvalue_type is not None + ): + lvalue.node.type = remove_instance_last_known_values(lvalue_type) + elif ( + self.options.allow_redefinition + and lvalue_type is not None + and not isinstance(lvalue_type, PartialType) + # Note that `inferred is not None` is not a reliable check here, because + # simple assignments like x = "a" are inferred during semantic analysis. + and isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and lvalue.node.is_inferred + ): + # TODO: Can we use put() here? + self.binder.assign_type(lvalue, lvalue_type, lvalue_type) + + elif index_lvalue: + self.check_indexed_assignment(index_lvalue, rvalue, lvalue) + + if inferred: + type_context = self.get_variable_type_context(inferred, rvalue) + rvalue_type = self.expr_checker.accept(rvalue, type_context=type_context) + if not ( + inferred.is_final + or inferred.is_index_var + or (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") + ): + rvalue_type = remove_instance_last_known_values(rvalue_type) + self.infer_variable_type(inferred, lvalue, rvalue_type, rvalue) + self.check_assignment_to_slots(lvalue) + if ( + isinstance(lvalue, RefExpr) + and not (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") + and not self.can_skip_diagnostics + ): + # We check override here at the end after storing the inferred type, since + # override check will try to access the current attribute via symbol tables + # (like a regular attribute access). + self.check_compatibility_all_supers(lvalue, rvalue) + + # (type, operator) tuples for augmented assignments supported with partial types + partial_type_augmented_ops: Final = {("builtins.list", "+"), ("builtins.set", "|")} + + def get_variable_type_context(self, inferred: Var, rvalue: Expression) -> Type | None: + type_contexts = [] + if inferred.info: + for base in inferred.info.mro[1:]: + if inferred.name not in base.names: + continue + # For inference within class body, get supertype attribute as it would look on + # a class object for lambdas overriding methods, etc. + base_node = base.names[inferred.name].node + base_type, _ = self.node_type_from_base( + inferred.name, + base, + inferred, + is_class=is_method(base_node) + or isinstance(base_node, Var) + and not is_instance_var(base_node), + ) + if ( + base_type + and not (isinstance(base_node, Var) and base_node.invalid_partial_type) + and not isinstance(base_type, PartialType) + ): + type_contexts.append(base_type) + # Use most derived supertype as type context if available. + if not type_contexts: + if inferred.name == "__slots__" and self.scope.active_class() is not None: + str_type = self.named_type("builtins.str") + return self.named_generic_type("typing.Iterable", [str_type]) + if inferred.name == "__all__" and self.scope.is_top_level(): + str_type = self.named_type("builtins.str") + return self.named_generic_type("typing.Sequence", [str_type]) + return None + candidate = type_contexts[0] + for other in type_contexts: + if is_proper_subtype(other, candidate): + candidate = other + elif not is_subtype(candidate, other): + # Multiple incompatible candidates, cannot use any of them as context. + return None + return candidate + + def try_infer_partial_generic_type_from_assignment( + self, lvalue: Lvalue, rvalue: Expression, op: str + ) -> None: + """Try to infer a precise type for partial generic type from assignment. + + 'op' is '=' for normal assignment and a binary operator ('+', ...) for + augmented assignment. + + Example where this happens: + + x = [] + if foo(): + x = [1] # Infer List[int] as type of 'x' + """ + var = None + if ( + isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and isinstance(lvalue.node.type, PartialType) + ): + var = lvalue.node + elif isinstance(lvalue, MemberExpr): + var = self.expr_checker.get_partial_self_var(lvalue) + if var is not None: + typ = var.type + assert isinstance(typ, PartialType) + if typ.type is None: + return + # Return if this is an unsupported augmented assignment. + if op != "=" and (typ.type.fullname, op) not in self.partial_type_augmented_ops: + return + # TODO: some logic here duplicates the None partial type counterpart + # inlined in check_assignment(), see #8043. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + rvalue_type = self.expr_checker.accept(rvalue) + rvalue_type = get_proper_type(rvalue_type) + if isinstance(rvalue_type, Instance): + if rvalue_type.type == typ.type and is_valid_inferred_type( + rvalue_type, self.options + ): + var.type = rvalue_type + del partial_types[var] + elif isinstance(rvalue_type, AnyType): + var.type = fill_typevars_with_any(typ.type) + del partial_types[var] + + def check_compatibility_all_supers(self, lvalue: RefExpr, rvalue: Expression) -> None: + lvalue_node = lvalue.node + # Check if we are a class variable with at least one base class + if ( + isinstance(lvalue_node, Var) + # If we have explicit annotation, there is no point in checking the override + # for each assignment, so we check only for the first one. + # TODO: for some reason annotated attributes on self are stored as inferred vars. + and ( + lvalue_node.line == lvalue.line + or lvalue_node.is_inferred + and not lvalue_node.explicit_self_type + ) + and lvalue.kind in (MDEF, None) # None for Vars defined via self + and len(lvalue_node.info.bases) > 0 + ): + for base in lvalue_node.info.mro[1:]: + tnode = base.names.get(lvalue_node.name) + if tnode is not None: + if not self.check_compatibility_classvar_super(lvalue_node, base, tnode.node): + # Show only one error per variable + break + + if not self.check_compatibility_final_super(lvalue_node, base, tnode.node): + # Show only one error per variable + break + + direct_bases = lvalue_node.info.direct_base_classes() + last_immediate_base = direct_bases[-1] if direct_bases else None + + # The historical behavior for inferred vars was to compare rvalue type against + # the type declared in a superclass. To preserve this behavior, we temporarily + # store the rvalue type on the variable. + actual_lvalue_type = None + if lvalue_node.is_inferred and not lvalue_node.explicit_self_type: + # Don't use partial types as context, similar to regular code path. + ctx = lvalue_node.type if not isinstance(lvalue_node.type, PartialType) else None + rvalue_type = self.expr_checker.accept(rvalue, ctx) + actual_lvalue_type = lvalue_node.type + lvalue_node.type = rvalue_type + lvalue_type, _ = self.node_type_from_base(lvalue_node.name, lvalue_node.info, lvalue) + if lvalue_node.is_inferred and not lvalue_node.explicit_self_type: + lvalue_node.type = actual_lvalue_type + + if not lvalue_type: + return + + for base in lvalue_node.info.mro[1:]: + # The type of "__slots__" and some other attributes usually doesn't need to + # be compatible with a base class. We'll still check the type of "__slots__" + # against "object" as an exception. + if lvalue_node.allow_incompatible_override and not ( + lvalue_node.name == "__slots__" and base.fullname == "builtins.object" + ): + continue + + if is_private(lvalue_node.name): + continue + + base_type, base_node = self.node_type_from_base(lvalue_node.name, base, lvalue) + # TODO: if the r.h.s. is a descriptor, we should check setter override as well. + custom_setter = is_custom_settable_property(base_node) + if isinstance(base_type, PartialType): + base_type = None + + if base_type: + assert base_node is not None + if not self.check_compatibility_super( + lvalue_type, + rvalue, + base, + base_type, + base_node, + always_allow_covariant=custom_setter, + ): + # Only show one error per variable; even if other + # base classes are also incompatible + return + if lvalue_type and custom_setter: + base_type, _ = self.node_type_from_base( + lvalue_node.name, base, lvalue, setter_type=True + ) + # Setter type for a custom property must be ready if + # the getter type is ready. + assert base_type is not None + if not is_subtype(base_type, lvalue_type): + self.msg.incompatible_setter_override( + lvalue, lvalue_type, base_type, base + ) + return + if base is last_immediate_base: + # At this point, the attribute was found to be compatible with all + # immediate parents. + break + + def check_compatibility_super( + self, + compare_type: Type, + rvalue: Expression, + base: TypeInfo, + base_type: Type, + base_node: Node, + always_allow_covariant: bool, + ) -> bool: + # TODO: check __set__() type override for custom descriptors. + # TODO: for descriptors check also class object access override. + ok = self.check_subtype( + compare_type, + base_type, + rvalue, + message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + "expression has type", + f'base class "{base.name}" defined the type as', + ) + if ( + ok + and codes.MUTABLE_OVERRIDE in self.options.enabled_error_codes + and self.is_writable_attribute(base_node) + and not always_allow_covariant + ): + ok = self.check_subtype( + base_type, + compare_type, + rvalue, + message_registry.COVARIANT_OVERRIDE_OF_MUTABLE_ATTRIBUTE, + f'base class "{base.name}" defined the type as', + "expression has type", + ) + return ok + + def node_type_from_base( + self, + name: str, + base: TypeInfo, + context: Context, + *, + setter_type: bool = False, + is_class: bool = False, + current_class: TypeInfo | None = None, + ) -> tuple[Type | None, SymbolNode | None]: + """Find a type for a name in base class. + + Return the type found and the corresponding node defining the name or None + for both if the name is not defined in base or the node type is not known (yet). + The type returned is already properly mapped/bound to the subclass. + If setter_type is True, return setter types for settable properties (otherwise the + getter type is returned). + """ + base_node = base.names.get(name) + + # TODO: defer current node if the superclass node is not ready. + if ( + not base_node + or isinstance(base_node.node, (Var, Decorator)) + and not base_node.type + or isinstance(base_node.type, PartialType) + and base_node.type.type is not None + ): + return None, None + + if current_class is None: + self_type = self.scope.current_self_type() + else: + self_type = fill_typevars(current_class) + assert self_type is not None, "Internal error: base lookup outside class" + if isinstance(self_type, TupleType): + instance = tuple_fallback(self_type) + else: + instance = self_type + + mx = MemberContext( + is_lvalue=setter_type, + is_super=False, + is_operator=mypy.checkexpr.is_operator_method(name), + original_type=self_type, + context=context, + chk=self, + suppress_errors=True, + ) + # TODO: we should not filter "cannot determine type" errors here. + with self.msg.filter_errors(filter_deprecated=True): + if is_class: + fallback = instance.type.metaclass_type or mx.named_type("builtins.type") + base_type = analyze_class_attribute_access( + instance, name, mx, mcs_fallback=fallback, override_info=base + ) + else: + base_type = analyze_instance_member_access(name, instance, mx, base) + return base_type, base_node.node + + def check_compatibility_classvar_super( + self, node: Var, base: TypeInfo, base_node: Node | None + ) -> bool: + if not isinstance(base_node, Var): + return True + if node.is_classvar and not base_node.is_classvar: + self.fail(message_registry.CANNOT_OVERRIDE_INSTANCE_VAR.format(base.name), node) + return False + elif not node.is_classvar and base_node.is_classvar: + self.fail(message_registry.CANNOT_OVERRIDE_CLASS_VAR.format(base.name), node) + return False + return True + + def check_compatibility_final_super( + self, node: Var, base: TypeInfo, base_node: Node | None + ) -> bool: + """Check if an assignment overrides a final attribute in a base class. + + This only checks situations where either a node in base class is not a variable + but a final method, or where override is explicitly declared as final. + In these cases we give a more detailed error message. In addition, we check that + a final variable doesn't override writeable attribute, which is not safe. + + Other situations are checked in `check_final()`. + """ + if not isinstance(base_node, (Var, FuncBase, Decorator)): + return True + if is_private(node.name): + return True + if base_node.is_final and (node.is_final or not isinstance(base_node, Var)): + # Give this error only for explicit override attempt with `Final`, or + # if we are overriding a final method with variable. + # Other override attempts will be flagged as assignment to constant + # in `check_final()`. + self.msg.cant_override_final(node.name, base.name, node) + return False + if node.is_final: + if base.fullname in ENUM_BASES or node.name in ENUM_SPECIAL_PROPS: + return True + self.check_if_final_var_override_writable(node.name, base_node, node) + return True + + def check_if_final_var_override_writable( + self, name: str, base_node: Node | None, ctx: Context + ) -> None: + """Check that a final variable doesn't override writeable attribute. + + This is done to prevent situations like this: + class C: + attr = 1 + class D(C): + attr: Final = 2 + + x: C = D() + x.attr = 3 # Oops! + """ + writable = True + if base_node: + writable = self.is_writable_attribute(base_node) + if writable: + self.msg.final_cant_override_writable(name, ctx) + + def get_final_context(self) -> bool: + """Check whether we a currently checking a final declaration.""" + return self._is_final_def + + @contextmanager + def enter_final_context(self, is_final_def: bool) -> Iterator[None]: + """Store whether the current checked assignment is a final declaration.""" + old_ctx = self._is_final_def + self._is_final_def = is_final_def + try: + yield + finally: + self._is_final_def = old_ctx + + def check_final(self, s: AssignmentStmt | OperatorAssignmentStmt | AssignmentExpr) -> None: + """Check if this assignment does not assign to a final attribute. + + This function performs the check only for name assignments at module + and class scope. The assignments to `obj.attr` and `Cls.attr` are checked + in checkmember.py. + """ + if isinstance(s, AssignmentStmt): + lvs = self.flatten_lvalues(s.lvalues) + elif isinstance(s, AssignmentExpr): + lvs = [s.target] + else: + lvs = [s.lvalue] + is_final_decl = s.is_final_def if isinstance(s, AssignmentStmt) else False + if is_final_decl and (active_class := self.scope.active_class()): + lv = lvs[0] + assert isinstance(lv, RefExpr) + if lv.node is not None: + assert isinstance(lv.node, Var) + if ( + lv.node.final_unset_in_class + and not lv.node.final_set_in_init + and not self.is_stub # It is OK to skip initializer in stub files. + and + # Avoid extra error messages, if there is no type in Final[...], + # then we already reported the error about missing r.h.s. + isinstance(s, AssignmentStmt) + and s.type is not None + # Avoid extra error message for NamedTuples, + # they were reported during semanal + and not active_class.is_named_tuple + ): + self.msg.final_without_value(s) + for lv in lvs: + if isinstance(lv, RefExpr) and isinstance(lv.node, Var): + name = lv.node.name + cls = self.scope.active_class() + if cls is not None: + # These additional checks exist to give more error messages + # even if the final attribute was overridden with a new symbol + # (which is itself an error)... + for base in cls.mro[1:]: + sym = base.names.get(name) + # We only give this error if base node is variable, + # overriding final method will be caught in + # `check_compatibility_final_super()`. + if sym and isinstance(sym.node, Var): + if sym.node.is_final and not is_final_decl: + self.msg.cant_assign_to_final(name, sym.node.info is None, s) + # ...but only once + break + if lv.node.is_final and not is_final_decl: + self.msg.cant_assign_to_final(name, lv.node.info is None, s) + + def check_assignment_to_slots(self, lvalue: Lvalue) -> None: + if not isinstance(lvalue, MemberExpr): + return + + inst = get_proper_type(self.expr_checker.accept(lvalue.expr)) + if isinstance(inst, TypeVarType) and inst.id.is_self(): + # Unwrap self type + inst = get_proper_type(inst.upper_bound) + if not isinstance(inst, Instance): + return + if inst.type.slots is None: + return # Slots do not exist, we can allow any assignment + if lvalue.name in inst.type.slots: + return # We are assigning to an existing slot + for base_info in inst.type.mro[:-1]: + if base_info.names.get("__setattr__") is not None: + # When type has `__setattr__` defined, + # we can assign any dynamic value. + # We exclude object, because it always has `__setattr__`. + return + + definition = inst.type.get(lvalue.name) + if definition is None: + # We don't want to duplicate + # `"SomeType" has no attribute "some_attr"` + # error twice. + return + if self.is_assignable_slot(lvalue, definition.type): + return + + self.fail( + message_registry.NAME_NOT_IN_SLOTS.format(lvalue.name, inst.type.fullname), lvalue + ) + + def is_assignable_slot(self, lvalue: Lvalue, typ: Type | None) -> bool: + if getattr(lvalue, "node", None): + return False # This is a definition + + typ = get_proper_type(typ) + if typ is None or isinstance(typ, AnyType): + return True # Any can be literally anything, like `@property` + if isinstance(typ, Instance): + # When working with instances, we need to know if they contain + # `__set__` special method. Like `@property` does. + # This makes assigning to properties possible, + # even without extra slot spec. + return typ.type.get("__set__") is not None + if isinstance(typ, FunctionLike): + return True # Can be a property, or some other magic + if isinstance(typ, UnionType): + return all(self.is_assignable_slot(lvalue, u) for u in typ.items) + return False + + def flatten_rvalues(self, rvalues: list[Expression]) -> list[Expression]: + """Flatten expression list by expanding those * items that have tuple type. + + For each regular type item in the tuple type use a TempNode(), for an Unpack + item use a corresponding StarExpr(TempNode()). + """ + new_rvalues = [] + for rv in rvalues: + if not isinstance(rv, StarExpr): + new_rvalues.append(rv) + continue + typ = get_proper_type(self.expr_checker.accept(rv.expr)) + if not isinstance(typ, TupleType): + new_rvalues.append(rv) + continue + for t in typ.items: + if not isinstance(t, UnpackType): + new_rvalues.append(TempNode(t)) + else: + unpacked = get_proper_type(t.type) + if isinstance(unpacked, TypeVarTupleType): + fallback = unpacked.upper_bound + else: + assert ( + isinstance(unpacked, Instance) + and unpacked.type.fullname == "builtins.tuple" + ) + fallback = unpacked + new_rvalues.append(StarExpr(TempNode(fallback))) + return new_rvalues + + def check_assignment_to_multiple_lvalues( + self, + lvalues: list[Lvalue], + rvalue: Expression, + context: Context, + infer_lvalue_type: bool = True, + ) -> None: + if isinstance(rvalue, (TupleExpr, ListExpr)): + # Recursively go into Tuple or List expression rhs instead of + # using the type of rhs, because this allows more fine-grained + # control in cases like: a, b = [int, str] where rhs would get + # type List[object] + rvalues: list[Expression] = [] + iterable_type: Type | None = None + last_idx: int | None = None + for idx_rval, rval in enumerate(self.flatten_rvalues(rvalue.items)): + if isinstance(rval, StarExpr): + typs = get_proper_type(self.expr_checker.accept(rval.expr)) + if self.type_is_iterable(typs) and isinstance(typs, Instance): + if iterable_type is not None and iterable_type != self.iterable_item_type( + typs, rvalue + ): + self.fail(message_registry.CONTIGUOUS_ITERABLE_EXPECTED, context) + else: + if last_idx is None or last_idx + 1 == idx_rval: + rvalues.append(rval) + last_idx = idx_rval + iterable_type = self.iterable_item_type(typs, rvalue) + else: + self.fail(message_registry.CONTIGUOUS_ITERABLE_EXPECTED, context) + else: + self.fail(message_registry.ITERABLE_TYPE_EXPECTED.format(typs), context) + else: + rvalues.append(rval) + iterable_start: int | None = None + iterable_end: int | None = None + for i, rval in enumerate(rvalues): + if isinstance(rval, StarExpr): + typs = get_proper_type(self.expr_checker.accept(rval.expr)) + if self.type_is_iterable(typs) and isinstance(typs, Instance): + if iterable_start is None: + iterable_start = i + iterable_end = i + if ( + iterable_start is not None + and iterable_end is not None + and iterable_type is not None + ): + iterable_num = iterable_end - iterable_start + 1 + rvalue_needed = len(lvalues) - (len(rvalues) - iterable_num) + if rvalue_needed > 0: + rvalues = ( + rvalues[0:iterable_start] + + [TempNode(iterable_type, context=rval) for _ in range(rvalue_needed)] + + rvalues[iterable_end + 1 :] + ) + + if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): + star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) + ) + + left_lvs = lvalues[:star_index] + star_lv = ( + cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + ) + right_lvs = lvalues[star_index + 1 :] + + left_rvs, star_rvs, right_rvs = self.split_around_star( + rvalues, star_index, len(lvalues) + ) + + lr_pairs = list(zip(left_lvs, left_rvs)) + if star_lv: + rv_list = ListExpr(star_rvs) + rv_list.set_line(rvalue) + lr_pairs.append((star_lv.expr, rv_list)) + lr_pairs.extend(zip(right_lvs, right_rvs)) + + for lv, rv in lr_pairs: + self.check_assignment(lv, rv, infer_lvalue_type) + else: + self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) + + def check_rvalue_count_in_assignment( + self, + lvalues: list[Lvalue], + rvalue_count: int, + context: Context, + rvalue_unpack: int | None = None, + ) -> bool: + if rvalue_unpack is not None: + if not any(isinstance(e, StarExpr) for e in lvalues): + self.fail("Variadic tuple unpacking requires a star target", context) + return False + if len(lvalues) > rvalue_count: + self.fail(message_registry.TOO_MANY_TARGETS_FOR_VARIADIC_UNPACK, context) + return False + left_star_index = next(i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)) + left_prefix = left_star_index + left_suffix = len(lvalues) - left_star_index - 1 + right_prefix = rvalue_unpack + right_suffix = rvalue_count - rvalue_unpack - 1 + if left_suffix > right_suffix or left_prefix > right_prefix: + # Case of asymmetric unpack like: + # rv: tuple[int, *Ts, int, int] + # x, y, *xs, z = rv + # it is technically valid, but is tricky to reason about. + # TODO: support this (at least if the r.h.s. unpack is a homogeneous tuple). + self.fail(message_registry.TOO_MANY_TARGETS_FOR_VARIADIC_UNPACK, context) + return True + if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): + if len(lvalues) - 1 > rvalue_count: + self.msg.wrong_number_values_to_unpack(rvalue_count, len(lvalues) - 1, context) + return False + elif rvalue_count != len(lvalues): + self.msg.wrong_number_values_to_unpack(rvalue_count, len(lvalues), context) + return False + return True + + def check_multi_assignment( + self, + lvalues: list[Lvalue], + rvalue: Expression, + context: Context, + infer_lvalue_type: bool = True, + rv_type: Type | None = None, + undefined_rvalue: bool = False, + ) -> None: + """Check the assignment of one rvalue to a number of lvalues.""" + + # Infer the type of an ordinary rvalue expression. + # TODO: maybe elsewhere; redundant. + rvalue_type = get_proper_type(rv_type or self.expr_checker.accept(rvalue)) + + if isinstance(rvalue_type, TypeVarLikeType): + rvalue_type = get_proper_type(rvalue_type.upper_bound) + + if isinstance(rvalue_type, UnionType): + # If this is an Optional type in non-strict Optional code, unwrap it. + relevant_items = rvalue_type.relevant_items() + if len(relevant_items) == 1: + rvalue_type = get_proper_type(relevant_items[0]) + + if ( + isinstance(rvalue_type, TupleType) + and find_unpack_in_list(rvalue_type.items) is not None + ): + # Normalize for consistent handling with "old-style" homogeneous tuples. + rvalue_type = expand_type(rvalue_type, {}) + + if isinstance(rvalue_type, AnyType): + for lv in lvalues: + if isinstance(lv, StarExpr): + lv = lv.expr + temp_node = self.temp_node( + AnyType(TypeOfAny.from_another_any, source_any=rvalue_type), context + ) + self.check_assignment(lv, temp_node, infer_lvalue_type) + elif isinstance(rvalue_type, TupleType): + self.check_multi_assignment_from_tuple( + lvalues, rvalue, rvalue_type, context, undefined_rvalue, infer_lvalue_type + ) + elif isinstance(rvalue_type, UnionType): + self.check_multi_assignment_from_union( + lvalues, rvalue, rvalue_type, context, infer_lvalue_type + ) + else: + rvalue_literal = rvalue_type + if isinstance(rvalue_type, Instance) and rvalue_type.type.fullname == "builtins.str": + if rvalue_type.last_known_value is None: + self.msg.unpacking_strings_disallowed(context) + else: + rvalue_literal = rvalue_type.last_known_value + if ( + isinstance(rvalue_literal, LiteralType) + and isinstance(rvalue_literal.value, str) + and len(lvalues) != len(rvalue_literal.value) + ): + self.msg.unpacking_strings_disallowed(context) + + self.check_multi_assignment_from_iterable( + lvalues, rvalue_type, context, infer_lvalue_type + ) + + def check_multi_assignment_from_union( + self, + lvalues: list[Expression], + rvalue: Expression, + rvalue_type: UnionType, + context: Context, + infer_lvalue_type: bool, + ) -> None: + """Check assignment to multiple lvalue targets when rvalue type is a Union[...]. + For example: + + t: Union[Tuple[int, int], Tuple[str, str]] + x, y = t + reveal_type(x) # Union[int, str] + + The idea in this case is to process the assignment for every item of the union. + Important note: the types are collected in two places, 'union_types' contains + inferred types for first assignments, 'assignments' contains the narrowed types + for binder. + """ + self.no_partial_types = True + transposed: tuple[list[Type], ...] = tuple([] for _ in self.flatten_lvalues(lvalues)) + # Notify binder that we want to defer bindings and instead collect types. + with self.binder.accumulate_type_assignments() as assignments: + for item in rvalue_type.items: + # Type check the assignment separately for each union item and collect + # the inferred lvalue types for each union item. + self.check_multi_assignment( + lvalues, + rvalue, + context, + infer_lvalue_type=infer_lvalue_type, + rv_type=item, + undefined_rvalue=True, + ) + for t, lv in zip(transposed, self.flatten_lvalues(lvalues)): + # We can access _type_maps directly since temporary type maps are + # only created within expressions. + t.append(self._type_maps[-1].pop(lv, AnyType(TypeOfAny.special_form))) + union_types = tuple(make_simplified_union(col) for col in transposed) + for expr, items in assignments.items(): + # Bind a union of types collected in 'assignments' to every expression. + if isinstance(expr, StarExpr): + expr = expr.expr + + # TODO: See comment in binder.py, ConditionalTypeBinder.assign_type + # It's unclear why the 'declared_type' param is sometimes 'None' + clean_items: list[tuple[Type, Type]] = [] + for type, declared_type in items: + assert declared_type is not None + clean_items.append((type, declared_type)) + + types, declared_types = zip(*clean_items) + self.binder.assign_type( + expr, + make_simplified_union(list(types)), + make_simplified_union(list(declared_types)), + ) + for union, lv in zip(union_types, self.flatten_lvalues(lvalues)): + # Properly store the inferred types. + _1, _2, inferred = self.check_lvalue(lv) + if inferred: + self.set_inferred_type(inferred, lv, union) + else: + self.store_type(lv, union) + self.no_partial_types = False + + def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: + res: list[Expression] = [] + for lv in lvalues: + if isinstance(lv, (TupleExpr, ListExpr)): + res.extend(self.flatten_lvalues(lv.items)) + if isinstance(lv, StarExpr): + # Unwrap StarExpr, since it is unwrapped by other helpers. + lv = lv.expr + res.append(lv) + return res + + def adjust_rvalue_type_if_possible( + self, rvalue_type: TupleType, lvalues: list[Lvalue] + ) -> TupleType: + """Adjust type of rvalue to match the shape/structure of lvalues. + + Currently, we only allow this if the rvalue type has contains *tuple[Any, ...]. + """ + right_variadic = get_variadic_item(rvalue_type) + if right_variadic is None: + return rvalue_type + right_unpack_index, right_item = right_variadic + if not isinstance(get_proper_type(right_item), AnyType): + return rvalue_type + left_star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), None + ) + if left_star_index is None: + extra = len(lvalues) - len(rvalue_type.items) + 1 + if extra < 0: + return rvalue_type + return rvalue_type.copy_modified( + items=rvalue_type.items[:right_unpack_index] + + [right_item] * extra + + rvalue_type.items[right_unpack_index + 1 :] + ) + new_items = extend_args_for_prefix_and_suffix( + tuple(rvalue_type.items), left_star_index, len(lvalues) - left_star_index - 1 + ) + return rvalue_type.copy_modified(items=list(new_items)) + + def check_multi_assignment_from_tuple( + self, + lvalues: list[Lvalue], + rvalue: Expression, + rvalue_type: TupleType, + context: Context, + undefined_rvalue: bool, + infer_lvalue_type: bool = True, + ) -> None: + rvalue_unpack = find_unpack_in_list(rvalue_type.items) + if rvalue_unpack is not None: + rvalue_type = self.adjust_rvalue_type_if_possible(rvalue_type, lvalues) + rvalue_unpack = find_unpack_in_list(rvalue_type.items) + if self.check_rvalue_count_in_assignment( + lvalues, len(rvalue_type.items), context, rvalue_unpack=rvalue_unpack + ): + star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) + ) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1 :] + + if not undefined_rvalue: + # Infer rvalue again, now in the correct type context. + lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) + reinferred_rvalue_type = get_proper_type( + self.expr_checker.accept(rvalue, lvalue_type) + ) + + if isinstance(reinferred_rvalue_type, TypeVarLikeType): + reinferred_rvalue_type = get_proper_type(reinferred_rvalue_type.upper_bound) + if isinstance(reinferred_rvalue_type, UnionType): + # If this is an Optional type in non-strict Optional code, unwrap it. + relevant_items = reinferred_rvalue_type.relevant_items() + if len(relevant_items) == 1: + reinferred_rvalue_type = get_proper_type(relevant_items[0]) + if isinstance(reinferred_rvalue_type, UnionType): + self.check_multi_assignment_from_union( + lvalues, rvalue, reinferred_rvalue_type, context, infer_lvalue_type + ) + return + if isinstance(reinferred_rvalue_type, AnyType): + # We can get Any if the current node is + # deferred. Doing more inference in deferred nodes + # is hard, so give up for now. We can also get + # here if reinferring types above changes the + # inferred return type for an overloaded function + # to be ambiguous. + return + if isinstance(reinferred_rvalue_type, TupleType): + # This branch will usually be taken, but in some cases context can + # e.g. select a different overload + # TODO: reinferred tuple may be of a different (invalid) shape. + rvalue_type = reinferred_rvalue_type + + # Reinferring the type can undo the shape adjustment, so do it again. + rvalue_unpack = find_unpack_in_list(rvalue_type.items) + if rvalue_unpack is not None: + rvalue_type = self.adjust_rvalue_type_if_possible(rvalue_type, lvalues) + + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues) + ) + + for lv, rv_type in zip(left_lvs, left_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + if star_lv: + list_expr = ListExpr( + [ + ( + self.temp_node(rv_type, context) + if not isinstance(rv_type, UnpackType) + else StarExpr(self.temp_node(rv_type.type, context)) + ) + for rv_type in star_rv_types + ] + ) + list_expr.set_line(context) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + for lv, rv_type in zip(right_lvs, right_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + else: + # Store meaningful Any types for lvalues, errors are already given + # by check_rvalue_count_in_assignment() + if infer_lvalue_type: + for lv in lvalues: + if ( + isinstance(lv, NameExpr) + and isinstance(lv.node, Var) + and lv.node.type is None + ): + lv.node.type = AnyType(TypeOfAny.from_error) + elif isinstance(lv, StarExpr): + if ( + isinstance(lv.expr, NameExpr) + and isinstance(lv.expr.node, Var) + and lv.expr.node.type is None + ): + lv.expr.node.type = self.named_generic_type( + "builtins.list", [AnyType(TypeOfAny.from_error)] + ) + + def lvalue_type_for_inference(self, lvalues: list[Lvalue], rvalue_type: TupleType) -> Type: + star_index = next( + (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) + ) + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1 :] + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues) + ) + + type_parameters: list[Type] = [] + + def append_types_for_inference(lvs: list[Expression], rv_types: list[Type]) -> None: + for lv, rv_type in zip(lvs, rv_types): + sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) + if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): + type_parameters.append(sub_lvalue_type) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.append(rv_type) + + append_types_for_inference(left_lvs, left_rv_types) + + if star_lv: + sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) + if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): + type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.extend(star_rv_types) + + append_types_for_inference(right_lvs, right_rv_types) + + return TupleType(type_parameters, self.named_type("builtins.tuple")) + + def split_around_star( + self, items: list[T], star_index: int, length: int + ) -> tuple[list[T], list[T], list[T]]: + """Splits a list of items in three to match another list of length 'length' + that contains a starred expression at 'star_index' in the following way: + + star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] + returns in: ([1,2], [3,4,5], [6,7]) + """ + nr_right_of_star = length - star_index - 1 + right_index = -nr_right_of_star if nr_right_of_star != 0 else len(items) + left = items[:star_index] + star = items[star_index:right_index] + right = items[right_index:] + return left, star, right + + def type_is_iterable(self, type: Type) -> bool: + type = get_proper_type(type) + if isinstance(type, FunctionLike) and type.is_type_obj(): + type = type.fallback + return is_subtype( + type, self.named_generic_type("typing.Iterable", [AnyType(TypeOfAny.special_form)]) + ) + + def check_multi_assignment_from_iterable( + self, + lvalues: list[Lvalue], + rvalue_type: Type, + context: Context, + infer_lvalue_type: bool = True, + ) -> None: + rvalue_type = get_proper_type(rvalue_type) + if self.type_is_iterable(rvalue_type): + item_type = self.iterable_item_type(rvalue_type, context) + for lv in lvalues: + if isinstance(lv, StarExpr): + items_type = self.named_generic_type("builtins.list", [item_type]) + self.check_assignment( + lv.expr, self.temp_node(items_type, context), infer_lvalue_type + ) + else: + self.check_assignment( + lv, self.temp_node(item_type, context), infer_lvalue_type + ) + else: + self.msg.type_not_iterable(rvalue_type, context) + + def check_lvalue( + self, lvalue: Lvalue, rvalue: Expression | None = None + ) -> tuple[Type | None, IndexExpr | None, Var | None]: + lvalue_type = None + index_lvalue = None + inferred = None + + # When revisiting the initial assignment (for example in a loop), + # treat is as regular if redefinitions are allowed. + skip_definition = ( + self.options.allow_redefinition + and isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and lvalue.node.is_inferred + and lvalue.node.type is not None + and not isinstance(lvalue.node.type, PartialType) + # Indexes in for loops require special handling, we need to reset them to + # a literal value on each loop, but binder doesn't work well with literals. + and not lvalue.node.is_index_var + ) + + if ( + self.is_definition(lvalue) + and (not isinstance(lvalue, NameExpr) or isinstance(lvalue.node, Var)) + and not skip_definition + ): + if isinstance(lvalue, NameExpr): + assert isinstance(lvalue.node, Var) + inferred = lvalue.node + else: + assert isinstance(lvalue, MemberExpr) + self.expr_checker.accept(lvalue.expr) + inferred = lvalue.def_var + elif isinstance(lvalue, IndexExpr): + index_lvalue = lvalue + elif isinstance(lvalue, MemberExpr): + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, True, rvalue) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, NameExpr): + lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) + if ( + self.options.allow_redefinition + and isinstance(lvalue.node, Var) + # We allow redefinition for function arguments inside function body. + # Although we normally do this for variables without annotation, users + # don't have a choice to leave a function argument without annotation. + and (lvalue.node.is_inferred or lvalue.node.is_argument) + ): + inferred = lvalue.node + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, (TupleExpr, ListExpr)): + types = [ + self.check_lvalue(sub_expr)[0] or + # This type will be used as a context for further inference of rvalue, + # we put Uninhabited if there is no information available from lvalue. + UninhabitedType(ambiguous=True) + for sub_expr in lvalue.items + ] + lvalue_type = TupleType(types, self.named_type("builtins.tuple")) + elif isinstance(lvalue, StarExpr): + lvalue_type, _, _ = self.check_lvalue(lvalue.expr) + else: + lvalue_type = self.expr_checker.accept(lvalue) + + return lvalue_type, index_lvalue, inferred + + def is_definition(self, s: Lvalue) -> bool: + if isinstance(s, NameExpr): + if s.is_inferred_def: + return True + # If the node type is not defined, this must the first assignment + # that we process => this is a definition, even though the semantic + # analyzer did not recognize this as such. This can arise in code + # that uses isinstance checks, if type checking of the primary + # definition is skipped due to an always False type check. + node = s.node + if isinstance(node, Var): + return node.type is None + elif isinstance(s, MemberExpr): + return s.is_inferred_def + return False + + def infer_variable_type( + self, name: Var, lvalue: Lvalue, init_type: Type, context: Context + ) -> None: + """Infer the type of initialized variables from initializer type.""" + if isinstance(init_type, DeletedType): + self.msg.deleted_as_rvalue(init_type, context) + elif ( + not is_valid_inferred_type( + init_type, + self.options, + is_lvalue_final=name.is_final, + is_lvalue_member=isinstance(lvalue, MemberExpr), + ) + and not ( + # Trust None assignments to dunder methods + # This is a bit ad-hoc, but it improves protocol + # (non-)assignability, for instance `__hash__ = None` + self.scope.active_class() + and is_dunder(name.name) + and isinstance(get_proper_type(init_type), NoneType) + ) + and not self.no_partial_types + ): + # We cannot use the type of the initialization expression for full type + # inference (it's not specific enough), but we might be able to give + # partial type which will be made more specific later. A partial type + # gets generated in assignment like 'x = []' where item type is not known. + if name.name != "_" and not self.infer_partial_type(name, lvalue, init_type): + self.msg.need_annotation_for_var(name, context, self.options) + self.set_inference_error_fallback_type(name, lvalue, init_type) + elif ( + isinstance(lvalue, MemberExpr) + and self.inferred_attribute_types is not None + and lvalue.def_var + and lvalue.def_var in self.inferred_attribute_types + and not is_same_type(self.inferred_attribute_types[lvalue.def_var], init_type) + ): + # Multiple, inconsistent types inferred for an attribute. + self.msg.need_annotation_for_var(name, context, self.options) + name.type = AnyType(TypeOfAny.from_error) + else: + # Infer type of the target. + + # Make the type more general (strip away function names etc.). + init_type = strip_type(init_type) + + self.set_inferred_type(name, lvalue, init_type) + if self.options.allow_redefinition: + self.binder.assign_type(lvalue, init_type, init_type) + + def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool: + init_type = get_proper_type(init_type) + if isinstance(init_type, NoneType) and ( + isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition + ): + # When using --allow-redefinition, None types aren't special + # when inferring simple variable types. + partial_type = PartialType(None, name) + elif isinstance(init_type, Instance): + fullname = init_type.type.fullname + is_ref = isinstance(lvalue, RefExpr) + if ( + is_ref + and ( + fullname == "builtins.list" + or fullname == "builtins.set" + or fullname == "builtins.dict" + or fullname == "collections.OrderedDict" + ) + and all( + isinstance(t, (NoneType, UninhabitedType)) + for t in get_proper_types(init_type.args) + ) + ): + partial_type = PartialType(init_type.type, name) + elif is_ref and fullname == "collections.defaultdict": + arg0 = get_proper_type(init_type.args[0]) + arg1 = get_proper_type(init_type.args[1]) + if isinstance( + arg0, (NoneType, UninhabitedType) + ) and self.is_valid_defaultdict_partial_value_type(arg1): + arg1 = erase_type(arg1) + assert isinstance(arg1, Instance) + partial_type = PartialType(init_type.type, name, arg1) + else: + return False + else: + return False + else: + return False + self.set_inferred_type(name, lvalue, partial_type) + self.partial_types[-1].map[name] = lvalue + return True + + def is_valid_defaultdict_partial_value_type(self, t: ProperType) -> bool: + """Check if t can be used as the basis for a partial defaultdict value type. + + Examples: + + * t is 'int' --> True + * t is 'list[Never]' --> True + * t is 'dict[...]' --> False (only generic types with a single type + argument supported) + """ + if not isinstance(t, Instance): + return False + if len(t.args) == 0: + return True + if len(t.args) == 1: + arg = get_proper_type(t.args[0]) + if self.options.old_type_inference: + # Allow leaked TypeVars for legacy inference logic. + allowed = isinstance(arg, (UninhabitedType, NoneType, TypeVarType)) + else: + allowed = isinstance(arg, (UninhabitedType, NoneType)) + if allowed: + return True + return False + + def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: + """Store inferred variable type. + + Store the type to both the variable node and the expression node that + refers to the variable (lvalue). If var is None, do nothing. + """ + if var and not self.current_node_deferred: + var.type = type + var.is_inferred = True + var.is_ready = True + if var not in self.var_decl_frames: + # Used for the hack to improve optional type inference in conditionals + self.var_decl_frames[var] = {frame.id for frame in self.binder.frames} + if isinstance(lvalue, MemberExpr) and self.inferred_attribute_types is not None: + # Store inferred attribute type so that we can check consistency afterwards. + if lvalue.def_var is not None: + self.inferred_attribute_types[lvalue.def_var] = type + self.store_type(lvalue, type) + p_type = get_proper_type(type) + definition = None + if isinstance(p_type, CallableType): + definition = p_type.definition + elif isinstance(p_type, Overloaded): + # Randomly select first item, if items are different, there will + # be an error during semantic analysis. + definition = p_type.items[0].definition + if definition: + if is_node_static(definition): + var.is_staticmethod = True + elif is_classmethod_node(definition): + var.is_classmethod = True + elif is_property(definition): + var.is_property = True + if isinstance(p_type, Overloaded): + # TODO: in theory we can have a property with a deleter only. + var.is_settable_property = True + assert isinstance(definition, Decorator), definition + var.setter_type = definition.var.setter_type + + def set_inference_error_fallback_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: + """Store best known type for variable if type inference failed. + + If a program ignores error on type inference error, the variable should get some + inferred type so that it can be used later on in the program. Example: + + x = [] # type: ignore + x.append(1) # Should be ok! + + We implement this here by giving x a valid type (replacing inferred Never with Any). + """ + fallback = self.inference_error_fallback_type(type) + self.set_inferred_type(var, lvalue, fallback) + + def inference_error_fallback_type(self, type: Type) -> Type: + fallback = type.accept(SetNothingToAny()) + # Type variables may leak from inference, see https://github.com/python/mypy/issues/5738, + # we therefore need to erase them. + return erase_typevars(fallback) + + def simple_rvalue(self, rvalue: Expression) -> bool: + """Returns True for expressions for which inferred type should not depend on context. + + Note that this function can still return False for some expressions where inferred type + does not depend on context. It only exists for performance optimizations. + """ + if isinstance(rvalue, (IntExpr, StrExpr, BytesExpr, FloatExpr, RefExpr)): + return True + if isinstance(rvalue, CallExpr): + if isinstance(rvalue.callee, RefExpr) and isinstance( + rvalue.callee.node, SYMBOL_FUNCBASE_TYPES + ): + typ = rvalue.callee.node.type + if isinstance(typ, CallableType): + return not typ.variables + elif isinstance(typ, Overloaded): + return not any(item.variables for item in typ.items) + return False + + def infer_rvalue_with_fallback_context( + self, + lvalue_type: Type | None, + rvalue: Expression, + preferred_context: Type | None, + fallback_context: Type | None, + inferred: Var | None, + always_allow_any: bool, + ) -> Type: + """Infer rvalue type in two type context and select the best one. + + See comments below for supported fallback scenarios. + """ + assert fallback_context is not preferred_context + # TODO: make assignment checking correct in presence of walrus in r.h.s. + # We may accept r.h.s. twice. In presence of walrus this can lead to weird + # false negatives and "back action". A proper solution would be to use + # binder.accumulate_type_assignments() and assign the types inferred for type + # context that is ultimately used. This is however tricky with redefinitions. + # For now we simply disable second accept in cases known to cause problems, + # see e.g. testAssignToOptionalTupleWalrus. + assignment_expression_effect = self.assignment_expression_effect + + fallback_context_used = False + with ( + self.msg.filter_errors(save_filtered_errors=True) as local_errors, + self.local_type_map as type_map, + ): + rvalue_type = self.expr_checker.accept( + rvalue, type_context=preferred_context, always_allow_any=always_allow_any + ) + + # There are two cases where we want to try re-inferring r.h.s. in a fallback + # type context. First case is when redefinitions are allowed, and we got + # invalid type when using the preferred (empty) type context. + redefinition_fallback = ( + inferred is not None + and not inferred.is_argument + and not is_valid_inferred_type(rvalue_type, self.options) + ) + # For function arguments the preference order is opposite, and we use errors + # during type-checking as the fallback trigger. + argument_redefinition_fallback = ( + inferred is not None and inferred.is_argument and local_errors.has_new_errors() + ) + # Try re-inferring r.h.s. in empty context for union with explicit annotation, + # and use it results in a narrower type. This helps with various practical + # examples, see e.g. testOptionalTypeNarrowedByGenericCall. + union_fallback = ( + preferred_context is not None + and isinstance(get_proper_type(lvalue_type), UnionType) + and assignment_expression_effect == self.assignment_expression_effect + ) + + # Skip literal types, as they have special logic (for better errors). + try_fallback = redefinition_fallback or union_fallback or argument_redefinition_fallback + if try_fallback and not is_literal_type_like(rvalue_type): + with ( + self.msg.filter_errors(save_filtered_errors=True) as alt_local_errors, + self.local_type_map as alt_type_map, + ): + alt_rvalue_type = self.expr_checker.accept( + rvalue, fallback_context, always_allow_any=always_allow_any + ) + if ( + not alt_local_errors.has_new_errors() + and is_valid_inferred_type(alt_rvalue_type, self.options) + and ( + # For redefinition fallbacks we are fine getting not a subtype. + redefinition_fallback + or argument_redefinition_fallback + # Skip Any type, since it is special cased in binder. + or not isinstance(get_proper_type(alt_rvalue_type), AnyType) + and is_proper_subtype(alt_rvalue_type, rvalue_type) + ) + ): + fallback_context_used = True + rvalue_type = alt_rvalue_type + self.store_types(alt_type_map) + + if not fallback_context_used: + self.msg.add_errors(local_errors.filtered_errors()) + self.store_types(type_map) + return rvalue_type + + def check_simple_assignment( + self, + lvalue_type: Type | None, + rvalue: Expression, + context: Context, + msg: ErrorMessage = message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + lvalue_name: str = "variable", + rvalue_name: str = "expression", + *, + notes: list[str] | None = None, + lvalue: Expression | None = None, + inferred: Var | None = None, + ) -> tuple[Type, Type | None]: + if self.is_stub and isinstance(rvalue, EllipsisExpr): + # '...' is always a valid initializer in a stub. + return AnyType(TypeOfAny.special_form), lvalue_type + else: + always_allow_any = lvalue_type is not None and not isinstance( + get_proper_type(lvalue_type), AnyType + ) + + # If redefinitions are allowed (i.e. we have --allow-redefinition + # and a variable without annotation) or if a variable has union type we + # try inferring r.h.s. twice with a fallback type context. The only exception + # is TypedDicts, they are often useless without context. + try_fallback = ( + inferred is not None or isinstance(get_proper_type(lvalue_type), UnionType) + ) and not self.simple_rvalue(rvalue) + + if not try_fallback or lvalue_type is None or is_typeddict_type_context(lvalue_type): + rvalue_type = self.expr_checker.accept( + rvalue, type_context=lvalue_type, always_allow_any=always_allow_any + ) + else: + # Prefer full type context for function arguments as this reduces + # false positives, see issue #19918 for discussion. + if inferred is not None and not inferred.is_argument: + preferred = None + fallback = lvalue_type + else: + preferred = lvalue_type + fallback = None + + rvalue_type = self.infer_rvalue_with_fallback_context( + lvalue_type, rvalue, preferred, fallback, inferred, always_allow_any + ) + + if ( + inferred is not None + and not is_valid_inferred_type(rvalue_type, self.options) + and ( + not inferred.type + or isinstance(inferred.type, PartialType) + # This additional check is to give an error instead of inferring + # a useless type like None | list[Never] in case of "double-partial" + # types that are not supported yet, see issue #20257. + or not is_subtype(rvalue_type, inferred.type) + ) + ): + self.msg.need_annotation_for_var(inferred, inferred, self.options) + rvalue_type = rvalue_type.accept(SetNothingToAny()) + + if ( + isinstance(lvalue, NameExpr) + and inferred is not None + and inferred.type is not None + and not inferred.is_final + ): + new_inferred = remove_instance_last_known_values(rvalue_type) + # Should we widen the inferred type or the lvalue? Variables defined + # at module level or class bodies can't be widened in functions, or + # in another module. + if ( + not self.refers_to_different_scope(lvalue) + and not isinstance(inferred.type, PartialType) + and not is_proper_subtype(new_inferred, inferred.type) + and self.can_widen_in_scope(lvalue, inferred.type) + ): + lvalue_type = make_simplified_union([inferred.type, new_inferred]) + # Widen the type to the union of original and new type. + if not inferred.is_index_var: + # Skip index variables as they are reset on each loop. + self.widened_vars.append(inferred.name) + self.set_inferred_type(inferred, lvalue, lvalue_type) + if lvalue.kind == GDEF and self.scope.top_level_function() is not None: + # Widening a global inside a function -- record for + # propagation to the module-level binder afterwards. + self._globals_widened_in_func.append((lvalue, lvalue_type)) + self.binder.put(lvalue, rvalue_type) + # TODO: A bit hacky, maybe add a binder method that does put and + # updates declaration? + lit = literal_hash(lvalue) + if lit is not None: + self.binder.declarations[lit] = lvalue_type + + if isinstance(rvalue_type, DeletedType): + self.msg.deleted_as_rvalue(rvalue_type, context) + if isinstance(lvalue_type, DeletedType): + self.msg.deleted_as_lvalue(lvalue_type, context) + elif lvalue_type: + self.check_subtype( + # Preserve original aliases for error messages when possible. + rvalue_type, + lvalue_type, + context, + msg, + f"{rvalue_name} has type", + f"{lvalue_name} has type", + notes=notes, + ) + return rvalue_type, lvalue_type + + def refers_to_different_scope(self, name: NameExpr) -> bool: + if name.kind == LDEF: + # TODO: Consider reference to outer function as a different scope? + return False + elif self.scope.top_level_function() is not None and name.kind != GDEF: + # A non-local reference from within a function must refer to a different scope + return True + elif name.kind == GDEF and name.fullname.rpartition(".")[0] != self.tree.fullname: + # Reference to global definition from another module + return True + return False + + def can_widen_in_scope(self, name: NameExpr, orig_type: Type) -> bool: + """Can a variable type be widened via assignment in the current scope? + + Globals can only be widened from within a function if the original type + is None (backward compat with partial type handling of `x = None`). + + See test cases testNewRedefineGlobalVariableNoneInit[1-4], for example. + """ + if ( + name.kind == GDEF + and self.scope.top_level_function() is not None + and not isinstance(get_proper_type(orig_type), NoneType) + ): + return False + return True + + def check_member_assignment( + self, + lvalue: MemberExpr, + instance_type: Type, + set_lvalue_type: Type, + rvalue: Expression, + context: Context, + ) -> tuple[Type, Type, bool]: + """Type member assignment. + + This defers to check_simple_assignment, unless the member expression + is a descriptor, in which case this checks descriptor semantics as well. + + Return the inferred rvalue_type, inferred lvalue_type, and whether to use the binder + for this assignment. + """ + instance_type = get_proper_type(instance_type) + # Descriptors don't participate in class-attribute access + if (isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or isinstance( + instance_type, TypeType + ): + rvalue_type, _ = self.check_simple_assignment(set_lvalue_type, rvalue, context) + return rvalue_type, set_lvalue_type, True + + with self.msg.filter_errors(filter_deprecated=True): + get_lvalue_type = self.expr_checker.analyze_ordinary_member_access( + lvalue, is_lvalue=False + ) + + # Special case: if the rvalue_type is a subtype of '__get__' type, and + # '__get__' type is narrower than '__set__', then we invoke the binder to narrow type + # by this assignment. Technically, this is not safe, but in practice this is + # what a user expects. + rvalue_type, _ = self.check_simple_assignment(set_lvalue_type, rvalue, context) + rvalue_type = rvalue_type if is_subtype(rvalue_type, get_lvalue_type) else get_lvalue_type + return rvalue_type, set_lvalue_type, is_subtype(get_lvalue_type, set_lvalue_type) + + def check_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Expression, context: Context + ) -> None: + """Type check indexed assignment base[index] = rvalue. + + The lvalue argument is the base[index] expression. + """ + self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) + basetype = get_proper_type(self.expr_checker.accept(lvalue.base)) + method_type = self.expr_checker.analyze_external_member_access( + "__setitem__", basetype, lvalue + ) + + lvalue.method_type = method_type + res_type, _ = self.expr_checker.check_method_call( + "__setitem__", + basetype, + method_type, + [lvalue.index, rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context, + ) + res_type = get_proper_type(res_type) + if isinstance(res_type, UninhabitedType) and not res_type.ambiguous: + self.binder.unreachable() + + def replace_partial_type( + self, var: Var, new_type: Type, partial_types: dict[Var, Context] + ) -> None: + """Replace the partial type of var with a non-partial type.""" + var.type = new_type + # Updating a partial type should invalidate expression caches. + self.binder.version += 1 + del partial_types[var] + if self.options.allow_redefinition: + # When using --allow-redefinition, binder tracks all types of + # simple variables. + n = NameExpr(var.name) + n.node = var + self.binder.assign_type(n, new_type, new_type) + + def try_infer_partial_type_from_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Expression + ) -> None: + # TODO: Should we share some of this with try_infer_partial_type? + var = None + if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): + var = lvalue.base.node + elif isinstance(lvalue.base, MemberExpr): + var = self.expr_checker.get_partial_self_var(lvalue.base) + if isinstance(var, Var): + if isinstance(var.type, PartialType): + type_type = var.type.type + if type_type is None: + return # The partial type is None. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + typename = type_type.fullname + if ( + typename == "builtins.dict" + or typename == "collections.OrderedDict" + or typename == "collections.defaultdict" + ): + # TODO: Don't infer things twice. + key_type = self.expr_checker.accept(lvalue.index) + value_type = self.expr_checker.accept(rvalue) + if ( + is_valid_inferred_type(key_type, self.options) + and is_valid_inferred_type(value_type, self.options) + and not self.current_node_deferred + and not ( + typename == "collections.defaultdict" + and var.type.value_type is not None + and not is_equivalent(value_type, var.type.value_type) + ) + ): + new_type = self.named_generic_type(typename, [key_type, value_type]) + self.replace_partial_type(var, new_type, partial_types) + + def type_requires_usage(self, typ: Type, s: ExpressionStmt) -> tuple[str, ErrorCode] | None: + """Some types require usage in basically all cases. The classic + example is an unused coroutine. + + In the case that it does require usage, returns a note to attach + to the error message. We special case somethings that return + awaitables because in those particular cases we can guarantee + it's safe. + """ + proper_type = get_proper_type(typ) + if isinstance(proper_type, Instance): + # We use different error codes for generic awaitable vs coroutine. + # Coroutines are on by default, whereas generic awaitables are not. + if proper_type.type.fullname == "typing.Coroutine": + return ("Are you missing an await?", UNUSED_COROUTINE) + if proper_type.type.get("__await__") is not None: + # this is quite ad-hoc, but there's no good way around + # this. the alternative is a hardcoded list of + # TaskGroups and their respective functions, but that's + # a lot of maintenance! + if isinstance(s.expr, CallExpr) and isinstance(s.expr.callee, MemberExpr): + called_on = get_proper_type(self.expr_checker.accept(s.expr.callee.expr)) + is_a_taskgroup = isinstance( + called_on, Instance + ) and called_on.type.fullname.endswith(".TaskGroup") + else: + is_a_taskgroup = False + if not is_a_taskgroup: + return ("Are you missing an await?", UNUSED_AWAITABLE) + return None + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + expr_type = self.expr_checker.accept(s.expr, allow_none_return=True, always_allow_any=True) + error_note_and_code = self.type_requires_usage(expr_type, s) + if error_note_and_code: + error_note, code = error_note_and_code + self.fail( + message_registry.TYPE_MUST_BE_USED.format(format_type(expr_type, self.options)), + s, + code=code, + ) + self.note(error_note, s, code=code) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + """Type check a return statement.""" + self.check_return_stmt(s) + self.binder.unreachable() + + def infer_context_dependent( + self, expr: Expression, type_ctx: Type, allow_none_func_call: bool + ) -> ProperType: + """Infer type of expression with fallback to empty type context.""" + with self.msg.filter_errors(filter_deprecated=True, save_filtered_errors=True) as msg: + with ( + self.local_type_map as type_map, + # Prevent any narrowing (e.g. from walrus) to have effect during second accept. + self.binder.frame_context(can_skip=False, discard=True), + ): + typ = get_proper_type( + self.expr_checker.accept( + expr, type_ctx, allow_none_return=allow_none_func_call + ) + ) + if not msg.has_new_errors(): + self.store_types(type_map) + return typ + + # If there are errors with the original type context, try re-inferring in empty context. + original_messages = msg.filtered_errors() + original_type_map = type_map + with self.msg.filter_errors(filter_deprecated=True, save_filtered_errors=True) as msg: + with self.local_type_map as type_map: + alt_typ = get_proper_type( + self.expr_checker.accept(expr, None, allow_none_return=allow_none_func_call) + ) + if not msg.has_new_errors() and is_subtype(alt_typ, type_ctx): + self.store_types(type_map) + return alt_typ + + # If empty fallback didn't work, use results from the original type context. + self.msg.add_errors(original_messages) + self.store_types(original_type_map) + return typ + + def check_return_stmt(self, s: ReturnStmt) -> None: + defn = self.scope.current_function() + if defn is not None: + if defn.is_generator: + return_type = self.get_generator_return_type( + self.return_types[-1], defn.is_coroutine + ) + elif defn.is_coroutine: + return_type = self.get_coroutine_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + return_type = get_proper_type(return_type) + + is_lambda = isinstance(defn, LambdaExpr) + if isinstance(return_type, UninhabitedType): + # Avoid extra error messages for failed inference in lambdas + if not is_lambda and not return_type.ambiguous: + self.fail(message_registry.NO_RETURN_EXPECTED, s) + return + + if s.expr: + declared_none_return = isinstance(return_type, NoneType) + declared_any_return = isinstance(return_type, AnyType) + + # This controls whether or not we allow a function call that + # returns None as the expression of this return statement. + # E.g. `return f()` for some `f` that returns None. We allow + # this only if we're in a lambda or in a function that returns + # `None` or `Any`. + allow_none_func_call = is_lambda or declared_none_return or declared_any_return + + # Return with a value. + if ( + isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)) + or isinstance(s.expr, AwaitExpr) + and isinstance(s.expr.expr, CallExpr) + ): + # For expressions that (strongly) depend on type context (i.e. those that + # are handled like a function call), we allow fallback to empty type context + # in case of errors, this improves user experience in some cases, + # see e.g. testReturnFallbackInference. + typ = self.infer_context_dependent(s.expr, return_type, allow_none_func_call) + else: + typ = get_proper_type( + self.expr_checker.accept( + s.expr, return_type, allow_none_return=allow_none_func_call + ) + ) + # Treat NotImplemented as having type Any, consistent with its + # definition in typeshed prior to python/typeshed#4222. + if isinstance(typ, Instance) and typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES: + typ = AnyType(TypeOfAny.special_form) + + if defn.is_async_generator: + self.fail(message_registry.RETURN_IN_ASYNC_GENERATOR, s) + return + # Returning a value of type Any is always fine. + if isinstance(typ, AnyType): + # (Unless you asked to be warned in that case, and the + # function is not declared to return Any) + if ( + self.options.warn_return_any + and not self.current_node_deferred + and not is_proper_subtype(AnyType(TypeOfAny.special_form), return_type) + and not ( + defn.name in BINARY_MAGIC_METHODS + and is_literal_not_implemented(s.expr) + ) + and not ( + isinstance(return_type, Instance) + and return_type.type.fullname == "builtins.object" + ) + and not is_lambda + ): + self.msg.incorrectly_returning_any(return_type, s) + return + + # Disallow return expressions in functions declared to return + # None, subject to two exceptions below. + if declared_none_return: + # Lambdas are allowed to have None returns. + # Functions returning a value of type None are allowed to have a None return. + if is_lambda or isinstance(typ, NoneType): + return + self.fail(message_registry.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + subtype_label="got", + subtype=typ, + supertype_label="expected", + supertype=return_type, + context=s.expr, + outer_context=s, + msg=message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE, + ) + else: + # Empty returns are valid in Generators with Any typed returns, but not in + # coroutines. + if ( + defn.is_generator + and not defn.is_coroutine + and isinstance(return_type, AnyType) + ): + return + + if isinstance(return_type, (NoneType, AnyType)): + return + + if self.in_checked_function(): + self.fail(message_registry.RETURN_VALUE_EXPECTED, s) + + def visit_if_stmt(self, s: IfStmt) -> None: + """Type check an if statement.""" + # This frame records the knowledge from previous if/elif clauses not being taken. + # Fall-through to the original frame is handled explicitly in each block. + with self.binder.frame_context(can_skip=False, conditional_frame=True, fall_through=0): + for e, b in zip(s.expr, s.body): + t = get_proper_type(self.expr_checker.accept(e)) + + if isinstance(t, DeletedType): + self.msg.deleted_as_rvalue(t, s) + + if_map, else_map = self.find_isinstance_check(e) + + s.unreachable_else = is_unreachable_map(else_map) + + # XXX Issue a warning if condition is always False? + with self.binder.frame_context(can_skip=True, fall_through=2): + self.push_type_map(if_map, from_assignment=False) + self.accept(b) + + # XXX Issue a warning if condition is always True? + self.push_type_map(else_map, from_assignment=False) + + with self.binder.frame_context(can_skip=False, fall_through=2): + if s.else_body: + self.accept(s.else_body) + + def visit_while_stmt(self, s: WhileStmt) -> None: + """Type check a while statement.""" + if_stmt = IfStmt([s.expr], [s.body], None) + if_stmt.set_line(s) + self.accept_loop(if_stmt, s.else_body, exit_condition=s.expr) + + def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: + """Type check an operator assignment statement, e.g. x += 1.""" + self.try_infer_partial_generic_type_from_assignment(s.lvalue, s.rvalue, s.op) + if isinstance(s.lvalue, MemberExpr): + # Special case, some additional errors may be given for + # assignments to read-only or final attributes. + lvalue_type = self.expr_checker.visit_member_expr(s.lvalue, True) + else: + lvalue_type = self.expr_checker.accept(s.lvalue) + inplace, method = infer_operator_assignment_method(lvalue_type, s.op) + if inplace: + # There is __ifoo__, treat as x = x.__ifoo__(y) + rvalue_type, _ = self.expr_checker.check_op(method, lvalue_type, s.rvalue, s) + if not is_subtype(rvalue_type, lvalue_type): + self.msg.incompatible_operator_assignment(s.op, s) + else: + # There is no __ifoo__, treat as x = x y + expr = OpExpr(s.op, s.lvalue, s.rvalue) + expr.set_line(s) + self.check_assignment(lvalue=s.lvalue, rvalue=expr, infer_lvalue_type=True) + self.check_final(s) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + self.expr_checker.accept(s.expr) + + if isinstance(s.expr, TupleExpr) and len(s.expr.items) > 0: + self.fail(message_registry.MALFORMED_ASSERT, s) + + # If this is asserting some isinstance check, bind that type in the following code + true_map, else_map = self.find_isinstance_check(s.expr) + if s.msg is not None: + self.expr_checker.analyze_cond_branch( + else_map, s.msg, None, suppress_unreachable_errors=False + ) + self.push_type_map(true_map) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + """Type check a raise statement.""" + if s.expr: + self.type_check_raise(s.expr, s) + if s.from_expr: + self.type_check_raise(s.from_expr, s, optional=True) + self.binder.unreachable() + + def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) -> None: + typ = get_proper_type(self.expr_checker.accept(e)) + if isinstance(typ, DeletedType): + self.msg.deleted_as_rvalue(typ, e) + return + + exc_type = self.named_type("builtins.BaseException") + expected_type_items = [exc_type, TypeType(exc_type)] + if optional: + # This is used for `x` part in a case like `raise e from x`, + # where we allow `raise e from None`. + expected_type_items.append(NoneType()) + + self.check_subtype( + typ, UnionType.make_union(expected_type_items), s, message_registry.INVALID_EXCEPTION + ) + + if isinstance(typ, FunctionLike): + # https://github.com/python/mypy/issues/11089 + self.expr_checker.check_call(typ, [], [], e) + + if (isinstance(typ, Instance) and typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES) or ( + isinstance(e, CallExpr) + and isinstance(e.callee, RefExpr) + and e.callee.fullname == "builtins.NotImplemented" + ): + self.fail( + message_registry.INVALID_EXCEPTION.with_additional_msg( + '; did you mean "NotImplementedError"?' + ), + s, + ) + + def visit_try_stmt(self, s: TryStmt) -> None: + """Type check a try statement.""" + + iter_errors = None + + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible states after the try block exited abnormally + # (by exception, return, break, etc.) + with self.binder.frame_context(can_skip=False, fall_through=0): + # Not only might the body of the try statement exit + # abnormally, but so might an exception handler or else + # clause. The finally clause runs in *all* cases, so we + # need an outer try frame to catch all intermediate states + # in case an exception is raised during an except or else + # clause. As an optimization, only create the outer try + # frame when there actually is a finally clause. + self.visit_try_without_finally(s, try_frame=bool(s.finally_body)) + if s.finally_body: + # First we check finally_body is type safe on all abnormal exit paths + iter_errors = IterationDependentErrors() + with IterationErrorWatcher(self.msg.errors, iter_errors): + self.accept(s.finally_body) + + if s.finally_body: + previously_suppressed = self.binder.is_unreachable_warning_suppressed() + # Then we try again for the more restricted set of options + # that can fall through. (Why do we need to check the + # finally clause twice? Depending on whether the finally + # clause was reached by the try clause falling off the end + # or exiting abnormally, after completing the finally clause + # either flow will continue to after the entire try statement + # or the exception/return/etc. will be processed and control + # flow will escape. We need to check that the finally clause + # type checks in both contexts, but only the resulting types + # from the latter context affect the type state in the code + # that follows the try statement.) + assert iter_errors is not None + if not self.binder.is_unreachable(): + with IterationErrorWatcher(self.msg.errors, iter_errors): + self.accept(s.finally_body) + self.msg.iteration_dependent_errors(iter_errors) + + if not previously_suppressed: + # The finally body might have warned about unreachability, + # but we still want anything afterwards to warn too. + self.binder.frames[-1].suppress_unreachable_warnings = False + + def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: + """Type check a try statement, ignoring the finally block. + + On entry, the top frame should receive all flow that exits the + try block abnormally (i.e., such that the else block does not + execute), and its parent should receive all flow that exits + the try block normally. + """ + # This frame will run the else block if the try fell through. + # In that case, control flow continues to the parent of what + # was the top frame on entry. + with self.binder.frame_context(can_skip=False, fall_through=2, try_frame=try_frame): + # This frame receives exit via exception, and runs exception handlers + with self.binder.frame_context(can_skip=False, conditional_frame=True, fall_through=2): + # Finally, the body of the try statement + with self.binder.frame_context(can_skip=False, fall_through=2, try_frame=True): + self.accept(s.body) + for i in range(len(s.handlers)): + with self.binder.frame_context(can_skip=True, fall_through=4): + typ = s.types[i] + if typ: + t = self.check_except_handler_test(typ, s.is_star) + var = s.vars[i] + if var: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + var.is_inferred_def = True + self.check_assignment(var, self.temp_node(t, var)) + self.accept(s.handlers[i]) + var = s.vars[i] + if var: + # Exception variables are deleted. + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + source = var.name + if isinstance(var.node, Var): + new_type = DeletedType(source=source) + var.node.type = new_type + if self.options.allow_redefinition: + # TODO: Should we use put() here? + self.binder.assign_type(var, new_type, new_type) + if not self.options.allow_redefinition: + self.binder.cleanse(var) + if s.else_body: + self.accept(s.else_body) + + def check_except_handler_test(self, n: Expression, is_star: bool) -> Type: + """Type check an exception handler test clause.""" + typ = self.expr_checker.accept(n) + + all_types: list[Type] = [] + test_types = self.get_types_from_except_handler(typ, n) + + for ttype in get_proper_types(test_types): + if isinstance(ttype, AnyType): + all_types.append(ttype) + continue + if isinstance(ttype, UninhabitedType): + continue + + if isinstance(ttype, FunctionLike): + item = ttype.items[0] + if not item.is_type_obj(): + self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) + return self.default_exception_type(is_star) + exc_type = erase_typevars(item.get_instance_type()) + elif isinstance(ttype, TypeType): + exc_type = ttype.item + else: + self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) + return self.default_exception_type(is_star) + + if not is_subtype(exc_type, self.named_type("builtins.BaseException")): + self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) + return self.default_exception_type(is_star) + + all_types.append(exc_type) + + if is_star: + new_all_types: list[Type] = [] + for typ in all_types: + if is_proper_subtype(typ, self.named_type("builtins.BaseExceptionGroup")): + self.fail(message_registry.INVALID_EXCEPTION_GROUP, n) + new_all_types.append(AnyType(TypeOfAny.from_error)) + else: + new_all_types.append(typ) + return self.wrap_exception_group(new_all_types) + return make_simplified_union(all_types) + + def default_exception_type(self, is_star: bool) -> Type: + """Exception type to return in case of a previous type error.""" + any_type = AnyType(TypeOfAny.from_error) + if is_star: + return self.named_generic_type("builtins.ExceptionGroup", [any_type]) + return any_type + + def wrap_exception_group(self, types: Sequence[Type]) -> Type: + """Transform except* variable type into an appropriate exception group.""" + arg = make_simplified_union(types) + if is_subtype(arg, self.named_type("builtins.Exception")): + base = "builtins.ExceptionGroup" + else: + base = "builtins.BaseExceptionGroup" + return self.named_generic_type(base, [arg]) + + def get_types_from_except_handler(self, typ: Type, n: Expression) -> list[Type]: + """Helper for check_except_handler_test to retrieve handler types.""" + typ = get_proper_type(typ) + if isinstance(typ, TupleType): + merged_type = make_simplified_union(typ.items) + if isinstance(merged_type, UnionType): + return merged_type.relevant_items() + return [merged_type] + elif is_named_instance(typ, "builtins.tuple"): + # variadic tuple + merged_type = make_simplified_union((typ.args[0],)) + if isinstance(merged_type, UnionType): + return merged_type.relevant_items() + return [merged_type] + elif isinstance(typ, UnionType): + return [ + union_typ + for item in typ.relevant_items() + for union_typ in self.get_types_from_except_handler(item, n) + ] + else: + return [typ] + + def visit_for_stmt(self, s: ForStmt) -> None: + """Type check a for statement.""" + if s.is_async: + iterator_type, item_type = self.analyze_async_iterable_item_type(s.expr) + else: + iterator_type, item_type = self.analyze_iterable_item_type(s.expr) + s.inferred_item_type = item_type + s.inferred_iterator_type = iterator_type + + self.accept_loop( + s.body, + s.else_body, + on_enter_body=lambda: self.analyze_index_variables( + s.index, item_type, s.index_type is None, s + ), + ) + + def analyze_async_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: + """Analyse async iterable expression and return iterator and iterator item types.""" + echk = self.expr_checker + iterable = echk.accept(expr) + iterator = echk.check_method_call_by_name("__aiter__", iterable, [], [], expr)[0] + awaitable = echk.check_method_call_by_name("__anext__", iterator, [], [], expr)[0] + item_type = echk.check_awaitable_expr( + awaitable, expr, message_registry.INCOMPATIBLE_TYPES_IN_ASYNC_FOR + ) + return iterator, item_type + + def analyze_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: + """Analyse iterable expression and return iterator and iterator item types.""" + iterator, iterable = self.analyze_iterable_item_type_without_expression( + self.expr_checker.accept(expr), context=expr + ) + int_type = self.analyze_range_native_int_type(expr) + if int_type: + return iterator, int_type + return iterator, iterable + + def analyze_iterable_item_type_without_expression( + self, type: Type, context: Context + ) -> tuple[Type, Type]: + """Analyse iterable type and return iterator and iterator item types.""" + echk = self.expr_checker + iterable: Type + iterable = get_proper_type(type) + iterator = echk.check_method_call_by_name("__iter__", iterable, [], [], context)[0] + + if ( + isinstance(iterable, TupleType) + and iterable.partial_fallback.type.fullname == "builtins.tuple" + ): + return iterator, tuple_fallback(iterable).args[0] + else: + # Non-tuple iterable. + iterable = echk.check_method_call_by_name("__next__", iterator, [], [], context)[0] + return iterator, iterable + + def analyze_range_native_int_type(self, expr: Expression) -> Type | None: + """Try to infer native int item type from arguments to range(...). + + For example, return i64 if the expression is "range(0, i64(n))". + + Return None if unsuccessful. + """ + if ( + isinstance(expr, CallExpr) + and isinstance(expr.callee, RefExpr) + and expr.callee.fullname == "builtins.range" + and 1 <= len(expr.args) <= 3 + and all(kind == ARG_POS for kind in expr.arg_kinds) + ): + native_int: Type | None = None + ok = True + for arg in expr.args: + argt = get_proper_type(self.lookup_type(arg)) + if isinstance(argt, Instance) and argt.type.fullname in MYPYC_NATIVE_INT_NAMES: + if native_int is None: + native_int = argt + elif argt != native_int: + ok = False + if ok and native_int: + return native_int + return None + + def analyze_container_item_type(self, typ: Type) -> Type | None: + """Check if a type is a nominal container of a union of such. + + Return the corresponding container item type. + """ + typ = get_proper_type(typ) + if isinstance(typ, UnionType): + types: list[Type] = [] + for item in typ.items: + c_type = self.analyze_container_item_type(item) + if c_type: + types.append(c_type) + return UnionType.make_union(types) + if isinstance(typ, Instance) and typ.type.has_base("typing.Container"): + supertype = self.named_type("typing.Container").type + super_instance = map_instance_to_supertype(typ, supertype) + assert len(super_instance.args) == 1 + return super_instance.args[0] + if isinstance(typ, TupleType): + return self.analyze_container_item_type(tuple_fallback(typ)) + return None + + def analyze_index_variables( + self, index: Expression, item_type: Type, infer_lvalue_type: bool, context: Context + ) -> None: + """Type check or infer for loop or list comprehension index vars.""" + self.check_assignment(index, self.temp_node(item_type, context), infer_lvalue_type) + + def visit_del_stmt(self, s: DelStmt) -> None: + if isinstance(s.expr, IndexExpr): + e = s.expr + m = MemberExpr(e.base, "__delitem__") + m.line = s.line + m.column = s.column + c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) + c.line = s.line + c.column = s.column + self.expr_checker.accept(c, allow_none_return=True) + else: + s.expr.accept(self.expr_checker) + for elt in flatten(s.expr): + if isinstance(elt, NameExpr): + self.binder.assign_type( + elt, DeletedType(source=elt.name), get_declaration(elt) + ) + + def visit_decorator(self, e: Decorator) -> None: + if any(refers_to_fullname(d, "typing.no_type_check") for d in e.decorators): + e.var.type = AnyType(TypeOfAny.special_form) + e.var.is_ready = True + return + self.visit_decorator_inner(e) + + def visit_decorator_inner( + self, e: Decorator, allow_empty: bool = False, skip_first_item: bool = False + ) -> None: + # Fix the type if decorated with `@types.coroutine` or `@asyncio.coroutine`. + defn = e.func + if defn.is_awaitable_coroutine: + typ = self.function_type(defn) + assert isinstance(typ, CallableType) + # Update the return type to AwaitableGenerator (unless we already did). + # Note, this doesn't exist in typing.py, only in typing.pyi. + if not is_named_instance(typ.ret_type, "typing.AwaitableGenerator"): + t = typ.ret_type + c = defn.is_coroutine + ty = self.get_generator_yield_type(t, c) + tc = self.get_generator_receive_type(t, c) + if c: + tr = self.get_coroutine_return_type(t) + else: + tr = self.get_generator_return_type(t, c) + ret_type = self.named_generic_type("typing.AwaitableGenerator", [ty, tc, tr, t]) + defn.type = typ.copy_modified(ret_type=ret_type) + + # Type check initialization expressions as part of top-level. + if not self.can_skip_diagnostics: + self.check_default_params(defn) + + if self.recurse_into_functions or e.func.def_or_infer_vars: + with self.tscope.function_scope(e.func), self.set_recurse_into_functions(): + self.check_func_item(e.func, name=e.func.name, allow_empty=allow_empty) + + # Process decorators from the inside out to determine decorated signature, which + # may be different from the declared signature. + sig: Type = self.function_type(e.func) + non_trivial_decorator = False + # For settable properties skip the first decorator (that is @foo.setter). + for d in reversed(e.decorators[1:] if skip_first_item else e.decorators): + if refers_to_fullname(d, "abc.abstractmethod"): + # This is a hack to avoid spurious errors because of incomplete type + # of @abstractmethod in the test fixtures. + continue + if refers_to_fullname(d, OVERLOAD_NAMES): + if not allow_empty: + self.fail(message_registry.MULTIPLE_OVERLOADS_REQUIRED, e) + continue + non_trivial_decorator = True + dec = self.expr_checker.accept(d) + temp = self.temp_node(sig, context=d) + fullname = None + if isinstance(d, RefExpr): + fullname = d.fullname or None + # if this is an expression like @b.a where b is an object, get the type of b, + # so we can pass it the method hook in the plugins + object_type: Type | None = None + if fullname is None and isinstance(d, MemberExpr) and self.has_type(d.expr): + object_type = self.lookup_type(d.expr) + fullname = self.expr_checker.method_fullname(object_type, d.name) + self.check_for_untyped_decorator(e.func, dec, d) + sig, t2 = self.expr_checker.check_call( + dec, [temp], [nodes.ARG_POS], e, callable_name=fullname, object_type=object_type + ) + if non_trivial_decorator: + self.check_untyped_after_decorator(sig, e.func) + self.require_correct_self_argument(sig, e.func) + sig = set_callable_name(sig, e.func) + if isinstance(sig, CallableType): + sig.definition = e + e.var.type = sig + e.var.is_ready = True + if e.func.is_property: + if isinstance(sig, CallableType): + if len([k for k in sig.arg_kinds if k.is_required()]) > 1: + self.msg.fail("Too many arguments for property", e) + self.check_incompatible_property_override(e) + # For overloaded functions/properties we already checked override for overload as a whole. + if allow_empty or skip_first_item: + return + if e.func.info and not e.is_overload: + found_method_base_classes = self.check_method_override(e) + if ( + e.func.is_explicit_override + and not found_method_base_classes + and found_method_base_classes is not None + # If the class has Any fallback, we can't be certain that a method + # is really missing - it might come from unfollowed import. + and not e.func.info.fallback_to_any + ): + self.msg.no_overridable_method(e.func.name, e.func) + self.check_explicit_override_decorator(e.func, found_method_base_classes) + + if e.func.info and e.func.name in ("__init__", "__new__"): + if e.type and not isinstance(get_proper_type(e.type), (FunctionLike, AnyType)): + self.fail(message_registry.BAD_CONSTRUCTOR_TYPE, e) + + if e.func.original_def and isinstance(sig, FunctionLike): + # Function definition overrides function definition. + self.check_func_def_override(e.func, sig) + + def check_for_untyped_decorator( + self, func: FuncDef, dec_type: Type, dec_expr: Expression + ) -> None: + if ( + self.options.disallow_untyped_decorators + and is_typed_callable(func.type) + and is_untyped_decorator(dec_type) + and not self.current_node_deferred + ): + self.msg.typed_function_untyped_decorator(func.name, dec_expr) + + def check_incompatible_property_override(self, e: Decorator) -> None: + if not e.var.is_settable_property and e.func.info: + name = e.func.name + for base in e.func.info.mro[1:]: + base_attr = base.names.get(name) + if not base_attr: + continue + if ( + isinstance(base_attr.node, OverloadedFuncDef) + and base_attr.node.is_property + and cast(Decorator, base_attr.node.items[0]).var.is_settable_property + ): + self.fail(message_registry.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) + + def visit_with_stmt(self, s: WithStmt) -> None: + exceptions_maybe_suppressed = False + for expr, target in zip(s.expr, s.target): + if s.is_async: + exit_ret_type = self.check_async_with_item(expr, target, s.unanalyzed_type is None) + else: + exit_ret_type = self.check_with_item(expr, target, s.unanalyzed_type is None) + + # Based on the return type, determine if this context manager 'swallows' + # exceptions or not. We determine this using a heuristic based on the + # return type of the __exit__ method -- see the discussion in + # https://github.com/python/mypy/issues/7214 and the section about context managers + # in https://github.com/python/typeshed/blob/main/CONTRIBUTING.md#conventions + # for more details. + + exit_ret_type = get_proper_type(exit_ret_type) + if is_literal_type(exit_ret_type, "builtins.bool", False): + continue + + if is_literal_type(exit_ret_type, "builtins.bool", True) or ( + isinstance(exit_ret_type, Instance) + and exit_ret_type.type.fullname == "builtins.bool" + and state.strict_optional + ): + # Note: if strict-optional is disabled, this bool instance + # could actually be an Optional[bool]. + exceptions_maybe_suppressed = True + + if exceptions_maybe_suppressed: + # Treat this 'with' block in the same way we'd treat a 'try: BODY; except: pass' + # block. This means control flow can continue after the 'with' even if the 'with' + # block immediately returns. + with self.binder.frame_context(can_skip=True, try_frame=True): + self.accept(s.body) + else: + self.accept(s.body) + + def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None: + if not self.options.disallow_any_decorated or self.is_stub or self.current_node_deferred: + return + + if mypy.checkexpr.has_any_type(typ): + self.msg.untyped_decorated_function(typ, func) + + def check_async_with_item( + self, expr: Expression, target: Expression | None, infer_lvalue_type: bool + ) -> Type: + echk = self.expr_checker + ctx = echk.accept(expr) + obj = echk.check_method_call_by_name("__aenter__", ctx, [], [], expr)[0] + obj = echk.check_awaitable_expr( + obj, expr, message_registry.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AENTER + ) + if target: + self.check_assignment(target, self.temp_node(obj, expr), infer_lvalue_type) + arg = self.temp_node(AnyType(TypeOfAny.special_form), expr) + res, _ = echk.check_method_call_by_name( + "__aexit__", ctx, [arg] * 3, [nodes.ARG_POS] * 3, expr + ) + return echk.check_awaitable_expr( + res, expr, message_registry.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AEXIT + ) + + def check_with_item( + self, expr: Expression, target: Expression | None, infer_lvalue_type: bool + ) -> Type: + echk = self.expr_checker + ctx = echk.accept(expr) + obj = echk.check_method_call_by_name("__enter__", ctx, [], [], expr)[0] + if target: + self.check_assignment(target, self.temp_node(obj, expr), infer_lvalue_type) + arg = self.temp_node(AnyType(TypeOfAny.special_form), expr) + res, _ = echk.check_method_call_by_name( + "__exit__", ctx, [arg] * 3, [nodes.ARG_POS] * 3, expr + ) + return res + + def visit_break_stmt(self, s: BreakStmt) -> None: + self.binder.handle_break() + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + self.binder.handle_continue() + return + + def visit_match_stmt(self, s: MatchStmt) -> None: + # In sync with similar actions elsewhere, narrow the target if + # we are matching an AssignmentExpr + unwrapped_subject = collapse_walrus(s.subject) + named_subject = self._make_named_statement_for_match(s, unwrapped_subject) + with self.binder.frame_context(can_skip=False, fall_through=0): + subject_type = get_proper_type(self.expr_checker.accept(s.subject)) + + if isinstance(subject_type, DeletedType): + self.msg.deleted_as_rvalue(subject_type, s) + + # We infer types of patterns twice. The first pass is used + # to infer the types of capture variables. The type of a + # capture variable may depend on multiple patterns (it + # will be a union of all capture types). This pass ignores + # guard expressions. + pattern_types = [self.pattern_checker.accept(p, subject_type) for p in s.patterns] + type_maps: list[TypeMap] = [t.captures for t in pattern_types] + inferred_types = self.infer_variable_types_from_type_maps(type_maps) + + # The second pass narrows down the types and type checks bodies. + unmatched_types: TypeMap = {s.subject: UninhabitedType()} + for p, g, b in zip(s.patterns, s.guards, s.bodies): + current_subject_type = self.expr_checker.narrow_type_from_binder( + named_subject, subject_type + ) + pattern_type = self.pattern_checker.accept(p, current_subject_type) + with self.binder.frame_context(can_skip=True, fall_through=2): + pattern_map, else_map = conditional_types_to_typemaps( + named_subject, pattern_type.type, pattern_type.rest_type + ) + # Maybe the subject type can be inferred from constraints on + # its attribute/item? + if named_subject in pattern_map: + pattern_map[unwrapped_subject] = pattern_map[named_subject] + if named_subject in else_map: + else_map[unwrapped_subject] = else_map[named_subject] + pattern_map = self.propagate_up_typemap_info(pattern_map) + else_map = self.propagate_up_typemap_info(else_map) + self.check_and_remove_capture_conflicts(pattern_type.captures, inferred_types) + self.push_type_map(pattern_map, from_assignment=False) + if pattern_map: + for expr, typ in pattern_map.items(): + self.push_type_map( + self._get_recursive_sub_patterns_map(expr, typ), + from_assignment=False, + ) + self.push_type_map(pattern_type.captures, from_assignment=False) + if g is not None: + with self.binder.frame_context(can_skip=False, fall_through=3): + gt = get_proper_type(self.expr_checker.accept(g)) + + if isinstance(gt, DeletedType): + self.msg.deleted_as_rvalue(gt, s) + + guard_map, guard_else_map = self.find_isinstance_check(g) + else_map = or_conditional_maps(else_map, guard_else_map) + + # If the guard narrowed the subject, copy the narrowed types over + if isinstance(p, AsPattern): + case_target = p.pattern or p.name + if isinstance(case_target, NameExpr): + for type_map in (guard_map, else_map): + for expr in list(type_map): + if not ( + isinstance(expr, NameExpr) + and expr.fullname == case_target.fullname + ): + continue + type_map[named_subject] = type_map[expr] + + self.push_type_map(guard_map, from_assignment=False) + self.accept(b) + else: + self.accept(b) + self.push_type_map(else_map, from_assignment=False) + unmatched_types = else_map + + if not is_unreachable_map(unmatched_types) and not self.current_node_deferred: + for typ in unmatched_types.values(): + self.msg.match_statement_inexhaustive_match(typ, s) + + # This is needed due to a quirk in frame_context. Without it types will stay narrowed + # after the match. + with self.binder.frame_context(can_skip=False, fall_through=2): + pass + + def _make_named_statement_for_match(self, s: MatchStmt, subject: Expression) -> Expression: + """Construct a fake NameExpr for inference if a match clause is complex.""" + if self.binder.can_put_directly(subject): + # Already named - we should infer type of it as given + return subject + elif s.subject_dummy is not None: + return s.subject_dummy + else: + # Create a dummy subject expression to handle cases where a match statement's subject + # is not a literal value. This lets us correctly narrow types and check exhaustivity + # This is hack! + name = self.new_unique_dummy_name("match") + v = Var(name) + named_subject = NameExpr(name) + named_subject.node = v + s.subject_dummy = named_subject + return named_subject + + def _get_recursive_sub_patterns_map( + self, expr: Expression, typ: Type + ) -> dict[Expression, Type]: + sub_patterns_map: dict[Expression, Type] = {} + typ_ = get_proper_type(typ) + if isinstance(expr, TupleExpr) and isinstance(typ_, TupleType): + if any(isinstance(item, StarExpr) for item in expr.items): + # For a starred item (e.g. `*bar`), there's no one-to-one correspondence + # between expr.items and typ_.items, we are returning early to avoid an + # assertion crash. See: https://github.com/python/mypy/issues/21468. + return sub_patterns_map + + # When matching a tuple expression with a sequence pattern, narrow individual tuple items + assert len(expr.items) == len(typ_.items) + for item_expr, item_typ in zip(expr.items, typ_.items): + sub_patterns_map[item_expr] = item_typ + sub_patterns_map.update(self._get_recursive_sub_patterns_map(item_expr, item_typ)) + + return sub_patterns_map + + def infer_variable_types_from_type_maps( + self, type_maps: list[TypeMap] + ) -> dict[SymbolNode, Type]: + # Type maps may contain variables inherited from previous code which are not + # necessary `Var`s (e.g. a function defined earlier with the same name). + all_captures: dict[SymbolNode, list[tuple[NameExpr, Type]]] = defaultdict(list) + for tm in type_maps: + if not is_unreachable_map(tm): + for expr, typ in tm.items(): + if isinstance(expr, NameExpr): + node = expr.node + assert node is not None + all_captures[node].append((expr, typ)) + + inferred_types: dict[SymbolNode, Type] = {} + for var, captures in all_captures.items(): + already_exists = False + types: list[Type] = [] + for expr, typ in captures: + types.append(typ) + + previous_type, _, _ = self.check_lvalue(expr) + if previous_type is not None: + already_exists = True + if isinstance(expr.node, Var) and expr.node.is_final: + self.msg.cant_assign_to_final(expr.name, False, expr) + # We'll check compatibility in check_and_remove_capture_conflicts + inferred_types[var] = previous_type + + if not already_exists: + new_type = UnionType.make_union(types) + # Infer the union type at the first occurrence + first_occurrence, _ = captures[0] + # If it didn't exist before ``match``, it's a Var. + assert isinstance(var, Var) + inferred_types[var] = new_type + self.infer_variable_type(var, first_occurrence, new_type, first_occurrence) + return inferred_types + + def check_and_remove_capture_conflicts( + self, type_map: TypeMap, inferred_types: dict[SymbolNode, Type] + ) -> None: + if is_unreachable_map(type_map): + return + for expr, typ in list(type_map.items()): + if not isinstance(expr, NameExpr): + continue + node = expr.node + if node not in inferred_types or not self.check_subtype( + typ, + inferred_types[node], + expr, + msg=message_registry.INCOMPATIBLE_TYPES_IN_CAPTURE, + subtype_label="pattern captures type", + supertype_label="variable has type", + ): + del type_map[expr] + + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + if o.alias_node: + self.check_typevar_defaults(o.alias_node.alias_tvars) + + with self.msg.filter_errors(): + self.expr_checker.accept(o.value) + + def make_fake_typeinfo( + self, + curr_module_fullname: str, + class_gen_name: str, + class_short_name: str, + bases: list[Instance], + ) -> tuple[ClassDef, TypeInfo]: + # Build the fake ClassDef and TypeInfo together. + # The ClassDef is full of lies and doesn't actually contain a body. + # Use format_bare to generate a nice name for error messages. + # We skip fully filling out a handful of TypeInfo fields because they + # should be irrelevant for a generated type like this: + # is_protocol, protocol_members, is_abstract + cdef = ClassDef(class_short_name, Block([])) + cdef.fullname = curr_module_fullname + "." + class_gen_name + info = TypeInfo(SymbolTable(), cdef, curr_module_fullname) + cdef.info = info + info.bases = bases + calculate_mro(info) + info.metaclass_type = info.calculate_metaclass_type() + return cdef, info + + def intersect_instances( + self, instances: tuple[Instance, Instance], errors: list[tuple[str, str]] + ) -> Instance | None: + """Try creating an ad-hoc intersection of the given instances. + + Note that this function does *not* try and create a full-fledged + intersection type. Instead, it returns an instance of a new ad-hoc + subclass of the given instances. + + This is mainly useful when you need a way of representing some + theoretical subclass of the instances the user may be trying to use + the generated intersection can serve as a placeholder. + + This function will create a fresh subclass the first time you call it. + So this means calling `self.intersect_intersection([inst_1, inst_2], ctx)` + twice will return the same subclass of inst_1 and inst_2. + + Returns None if creating the subclass is impossible (e.g. due to + MRO errors or incompatible signatures). If we do successfully create + a subclass, its TypeInfo will automatically be added to the global scope. + """ + curr_module = self.scope.stack[0] + assert isinstance(curr_module, MypyFile) + + # First, retry narrowing while allowing promotions (they are disabled by default + # for isinstance() checks, etc). This way we will still type-check branches like + # x: complex = 1 + # if isinstance(x, int): + # ... + left, right = instances + if is_proper_subtype(left, right, ignore_promotions=False): + return left + if is_proper_subtype(right, left, ignore_promotions=False): + return right + + def _get_base_classes(instances_: tuple[Instance, Instance]) -> list[Instance]: + base_classes_ = [] + for inst in instances_: + if inst.type.is_intersection: + expanded = inst.type.bases + else: + expanded = [inst] + + for expanded_inst in expanded: + base_classes_.append(expanded_inst) + return base_classes_ + + def _make_fake_typeinfo_and_full_name( + base_classes_: list[Instance], curr_module_: MypyFile, options: Options + ) -> tuple[TypeInfo, str]: + names = [format_type_bare(x, options=options, verbosity=2) for x in base_classes_] + name = f"" + if (symbol := curr_module_.names.get(name)) is not None: + assert isinstance(symbol.node, TypeInfo) + return symbol.node, name + cdef, info_ = self.make_fake_typeinfo(curr_module_.fullname, name, name, base_classes_) + return info_, name + + base_classes = _get_base_classes(instances) + # We use the pretty_names_list for error messages but for the real name that goes + # into the symbol table because it is not specific enough. + pretty_names_list = pretty_seq( + format_type_distinctly(*base_classes, options=self.options, bare=True), "and" + ) + + if not can_have_shared_disjoint_base(base_classes): + errors.append((pretty_names_list, "have distinct disjoint bases")) + return None + + new_errors = [] + for base in base_classes: + if base.type.is_final: + new_errors.append((pretty_names_list, f'"{base.type.name}" is final')) + if new_errors: + errors.extend(new_errors) + return None + + try: + info, full_name = _make_fake_typeinfo_and_full_name( + base_classes, curr_module, self.options + ) + with self.msg.filter_errors() as local_errors: + self.check_multiple_inheritance(info) + if local_errors.has_new_errors(): + # "class A(B, C)" unsafe, now check "class A(C, B)": + base_classes = _get_base_classes(instances[::-1]) + info, full_name = _make_fake_typeinfo_and_full_name( + base_classes, curr_module, self.options + ) + with self.msg.filter_errors() as local_errors: + self.check_multiple_inheritance(info) + info.is_intersection = True + except MroError: + errors.append((pretty_names_list, "would have inconsistent method resolution order")) + return None + if local_errors.has_new_errors(): + errors.append((pretty_names_list, "would have incompatible method signatures")) + return None + + curr_module.names[full_name] = SymbolTableNode(GDEF, info, False, module_hidden=True) + return Instance(info, [], extra_attrs=instances[0].extra_attrs or instances[1].extra_attrs) + + def intersect_instance_callable(self, typ: Instance, callable_type: CallableType) -> Instance: + """Creates a fake type that represents the intersection of an Instance and a CallableType. + + It operates by creating a bare-minimum dummy TypeInfo that + subclasses type and adds a __call__ method matching callable_type. + """ + + # In order for this to work in incremental mode, the type we generate needs to + # have a valid fullname and a corresponding entry in a symbol table. We generate + # a unique name inside the symbol table of the current module. + cur_module = self.scope.stack[0] + assert isinstance(cur_module, MypyFile) + gen_name = gen_unique_name(f"", cur_module.names) + + # Synthesize a fake TypeInfo + short_name = format_type_bare(typ, self.options) + cdef, info = self.make_fake_typeinfo(cur_module.fullname, gen_name, short_name, [typ]) + + # Build up a fake FuncDef, so we can populate the symbol table. + func_def = FuncDef("__call__", [], Block([]), callable_type) + func_def._fullname = cdef.fullname + ".__call__" + func_def.info = info + sym = SymbolTableNode(MDEF, func_def) + sym.plugin_generated = True + info.names["__call__"] = sym + + cur_module.names[gen_name] = SymbolTableNode(GDEF, info) + + return Instance(info, [], extra_attrs=typ.extra_attrs) + + def make_fake_callable(self, typ: Instance) -> Instance: + """Produce a new type that makes type Callable with a generic callable type.""" + + fallback = self.named_type("builtins.function") + callable_type = CallableType( + [AnyType(TypeOfAny.explicit), AnyType(TypeOfAny.explicit)], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(TypeOfAny.explicit), + fallback=fallback, + is_ellipsis_args=True, + ) + + return self.intersect_instance_callable(typ, callable_type) + + def partition_by_callable( + self, typ: Type, unsound_partition: bool + ) -> tuple[list[Type], list[Type]]: + """Partitions a type into callable subtypes and uncallable subtypes. + + Thus, given: + `callables, uncallables = partition_by_callable(type)` + + If we assert `callable(type)` then `type` has type Union[*callables], and + If we assert `not callable(type)` then `type` has type Union[*uncallables] + + If unsound_partition is set, assume that anything that is not + clearly callable is in fact not callable. Otherwise we generate a + new subtype that *is* callable. + + Guaranteed to not return [], []. + """ + typ = get_proper_type(typ) + + if isinstance(typ, (FunctionLike, TypeType)): + return [typ], [] + + if isinstance(typ, AnyType): + return [typ], [typ] + + if isinstance(typ, NoneType): + return [], [typ] + + if isinstance(typ, UnionType): + callables = [] + uncallables: list[Type] = [] + for subtype in typ.items: + # Use unsound_partition when handling unions in order to + # allow the expected type discrimination. + subcallables, subuncallables = self.partition_by_callable( + subtype, unsound_partition=True + ) + callables.extend(subcallables) + uncallables.extend(subuncallables) + return callables, uncallables + + if isinstance(typ, TypeVarType): + # We could do better probably? + # Refine the type variable's bound as our type in the case that + # callable() is true. This unfortunately loses the information that + # the type is a type variable in that branch. + # This matches what is done for isinstance, but it may be possible to + # do better. + # If it is possible for the false branch to execute, return the original + # type to avoid losing type information. + callables, uncallables = self.partition_by_callable( + erase_to_union_or_bound(typ), unsound_partition + ) + uncallables = [typ] if uncallables else [] + return callables, uncallables + + # A TupleType is callable if its fallback is, but needs special handling + # when we dummy up a new type. + ityp = typ + if isinstance(typ, TupleType): + ityp = tuple_fallback(typ) + + if isinstance(ityp, Instance): + method = ityp.type.get_method("__call__") + if method and method.type: + callables, uncallables = self.partition_by_callable( + method.type, unsound_partition=False + ) + if callables and not uncallables: + # Only consider the type callable if its __call__ method is + # definitely callable. + return [typ], [] + + if not unsound_partition: + fake = self.make_fake_callable(ityp) + if isinstance(typ, TupleType): + fake.type.tuple_type = TupleType(typ.items, fake) + return [fake.type.tuple_type], [typ] + return [fake], [typ] + + if unsound_partition: + return [], [typ] + else: + # We don't know how properly make the type callable. + return [typ], [typ] + + def conditional_callable_type_map( + self, expr: Expression, current_type: Type | None + ) -> tuple[TypeMap, TypeMap]: + """Takes in an expression and the current type of the expression. + + Returns a 2-tuple: The first element is a map from the expression to + the restricted type if it were callable. The second element is a + map from the expression to the type it would hold if it weren't + callable. + """ + if not current_type: + return {}, {} + + if isinstance(get_proper_type(current_type), AnyType): + return {}, {} + + callables, uncallables = self.partition_by_callable(current_type, unsound_partition=False) + + if callables and uncallables: + callable_map = {expr: UnionType.make_union(callables)} + uncallable_map = {expr: UnionType.make_union(uncallables)} + return callable_map, uncallable_map + + elif callables: + return {}, {expr: UninhabitedType()} + + return {expr: UninhabitedType()}, {} + + def conditional_types_for_iterable( + self, item_type: Type, iterable_type: Type + ) -> tuple[Type, Type]: + """ + Narrows the type of `iterable_type` based on the type of `item_type`. + For now, we only support narrowing unions of TypedDicts, and TypeVars with TypedDict + bounds, based on left operand being literal string(s). + """ + if_types: list[Type] = [] + else_types: list[Type] = [] + + iterable_type = get_proper_type(iterable_type) + if isinstance(iterable_type, UnionType): + possible_iterable_types = get_proper_types(iterable_type.relevant_items()) + else: + possible_iterable_types = [iterable_type] + + item_str_literals = try_getting_str_literals_from_type(item_type) + + for possible_iterable_type in possible_iterable_types: + bound = ( + get_proper_type(possible_iterable_type.upper_bound) + if isinstance(possible_iterable_type, TypeVarType) + else possible_iterable_type + ) + + if item_str_literals and isinstance(bound, TypedDictType): + for key in item_str_literals: + if key in bound.required_keys: + if_types.append(possible_iterable_type) + elif key in bound.items and isinstance( + get_proper_type(bound.items[key]), UninhabitedType + ): + # If an item is explicitly declared uninhabited, we can exclude it from + # if_types; see testOperatorContainsNarrowsTypedDicts_closed + else_types.append(possible_iterable_type) + elif key not in bound.items and (bound.is_closed or bound.is_final): + # If an item is missing and the type is closed, we can exclude it from + # if_types; see testOperatorContainsNarrowsTypedDicts_closed + # We also support "final" as a legacy way of expressing "closed" in this + # specific case; see testOperatorContainsNarrowsTypedDicts_final + else_types.append(possible_iterable_type) + else: + if_types.append(possible_iterable_type) + else_types.append(possible_iterable_type) + else: + if_types.append(possible_iterable_type) + else_types.append(possible_iterable_type) + + return UnionType.make_union(if_types), UnionType.make_union(else_types) + + def _is_truthy_type(self, t: ProperType) -> bool: + return ( + ( + isinstance(t, Instance) + and bool(t.type) + and not t.type.has_readable_member("__bool__") + and not t.type.has_readable_member("__len__") + and t.type.fullname != "builtins.object" + ) + or isinstance(t, FunctionLike) + or ( + isinstance(t, UnionType) + and all(self._is_truthy_type(t) for t in get_proper_types(t.items)) + ) + ) + + def check_for_truthy_type(self, t: Type, expr: Expression) -> None: + """ + Check if a type can have a truthy value. + + Used in checks like:: + + if x: # <--- + + not x # <--- + """ + if not state.strict_optional: + return # if everything can be None, all bets are off + + t = get_proper_type(t) + if not self._is_truthy_type(t): + return + + def format_expr_type() -> str: + typ = format_type(t, self.options) + if isinstance(expr, MemberExpr): + return f'Member "{expr.name}" has type {typ}' + elif isinstance(expr, RefExpr) and expr.fullname: + return f'"{expr.fullname}" has type {typ}' + elif isinstance(expr, CallExpr): + if isinstance(expr.callee, MemberExpr): + return f'"{expr.callee.name}" returns {typ}' + elif isinstance(expr.callee, RefExpr) and expr.callee.fullname: + return f'"{expr.callee.fullname}" returns {typ}' + return f"Call returns {typ}" + else: + return f"Expression has type {typ}" + + def get_expr_name() -> str: + if isinstance(expr, (NameExpr, MemberExpr)): + return f'"{expr.name}"' + else: + # return type if expr has no name + return format_type(t, self.options) + + if isinstance(t, FunctionLike): + self.fail(message_registry.FUNCTION_ALWAYS_TRUE.format(get_expr_name()), expr) + elif isinstance(t, UnionType): + self.fail(message_registry.TYPE_ALWAYS_TRUE_UNIONTYPE.format(format_expr_type()), expr) + elif isinstance(t, Instance) and t.type.fullname == "typing.Iterable": + _, info = self.make_fake_typeinfo("typing", "Collection", "Collection", []) + self.fail( + message_registry.ITERABLE_ALWAYS_TRUE.format( + format_expr_type(), format_type(Instance(info, t.args), self.options) + ), + expr, + ) + else: + self.fail(message_registry.TYPE_ALWAYS_TRUE.format(format_expr_type()), expr) + + def find_isinstance_check( + self, node: Expression, *, in_boolean_context: bool = True + ) -> tuple[TypeMap, TypeMap]: + """Find any isinstance checks (within a chain of ands). Includes + implicit and explicit checks for None and calls to callable. + Also includes TypeGuard and TypeIs functions. + + Return value is a map of variables to their types if the condition + is true and a map of variables to their types if the condition is false. + + If either of the values in the tuple is None, then that particular + branch can never occur. + + If `in_boolean_context=True` is passed, it means that we handle + a walrus expression. We treat rhs values + in expressions like `(a := A())` specially: + for example, some errors are suppressed. + + May return {}, {}. + Can return None, None in situations involving NoReturn. + """ + if_map, else_map = self.find_isinstance_check_helper( + node, in_boolean_context=in_boolean_context + ) + new_if_map = self.propagate_up_typemap_info(if_map) + new_else_map = self.propagate_up_typemap_info(else_map) + return new_if_map, new_else_map + + def find_isinstance_check_helper( + self, node: Expression, *, in_boolean_context: bool = True + ) -> tuple[TypeMap, TypeMap]: + if is_true_literal(node): + return {}, {node: UninhabitedType()} + if is_false_literal(node): + return {node: UninhabitedType()}, {} + + if isinstance(node, CallExpr) and len(node.args) != 0: + expr = collapse_walrus(node.args[0]) + if refers_to_fullname(node.callee, "builtins.isinstance"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE: + return conditional_types_to_typemaps( + expr, + *self.conditional_types_with_intersection( + self.lookup_type(expr), self.get_isinstance_type(node.args[1]), expr + ), + ) + elif refers_to_fullname(node.callee, "builtins.issubclass"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE: + return self.infer_issubclass_maps(node, expr) + elif refers_to_fullname(node.callee, "builtins.callable"): + if len(node.args) != 1: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE: + vartype = self.lookup_type(expr) + return self.conditional_callable_type_map(expr, vartype) + elif refers_to_fullname(node.callee, "builtins.hasattr"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + attr = try_getting_str_literals(node.args[1], self.lookup_type(node.args[1])) + if literal(expr) == LITERAL_TYPE and attr and len(attr) == 1: + return self.hasattr_type_maps(expr, self.lookup_type(expr), attr[0]) + else: + type_is, type_guard = None, None + called_type = self.lookup_type_or_none(node.callee) + if called_type is not None: + called_type = get_proper_type(called_type) + # TODO: there are some more cases in check_call() to handle. + # If the callee is an instance, try to extract TypeGuard/TypeIs from its __call__ method. + if isinstance(called_type, Instance): + call = find_member("__call__", called_type, called_type, is_operator=True) + if call is not None: + called_type = get_proper_type(call) + if isinstance(called_type, CallableType): + type_is, type_guard = called_type.type_is, called_type.type_guard + + # If the callee is a RefExpr, extract TypeGuard/TypeIs directly. + if isinstance(node.callee, RefExpr): + type_is, type_guard = node.callee.type_is, node.callee.type_guard + if type_guard is not None or type_is is not None: + # TODO: Follow *args, **kwargs + if node.arg_kinds[0] != nodes.ARG_POS: + # *assuming* the overloaded function is correct, there's a couple cases: + # 1) The first argument has different names, but is pos-only. We don't + # care about this case, the argument must be passed positionally. + # 2) The first argument allows keyword reference, therefore must be the + # same between overloads. + if isinstance(called_type, (CallableType, Overloaded)): + name = called_type.items[0].arg_names[0] + if name in node.arg_names: + idx = node.arg_names.index(name) + # we want the idx-th variable to be narrowed + expr = collapse_walrus(node.args[idx]) + else: + kind = "guard" if type_guard is not None else "narrower" + self.fail( + message_registry.TYPE_GUARD_POS_ARG_REQUIRED.format(kind), node + ) + return {}, {} + if literal(expr) == LITERAL_TYPE: + # Note: we wrap the target type, so that we can special case later. + # Namely, for isinstance() we use a normal meet, while TypeGuard is + # considered "always right" (i.e. even if the types are not overlapping). + # Also note that a care must be taken to unwrap this back at read places + # where we use this to narrow down declared type. + if type_guard is not None: + return {expr: TypeGuardedType(type_guard)}, {} + else: + assert type_is is not None + return conditional_types_to_typemaps( + expr, + *self.conditional_types_with_intersection( + self.lookup_type(expr), + [TypeRange(type_is, is_upper_bound=False)], + expr, + consider_runtime_isinstance=False, + ), + ) + elif isinstance(node, ComparisonExpr): + return self.comparison_type_narrowing_helper(node) + elif isinstance(node, AssignmentExpr): + if_map: TypeMap + else_map: TypeMap + if_map = {} + else_map = {} + + if_assignment_map, else_assignment_map = self.find_isinstance_check(node.target) + if_map.update(if_assignment_map) + else_map.update(else_assignment_map) + + if_condition_map, else_condition_map = self.find_isinstance_check( + node.value, in_boolean_context=False + ) + + if_map.update(if_condition_map) + else_map.update(else_condition_map) + + return if_map, else_map + elif isinstance(node, OpExpr) and node.op == "and": + left_if_vars, left_else_vars = self.find_isinstance_check(node.left) + right_if_vars, right_else_vars = self.find_isinstance_check(node.right) + + # (e1 and e2) is true if both e1 and e2 are true, + # and false if at least one of e1 and e2 is false. + return ( + and_conditional_maps(left_if_vars, right_if_vars), + # Note that if left else type is Any, we can't add any additional + # types to it, since the right maps were computed assuming + # the left is True, which may be not the case in the else branch. + or_conditional_maps(left_else_vars, right_else_vars, coalesce_any=True), + ) + elif isinstance(node, OpExpr) and node.op == "or": + left_if_vars, left_else_vars = self.find_isinstance_check(node.left) + right_if_vars, right_else_vars = self.find_isinstance_check(node.right) + + # (e1 or e2) is true if at least one of e1 or e2 is true, + # and false if both e1 and e2 are false. + return ( + or_conditional_maps(left_if_vars, right_if_vars), + and_conditional_maps(left_else_vars, right_else_vars), + ) + elif isinstance(node, UnaryExpr) and node.op == "not": + left, right = self.find_isinstance_check(node.expr) + return right, left + elif ( + literal(node) == LITERAL_TYPE + and self.has_type(node) + and self.can_be_narrowed_with_len(self.lookup_type(node)) + # Only translate `if x` to `if len(x) > 0` when possible. + and not custom_special_method(self.lookup_type(node), "__bool__") + and self.options.strict_optional + ): + # Combine a `len(x) > 0` check with the default logic below. + yes_type, no_type = self.narrow_with_len(self.lookup_type(node), ">", 0) + if_map = {node: true_only(yes_type)} + else_map = {node: false_only(no_type)} + return if_map, else_map + + # Restrict the type of the variable to True-ish/False-ish in the if and else branches + # respectively + original_vartype = self.lookup_type(node) + if in_boolean_context: + # We don't check `:=` values in expressions like `(a := A())`, + # because they produce two error messages. + self.check_for_truthy_type(original_vartype, node) + vartype = try_expanding_sum_type_to_union(original_vartype, "builtins.bool") + + if_map = {node: true_only(vartype)} + else_map = {node: false_only(vartype)} + return if_map, else_map + + def comparison_type_narrowing_helper(self, node: ComparisonExpr) -> tuple[TypeMap, TypeMap]: + """Infer type narrowing from a comparison expression.""" + # Step 1: Obtain the types of each operand and whether or not we can + # narrow their types. (For example, we shouldn't try narrowing the + # types of literal string or enum expressions). + + operands = [collapse_walrus(x) for x in node.operands] + operand_types = [] + narrowable_operand_index_to_hash = {} + narrowable_operand_hash_to_index = {} + for i, expr in enumerate(operands): + if not self.has_type(expr): + return {}, {} + expr_type = self.lookup_type(expr) + operand_types.append(expr_type) + + if ( + literal(expr) == LITERAL_TYPE + and not is_literal_none(expr) + and not is_literal_not_implemented(expr) + and not is_false_literal(expr) + and not is_true_literal(expr) + and not self.is_literal_enum(expr) + # CallableType type objects are usually already maximally specific + and not ( + isinstance(p_expr := get_proper_type(expr_type), FunctionLike) + and p_expr.is_type_obj() + ) + # This is a little ad hoc, in the absence of intersection types + and not (isinstance(p_expr, TypeType) and isinstance(p_expr.item, TypeVarType)) + ): + h = literal_hash(expr) + if h is not None: + narrowable_operand_index_to_hash[i] = h + narrowable_operand_hash_to_index[h] = i + + # Step 2: Group operands chained by either the 'is' or '==' operands + # together. For all other operands, we keep them in groups of size 2. + # So the expression: + # + # x0 == x1 == x2 < x3 < x4 is x5 is x6 is not x7 is not x8 + # + # ...is converted into the simplified operator list: + # + # [("==", [0, 1, 2]), ("<", [2, 3]), ("<", [3, 4]), + # ("is", [4, 5, 6]), ("is not", [6, 7]), ("is not", [7, 8])] + # + # We group identity/equality expressions so we can propagate information + # we discover about one operand across the entire chain. We don't bother + # handling 'is not' and '!=' chains in a special way: those are very rare + # in practice. + + simplified_operator_list = group_comparison_operands( + node.pairwise(), narrowable_operand_index_to_hash, {"==", "is"} + ) + + # Step 3: Analyze each group and infer more precise type maps for each + # assignable operand, if possible. We combine these type maps together + # in the final step. + + partial_type_maps = [] + for operator, expr_indices in simplified_operator_list: + if_map: TypeMap + else_map: TypeMap + + if operator in {"is", "is not", "==", "!="}: + if_map, else_map = self.narrow_type_by_identity_equality( + operator, + operands, + operand_types, + expr_indices, + narrowable_indices=narrowable_operand_index_to_hash.keys(), + ) + elif operator in {"in", "not in"}: + assert len(expr_indices) == 2 + left_index, right_index = expr_indices + item_type = operand_types[left_index] + iterable_type = operand_types[right_index] + + if_map = {} + else_map = {} + + if left_index in narrowable_operand_index_to_hash: + p_iterable_type = get_proper_type(iterable_type) + container_item_types = None + + # Can we statically determine container contents? + if ( + isinstance(p_iterable_type, TupleType) + and find_unpack_in_list(p_iterable_type.items) is None + ): + container_item_types = p_iterable_type.items + else: + container_expr = collapse_walrus(operands[right_index]) + if isinstance(container_expr, (ListExpr, SetExpr)): + if all(not isinstance(i, StarExpr) for i in container_expr.items): + container_item_types = [ + self.lookup_type(e) for e in container_expr.items + ] + elif isinstance(container_expr, DictExpr): + if all(k is not None for k, v in container_expr.items): + container_item_types = [ + self.lookup_type(cast(Expression, k)) + for k, v in container_expr.items + ] + + if container_item_types is not None: + # If we know the exact contents, we can potentially do negative narrowing, + # e.g. `x not in (None,)` + all_if_maps = [] + all_else_maps = [] + for known_item in container_item_types: + # Match the should_coerce_literals logic from narrow_type_by_identity_equality + p_known_item = get_proper_type(known_item) + if ( + is_literal_type_like(p_known_item) + or ( + isinstance(p_known_item, Instance) + and p_known_item.type.is_enum + ) + ) and not has_custom_eq_checks(p_known_item): + known_item = coerce_to_literal(known_item) + if_map, else_map = self.narrow_type_by_identity_equality( + "==", + operands=[operands[left_index], operands[right_index]], + operand_types=[item_type, known_item], + expr_indices=[0, 1], + narrowable_indices={0}, + ) + all_if_maps.append(if_map) + if is_singleton_equality_type(get_proper_type(known_item)): + all_else_maps.append(else_map) + if_map = reduce_or_conditional_type_maps(all_if_maps) + else_map = reduce_and_conditional_type_maps(all_else_maps, use_meet=True) + else: + collection_item_type = get_proper_type(builtin_item_type(p_iterable_type)) + if collection_item_type is not None: + if_map, else_map = self.narrow_type_by_identity_equality( + "==", + operands=[operands[left_index], operands[right_index]], + operand_types=[item_type, collection_item_type], + expr_indices=[0, 1], + narrowable_indices={0}, + ) + # We can't do negative narrowing, since e.g. the container could + # just be empty. + else_map = {} + + if right_index in narrowable_operand_index_to_hash: + # E.g. narrows the right operand in `if "key" in typed_dict` + if_type, else_type = self.conditional_types_for_iterable( + item_type, iterable_type + ) + expr = operands[right_index] + if_map[expr] = if_type + else_map[expr] = else_type + + else: + if_map = {} + else_map = {} + + if operator in {"is not", "!=", "not in"}: + if_map, else_map = else_map, if_map + + partial_type_maps.append((if_map, else_map)) + + # Chained comparisons are conjunctions evaluated left-to-right. Feed what we learned + # from earlier true comparisons into later comparisons, similarly to `and`. + if len(simplified_operator_list) > 1: + for expr, expr_type in if_map.items(): + h = literal_hash(expr) + if h is None or h not in narrowable_operand_hash_to_index: + continue + operand_index = narrowable_operand_hash_to_index[h] + operand_types[operand_index] = meet_types( + operand_types[operand_index], expr_type + ) + + # If we have found non-trivial restrictions from the regular comparisons, + # then return soon. Otherwise try to infer restrictions involving `len(x)`. + # TODO: support regular and len() narrowing in the same chain. + if any(len(m[0]) or len(m[1]) for m in partial_type_maps): + return reduce_conditional_maps(partial_type_maps, use_meet=True) + else: + # Use meet for `and` maps to get correct results for chained checks + # like `if 1 < len(x) < 4: ...` + return reduce_conditional_maps(self.find_tuple_len_narrowing(node), use_meet=True) + + def narrow_type_by_identity_equality( + self, + operator: str, + operands: list[Expression], + operand_types: list[Type], + expr_indices: list[int], + narrowable_indices: AbstractSet[int], + ) -> tuple[TypeMap, TypeMap]: + """ + Calculate type maps for '==', '!=', 'is' or 'is not' expression, ignoring `type(x)` checks. + + The 'operands' and 'operand_types' lists should be the full list of operands used + in the overall comparison expression. The 'chain_indices' list is the list of indices + actually used within this identity comparison chain. + + So if we have the expression: + + a <= b is c is d <= e + + ...then 'operands' and 'operand_types' would be lists of length 5 and 'chain_indices' + would be the list [1, 2, 3]. + + The 'narrowable_indices' parameter is the set of all indices we are allowed + to refine the types of: that is, all operands that will potentially be a part of + the output TypeMaps. + + """ + # is_target_for_value_narrowing: + # If the operator returns True when compared to this target, do we narrow in else branch? + # E.g. if operator is "==", then: + # - is_target_for_value_narrowing(str) == False + # - is_target_for_value_narrowing(Literal["asdf"]) == True + is_target_for_value_narrowing: Callable[[ProperType], bool] + + # should_coerce_literals: + # Ideally, we should always attempt to have this set to True. Unfortunately, for now, + # performing this coercion can sometimes result in overly aggressive narrowing when taking + # in the context of other type checker behaviour. + should_coerce_literals: bool + + # custom_eq_indices: + # Operands at these indices define a custom `__eq__`. These can do arbitrary things, so we + # have to be more careful about what narrowing we can conclude from a successful comparison + custom_eq_indices: set[int] + + # Equality can use value semantics, so `if x == Fruits.APPLE: ...` may also + # match non-enum values for IntEnum/StrEnum-like enums. Identity checks don't + # have this ambiguity. + is_identity_comparison = operator in {"is", "is not"} + + if is_identity_comparison: + is_target_for_value_narrowing = is_singleton_identity_type + should_coerce_literals = True + custom_eq_indices = set() + + elif operator in {"==", "!="}: + is_target_for_value_narrowing = is_singleton_equality_type + + should_coerce_literals = False + for i in expr_indices: + typ = get_proper_type(operand_types[i]) + if is_literal_type_like(typ) or (isinstance(typ, Instance) and typ.type.is_enum): + should_coerce_literals = True + break + + custom_eq_indices = {i for i in expr_indices if has_custom_eq_checks(operand_types[i])} + else: + raise AssertionError + + all_if_maps: list[TypeMap] = [] + all_else_maps: list[TypeMap] = [] + + # For each narrowable index, we see what we can narrow based on each relevant target + for i in expr_indices: + if i not in narrowable_indices: + continue + if i in custom_eq_indices: + # Handled later + continue + + expr_type = operand_types[i] + for j in expr_indices: + if i == j: + continue + if j in custom_eq_indices: + # We can't use types with custom __eq__ as targets for narrowing + # E.g. if (x: int | None) == (y: CustomEq | None), we cannot narrow x to None + continue + target_type = operand_types[j] + if should_coerce_literals: + target_type = coerce_to_literal(target_type) + + # Morally what we want to do is narrow for each branch based on: + # `if_type, else_type = conditional_types(expr_type, target)` + # What we actually do is first munge expr_type based on target_type to handle some + # special cased known `__eq__` implementations. + narrowable_expr_type, ambiguous_expr_type = partition_equality_ambiguous_types( + expr_type, target_type, is_identity=is_identity_comparison + ) + + if narrowable_expr_type is None: + if_type = else_type = ambiguous_expr_type + else: + narrowable_expr_type = try_expanding_sum_type_to_union( + coerce_to_literal(narrowable_expr_type), None + ) + if_type, else_type = conditional_types( + narrowable_expr_type, + [TypeRange(target_type, is_upper_bound=False)], + from_equality=True, + ) + if ambiguous_expr_type is not None: + if_type = make_simplified_union( + [if_type or narrowable_expr_type, ambiguous_expr_type] + ) + else_type = make_simplified_union( + [else_type or narrowable_expr_type, ambiguous_expr_type] + ) + + if_map, else_map = conditional_types_to_typemaps(operands[i], if_type, else_type) + if is_target_for_value_narrowing(get_proper_type(target_type)): + all_if_maps.append(if_map) + all_else_maps.append(else_map) + else: + # For value targets, it is safe to narrow in the negative case. + # e.g. if (x: Literal[5] | None) != (y: Literal[5]), we can narrow x to None + # However, for non-value targets, we cannot do this narrowing, + # and so we ignore else_map + # e.g. if (x: str | None) != (y: str), we cannot narrow x to None + + # It is correct to always narrow here. It improves behaviour on tests and + # detects many inaccurate type annotations on primer. + # However, because mypy does not currently check unreachable code, it feels + # risky to narrow to unreachable without --warn-unreachable or not + # at module level + # See also this specific primer comment, where I force primer to run with + # --warn-unreachable to see what code we would stop checking: + # https://github.com/python/mypy/pull/20660#issuecomment-3865794148 + if ( + self.options.warn_unreachable and len(self.scope.stack) != 1 + ) or not is_unreachable_map(if_map): + all_if_maps.append(if_map) + + # Handle narrowing for operands with custom __eq__ methods specially + # In most cases, we won't be able to do any narrowing + for i in custom_eq_indices: + if i not in narrowable_indices: + continue + union_expr_type = get_proper_type(operand_types[i]) + if not isinstance(union_expr_type, UnionType): + # Here we don't do any positive narrowing, because we can't conclude much + # from a custom __eq__ returning True. + # But we might be able to do some negative narrowing, since we can assume + # a custom __eq__ is reflexive. This should only apply to custom __eq__ enums, + # see testNarrowingEqualityCustomEqualityEnum + expr_type = operand_types[i] + for j in expr_indices: + if j in custom_eq_indices: + continue + target_type = operand_types[j] + if should_coerce_literals: + target_type = coerce_to_literal(target_type) + target = TypeRange(target_type, is_upper_bound=False) + if is_target_for_value_narrowing(get_proper_type(target_type)): + if_map, else_map = conditional_types_to_typemaps( + operands[i], + *conditional_types(expr_type, [target], from_equality=True), + ) + all_else_maps.append(else_map) + continue + + # If our operand with custom __eq__ is a union, where only some members of the union + # implement custom __eq__, then we can narrow down the other members as usual. + # This is basically the same logic as the main narrowing loop above. + or_if_maps: list[TypeMap] = [] + or_else_maps: list[TypeMap] = [] + for expr_type in union_expr_type.items: + if has_custom_eq_checks(expr_type): + # Always include the union items with custom __eq__ in the type + # we narrow to in the if_map + or_if_maps.append({operands[i]: expr_type}) + + expr_type = coerce_to_literal(try_expanding_sum_type_to_union(expr_type, None)) + for j in expr_indices: + if j in custom_eq_indices: + if i == j: + continue + # If we compare to a target with custom __eq__, we cannot narrow at all + if is_overlapping_none(expr_type) and not is_overlapping_none( + operand_types[j] + ): + # Narrow away from None. This is unsound, we're hoping that no one + # has a custom __eq__ that returns True for None. + or_if_maps.append({operands[i]: remove_optional(expr_type)}) + else: + or_if_maps.append({operands[i]: expr_type}) + continue + target_type = operand_types[j] + if should_coerce_literals: + target_type = coerce_to_literal(target_type) + + narrowable_expr_type, ambiguous_expr_type = partition_equality_ambiguous_types( + expr_type, target_type, is_identity=is_identity_comparison + ) + + if narrowable_expr_type is None: + if_type = else_type = ambiguous_expr_type + else: + narrowable_expr_type = coerce_to_literal( + try_expanding_sum_type_to_union(narrowable_expr_type, None) + ) + if_type, else_type = conditional_types( + narrowable_expr_type, + [TypeRange(target_type, is_upper_bound=False)], + default=narrowable_expr_type, + from_equality=True, + ) + if ambiguous_expr_type is not None: + if_type = make_simplified_union([if_type, ambiguous_expr_type]) + else_type = make_simplified_union([else_type, ambiguous_expr_type]) + + if_map, else_map = conditional_types_to_typemaps( + operands[i], if_type, else_type + ) + if not isinstance(get_proper_type(expr_type), AnyType): + # We avoid positive narrowing for Any here, for consistency with the + # non-union branch above and to preserve the gradual guarantee + or_if_maps.append(if_map) + if is_target_for_value_narrowing(get_proper_type(target_type)): + or_else_maps.append(else_map) + + all_if_maps.append(reduce_or_conditional_type_maps(or_if_maps)) + all_else_maps.append(reduce_or_conditional_type_maps(or_else_maps)) + + # Handle narrowing for comparisons that produce additional narrowing, like + # `type(x) == T` or `x.__class__ is T` + for i in expr_indices: + type_expr = operands[i] + if ( + isinstance(type_expr, CallExpr) + and refers_to_fullname(type_expr.callee, "builtins.type") + and len(type_expr.args) == 1 + ): + expr_in_type_expr = type_expr.args[0] + elif isinstance(type_expr, MemberExpr) and type_expr.name == "__class__": + expr_in_type_expr = type_expr.expr + else: + continue + + p_expr_type = get_proper_type(operand_types[i]) + if isinstance(p_expr_type, TypeType) and isinstance(p_expr_type.item, TypeVarType): + # This mirrors logic in comparison_type_narrowing_helper + # In theory, this is like `i not in narrowable_indices`, except that + # narrowable_indices filters all type(x) narrowing as it's a call + continue + + for j in expr_indices: + if i == j: + continue + expr = operands[j] + + current_type_range = self.get_type_range_of_type(operand_types[j]) + if current_type_range is None: + continue + if isinstance(get_proper_type(current_type_range.item), AnyType): + # Avoid widening to Any for checks like `type(x) is type(y: Any)`. + # We patch this here because it is desirable to widen to any for cases like + # isinstance(x, (y: Any)) + continue + + arg_type = self.lookup_type(expr_in_type_expr) + if_type, else_type = self.conditional_types_with_intersection( + arg_type, [current_type_range], expr_in_type_expr + ) + if if_type is not None and ( + not current_type_range.is_upper_bound + and not is_equivalent(if_type, current_type_range.item) + ): + # type(x) and x.__class__ checks must exact match + if_type = UninhabitedType() + + if_map, else_map = conditional_types_to_typemaps( + expr_in_type_expr, if_type, else_type + ) + + is_final = ( + expr.node.is_final + if isinstance(expr, RefExpr) and isinstance(expr.node, TypeInfo) + else False + ) + all_if_maps.append(if_map) + if is_final: + # We can only narrow `type(x) == T` in the negative case if T is final + all_else_maps.append(else_map) + + # We will not have duplicate entries in our type maps if we only have two operands, + # so we can skip running meets on the intersections + if_map = reduce_and_conditional_type_maps(all_if_maps, use_meet=len(operands) > 2) + else_map = reduce_or_conditional_type_maps(all_else_maps) + return if_map, else_map + + def propagate_up_typemap_info(self, new_types: TypeMap) -> TypeMap: + """Attempts refining parent expressions of any MemberExpr or IndexExprs in new_types. + + This function iterates through new_types and attempts to use the information to try + refining any parent types that happen to be unions. + + For example, suppose there are two types "A = Tuple[int, int]" and "B = Tuple[str, str]". + Next, suppose that 'new_types' specifies the expression 'foo[0]' has a refined type + of 'int' and that 'foo' was previously deduced to be of type Union[A, B]. + + Then, this function will observe that since A[0] is an int and B[0] is not, the type of + 'foo' can be further refined from Union[A, B] into just B. + + We perform this kind of "parent narrowing" for member lookup expressions and indexing + expressions into tuples, namedtuples, and typeddicts. We repeat this narrowing + recursively if the parent is also a "lookup expression". So for example, if we have + the expression "foo['bar'].baz[0]", we'd potentially end up refining types for the + expressions "foo", "foo['bar']", and "foo['bar'].baz". + + We return the newly refined map. This map is guaranteed to be a superset of 'new_types'. + """ + all_mappings = [new_types] + for expr, expr_type in new_types.items(): + all_mappings.append(self.refine_parent_types(expr, expr_type)) + return reduce_and_conditional_type_maps(all_mappings, use_meet=True) + + def refine_parent_types(self, expr: Expression, expr_type: Type) -> TypeMap: + """Checks if the given expr is a 'lookup operation' into a union and iteratively refines + the parent types based on the 'expr_type'. + + For example, if 'expr' is an expression like 'a.b.c.d', we'll potentially return refined + types for expressions 'a', 'a.b', and 'a.b.c'. + + For more details about what a 'lookup operation' is and how we use the expr_type to refine + the parent types of lookup_expr, see the docstring in 'propagate_up_typemap_info'. + """ + output: dict[Expression, Type] = {} + + # Note: parent_expr and parent_type are progressively refined as we crawl up the + # parent lookup chain. + while True: + # First, check if this expression is one that's attempting to + # "lookup" some key in the parent type. If so, save the parent type + # and create function that will try replaying the same lookup + # operation against arbitrary types. + if isinstance(expr, MemberExpr): + parent_expr = self._propagate_walrus_assignments(expr.expr, output) + parent_type = self.lookup_type_or_none(parent_expr) + member_name = expr.name + + def replay_lookup(new_parent_type: ProperType) -> Type | None: + with self.msg.filter_errors() as w: + member_type = analyze_member_access( + name=member_name, + typ=new_parent_type, + context=parent_expr, + is_lvalue=False, + is_super=False, + is_operator=False, + original_type=new_parent_type, + chk=self, + in_literal_context=False, + ) + if w.has_new_errors(): + return None + else: + return member_type + + elif isinstance(expr, IndexExpr): + parent_expr = self._propagate_walrus_assignments(expr.base, output) + parent_type = self.lookup_type_or_none(parent_expr) + + self._propagate_walrus_assignments(expr.index, output) + index_type = self.lookup_type_or_none(expr.index) + if index_type is None: + return output + + str_literals = try_getting_str_literals_from_type(index_type) + if str_literals is not None: + # Refactoring these two indexing replay functions is surprisingly + # tricky -- see https://github.com/python/mypy/pull/7917, which + # was blocked by https://github.com/mypyc/mypyc/issues/586 + def replay_lookup(new_parent_type: ProperType) -> Type | None: + if not isinstance(new_parent_type, TypedDictType): + return None + try: + assert str_literals is not None + member_types = [new_parent_type.items[key] for key in str_literals] + except KeyError: + return None + return make_simplified_union(member_types) + + else: + int_literals = try_getting_int_literals_from_type(index_type) + if int_literals is not None: + + def replay_lookup(new_parent_type: ProperType) -> Type | None: + if isinstance(new_parent_type, TupleType): + try: + assert int_literals is not None + member_types = [ + new_parent_type.items[key] for key in int_literals + ] + except IndexError: + return None + return make_simplified_union(member_types) + if ( + isinstance(new_parent_type, Instance) + and new_parent_type.type.fullname + in ("builtins.list", "builtins.tuple") + and new_parent_type.args + ): + return new_parent_type.args[0] + return None + + else: + return output + else: + return output + + # If we somehow didn't previously derive the parent type, abort completely + # with what we have so far: something went wrong at an earlier stage. + if parent_type is None: + return output + + # We currently only try refining the parent type if it's a Union. + # If not, there's no point in trying to refine any further parents + # since we have no further information we can use to refine the lookup + # chain, so we end early as an optimization. + parent_type = get_proper_type(parent_type) + if not isinstance(parent_type, UnionType): + return output + + # Take each element in the parent union and replay the original lookup procedure + # to figure out which parents are compatible. + new_parent_types = [] + for item in flatten_nested_unions(parent_type.items): + member_type = replay_lookup(get_proper_type(item)) + if member_type is None: + # We were unable to obtain the member type. So, we give up on refining this + # parent type entirely and abort. + return output + + if is_overlapping_types(member_type, expr_type): + new_parent_types.append(item) + + # If none of the parent types overlap (if we derived an empty union), something + # went wrong. We should never hit this case, but deriving the uninhabited type or + # reporting an error both seem unhelpful. So we abort. + if not new_parent_types: + return output + + expr = parent_expr + expr_type = output[parent_expr] = make_simplified_union(new_parent_types) + + def _propagate_walrus_assignments( + self, expr: Expression, type_map: dict[Expression, Type] + ) -> Expression: + """Add assignments from walrus expressions to inferred types. + + Only considers nested assignment exprs, does not recurse into other types. + This may be added later if necessary by implementing a dedicated visitor. + """ + if isinstance(expr, AssignmentExpr): + if isinstance(expr.value, AssignmentExpr): + self._propagate_walrus_assignments(expr.value, type_map) + assigned_type = self.lookup_type_or_none(expr.value) + parent_expr = collapse_walrus(expr) + if assigned_type is not None: + type_map[parent_expr] = assigned_type + return parent_expr + return expr + + def is_len_of_tuple(self, expr: Expression) -> bool: + """Is this expression a `len(x)` call where x is a tuple or union of tuples?""" + if not isinstance(expr, CallExpr): + return False + if not refers_to_fullname(expr.callee, "builtins.len"): + return False + if len(expr.args) != 1: + return False + expr = expr.args[0] + if literal(expr) != LITERAL_TYPE: + return False + if not self.has_type(expr): + return False + return self.can_be_narrowed_with_len(self.lookup_type(expr)) + + def can_be_narrowed_with_len(self, typ: Type) -> bool: + """Is this a type that can benefit from length check type restrictions? + + Currently supported types are TupleTypes, Instances of builtins.tuple, and + unions involving such types. + """ + if custom_special_method(typ, "__len__"): + # If user overrides builtin behavior, we can't do anything. + return False + p_typ = get_proper_type(typ) + # Note: we are conservative about tuple subclasses, because some code may rely on + # the fact that tuple_type of fallback TypeInfo matches the original TupleType. + if isinstance(p_typ, TupleType): + if any(isinstance(t, UnpackType) for t in p_typ.items): + return p_typ.partial_fallback.type.fullname == "builtins.tuple" + return True + if isinstance(p_typ, Instance): + return p_typ.type.has_base("builtins.tuple") + if isinstance(p_typ, UnionType): + return any(self.can_be_narrowed_with_len(t) for t in p_typ.items) + return False + + def literal_int_expr(self, expr: Expression) -> int | None: + """Is this expression an int literal, or a reference to an int constant? + + If yes, return the corresponding int value, otherwise return None. + """ + if not self.has_type(expr): + return None + expr_type = self.lookup_type(expr) + expr_type = coerce_to_literal(expr_type) + proper_type = get_proper_type(expr_type) + if not isinstance(proper_type, LiteralType): + return None + if not isinstance(proper_type.value, int): + return None + return proper_type.value + + def find_tuple_len_narrowing(self, node: ComparisonExpr) -> list[tuple[TypeMap, TypeMap]]: + """Top-level logic to find type restrictions from a length check on tuples. + + We try to detect `if` checks like the following: + x: tuple[int, int] | tuple[int, int, int] + y: tuple[int, int] | tuple[int, int, int] + if len(x) == len(y) == 2: + a, b = x # OK + c, d = y # OK + + z: tuple[int, ...] + if 1 < len(z) < 4: + x = z # OK + and report corresponding type restrictions to the binder. + """ + # First step: group consecutive `is` and `==` comparisons together. + # This is essentially a simplified version of group_comparison_operands(), + # tuned to the len()-like checks. Note that we don't propagate indirect + # restrictions like e.g. `len(x) > foo() > 1` yet, since it is tricky. + # TODO: propagate indirect len() comparison restrictions. + chained = [] + last_group = set() + for op, left, right in node.pairwise(): + if isinstance(left, AssignmentExpr): + left = left.value + if isinstance(right, AssignmentExpr): + right = right.value + if op in ("is", "=="): + last_group.add(left) + last_group.add(right) + else: + if last_group: + chained.append(("==", list(last_group))) + last_group = set() + if op in {"is not", "!=", "<", "<=", ">", ">="}: + chained.append((op, [left, right])) + if last_group: + chained.append(("==", list(last_group))) + + yes_map: TypeMap + no_map: TypeMap + + # Second step: infer type restrictions from each group found above. + type_maps = [] + for op, items in chained: + # TODO: support unions of literal types as len() comparison targets. + if not any(self.literal_int_expr(it) is not None for it in items): + continue + if not any(self.is_len_of_tuple(it) for it in items): + continue + + # At this step we know there is at least one len(x) and one literal in the group. + if op in ("is", "=="): + literal_values = set() + tuples = [] + for it in items: + lit = self.literal_int_expr(it) + if lit is not None: + literal_values.add(lit) + continue + if self.is_len_of_tuple(it): + assert isinstance(it, CallExpr) + tuples.append(it.args[0]) + if len(literal_values) > 1: + # More than one different literal value found, like 1 == len(x) == 2, + # so the corresponding branch is unreachable. + yes_map = {tpl: UninhabitedType() for tpl in tuples} + return [(yes_map, {})] + size = literal_values.pop() + if size > MAX_PRECISE_TUPLE_SIZE: + # Avoid creating huge tuples from checks like if len(x) == 300. + continue + for tpl in tuples: + yes_type, no_type = self.narrow_with_len(self.lookup_type(tpl), op, size) + yes_map = {tpl: yes_type} + no_map = {tpl: no_type} + type_maps.append((yes_map, no_map)) + else: + left, right = items + if self.is_len_of_tuple(right): + # Normalize `1 < len(x)` and similar as `len(x) > 1`. + left, right = right, left + op = flip_ops.get(op, op) + r_size = self.literal_int_expr(right) + assert r_size is not None + if r_size > MAX_PRECISE_TUPLE_SIZE: + # Avoid creating huge unions from checks like if len(x) > 300. + continue + assert isinstance(left, CallExpr) + yes_type, no_type = self.narrow_with_len( + self.lookup_type(left.args[0]), op, r_size + ) + yes_map = {left.args[0]: yes_type} + no_map = {left.args[0]: no_type} + type_maps.append((yes_map, no_map)) + return type_maps + + def narrow_with_len(self, typ: Type, op: str, size: int) -> tuple[Type, Type]: + """Dispatch tuple type narrowing logic depending on the kind of type we got.""" + typ = get_proper_type(typ) + if isinstance(typ, TupleType): + return self.refine_tuple_type_with_len(typ, op, size) + elif isinstance(typ, Instance): + return self.refine_instance_type_with_len(typ, op, size) + elif isinstance(typ, UnionType): + yes_types = [] + no_types = [] + other_types = [] + for t in typ.items: + if not self.can_be_narrowed_with_len(t): + other_types.append(t) + continue + yt, nt = self.narrow_with_len(t, op, size) + yes_types.append(yt) + no_types.append(nt) + yes_types += other_types + no_types += other_types + yes_type = make_simplified_union(yes_types) + no_type = make_simplified_union(no_types) + return yes_type, no_type + else: + assert False, "Unsupported type for len narrowing" + + def refine_tuple_type_with_len(self, typ: TupleType, op: str, size: int) -> tuple[Type, Type]: + """Narrow a TupleType using length restrictions.""" + unpack_index = find_unpack_in_list(typ.items) + if unpack_index is None: + # For fixed length tuple situation is trivial, it is either reachable or not, + # depending on the current length, expected length, and the comparison op. + method = int_op_to_method[op] + if method(typ.length(), size): + return typ, UninhabitedType() + return UninhabitedType(), typ + unpack = typ.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if isinstance(unpacked, TypeVarTupleType): + # For tuples involving TypeVarTuple unpack we can't do much except + # inferring reachability, and recording the restrictions on TypeVarTuple + # for further "manual" use elsewhere. + min_len = typ.length() - 1 + unpacked.min_len + if op in ("==", "is"): + if min_len <= size: + return typ, typ + return UninhabitedType(), typ + elif op in ("<", "<="): + if op == "<=": + size += 1 + if min_len < size: + prefix = typ.items[:unpack_index] + suffix = typ.items[unpack_index + 1 :] + # TODO: also record max_len to avoid false negatives? + unpack = UnpackType(unpacked.copy_modified(min_len=size - typ.length() + 1)) + return typ, typ.copy_modified(items=prefix + [unpack] + suffix) + return UninhabitedType(), typ + else: + yes_type, no_type = self.refine_tuple_type_with_len(typ, neg_ops[op], size) + return no_type, yes_type + # Homogeneous variadic item is the case where we are most flexible. Essentially, + # we adjust the variadic item by "eating away" from it to satisfy the restriction. + assert isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + min_len = typ.length() - 1 + arg = unpacked.args[0] + prefix = typ.items[:unpack_index] + suffix = typ.items[unpack_index + 1 :] + if op in ("==", "is"): + if min_len <= size: + # TODO: return fixed union + prefixed variadic tuple for no_type? + return typ.copy_modified(items=prefix + [arg] * (size - min_len) + suffix), typ + return UninhabitedType(), typ + elif op in ("<", "<="): + if op == "<=": + size += 1 + if min_len < size: + # Note: there is some ambiguity w.r.t. to where to put the additional + # items: before or after the unpack. However, such types are equivalent, + # so we always put them before for consistency. + no_type = typ.copy_modified( + items=prefix + [arg] * (size - min_len) + [unpack] + suffix + ) + yes_items = [] + for n in range(size - min_len): + yes_items.append(typ.copy_modified(items=prefix + [arg] * n + suffix)) + return UnionType.make_union(yes_items, typ.line, typ.column), no_type + return UninhabitedType(), typ + else: + yes_type, no_type = self.refine_tuple_type_with_len(typ, neg_ops[op], size) + return no_type, yes_type + + def refine_instance_type_with_len( + self, typ: Instance, op: str, size: int + ) -> tuple[Type, Type]: + """Narrow a homogeneous tuple using length restrictions.""" + base = map_instance_to_supertype(typ, self.lookup_typeinfo("builtins.tuple")) + arg = base.args[0] + # Again, we are conservative about subclasses until we gain more confidence. + allow_precise = ( + PRECISE_TUPLE_TYPES in self.options.enable_incomplete_feature + ) and typ.type.fullname == "builtins.tuple" + if op in ("==", "is"): + # TODO: return fixed union + prefixed variadic tuple for no_type? + return TupleType(items=[arg] * size, fallback=typ), typ + elif op in ("<", "<="): + if op == "<=": + size += 1 + if allow_precise: + unpack = UnpackType(self.named_generic_type("builtins.tuple", [arg])) + no_type: Type = TupleType(items=[arg] * size + [unpack], fallback=typ) + else: + no_type = typ + if allow_precise: + items = [] + for n in range(size): + items.append(TupleType([arg] * n, fallback=typ)) + yes_type: Type = UnionType.make_union(items, typ.line, typ.column) + else: + yes_type = typ + return yes_type, no_type + else: + yes_type, no_type = self.refine_instance_type_with_len(typ, neg_ops[op], size) + return no_type, yes_type + + # + # Helpers + # + @overload + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: str, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + code: ErrorCode | None = None, + outer_context: Context | None = None, + ) -> bool: ... + + @overload + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: ErrorMessage, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + outer_context: Context | None = None, + ) -> bool: ... + + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: str | ErrorMessage, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + code: ErrorCode | None = None, + outer_context: Context | None = None, + ) -> bool: + """Generate an error if the subtype is not compatible with supertype.""" + if is_subtype(subtype, supertype, options=self.options): + return True + + if isinstance(msg, str): + msg = ErrorMessage(msg, code=code) + + if ( + self.msg.prefer_simple_messages() + or code is not None + and not self.errors.is_error_code_enabled(code) + ): + self.fail(msg, context) # Fast path -- skip all fancy logic + return False + + orig_subtype = subtype + subtype = get_proper_type(subtype) + orig_supertype = supertype + supertype = get_proper_type(supertype) + if self.msg.try_report_long_tuple_assignment_error( + subtype, supertype, context, msg, subtype_label, supertype_label + ): + return False + extra_info: list[str] = [] + note_msg = "" + notes = notes or [] + if subtype_label is not None or supertype_label is not None: + subtype_str, supertype_str = format_type_distinctly( + orig_subtype, orig_supertype, options=self.options + ) + if subtype_label is not None: + extra_info.append(subtype_label + " " + subtype_str) + if supertype_label is not None: + extra_info.append(supertype_label + " " + supertype_str) + note_msg = make_inferred_type_note( + outer_context or context, subtype, supertype, supertype_str + ) + if isinstance(subtype, Instance) and isinstance(supertype, Instance): + notes = append_invariance_notes(notes, subtype, supertype) + if isinstance(subtype, UnionType) and isinstance(supertype, UnionType): + notes = append_union_note(notes, subtype, supertype, self.options) + if extra_info: + msg = msg.with_additional_msg(" (" + ", ".join(extra_info) + ")") + + error = self.fail(msg, context) + for note in notes: + self.msg.note(note, context, code=msg.code) + if note_msg: + self.note(note_msg, context, code=msg.code) + self.msg.maybe_note_concatenate_pos_args(subtype, supertype, context, code=msg.code) + if ( + isinstance(supertype, Instance) + and supertype.type.is_protocol + and isinstance(subtype, (CallableType, Instance, TupleType, TypedDictType, TypeType)) + ): + self.msg.report_protocol_problems(subtype, supertype, context, parent_error=error) + if isinstance(supertype, CallableType) and isinstance(subtype, Instance): + call = find_member("__call__", subtype, subtype, is_operator=True) + if call: + self.msg.note_call(subtype, call, context, code=msg.code) + if isinstance(subtype, (CallableType, Overloaded)) and isinstance(supertype, Instance): + if supertype.type.is_protocol and "__call__" in supertype.type.protocol_members: + call = find_member("__call__", supertype, subtype, is_operator=True) + assert call is not None + if not is_subtype(subtype, call, options=self.options): + self.msg.note_call(supertype, call, context, code=msg.code) + self.check_possible_missing_await(subtype, supertype, context, code=msg.code) + return False + + def get_precise_awaitable_type(self, typ: Type, local_errors: ErrorWatcher) -> Type | None: + """If type implements Awaitable[X] with non-Any X, return X. + + In all other cases return None. This method must be called in context + of local_errors. + """ + if isinstance(get_proper_type(typ), PartialType): + # Partial types are special, ignore them here. + return None + try: + aw_type = self.expr_checker.check_awaitable_expr( + typ, Context(), "", ignore_binder=True + ) + except KeyError: + # This is a hack to speed up tests by not including Awaitable in all typing stubs. + return None + if local_errors.has_new_errors(): + return None + if isinstance(get_proper_type(aw_type), (AnyType, UnboundType)): + return None + return aw_type + + @contextmanager + def checking_await_set(self) -> Iterator[None]: + self.checking_missing_await = True + try: + yield + finally: + self.checking_missing_await = False + + def check_possible_missing_await( + self, subtype: Type, supertype: Type, context: Context, code: ErrorCode | None + ) -> None: + """Check if the given type becomes a subtype when awaited.""" + if self.checking_missing_await: + # Avoid infinite recursion. + return + with self.checking_await_set(), self.msg.filter_errors() as local_errors: + aw_type = self.get_precise_awaitable_type(subtype, local_errors) + if aw_type is None: + return + if not self.check_subtype( + aw_type, supertype, context, msg=message_registry.INCOMPATIBLE_TYPES + ): + return + self.msg.possible_missing_await(context, code) + + def named_type(self, name: str) -> Instance: + """Return an instance type with given name and implicit Any type args. + + For example, named_type('builtins.object') produces the 'object' type. + """ + if name == "builtins.str": + if instance_cache.str_type is None: + instance_cache.str_type = self._named_type(name) + return instance_cache.str_type + if name == "builtins.function": + if instance_cache.function_type is None: + instance_cache.function_type = self._named_type(name) + return instance_cache.function_type + if name == "builtins.int": + if instance_cache.int_type is None: + instance_cache.int_type = self._named_type(name) + return instance_cache.int_type + if name == "builtins.bool": + if instance_cache.bool_type is None: + instance_cache.bool_type = self._named_type(name) + return instance_cache.bool_type + if name == "builtins.object": + if instance_cache.object_type is None: + instance_cache.object_type = self._named_type(name) + return instance_cache.object_type + return self._named_type(name) + + def _named_type(self, name: str) -> Instance: + # Assume that the name refers to a type. + sym = self.lookup_qualified(name) + node = sym.node + if isinstance(node, TypeAlias): + assert isinstance(node.target, Instance) # type: ignore[misc] + node = node.target.type + assert isinstance(node, TypeInfo), node + any_type = AnyType(TypeOfAny.from_omitted_generics) + return Instance(node, [any_type] * len(node.defn.type_vars)) + + def named_generic_type(self, name: str, args: list[Type]) -> Instance: + """Return an instance with the given name and type arguments. + + Assume that the number of arguments is correct. Assume that + the name refers to a compatible generic type. + """ + info = self.lookup_typeinfo(name) + args = [remove_instance_last_known_values(arg) for arg in args] + # TODO: assert len(args) == len(info.defn.type_vars) + return Instance(info, args) + + def lookup_typeinfo(self, fullname: str) -> TypeInfo: + # Assume that the name refers to a class. + sym = self.lookup_qualified(fullname) + node = sym.node + assert isinstance(node, TypeInfo), node + return node + + def type_type(self) -> Instance: + """Return instance type 'type'.""" + return self.named_type("builtins.type") + + def str_type(self) -> Instance: + """Return instance type 'str'.""" + return self.named_type("builtins.str") + + def store_type(self, node: Expression, typ: Type) -> None: + """Store the type of a node in the type map.""" + self._type_maps[-1][node] = typ + + def has_type(self, node: Expression) -> bool: + return any(node in m for m in reversed(self._type_maps)) + + def lookup_type_or_none(self, node: Expression) -> Type | None: + for m in reversed(self._type_maps): + if node in m: + return m[node] + return None + + def lookup_type(self, node: Expression) -> Type: + for m in reversed(self._type_maps): + t = m.get(node) + if t is not None: + return t + raise KeyError(node) + + def store_types(self, d: dict[Expression, Type]) -> None: + self._type_maps[-1].update(d) + + def in_checked_function(self) -> bool: + """Should we type-check the current function? + + - Yes if --check-untyped-defs is set. + - Yes outside functions. + - Yes in annotated functions. + - No otherwise. + """ + return ( + self.options.check_untyped_defs or not self.dynamic_funcs or not self.dynamic_funcs[-1] + ) + + def lookup(self, name: str) -> SymbolTableNode: + """Look up a definition from the symbol table with the given name.""" + if name in self.globals: + return self.globals[name] + else: + b = self.globals.get("__builtins__", None) + if b: + assert isinstance(b.node, MypyFile) + table = b.node.names + if name in table: + return table[name] + raise KeyError(f"Failed lookup: {name}") + + def lookup_qualified(self, name: str) -> SymbolTableNode: + if "." not in name: + return self.lookup(name) + else: + parts = name.split(".") + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + sym = n.names.get(parts[i]) + assert sym is not None, "Internal error: attempted lookup of unknown name" + assert isinstance(sym.node, MypyFile) + n = sym.node + last = parts[-1] + if last in n.names: + return n.names[last] + elif len(parts) == 2 and parts[0] in ("builtins", "typing"): + fullname = ".".join(parts) + if fullname in SUGGESTED_TEST_FIXTURES: + suggestion = ", e.g. add '[{} fixtures/{}]' to your test".format( + parts[0], SUGGESTED_TEST_FIXTURES[fullname] + ) + else: + suggestion = "" + raise KeyError( + "Could not find builtin symbol '{}' (If you are running a " + "test case, use a fixture that " + "defines this symbol{})".format(last, suggestion) + ) + else: + msg = "Failed qualified lookup: '{}' (fullname = '{}')." + raise KeyError(msg.format(last, name)) + + @contextmanager + def enter_partial_types( + self, *, is_function: bool = False, is_class: bool = False + ) -> Iterator[None]: + """Enter a new scope for collecting partial types. + + Also report errors for (some) variables which still have partial + types, i.e. we couldn't infer a complete type. + """ + is_local = (self.partial_types and self.partial_types[-1].is_local) or is_function + self.partial_types.append(PartialTypeScope({}, is_function, is_local)) + yield + + # Don't complain about not being able to infer partials if it is + # at the toplevel (with allow_untyped_globals) or if it is in an + # untyped function being checked with check_untyped_defs. + permissive = (self.options.allow_untyped_globals and not is_local) or ( + self.options.check_untyped_defs and self.dynamic_funcs and self.dynamic_funcs[-1] + ) + + partial_types = self.partial_types.pop().map + if not self.current_node_deferred: + for var, context in partial_types.items(): + if isinstance(var.type, PartialType) and var.type.type is None and not permissive: + var.type = NoneType() + else: + if var not in self.partial_reported and not permissive: + self.msg.need_annotation_for_var(var, context, self.options) + self.partial_reported.add(var) + if var.type: + fixed = fixup_partial_type(var.type) + var.invalid_partial_type = fixed != var.type + var.type = fixed + + def handle_partial_var_type( + self, typ: PartialType, is_lvalue: bool, node: Var, context: Context + ) -> Type: + """Handle a reference to a partial type through a var. + + (Used by checkexpr and checkmember.) + """ + in_scope, is_local, partial_types = self.find_partial_types_in_all_scopes(node) + if typ.type is None and in_scope: + # 'None' partial type. It has a well-defined type. In an lvalue context + # we want to preserve the knowledge of it being a partial type. + if not is_lvalue: + return NoneType() + else: + return typ + else: + if partial_types is not None and not self.current_node_deferred: + if in_scope: + context = partial_types[node] + if is_local or not self.options.allow_untyped_globals: + self.msg.need_annotation_for_var(node, context, self.options) + self.partial_reported.add(node) + else: + # Defer the node -- we might get a better type in the outer scope + self.handle_cannot_determine_type(node.name, context) + return fixup_partial_type(typ) + + def is_defined_in_base_class(self, var: Var) -> bool: + if not var.info: + return False + return var.info.fallback_to_any or any( + base.get(var.name) is not None for base in var.info.mro[1:] + ) + + def find_partial_types(self, var: Var) -> dict[Var, Context] | None: + """Look for an active partial type scope containing variable. + + A scope is active if assignments in the current context can refine a partial + type originally defined in the scope. This is affected by the local_partial_types + configuration option. + """ + in_scope, _, partial_types = self.find_partial_types_in_all_scopes(var) + if in_scope: + return partial_types + return None + + def find_partial_types_in_all_scopes( + self, var: Var + ) -> tuple[bool, bool, dict[Var, Context] | None]: + """Look for partial type scope containing variable. + + Return tuple (is the scope active, is the scope a local scope, scope). + """ + for scope in reversed(self.partial_types): + if var in scope.map: + # All scopes within the outermost function are active. Scopes out of + # the outermost function are inactive to allow local reasoning (important + # for fine-grained incremental mode). + disallow_other_scopes = self.options.local_partial_types + + # There are two exceptions: + if isinstance(var.type, PartialType) and var.type.type is not None and var.info: + # We always prohibit non-None partial types at class scope + # for historical reasons. + disallow_other_scopes = True + if isinstance(var.type, PartialType) and var.type.type is None: + # We always allow None partial types, since this is a common use case. + # It is special-cased in fine-grained incremental mode. + disallow_other_scopes = False + + scope_active = ( + not disallow_other_scopes or scope.is_local == self.partial_types[-1].is_local + ) + return scope_active, scope.is_local, scope.map + return False, False, None + + def temp_node(self, t: Type, context: Context | None = None) -> TempNode: + """Create a temporary node with the given, fixed type.""" + return TempNode(t, context=context) + + def fail( + self, msg: str | ErrorMessage, context: Context, *, code: ErrorCode | None = None + ) -> ErrorInfo: + """Produce an error message.""" + if isinstance(msg, ErrorMessage): + return self.msg.fail(msg.value, context, code=msg.code) + return self.msg.fail(msg, context, code=code) + + def note( + self, + msg: str | ErrorMessage, + context: Context, + offset: int = 0, + *, + code: ErrorCode | None = None, + ) -> None: + """Produce a note.""" + if isinstance(msg, ErrorMessage): + self.msg.note(msg.value, context, code=msg.code) + return + self.msg.note(msg, context, offset=offset, code=code) + + def iterable_item_type(self, it: ProperType, context: Context) -> Type: + if isinstance(it, Instance): + iterable = map_instance_to_supertype(it, self.lookup_typeinfo("typing.Iterable")) + item_type = iterable.args[0] + if not isinstance(get_proper_type(item_type), AnyType): + # This relies on 'map_instance_to_supertype' returning 'Iterable[Any]' + # in case there is no explicit base class. + return item_type + # Try also structural typing. + return self.analyze_iterable_item_type_without_expression(it, context)[1] + + def function_type(self, func: FuncBase) -> FunctionLike: + typ = function_type(func, self.named_type("builtins.function")) + if ( + isinstance(func, FuncItem) + and func.is_coroutine + and not func.is_async_generator + and func.type is None + and isinstance(typ, CallableType) + ): + any_type = AnyType(TypeOfAny.special_form) + ret_type = self.named_generic_type( + "typing.Coroutine", [any_type, any_type, typ.ret_type] + ) + return typ.copy_modified(ret_type=ret_type) + return typ + + def push_type_map(self, type_map: TypeMap, *, from_assignment: bool = True) -> None: + if is_unreachable_map(type_map): + self.binder.unreachable() + else: + for expr, type in type_map.items(): + self.binder.put(expr, type, from_assignment=from_assignment) + + def infer_issubclass_maps(self, node: CallExpr, expr: Expression) -> tuple[TypeMap, TypeMap]: + """Infer type restrictions for an expression in issubclass call.""" + vartype = self.lookup_type(expr) + type = self.get_isinstance_type(node.args[1]) + if isinstance(vartype, TypeVarType): + vartype = vartype.upper_bound + vartype = get_proper_type(vartype) + if isinstance(vartype, UnionType): + union_list = [] + for t in get_proper_types(vartype.items): + if isinstance(t, TypeType): + union_list.append(t.item) + else: + # This is an error that should be reported earlier + # if we reach here, we refuse to do any type inference. + return {}, {} + vartype = UnionType(union_list) + elif isinstance(vartype, TypeType): + vartype = vartype.item + elif isinstance(vartype, Instance) and vartype.type.is_metaclass(): + vartype = self.named_type("builtins.object") + else: + # Any other object whose type we don't know precisely + # for example, Any or a custom metaclass. + return {}, {} # unknown type + yes_type, no_type = self.conditional_types_with_intersection(vartype, type, expr) + yes_map, no_map = conditional_types_to_typemaps(expr, yes_type, no_type) + yes_map, no_map = map(convert_to_typetype, (yes_map, no_map)) + return yes_map, no_map + + @overload + def conditional_types_with_intersection( + self, + expr_type: Type, + type_ranges: list[TypeRange] | None, + ctx: Context, + default: None = None, + *, + consider_runtime_isinstance: bool = True, + ) -> tuple[Type | None, Type | None]: ... + + @overload + def conditional_types_with_intersection( + self, + expr_type: Type, + type_ranges: list[TypeRange] | None, + ctx: Context, + default: Type, + *, + consider_runtime_isinstance: bool = True, + ) -> tuple[Type, Type]: ... + + def conditional_types_with_intersection( + self, + expr_type: Type, + type_ranges: list[TypeRange] | None, + ctx: Context, + default: Type | None = None, + *, + consider_runtime_isinstance: bool = True, + ) -> tuple[Type | None, Type | None]: + initial_types = conditional_types( + expr_type, + type_ranges, + default, + consider_runtime_isinstance=consider_runtime_isinstance, + ) + # For some reason, doing "yes_map, no_map = conditional_types_to_typemaps(...)" + # doesn't work: mypyc will decide that 'yes_map' is of type None if we try. + yes_type: Type | None = initial_types[0] + no_type: Type | None = initial_types[1] + + if not isinstance(get_proper_type(yes_type), UninhabitedType) or type_ranges is None: + return yes_type, no_type + + # If conditional_types was unable to successfully narrow the expr_type + # using the type_ranges and concluded if-branch is unreachable, we try + # computing it again using a different algorithm that tries to generate + # an ad-hoc intersection between the expr_type and the type_ranges. + proper_type = get_proper_type(expr_type) + if isinstance(proper_type, UnionType): + possible_expr_types = get_proper_types(proper_type.relevant_items()) + else: + possible_expr_types = [proper_type] + + possible_target_types = [] + for tr in type_ranges: + item = get_proper_type(tr.item) + if isinstance(item, (Instance, NoneType)): + possible_target_types.append(item) + if not possible_target_types: + return yes_type, no_type + + out = [] + errors: list[tuple[str, str]] = [] + for v in possible_expr_types: + if not isinstance(v, Instance): + return yes_type, no_type + for t in possible_target_types: + if isinstance(t, NoneType): + errors.append((f'"{v.type.name}" and "NoneType"', '"NoneType" is final')) + continue + intersection = self.intersect_instances((v, t), errors) + if intersection is None: + continue + out.append(intersection) + if not out: + # Only report errors if no element in the union worked. + if self.should_report_unreachable_issues(): + for types, reason in errors: + self.msg.impossible_intersection(types, reason, ctx) + return UninhabitedType(), expr_type + new_yes_type = make_simplified_union(out) + return new_yes_type, expr_type + + def is_writable_attribute(self, node: Node) -> bool: + """Check if an attribute is writable""" + if isinstance(node, Var): + if node.is_property and not node.is_settable_property: + return False + return True + elif isinstance(node, OverloadedFuncDef) and node.is_property: + first_item = node.items[0] + assert isinstance(first_item, Decorator) + return first_item.var.is_settable_property + return False + + def get_isinstance_type( + self, expr: Expression, flatten_tuples: bool = True + ) -> list[TypeRange] | None: + """Get the type(s) resulting from an isinstance check. + + Returns an empty list for isinstance(x, ()). + """ + if isinstance(expr, OpExpr) and expr.op == "|": + left: list[TypeRange] | None + right: list[TypeRange] | None + if is_literal_none(expr.left): + left = [TypeRange(NoneType(), is_upper_bound=False)] + else: + left = self.get_isinstance_type(expr.left, flatten_tuples=False) + if is_literal_none(expr.right): + right = [TypeRange(NoneType(), is_upper_bound=False)] + else: + right = self.get_isinstance_type(expr.right, flatten_tuples=False) + if left is None or right is None: + return None + return left + right + + if flatten_tuples: + type_ranges = [] + for typ in flatten_types_if_tuple(self.lookup_type(expr)): + type_range = self.get_type_range_of_type(typ) + if type_range is None: + return None + type_ranges.append(type_range) + return type_ranges + + else: + type_range = self.get_type_range_of_type(self.lookup_type(expr)) + if type_range is None: + return None + return [type_range] + + def get_type_range_of_type(self, typ: Type) -> TypeRange | None: + typ = get_proper_type(typ) + if isinstance(typ, TypeVarType): + typ = get_proper_type(typ.upper_bound) + + if isinstance(typ, UnionType): + type_ranges = [self.get_type_range_of_type(item) for item in typ.items] + item = make_simplified_union([t.item for t in type_ranges if t is not None]) + return TypeRange(item, is_upper_bound=True) + if isinstance(typ, FunctionLike) and typ.is_type_obj(): + # If a type is generic, `isinstance` can only narrow its variables to Any. + any_parameterized = fill_typevars_with_any(typ.type_object()) + # Tuples may have unattended type variables among their items + if isinstance(any_parameterized, TupleType): + erased_type = erase_typevars(any_parameterized) + else: + erased_type = any_parameterized + return TypeRange(erased_type, is_upper_bound=False) + if isinstance(typ, TypeType): + # Type[A] means "any type that is a subtype of A" rather than "precisely type A" + # we indicate this by setting is_upper_bound flag + is_upper_bound = True + if isinstance(typ.item, NoneType): + # except for Type[None], because "'NoneType' is not an acceptable base type" + is_upper_bound = False + if isinstance(typ.item, Instance) and typ.item.type.is_final: + is_upper_bound = False + return TypeRange(typ.item, is_upper_bound=is_upper_bound) + if isinstance(typ, AnyType): + return TypeRange(typ, is_upper_bound=False) + if isinstance(typ, Instance) and typ.type.fullname == "builtins.type": + object_type = Instance(typ.type.mro[-1], []) + return TypeRange(object_type, is_upper_bound=True) + if isinstance(typ, Instance) and typ.type.fullname == "types.UnionType" and typ.args: + return TypeRange(UnionType(typ.args), is_upper_bound=False) + if isinstance(typ, Instance) and typ.type.fullname == "typing._SpecialForm": + # This is probably an alias to a Union object. We don't have the args here so we can't + # conclude anything + return None + if not is_subtype(self.named_type("builtins.type"), typ): + # We saw something, but it couldn't possibly be valid + return TypeRange(UninhabitedType(), is_upper_bound=False) + + # This is e.g. a variable of type object, so we can't conclude anything + return None + + def is_literal_enum(self, n: Expression) -> bool: + """Returns true if this expression (with the given type context) is an Enum literal. + + For example, if we had an enum: + + class Foo(Enum): + A = 1 + B = 2 + + ...and if the expression 'Foo' referred to that enum within the current type context, + then the expression 'Foo.A' would be a literal enum. However, if we did 'a = Foo.A', + then the variable 'a' would *not* be a literal enum. + + We occasionally special-case expressions like 'Foo.A' and treat them as a single primitive + unit for the same reasons we sometimes treat 'True', 'False', or 'None' as a single + primitive unit. + """ + if not isinstance(n, MemberExpr) or not isinstance(n.expr, NameExpr): + return False + + parent_type = self.lookup_type_or_none(n.expr) + member_type = self.lookup_type_or_none(n) + if member_type is None or parent_type is None: + return False + + parent_type = get_proper_type(parent_type) + member_type = get_proper_type(coerce_to_literal(member_type)) + if not isinstance(parent_type, FunctionLike) or not isinstance(member_type, LiteralType): + return False + + if not parent_type.is_type_obj(): + return False + + return ( + member_type.is_enum_literal() + and member_type.fallback.type == parent_type.type_object() + ) + + def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: + """Inject an extra attribute with Any type using fallbacks.""" + orig_typ = typ + typ = get_proper_type(typ) + any_type = AnyType(TypeOfAny.unannotated) + if isinstance(typ, Instance): + result = typ.copy_with_extra_attr(name, any_type) + # For instances, we erase the possible module name, so that restrictions + # become anonymous types.ModuleType instances, allowing hasattr() to + # have effect on modules. + assert result.extra_attrs is not None + result.extra_attrs.mod_name = None + return result + if isinstance(typ, TupleType): + fallback = typ.partial_fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, CallableType): + fallback = typ.fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, TypeType) and isinstance(typ.item, Instance): + return TypeType.make_normalized( + self.add_any_attribute_to_type(typ.item, name), is_type_form=typ.is_type_form + ) + if isinstance(typ, TypeVarType): + return typ.copy_modified( + upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name), + values=[self.add_any_attribute_to_type(v, name) for v in typ.values], + ) + if isinstance(typ, UnionType): + with_attr, without_attr = self.partition_union_by_attr(typ, name) + return make_simplified_union( + with_attr + [self.add_any_attribute_to_type(typ, name) for typ in without_attr] + ) + return orig_typ + + def hasattr_type_maps( + self, expr: Expression, source_type: Type, name: str + ) -> tuple[TypeMap, TypeMap]: + """Simple support for hasattr() checks. + + Essentially the logic is following: + * In the if branch, keep types that already has a valid attribute as is, + for other inject an attribute with `Any` type. + * In the else branch, remove types that already have a valid attribute, + while keeping the rest. + """ + if self.has_valid_attribute(source_type, name): + return {expr: source_type}, {} + + source_type = get_proper_type(source_type) + if isinstance(source_type, UnionType): + _, without_attr = self.partition_union_by_attr(source_type, name) + yes_map = {expr: self.add_any_attribute_to_type(source_type, name)} + return yes_map, {expr: make_simplified_union(without_attr)} + + type_with_attr = self.add_any_attribute_to_type(source_type, name) + if type_with_attr != source_type: + return {expr: type_with_attr}, {} + return {}, {} + + def partition_union_by_attr( + self, source_type: UnionType, name: str + ) -> tuple[list[Type], list[Type]]: + with_attr = [] + without_attr = [] + for item in source_type.items: + if self.has_valid_attribute(item, name): + with_attr.append(item) + else: + without_attr.append(item) + return with_attr, without_attr + + def has_valid_attribute(self, typ: Type, name: str) -> bool: + p_typ = get_proper_type(typ) + if isinstance(p_typ, AnyType): + return False + if isinstance(p_typ, Instance) and p_typ.extra_attrs and p_typ.extra_attrs.mod_name: + # Presence of module_symbol_table means this check will skip ModuleType.__getattr__ + module_symbol_table = p_typ.type.names + else: + module_symbol_table = None + with self.msg.filter_errors() as watcher: + analyze_member_access( + name, + typ, + TempNode(AnyType(TypeOfAny.special_form)), + is_lvalue=False, + is_super=False, + is_operator=False, + original_type=typ, + chk=self, + # This is not a real attribute lookup so don't mess with deferring nodes. + no_deferral=True, + module_symbol_table=module_symbol_table, + ) + return not watcher.has_new_errors() + + def get_expression_type(self, node: Expression, type_context: Type | None = None) -> Type: + return self.expr_checker.accept(node, type_context=type_context) + + def is_defined_in_stub(self, typ: Instance, /) -> bool: + return self.modules[typ.type.module_name].is_stub + + def check_deprecated(self, node: Node | None, context: Context) -> None: + """Warn if deprecated and not directly imported with a `from` statement.""" + if isinstance(node, Decorator): + node = node.func + if isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) and ( + node.deprecated is not None + ): + for imp in self.tree.imports: + if isinstance(imp, ImportFrom) and any(node.name == n[0] for n in imp.names): + break + else: + self.warn_deprecated(node, context) + + def warn_deprecated(self, node: Node | None, context: Context) -> None: + """Warn if deprecated.""" + if isinstance(node, Decorator): + node = node.func + if ( + isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) + and (deprecated := node.deprecated) is not None + and not self.is_typeshed_stub + and not any( + node.fullname == p or node.fullname.startswith(f"{p}.") + for p in self.options.deprecated_calls_exclude + ) + ): + warn = self.msg.note if self.options.report_deprecated_as_note else self.msg.fail + warn(deprecated, context, code=codes.DEPRECATED) + + def new_unique_dummy_name(self, namespace: str) -> str: + """Generate a name that is guaranteed to be unique for this TypeChecker instance.""" + name = f"dummy-{namespace}-{self._unique_id}" + self._unique_id += 1 + return name + + # leafs + + def visit_pass_stmt(self, o: PassStmt, /) -> None: + return None + + def visit_nonlocal_decl(self, o: NonlocalDecl, /) -> None: + return None + + def visit_global_decl(self, o: GlobalDecl, /) -> None: + if self.options.allow_redefinition: + # Add names to binder, since their types could be widened + for name in o.names: + sym = self.globals.get(name) + if sym and isinstance(sym.node, Var) and sym.node.type is not None: + n = NameExpr(name) + n.node = sym.node + n.kind = GDEF + n.fullname = sym.node.fullname + typ = get_declaration(n) + if typ is not None: + self.binder.assign_type(n, typ, typ) + + +class TypeCheckerAsSemanticAnalyzer(SemanticAnalyzerCoreInterface): + """ + Adapts TypeChecker to the SemanticAnalyzerCoreInterface, + allowing most type expressions to be parsed during the TypeChecker pass. + + See ExpressionChecker.try_parse_as_type_expression() to understand how this + class is used. + """ + + _chk: TypeChecker + _names: dict[str, SymbolTableNode] + did_fail: bool + + def __init__(self, chk: TypeChecker, names: dict[str, SymbolTableNode]) -> None: + self._chk = chk + self._names = names + self.did_fail = False + + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + sym = self._names.get(name) + # All names being looked up should have been previously gathered, + # even if the related SymbolTableNode does not refer to a valid SymbolNode + assert sym is not None, name + return sym + + def lookup_fully_qualified(self, fullname: str, /) -> SymbolTableNode: + ret = self.lookup_fully_qualified_or_none(fullname) + assert ret is not None, fullname + return ret + + def lookup_fully_qualified_or_none(self, fullname: str, /) -> SymbolTableNode | None: + try: + return self._chk.lookup_qualified(fullname) + except KeyError: + return None + + def record_fixed_type(self, fixed: TypeInfo | TypeAlias) -> None: + pass + + def fail( + self, + msg: str, + ctx: Context, + serious: bool = False, + *, + blocker: bool = False, + code: ErrorCode | None = None, + ) -> None: + self.did_fail = True + + def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + pass + + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + if feature not in self._chk.options.enable_incomplete_feature: + self.fail("__ignored__", ctx) + return False + return True + + def record_incomplete_ref(self) -> None: + pass + + def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None: + pass + + def is_incomplete_namespace(self, fullname: str) -> bool: + return False + + @property + def final_iteration(self) -> bool: + return True + + def is_future_flag_set(self, flag: str) -> bool: + return self._chk.tree.is_future_flag_set(flag) + + @property + def is_stub_file(self) -> bool: + return self._chk.tree.is_stub + + def is_func_scope(self) -> bool: + # Return arbitrary value. + # + # This method is currently only used to decide whether to pair + # a fail() message with a note() message or not. Both of those + # message types are ignored. + return False + + def is_nested_within_func_scope(self) -> bool: + return self._chk.scope.top_level_function() is not None + + @property + def type(self) -> TypeInfo | None: + return self._chk.scope.current_class() + + +class CollectArgTypeVarTypes(TypeTraverserVisitor): + """Collects the non-nested argument types in a set.""" + + def __init__(self) -> None: + self.arg_types: set[TypeVarType] = set() + + def visit_type_var(self, t: TypeVarType) -> None: + self.arg_types.add(t) + + +@overload +def conditional_types( + current_type: Type, + proposed_type_ranges: list[TypeRange] | None, + default: None = None, + *, + consider_runtime_isinstance: bool = True, + from_equality: bool = False, +) -> tuple[Type | None, Type | None]: ... + + +@overload +def conditional_types( + current_type: Type, + proposed_type_ranges: list[TypeRange] | None, + default: Type, + *, + consider_runtime_isinstance: bool = True, + from_equality: bool = False, +) -> tuple[Type, Type]: ... + + +def conditional_types( + current_type: Type, + proposed_type_ranges: list[TypeRange] | None, + default: Type | None = None, + *, + consider_runtime_isinstance: bool = True, + from_equality: bool = False, +) -> tuple[Type | None, Type | None]: + """Takes in the current type and a proposed type of an expression. + + Returns a 2-tuple: + The first element is the proposed type, if the expression can be the proposed type. + (or default, if default is set and the expression is a subtype of the proposed type). + The second element is the type it would hold if it was not the proposed type, if any. + (or default, if default is set and the expression is not a subtype of the proposed type). + + UninhabitedType means unreachable. + None means no new information can be inferred. + """ + if proposed_type_ranges is None: + # An isinstance check, but we don't understand the type + return current_type, default + + if not proposed_type_ranges: + # This is the case for `if isinstance(x, ())` which always returns False. + return UninhabitedType(), default + + if len(proposed_type_ranges) == 1: + # expand e.g. bool -> Literal[True] | Literal[False] + target = proposed_type_ranges[0].item + target = get_proper_type(target) + if isinstance(target, LiteralType) and ( + target.is_enum_literal() or isinstance(target.value, bool) + ): + enum_name = target.fallback.type.fullname + current_type = try_expanding_sum_type_to_union(current_type, enum_name) + + proposed_type: Type + remaining_type: Type + + p_current_type = get_proper_type(current_type) + # factorize over union types: isinstance(A|B, C) -> yes = A_yes | B_yes + if isinstance(p_current_type, UnionType): + yes_items: list[Type] = [] + no_items: list[Type] = [] + for union_item in p_current_type.items: + yes_type, no_type = conditional_types( + union_item, + proposed_type_ranges, + default=union_item, + consider_runtime_isinstance=consider_runtime_isinstance, + from_equality=from_equality, + ) + yes_items.append(yes_type) + no_items.append(no_type) + + proposed_type = make_simplified_union(yes_items) + remaining_type = make_simplified_union(no_items) + return proposed_type, remaining_type + + proposed_type = make_simplified_union([type_range.item for type_range in proposed_type_ranges]) + items = proposed_type.items if isinstance(proposed_type, UnionType) else [proposed_type] + for i in range(len(items)): + item = get_proper_type(items[i]) + # Avoid ever narrowing to a NewType. The principle is values of NewType should only be + # produce by explicit wrapping + while isinstance(item, Instance) and item.type.is_newtype: + item = item.type.bases[0] + items[i] = item + proposed_type = get_proper_type(UnionType.make_union(items)) + + if isinstance(p_current_type, AnyType): + return proposed_type, current_type + if isinstance(proposed_type, AnyType): + # We don't really know much about the proposed type, so we shouldn't + # attempt to narrow anything. Instead, we broaden the expr to Any to + # avoid false positives + return proposed_type, default + if not any(type_range.is_upper_bound for type_range in proposed_type_ranges): + # concrete subtype + if is_proper_subtype(current_type, proposed_type, ignore_promotions=True): + return default, UninhabitedType() + + # structural subtypes + if ( + isinstance(proposed_type, CallableType) + or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) + ) and is_subtype(current_type, proposed_type, ignore_promotions=True): + # Note: It's possible that current_type=`Any | Proto` while proposed_type=`Proto` + # so we cannot return `Never` for the else branch + remainder = restrict_subtype_away( + current_type, + default if default is not None else proposed_type, + consider_runtime_isinstance=consider_runtime_isinstance, + ) + return default, remainder + + if from_equality: + # We erase generic args because values with different generic types can compare equal + # For instance, cast(list[str], []) and cast(list[int], []) + proposed_type = shallow_erase_type_for_equality(proposed_type) + + if not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): + # Expression is never of any type in proposed_type_ranges + return UninhabitedType(), default + + # we can only restrict when the type is precise, not bounded + proposed_precise_type = UnionType.make_union( + [type_range.item for type_range in proposed_type_ranges if not type_range.is_upper_bound] + ) + remaining_type = restrict_subtype_away( + current_type, + proposed_precise_type, + consider_runtime_isinstance=consider_runtime_isinstance, + ) + + # Avoid widening the type + if is_proper_subtype(p_current_type, proposed_type, ignore_promotions=True): + proposed_type = default if default is not None else current_type + + return proposed_type, remaining_type + + +def conditional_types_to_typemaps( + expr: Expression, yes_type: Type | None, no_type: Type | None +) -> tuple[TypeMap, TypeMap]: + expr = collapse_walrus(expr) + + yes_map = {} if yes_type is None else {expr: yes_type} + no_map = {} if no_type is None else {expr: no_type} + return yes_map, no_map + + +def gen_unique_name(base: str, table: SymbolTable) -> str: + """Generate a name that does not appear in table by appending numbers to base.""" + if base not in table: + return base + i = 1 + while base + str(i) in table: + i += 1 + return base + str(i) + + +def is_true_literal(n: Expression) -> bool: + """Returns true if this expression is the 'True' literal/keyword.""" + return refers_to_fullname(n, "builtins.True") or isinstance(n, IntExpr) and n.value != 0 + + +def is_false_literal(n: Expression) -> bool: + """Returns true if this expression is the 'False' literal/keyword.""" + return refers_to_fullname(n, "builtins.False") or isinstance(n, IntExpr) and n.value == 0 + + +def is_literal_none(n: Expression) -> bool: + """Returns true if this expression is the 'None' literal/keyword.""" + return isinstance(n, NameExpr) and n.fullname == "builtins.None" + + +def is_literal_not_implemented(n: Expression | None) -> bool: + return isinstance(n, NameExpr) and n.fullname == "builtins.NotImplemented" + + +def _is_empty_generator_function(func: FuncItem) -> bool: + """ + Checks whether a function's body is 'return; yield' (the yield being added only + to promote the function into a generator function). + """ + body = func.body.body + return ( + len(body) == 2 + and isinstance(ret_stmt := body[0], ReturnStmt) + and (ret_stmt.expr is None or is_literal_none(ret_stmt.expr)) + and isinstance(expr_stmt := body[1], ExpressionStmt) + and isinstance(yield_expr := expr_stmt.expr, YieldExpr) + and (yield_expr.expr is None or is_literal_none(yield_expr.expr)) + ) + + +def builtin_item_type(tp: Type) -> Type | None: + """Get the item type of a builtin container. + + If 'tp' is not one of the built containers (these includes NamedTuple and TypedDict) + or if the container is not parameterized (like List or List[Any]) + return None. This function is used to narrow optional types in situations like this: + + x: Optional[int] + if x in (1, 2, 3): + x + 42 # OK + + Note: this is only OK for built-in containers, where we know the behavior + of __contains__. + """ + tp = get_proper_type(tp) + + if isinstance(tp, Instance): + if tp.type.fullname in [ + "builtins.list", + "builtins.tuple", + "builtins.dict", + "builtins.frozendict", + "builtins.set", + "builtins.frozenset", + "_collections_abc.dict_keys", + "typing.KeysView", + ]: + if not tp.args: + # TODO: fix tuple in lib-stub/builtins.pyi (it should be generic). + return None + if not isinstance(get_proper_type(tp.args[0]), AnyType): + return tp.args[0] + elif isinstance(tp, TupleType): + normalized_items = [] + for it in tp.items: + # This use case is probably rare, but not handling unpacks here can cause crashes. + if isinstance(it, UnpackType): + unpacked = get_proper_type(it.type) + if isinstance(unpacked, TypeVarTupleType): + unpacked = get_proper_type(unpacked.upper_bound) + assert ( + isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + ) + normalized_items.append(unpacked.args[0]) + else: + normalized_items.append(it) + if all(not isinstance(it, AnyType) for it in get_proper_types(normalized_items)): + return make_simplified_union(normalized_items) # this type is not externally visible + elif isinstance(tp, TypedDictType): + # TypedDict always has non-optional string keys. Find the key type from the Mapping + # base class. + for base in tp.fallback.type.mro: + if base.fullname == "typing.Mapping": + return map_instance_to_supertype(tp.fallback, base).args[0] + assert False, "No Mapping base class found for TypedDict fallback" + return None + + +def is_unreachable_map(map: TypeMap) -> bool: + return any(isinstance(get_proper_type(v), UninhabitedType) for v in map.values()) + + +def and_conditional_maps(m1: TypeMap, m2: TypeMap, *, use_meet: bool = False) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 and e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + # Both conditions can be true; combine the information. Anything + # we learn from either conditions' truth is valid. + result = m2.copy() + m2_exprs = {literal_hash(e2): e2 for e2 in m2} + for e1 in m1: + e1_hash = literal_hash(e1) + if e1_hash not in m2_exprs: + result[e1] = m1[e1] + continue + + if not use_meet: + # If use_meet=False and the same expression's type is refined by both conditions, + # we somewhat arbitrarily give precedence to m2, unless a) m1's value is Never, or + # b) m2's value is Any and m1 isn't a union that could have been narrowed to Any. + t1 = m1[e1] + pt1 = get_proper_type(t1) + if isinstance(pt1, UninhabitedType): + result[e1] = t1 + else: + e2 = m2_exprs[e1_hash] + pt2 = get_proper_type(m2[e2]) + if isinstance(pt2, AnyType) and not ( + isinstance(pt1, UnionType) + and any(isinstance(get_proper_type(item), AnyType) for item in pt1.items) + ): + result[e1] = t1 + + if use_meet: + # For now, meet common keys only if specifically requested. + # This is currently used for tuple types narrowing, where having + # a precise result is important. + for e1 in m1: + for e2 in m2: + if literal_hash(e1) == literal_hash(e2): + result[e1] = meet_types(m1[e1], m2[e2]) + return result + + +def or_conditional_maps(m1: TypeMap, m2: TypeMap, *, coalesce_any: bool = False) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 or e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. If coalesce_any is True, consider Any a supertype when + joining restrictions. + """ + + if is_unreachable_map(m1): + return m2 + if is_unreachable_map(m2): + return m1 + # Both conditions can be true. Combine information about + # expressions whose type is refined by both conditions. (We do not + # learn anything about expressions whose type is refined by only + # one condition.) + result: dict[Expression, Type] = {} + for n1 in m1: + for n2 in m2: + if literal_hash(n1) == literal_hash(n2): + if coalesce_any and isinstance(get_proper_type(m1[n1]), AnyType): + result[n1] = m1[n1] + else: + result[n1] = make_simplified_union([m1[n1], m2[n2]]) + return result + + +def reduce_conditional_maps( + type_maps: list[tuple[TypeMap, TypeMap]], use_meet: bool = False +) -> tuple[TypeMap, TypeMap]: + """Reduces a list containing pairs of if/else TypeMaps into a single pair. + + We "and" together all of the if TypeMaps and "or" together the else TypeMaps. So + for example, if we had the input: + + [ + ({x: TypeIfX, shared: TypeIfShared1}, {x: TypeElseX, shared: TypeElseShared1}), + ({y: TypeIfY, shared: TypeIfShared2}, {y: TypeElseY, shared: TypeElseShared2}), + ] + + ...we'd return the output: + + ( + {x: TypeIfX, y: TypeIfY, shared: PseudoIntersection[TypeIfShared1, TypeIfShared2]}, + {shared: Union[TypeElseShared1, TypeElseShared2]}, + ) + + ...where "PseudoIntersection[X, Y] == Y" because mypy actually doesn't understand intersections + yet, so we settle for just arbitrarily picking the right expr's type. + + We only retain the shared expression in the 'else' case because we don't actually know + whether x was refined or y was refined -- only just that one of the two was refined. + """ + if len(type_maps) == 0: + return {}, {} + elif len(type_maps) == 1: + return type_maps[0] + else: + final_if_map, final_else_map = type_maps[0] + for if_map, else_map in type_maps[1:]: + final_if_map = and_conditional_maps(final_if_map, if_map, use_meet=use_meet) + final_else_map = or_conditional_maps(final_else_map, else_map) + + return final_if_map, final_else_map + + +def reduce_or_conditional_type_maps(ms: list[TypeMap]) -> TypeMap: + """Reduces a list of TypeMaps into a single TypeMap by "or"-ing them together.""" + if len(ms) == 0: + return {} + if len(ms) == 1: + return ms[0] + result = ms[0] + for m in ms[1:]: + result = or_conditional_maps(result, m) + return result + + +def reduce_and_conditional_type_maps(ms: list[TypeMap], *, use_meet: bool) -> TypeMap: + """Reduces a list of TypeMaps into a single TypeMap by "and"-ing them together.""" + if len(ms) == 0: + return {} + if len(ms) == 1: + return ms[0] + result = ms[0] + for m in ms[1:]: + if not m: + continue # this is a micro-optimisation + result = and_conditional_maps(result, m, use_meet=use_meet) + return result + + +def has_custom_eq_checks(t: Type) -> bool: + return custom_special_method(t, "__eq__", check_all=False) or custom_special_method( + t, "__ne__", check_all=False + ) + + +def convert_to_typetype(type_map: TypeMap) -> TypeMap: + converted_type_map: dict[Expression, Type] = {} + for expr, typ in type_map.items(): + t = typ + if isinstance(t, TypeVarType): + t = t.upper_bound + t = get_proper_type(t) + + # TODO: should we only allow unions of instances as per PEP 484? + if isinstance(t, UninhabitedType): + converted_type_map[expr] = typ + elif isinstance(t, (UnionType, Instance, NoneType)): + converted_type_map[expr] = TypeType.make_normalized(typ) + else: + # unknown type; error was likely reported earlier + return {} + return converted_type_map + + +def flatten(t: Expression) -> list[Expression]: + """Flatten a nested sequence of tuples/lists into one list of nodes.""" + if isinstance(t, (TupleExpr, ListExpr)): + return [b for a in t.items for b in flatten(a)] + elif isinstance(t, StarExpr): + return flatten(t.expr) + else: + return [t] + + +def flatten_types_if_tuple(t: Type) -> list[Type]: + """Flatten a nested sequence of tuples into one list of nodes.""" + t = get_proper_type(t) + if isinstance(t, UnionType): + return [UnionType.make_union([b for a in t.items for b in flatten_types_if_tuple(a)])] + if isinstance(t, TupleType): + return [b for a in t.items for b in flatten_types_if_tuple(a)] + elif is_named_instance(t, "builtins.tuple"): + return [t.args[0]] + return [t] + + +def expand_func(defn: FuncItem, map: dict[TypeVarId, Type]) -> FuncItem: + visitor = TypeTransformVisitor(map) + ret = visitor.node(defn) + assert isinstance(ret, FuncItem) + return ret + + +class TypeTransformVisitor(TransformVisitor): + def __init__(self, map: dict[TypeVarId, Type]) -> None: + super().__init__() + self.map = map + + def type(self, type: Type) -> Type: + return expand_type(type, self.map) + + +def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool: + """Can a single call match both t and s, based just on positional argument counts?""" + min_args = max(t.min_args, s.min_args) + max_args = min(t.max_possible_positional_args(), s.max_possible_positional_args()) + return min_args <= max_args + + +def expand_callable_variants(c: CallableType) -> list[CallableType]: + """Expand a generic callable using all combinations of type variables' values/bounds.""" + for tv in c.variables: + # We need to expand self-type before other variables, because this is the only + # type variable that can have other type variables in the upper bound. + if tv.id.is_self(): + c = expand_type(c, {tv.id: tv.upper_bound}).copy_modified( + variables=[v for v in c.variables if not v.id.is_self()] + ) + break + + if not c.is_generic(): + # Fast path. + return [c] + + tvar_values = [] + for tvar in c.variables: + if isinstance(tvar, TypeVarType) and tvar.values: + tvar_values.append(tvar.values) + else: + tvar_values.append([tvar.upper_bound]) + + variants = [] + for combination in itertools.product(*tvar_values): + tvar_map = {tv.id: subst for (tv, subst) in zip(c.variables, combination)} + variants.append(expand_type(c, tvar_map).copy_modified(variables=[])) + return variants + + +def is_unsafe_overlapping_overload_signatures( + signature: CallableType, + other: CallableType, + class_type_vars: list[TypeVarLikeType], + partial_only: bool = True, +) -> bool: + """Check if two overloaded signatures are unsafely overlapping or partially overlapping. + + We consider two functions 's' and 't' to be unsafely overlapping if three + conditions hold: + + 1. s's parameters are partially overlapping with t's. i.e. there are calls that are + valid for both signatures. + 2. for these common calls, some of t's parameters types are wider that s's. + 3. s's return type is NOT a subset of t's. + + Note that we use subset rather than subtype relationship in these checks because: + * Overload selection happens at runtime, not statically. + * This results in more lenient behavior. + This can cause false negatives (e.g. if overloaded function returns an externally + visible attribute with invariant type), but such situations are rare. In general, + overloads in Python are generally unsafe, so we intentionally try to avoid giving + non-actionable errors (see more details in comments below). + + Assumes that 'signature' appears earlier in the list of overload + alternatives then 'other' and that their argument counts are overlapping. + """ + # Try detaching callables from the containing class so that all TypeVars + # are treated as being free, i.e. the signature is as seen from inside the class, + # where "self" is not yet bound to anything. + signature = detach_callable(signature, class_type_vars) + other = detach_callable(other, class_type_vars) + + # Note: We repeat this check twice in both directions compensate for slight + # asymmetries in 'is_callable_compatible'. + + other_expanded = expand_callable_variants(other) + for sig_variant in expand_callable_variants(signature): + for other_variant in other_expanded: + # Using only expanded callables may cause false negatives, we can add + # more variants (e.g. using inference between callables) in the future. + if is_subset_no_promote(sig_variant.ret_type, other_variant.ret_type): + continue + if not ( + is_callable_compatible( + sig_variant, + other_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=False, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(l, r), + allow_partial_overlap=True, + ) + or is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=True, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(r, l), + allow_partial_overlap=True, + ) + ): + continue + # Using the same `allow_partial_overlap` flag as before, can cause false + # negatives in case where star argument is used in a catch-all fallback overload. + # But again, practicality beats purity here. + if not partial_only or not is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_subset_no_promote, + check_args_covariantly=True, + is_proper_subtype=False, + ignore_return=True, + allow_partial_overlap=True, + ): + return True + return False + + +def detach_callable(typ: CallableType, class_type_vars: list[TypeVarLikeType]) -> CallableType: + """Ensures that the callable's type variables are 'detached' and independent of the context. + + A callable normally keeps track of the type variables it uses within its 'variables' field. + However, if the callable is from a method and that method is using a class type variable, + the callable will not keep track of that type variable since it belongs to the class. + """ + if not class_type_vars: + # Fast path, nothing to update. + return typ + return typ.copy_modified(variables=list(typ.variables) + class_type_vars) + + +def overload_can_never_match(signature: CallableType, other: CallableType) -> bool: + """Check if the 'other' method can never be matched due to 'signature'. + + This can happen if signature's parameters are all strictly broader then + other's parameters. + + Assumes that both signatures have overlapping argument counts. + """ + # The extra erasure is needed to prevent spurious errors + # in situations where an `Any` overload is used as a fallback + # for an overload with type variables. The spurious error appears + # because the type variables turn into `Any` during unification in + # the below subtype check and (surprisingly?) `is_proper_subtype(Any, Any)` + # returns `True`. + # TODO: find a cleaner solution instead of this ad-hoc erasure. + exp_signature = expand_type( + signature, {tvar.id: erase_def_to_union_or_bound(tvar) for tvar in signature.variables} + ) + return is_callable_compatible( + exp_signature, other, is_compat=is_more_precise, is_proper_subtype=True, ignore_return=True + ) + + +def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: + """Does t have wider arguments than s?""" + # TODO should an overload with additional items be allowed to be more + # general than one with fewer items (or just one item)? + if isinstance(t, CallableType): + if isinstance(s, CallableType): + return is_callable_compatible( + t, s, is_compat=is_proper_subtype, is_proper_subtype=True, ignore_return=True + ) + elif isinstance(t, FunctionLike): + if isinstance(s, FunctionLike): + if len(t.items) == len(s.items): + return all( + is_same_arg_prefix(items, itemt) for items, itemt in zip(t.items, s.items) + ) + return False + + +def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: + return is_callable_compatible( + t, + s, + is_compat=is_same_type, + is_proper_subtype=True, + ignore_return=True, + check_args_covariantly=True, + ignore_pos_arg_names=True, + ) + + +def infer_operator_assignment_method(typ: Type, operator: str) -> tuple[bool, str]: + """Determine if operator assignment on given value type is in-place, and the method name. + + For example, if operator is '+', return (True, '__iadd__') or (False, '__add__') + depending on which method is supported by the type. + """ + typ = get_proper_type(typ) + method = operators.op_methods[operator] + existing_method = None + if isinstance(typ, Instance): + existing_method = _find_inplace_method(typ, method, operator) + elif isinstance(typ, TypedDictType): + existing_method = _find_inplace_method(typ.fallback, method, operator) + + if existing_method is not None: + return True, existing_method + return False, method + + +def _find_inplace_method(inst: Instance, method: str, operator: str) -> str | None: + if operator in operators.ops_with_inplace_method: + inplace_method = "__i" + method[2:] + if inst.type.has_readable_member(inplace_method): + return inplace_method + return None + + +def is_valid_inferred_type( + typ: Type, options: Options, is_lvalue_final: bool = False, is_lvalue_member: bool = False +) -> bool: + """Is an inferred type valid and needs no further refinement? + + Examples of invalid types include the None type (when we are not assigning + None to a final lvalue) or List[]. + + When not doing strict Optional checking, all types containing None are + invalid. When doing strict Optional checking, only None and types that are + incompletely defined (i.e. contain UninhabitedType) are invalid. + """ + proper_type = get_proper_type(typ) + if isinstance(proper_type, NoneType): + # If the lvalue is final, we may immediately infer NoneType when the + # initializer is None. + # + # If not, we want to defer making this decision. The final inferred + # type could either be NoneType or an Optional type, depending on + # the context. This resolution happens in leave_partial_types when + # we pop a partial types scope. + return is_lvalue_final or (not is_lvalue_member and options.allow_redefinition) + elif isinstance(proper_type, UninhabitedType): + return False + return not typ.accept(InvalidInferredTypes()) + + +class InvalidInferredTypes(BoolTypeQuery): + """Find type components that are not valid for an inferred type. + + These include type, and any uninhabited types resulting from failed + (ambiguous) type inference. + """ + + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return t.ambiguous + + def visit_erased_type(self, t: ErasedType) -> bool: + # This can happen inside a lambda. + return True + + def visit_type_var(self, t: TypeVarType) -> bool: + # This is needed to prevent leaking into partial types during + # multi-step type inference. + return t.id.is_meta_var() + + def visit_tuple_type(self, t: TupleType, /) -> bool: + # Exclude fallback to avoid bogus "need type annotation" errors + return self.query_types(t.items) + + +class SetNothingToAny(TypeTranslator): + """Replace all ambiguous Uninhabited types with Any (to avoid spurious extra errors).""" + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if t.ambiguous: + return AnyType(TypeOfAny.from_error) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # Target of the alias cannot be an ambiguous UninhabitedType, so we just + # replace the arguments. + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + +def is_classmethod_node(node: SymbolNode | None) -> bool | None: + """Find out if a node describes a classmethod.""" + if isinstance(node, Decorator): + node = node.func + if isinstance(node, FuncDef): + return node.is_class + if isinstance(node, Var): + return node.is_classmethod + return None + + +def is_node_static(node: SymbolNode | None) -> bool | None: + """Find out if a node describes a static function method.""" + if isinstance(node, Decorator): + node = node.func + if isinstance(node, FuncDef): + return node.is_static + if isinstance(node, Var): + return node.is_staticmethod + return None + + +TKey = TypeVar("TKey") +TValue = TypeVar("TValue") + + +class DisjointDict(Generic[TKey, TValue]): + """An variation of the union-find algorithm/data structure where instead of keeping + track of just disjoint sets, we keep track of disjoint dicts -- keep track of multiple + Set[Key] -> Set[Value] mappings, where each mapping's keys are guaranteed to be disjoint. + + This data structure is currently used exclusively by 'group_comparison_operands' below + to merge chains of '==' and 'is' comparisons when two or more chains use the same expression + in best-case O(n), where n is the number of operands. + + Specifically, the `add_mapping()` function and `items()` functions will take on average + O(k + v) and O(n) respectively, where k and v are the number of keys and values we're adding + for a given chain. Note that k <= n and v <= n. + + We hit these average/best-case scenarios for most user code: e.g. when the user has just + a single chain like 'a == b == c == d == ...' or multiple disjoint chains like + 'a==b < c==d < e==f < ...'. (Note that a naive iterative merging would be O(n^2) for + the latter case). + + In comparison, this data structure will make 'group_comparison_operands' have a worst-case + runtime of O(n*log(n)): 'add_mapping()' and 'items()' are worst-case O(k*log(n) + v) and + O(k*log(n)) respectively. This happens only in the rare case where the user keeps repeatedly + making disjoint mappings before merging them in a way that persistently dodges the path + compression optimization in '_lookup_root_id', which would end up constructing a single + tree of height log_2(n). This makes root lookups no longer amoritized constant time when we + finally call 'items()'. + """ + + def __init__(self) -> None: + # Each key maps to a unique ID + self._key_to_id: dict[TKey, int] = {} + + # Each id points to the parent id, forming a forest of upwards-pointing trees. If the + # current id already is the root, it points to itself. We gradually flatten these trees + # as we perform root lookups: eventually all nodes point directly to its root. + self._id_to_parent_id: dict[int, int] = {} + + # Each root id in turn maps to the set of values. + self._root_id_to_values: dict[int, set[TValue]] = {} + + def add_mapping(self, keys: set[TKey], values: set[TValue]) -> None: + """Adds a 'Set[TKey] -> Set[TValue]' mapping. If there already exists a mapping + containing one or more of the given keys, we merge the input mapping with the old one. + + Note that the given set of keys must be non-empty -- otherwise, nothing happens. + """ + if not keys: + return + + subtree_roots = [self._lookup_or_make_root_id(key) for key in keys] + new_root = subtree_roots[0] + + root_values = self._root_id_to_values[new_root] + root_values.update(values) + for subtree_root in subtree_roots[1:]: + if subtree_root == new_root or subtree_root not in self._root_id_to_values: + continue + self._id_to_parent_id[subtree_root] = new_root + root_values.update(self._root_id_to_values.pop(subtree_root)) + + def items(self) -> list[tuple[set[TKey], set[TValue]]]: + """Returns all disjoint mappings in key-value pairs.""" + root_id_to_keys: dict[int, set[TKey]] = {} + for key in self._key_to_id: + root_id = self._lookup_root_id(key) + if root_id not in root_id_to_keys: + root_id_to_keys[root_id] = set() + root_id_to_keys[root_id].add(key) + + output = [] + for root_id, keys in root_id_to_keys.items(): + output.append((keys, self._root_id_to_values[root_id])) + + return output + + def _lookup_or_make_root_id(self, key: TKey) -> int: + if key in self._key_to_id: + return self._lookup_root_id(key) + else: + new_id = len(self._key_to_id) + self._key_to_id[key] = new_id + self._id_to_parent_id[new_id] = new_id + self._root_id_to_values[new_id] = set() + return new_id + + def _lookup_root_id(self, key: TKey) -> int: + i = self._key_to_id[key] + while i != self._id_to_parent_id[i]: + # Optimization: make keys directly point to their grandparents to speed up + # future traversals. This prevents degenerate trees of height n from forming. + new_parent = self._id_to_parent_id[self._id_to_parent_id[i]] + self._id_to_parent_id[i] = new_parent + i = new_parent + return i + + +def group_comparison_operands( + pairwise_comparisons: Iterable[tuple[str, Expression, Expression]], + operand_to_literal_hash: Mapping[int, Key], + operators_to_group: set[str], +) -> list[tuple[str, list[int]]]: + """Group a series of comparison operands together chained by any operand + in the 'operators_to_group' set. All other pairwise operands are kept in + groups of size 2. + + For example, suppose we have the input comparison expression: + + x0 == x1 == x2 < x3 < x4 is x5 is x6 is not x7 is not x8 + + If we get these expressions in a pairwise way (e.g. by calling ComparisonExpr's + 'pairwise()' method), we get the following as input: + + [('==', x0, x1), ('==', x1, x2), ('<', x2, x3), ('<', x3, x4), + ('is', x4, x5), ('is', x5, x6), ('is not', x6, x7), ('is not', x7, x8)] + + If `operators_to_group` is the set {'==', 'is'}, this function will produce + the following "simplified operator list": + + [("==", [0, 1, 2]), ("<", [2, 3]), ("<", [3, 4]), + ("is", [4, 5, 6]), ("is not", [6, 7]), ("is not", [7, 8])] + + Note that (a) we yield *indices* to the operands rather then the operand + expressions themselves and that (b) operands used in a consecutive chain + of '==' or 'is' are grouped together. + + If two of these chains happen to contain operands with the same underlying + literal hash (e.g. are assignable and correspond to the same expression), + we combine those chains together. For example, if we had: + + same == x < y == same + + ...and if 'operand_to_literal_hash' contained the same values for the indices + 0 and 3, we'd produce the following output: + + [("==", [0, 1, 2, 3]), ("<", [1, 2])] + + But if the 'operand_to_literal_hash' did *not* contain an entry, we'd instead + default to returning: + + [("==", [0, 1]), ("<", [1, 2]), ("==", [2, 3])] + + This function is currently only used to assist with type-narrowing refinements + and is extracted out to a helper function so we can unit test it. + """ + groups: dict[str, DisjointDict[Key, int]] = {op: DisjointDict() for op in operators_to_group} + + simplified_operator_list: list[tuple[str, list[int]]] = [] + last_operator: str | None = None + current_indices: set[int] = set() + current_hashes: set[Key] = set() + for i, (operator, left_expr, right_expr) in enumerate(pairwise_comparisons): + if last_operator is None: + last_operator = operator + + if current_indices and (operator != last_operator or operator not in operators_to_group): + # If some of the operands in the chain are assignable, defer adding it: we might + # end up needing to merge it with other chains that appear later. + if not current_hashes: + simplified_operator_list.append((last_operator, sorted(current_indices))) + else: + groups[last_operator].add_mapping(current_hashes, current_indices) + last_operator = operator + current_indices = set() + current_hashes = set() + + # Note: 'i' corresponds to the left operand index, so 'i + 1' is the + # right operand. + current_indices.add(i) + current_indices.add(i + 1) + + # We only ever want to combine operands/combine chains for these operators + if operator in operators_to_group: + left_hash = operand_to_literal_hash.get(i) + if left_hash is not None: + current_hashes.add(left_hash) + right_hash = operand_to_literal_hash.get(i + 1) + if right_hash is not None: + current_hashes.add(right_hash) + + if last_operator is not None: + if not current_hashes: + simplified_operator_list.append((last_operator, sorted(current_indices))) + else: + groups[last_operator].add_mapping(current_hashes, current_indices) + + # Now that we know which chains happen to contain the same underlying expressions + # and can be merged together, add in this info back to the output. + for operator, disjoint_dict in groups.items(): + for keys, indices in disjoint_dict.items(): + simplified_operator_list.append((operator, sorted(indices))) + + # For stability, reorder list by the first operand index to appear + simplified_operator_list.sort(key=lambda item: item[1][0]) + return simplified_operator_list + + +def is_typed_callable(c: Type | None) -> bool: + c = get_proper_type(c) + if not c or not isinstance(c, CallableType): + return False + return not all( + isinstance(t, AnyType) and t.type_of_any == TypeOfAny.unannotated + for t in get_proper_types(c.arg_types + [c.ret_type]) + ) + + +def is_untyped_decorator(typ: Type | None) -> bool: + typ = get_proper_type(typ) + if not typ: + return True + elif isinstance(typ, CallableType): + return not is_typed_callable(typ) + elif isinstance(typ, Instance): + method = typ.type.get_method("__call__") + if method: + if isinstance(method, Decorator): + return is_untyped_decorator(method.func.type) or is_untyped_decorator( + method.var.type + ) + + if isinstance(method.type, Overloaded): + return any(is_untyped_decorator(item) for item in method.type.items) + else: + return not is_typed_callable(method.type) + else: + return False + elif isinstance(typ, Overloaded): + return any(is_untyped_decorator(item) for item in typ.items) + return True + + +def is_static(func: FuncBase | Decorator) -> bool: + if isinstance(func, Decorator): + return is_static(func.func) + elif isinstance(func, FuncBase): + return func.is_static + assert False, f"Unexpected func type: {type(func)}" + + +def is_property(defn: SymbolNode) -> bool: + if isinstance(defn, FuncDef): + return defn.is_property + if isinstance(defn, Decorator): + return defn.func.is_property + if isinstance(defn, OverloadedFuncDef): + if defn.items and isinstance(defn.items[0], Decorator): + return defn.items[0].func.is_property + return False + + +def is_settable_property(defn: SymbolNode | None) -> TypeGuard[OverloadedFuncDef]: + if isinstance(defn, OverloadedFuncDef): + if defn.items and isinstance(defn.items[0], Decorator): + return defn.items[0].func.is_property + return False + + +def is_custom_settable_property(defn: SymbolNode | None) -> bool: + """Check if a node is a settable property with a non-trivial setter type. + + By non-trivial here we mean that it is known (i.e. definition was already type + checked), it is not Any, and it is different from the property getter type. + """ + if defn is None: + return False + if not is_settable_property(defn): + return False + first_item = defn.items[0] + assert isinstance(first_item, Decorator) + if not first_item.var.is_settable_property: + return False + var = first_item.var + if var.type is None or var.setter_type is None or isinstance(var.type, PartialType): + # The caller should defer in case of partial types or not ready variables. + return False + setter_type = var.setter_type.arg_types[1] + if isinstance(get_proper_type(setter_type), AnyType): + return False + return not is_same_type(get_property_type(get_proper_type(var.type)), setter_type) + + +def get_property_type(t: ProperType) -> ProperType: + if isinstance(t, CallableType): + return get_proper_type(t.ret_type) + if isinstance(t, Overloaded): + return get_proper_type(t.items[0].ret_type) + return t + + +def is_subset_no_promote(left: Type, right: Type) -> bool: + return is_subtype(left, right, ignore_promotions=True, always_covariant=True) + + +def is_overlapping_types_for_overload(left: Type, right: Type) -> bool: + # Note that among other effects 'overlap_for_overloads' flag will effectively + # ignore possible overlap between type variables and None. This is technically + # unsafe, but unsafety is tiny and this prevents some common use cases like: + # @overload + # def foo(x: None) -> None: .. + # @overload + # def foo(x: T) -> Foo[T]: ... + return is_overlapping_types(left, right, ignore_promotions=True, overlap_for_overloads=True) + + +def is_private(node_name: str) -> bool: + """Check if node is private to class definition.""" + return node_name.startswith("__") and not node_name.endswith("__") + + +def is_string_literal(typ: Type) -> bool: + strs = try_getting_str_literals_from_type(typ) + return strs is not None and len(strs) == 1 + + +def has_bool_item(typ: ProperType) -> bool: + """Return True if type is 'bool' or a union with a 'bool' item.""" + if is_named_instance(typ, "builtins.bool"): + return True + if isinstance(typ, UnionType): + return any(is_named_instance(item, "builtins.bool") for item in typ.items) + return False + + +def collapse_walrus(e: Expression) -> Expression: + """If an expression is an AssignmentExpr, pull out the assignment target. + + We don't make any attempt to pull out all the targets in code like `x := (y := z)`. + We could support narrowing those if that sort of code turns out to be common. + """ + if isinstance(e, AssignmentExpr): + return e.target + return e + + +def find_last_var_assignment_line(n: Node, v: Var) -> int: + """Find the highest line number of a potential assignment to variable within node. + + This supports local and global variables. + + Return -1 if no assignment was found. + """ + visitor = VarAssignVisitor(v) + n.accept(visitor) + return visitor.last_line + + +class VarAssignVisitor(TraverserVisitor): + def __init__(self, v: Var) -> None: + self.last_line = -1 + self.lvalue = False + self.var_node = v + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.lvalue = True + for lv in s.lvalues: + lv.accept(self) + self.lvalue = False + + def visit_name_expr(self, e: NameExpr) -> None: + if self.lvalue and e.node is self.var_node: + self.last_line = max(self.last_line, e.line) + + def visit_member_expr(self, e: MemberExpr) -> None: + old_lvalue = self.lvalue + self.lvalue = False + super().visit_member_expr(e) + self.lvalue = old_lvalue + + def visit_index_expr(self, e: IndexExpr) -> None: + old_lvalue = self.lvalue + self.lvalue = False + super().visit_index_expr(e) + self.lvalue = old_lvalue + + def visit_with_stmt(self, s: WithStmt) -> None: + self.lvalue = True + for lv in s.target: + if lv is not None: + lv.accept(self) + self.lvalue = False + s.body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.lvalue = True + s.index.accept(self) + self.lvalue = False + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_assignment_expr(self, e: AssignmentExpr) -> None: + self.lvalue = True + e.target.accept(self) + self.lvalue = False + e.value.accept(self) + + def visit_as_pattern(self, p: AsPattern) -> None: + if p.pattern is not None: + p.pattern.accept(self) + if p.name is not None: + self.lvalue = True + p.name.accept(self) + self.lvalue = False + + def visit_starred_pattern(self, p: StarredPattern) -> None: + if p.capture is not None: + self.lvalue = True + p.capture.accept(self) + self.lvalue = False + + +# Open domains also block cross-type narrowing for known domain members, but they +# don't provide an exhaustive union to narrow top types to. +OPEN_VALUE_EQUALITY_DOMAINS: Final = { + "builtins.str": "builtins.str", + "builtins.bool": "builtins.numeric", + "builtins.int": "builtins.numeric", + "builtins.float": "builtins.numeric", + "builtins.complex": "builtins.numeric", +} +OPEN_VALUE_EQUALITY_DOMAIN_NAMES: Final = frozenset(OPEN_VALUE_EQUALITY_DOMAINS.values()) + +# Closed domains also block ordinary cross-type narrowing within the domain. +CLOSED_VALUE_EQUALITY_DOMAINS: Final = { + "builtins.bytes": "builtins.bytes", + "builtins.bytearray": "builtins.bytes", + "builtins.memoryview": "builtins.bytes", + "typing.Mapping": "typing.Mapping", + "typing.AbstractSet": "typing.AbstractSet", +} + +VALUE_EQUALITY_DOMAINS: Final = {**OPEN_VALUE_EQUALITY_DOMAINS, **CLOSED_VALUE_EQUALITY_DOMAINS} + + +class EqualityDomainInfo: + def __init__(self, type_names: set[str], enum_type_names: set[str]) -> None: + self.type_names = type_names + self.enum_type_names = enum_type_names + + +class EqualityValueInfo: + def __init__(self, domains: dict[str, EqualityDomainInfo], is_top: bool) -> None: + self.domains = domains + self.is_top = is_top + + +def partition_equality_ambiguous_types( + current_type: Type, target_type: Type, *, is_identity: bool +) -> tuple[Type | None, Type | None]: + """Split current_type into ordinary-narrowable and equality-ambiguous pieces. + + Some values compare equal through a value domain broader than their nominal type. For + example, an IntEnum member can compare equal to an int, and a StrEnum member can compare + equal to a str. When narrowing `x: MyStrEnum | str` against `MyStrEnum.MEMBER`, we can + still narrow the enum portion of the union, but we must keep the str portion in both + branches. + """ + if is_identity: + return current_type, None + + typ = get_proper_type(current_type) + items = typ.relevant_items() if isinstance(typ, UnionType) else [current_type] + narrowable_items = [] + ambiguous_items = [] + for item in items: + if is_equality_ambiguous_for_narrowing(item, target_type): + ambiguous_items.append(item) + else: + narrowable_items.append(item) + return ( + UnionType.make_union(narrowable_items) if narrowable_items else None, + UnionType.make_union(ambiguous_items) if ambiguous_items else None, + ) + + +def is_equality_ambiguous_for_narrowing(left: Type, right: Type) -> bool: + """Can left compare equal to right through a value domain outside nominal overlap?""" + left_info = equality_value_info(left) + right_info = equality_value_info(right) + + if left_info.is_top or right_info.is_top: + # Only open-domain enum values can make a top-like type ambiguous. + # Closed domains can be narrowed to their complete known set instead. + other_info = right_info if left_info.is_top else left_info + return any( + domain in OPEN_VALUE_EQUALITY_DOMAIN_NAMES and domain_info.enum_type_names + for domain, domain_info in other_info.domains.items() + ) + + shared_domains = left_info.domains.keys() & right_info.domains.keys() + if not shared_domains: + return False + + for domain in shared_domains: + left_domain = left_info.domains[domain] + right_domain = right_info.domains[domain] + # Equality between two values from the same enum can still narrow by literal member. + if ( + left_domain.enum_type_names + and left_domain.enum_type_names == right_domain.enum_type_names + and left_domain.type_names == left_domain.enum_type_names + and right_domain.type_names == right_domain.enum_type_names + ): + continue + # Different domain-member types may compare equal, but nominal narrowing would + # otherwise treat them as disjoint. + if left_domain.type_names != right_domain.type_names: + return True + # Same domain-member types are only ambiguous if an enum value may compare equal to + # its underlying value type. + if left_domain.enum_type_names or right_domain.enum_type_names: + return True + + return False + + +def equality_value_info(t: Type) -> EqualityValueInfo: + t = get_proper_type(t) + if isinstance(t, UnionType): + return combine_equality_value_info(equality_value_info(item) for item in t.items) + if isinstance(t, TypeVarType): + if t.values: + return combine_equality_value_info(equality_value_info(item) for item in t.values) + return equality_value_info(t.upper_bound) + if isinstance(t, Instance) and t.last_known_value is not None: + return equality_value_info(t.last_known_value) + if isinstance(t, LiteralType): + return equality_value_info(t.fallback) + if isinstance(t, Instance): + if t.type.fullname == "builtins.object": + return EqualityValueInfo({}, is_top=True) + + enum_type_names = {t.type.fullname} if t.type.is_enum else set() + domains = {} + for base in t.type.mro: + if domain := VALUE_EQUALITY_DOMAINS.get(base.fullname): + domains[domain] = EqualityDomainInfo({t.type.fullname}, enum_type_names) + + return EqualityValueInfo(domains, is_top=False) + if isinstance(t, AnyType): + return EqualityValueInfo({}, is_top=True) + return EqualityValueInfo({}, is_top=False) + + +def combine_equality_value_info(infos: Iterable[EqualityValueInfo]) -> EqualityValueInfo: + domains: dict[str, EqualityDomainInfo] = {} + is_top = False + for info in infos: + for domain, domain_info in info.domains.items(): + existing_domain_info = domains.get(domain) + if existing_domain_info is None: + domains[domain] = EqualityDomainInfo( + set(domain_info.type_names), set(domain_info.enum_type_names) + ) + else: + existing_domain_info.type_names.update(domain_info.type_names) + existing_domain_info.enum_type_names.update(domain_info.enum_type_names) + is_top = is_top or info.is_top + return EqualityValueInfo(domains, is_top) + + +def is_typeddict_type_context(lvalue_type: Type) -> bool: + lvalue_type = get_proper_type(lvalue_type) + if isinstance(lvalue_type, TypedDictType): + return True + if isinstance(lvalue_type, UnionType): + for item in lvalue_type.items: + if is_typeddict_type_context(item): + return True + return False + + +def is_method(node: SymbolNode | None) -> bool: + if isinstance(node, OverloadedFuncDef): + return not node.is_property + if isinstance(node, Decorator): + return not node.var.is_property + return isinstance(node, FuncDef) diff --git a/mypy/nodes.py b/mypy/nodes.py new file mode 100644 index 000000000000..dc86550cebe4 --- /dev/null +++ b/mypy/nodes.py @@ -0,0 +1,5608 @@ +"""Abstract syntax tree node classes (i.e. parse tree).""" + +from __future__ import annotations + +import os +from abc import abstractmethod +from collections import defaultdict +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager +from enum import Enum, unique +from typing import ( + TYPE_CHECKING, + Any, + Final, + Optional, + TypeAlias as _TypeAlias, + TypedDict, + TypeGuard, + TypeVar, + Union, + cast, +) +from typing_extensions import NotRequired + +from librt.internal import ( + extract_symbol, + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) +from mypy_extensions import trait + +import mypy.strconv +from mypy.cache import ( + DICT_INT_GEN, + DICT_STR_GEN, + DT_SPEC, + END_TAG, + LIST_GEN, + LIST_STR, + LITERAL_COMPLEX, + LITERAL_FALSE, + LITERAL_NONE, + LITERAL_TRUE, + ReadBuffer, + Tag, + WriteBuffer, + read_bool, + read_bytes, + read_flags, + read_int, + read_int_list, + read_int_opt, + read_json, + read_literal, + read_str, + read_str_list, + read_str_opt, + read_str_opt_list, + read_tag, + write_bool, + write_bytes, + write_flags, + write_int, + write_int_list, + write_int_opt, + write_json, + write_literal, + write_str, + write_str_list, + write_str_opt, + write_str_opt_list, + write_tag, +) +from mypy.modules_state import modules_state +from mypy.options import Options +from mypy.util import is_sunder, is_typeshed_file, short_type +from mypy.visitor import ExpressionVisitor, NodeVisitor, StatementVisitor + +if TYPE_CHECKING: + from mypy.patterns import ConcretePattern + + +@unique +class NotParsed(Enum): + VALUE = "NotParsed" + + +class Context: + """Base type for objects that are valid as error message locations.""" + + __slots__ = ("line", "column", "end_line", "end_column") + + def __init__(self, line: int = -1, column: int = -1) -> None: + self.line = line + self.column = column + self.end_line: int | None = None + self.end_column: int | None = None + + def set_line( + self, + target: Context | int, + column: int | None = None, + end_line: int | None = None, + end_column: int | None = None, + ) -> None: + """If target is a node, pull line (and column) information + into this node. If column is specified, this will override any column + information coming from a node. + """ + if isinstance(target, int): + self.line = target + else: + self.line = target.line + self.column = target.column + self.end_line = target.end_line + self.end_column = target.end_column + + if column is not None: + self.column = column + + if end_line is not None: + self.end_line = end_line + + if end_column is not None: + self.end_column = end_column + + +if TYPE_CHECKING: + # break import cycle only needed for mypy + import mypy.types + + +T = TypeVar("T") + +JsonDict: _TypeAlias = dict[str, Any] + + +# Symbol table node kinds +# +# TODO rename to use more descriptive names + +LDEF: Final = 0 +GDEF: Final = 1 +MDEF: Final = 2 + +# Placeholder for a name imported via 'from ... import'. Second phase of +# semantic will replace this the actual imported reference. This is +# needed so that we can detect whether a name has been imported during +# XXX what? +UNBOUND_IMPORTED: Final = 3 + +# RevealExpr node kinds +REVEAL_TYPE: Final = 0 +REVEAL_LOCALS: Final = 1 + +# Kinds of 'literal' expressions. +# +# Use the function mypy.literals.literal to calculate these. +# +# TODO: Can we make these less confusing? +LITERAL_YES: Final = 2 # Value of expression known statically +LITERAL_TYPE: Final = 1 # Type of expression can be narrowed (e.g. variable reference) +LITERAL_NO: Final = 0 # None of the above + +node_kinds: Final = {LDEF: "Ldef", GDEF: "Gdef", MDEF: "Mdef", UNBOUND_IMPORTED: "UnboundImported"} +inverse_node_kinds: Final = {_kind: _name for _name, _kind in node_kinds.items()} + + +implicit_module_attrs: Final = { + "__name__": "__builtins__.str", + "__doc__": None, # depends on Python version, see semanal.py + "__path__": None, # depends on if the module is a package + "__file__": "__builtins__.str", + "__package__": "__builtins__.str", + "__annotations__": None, # dict[str, Any] bounded in add_implicit_module_attrs() + "__spec__": None, # importlib.machinery.ModuleSpec bounded in add_implicit_module_attrs() +} + + +# These aliases exist because built-in class objects are not subscriptable. +# For example `list[int]` fails at runtime. Instead List[int] should be used. +type_aliases: Final = { + "typing.List": "builtins.list", + "typing.Dict": "builtins.dict", + "typing.Set": "builtins.set", + "typing.FrozenSet": "builtins.frozenset", + "typing.ChainMap": "collections.ChainMap", + "typing.Counter": "collections.Counter", + "typing.DefaultDict": "collections.defaultdict", + "typing.Deque": "collections.deque", + "typing.OrderedDict": "collections.OrderedDict", + # HACK: a lie in lieu of actual support for PEP 675 + "typing.LiteralString": "builtins.str", +} + +# This keeps track of the oldest supported Python version where the corresponding +# alias source is available. +type_aliases_source_versions: Final = {"typing.LiteralString": (3, 11)} + +# This keeps track of aliases in `typing_extensions`, which we treat specially. +typing_extensions_aliases: Final = { + # See: https://github.com/python/mypy/issues/11528 + "typing_extensions.OrderedDict": "collections.OrderedDict", + # HACK: a lie in lieu of actual support for PEP 675 + "typing_extensions.LiteralString": "builtins.str", +} + +reverse_builtin_aliases: Final = { + "builtins.list": "typing.List", + "builtins.dict": "typing.Dict", + "builtins.set": "typing.Set", + "builtins.frozenset": "typing.FrozenSet", +} + + +RUNTIME_PROTOCOL_DECOS: Final = ( + "typing.runtime_checkable", + "typing_extensions.runtime", + "typing_extensions.runtime_checkable", +) + +LAMBDA_NAME: Final = "" + + +class Node(Context): + """Common base class for all non-type parse tree nodes.""" + + __slots__ = () + + def __str__(self) -> str: + return self.accept(mypy.strconv.StrConv(options=Options())) + + def str_with_options(self, options: Options) -> str: + a = self.accept(mypy.strconv.StrConv(options=options)) + assert a + return a + + def accept(self, visitor: NodeVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) + + +@trait +class Statement(Node): + """A statement node.""" + + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) + + +@trait +class Expression(Node): + """An expression node.""" + + __slots__ = () + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + raise RuntimeError("Not implemented", type(self)) + + +class FakeExpression(Expression): + """A dummy expression. + + We need a dummy expression in one place, and can't instantiate Expression + because it is a trait and mypyc barfs. + """ + + __slots__ = () + + +# TODO: +# Lvalue = Union['NameExpr', 'MemberExpr', 'IndexExpr', 'SuperExpr', 'StarExpr' +# 'TupleExpr']; see #1783. +Lvalue: _TypeAlias = Expression + + +@trait +class SymbolNode(Node): + """Nodes that can be stored in a symbol table.""" + + __slots__ = () + + @property + @abstractmethod + def name(self) -> str: + pass + + # Fully qualified name + @property + @abstractmethod + def fullname(self) -> str: + pass + + @abstractmethod + def serialize(self) -> JsonDict: + pass + + @classmethod + def deserialize(cls, data: JsonDict) -> SymbolNode: + classname = data[".class"] + method = deserialize_map.get(classname) + if method is not None: + return method(data) + raise NotImplementedError(f"unexpected .class {classname}") + + def write(self, data: WriteBuffer) -> None: + raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") + + @classmethod + def read(cls, data: ReadBuffer) -> SymbolNode: + raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") + + +# Items: fullname, related symbol table node, surrounding type (if any) +Definition: _TypeAlias = tuple[str, "SymbolTableNode", Optional["TypeInfo"]] + + +class ParseError(TypedDict): + line: int + column: int + message: str + blocker: NotRequired[bool] + code: NotRequired[str] + + +def write_parse_error(data: WriteBuffer, err: ParseError) -> None: + write_int(data, err["line"]) + write_int(data, err["column"]) + write_str(data, err["message"]) + if (blocker := err.get("blocker")) is not None: + write_bool(data, blocker) + else: + write_tag(data, LITERAL_NONE) + write_str_opt(data, err.get("code")) + + +def read_parse_error(data: ReadBuffer) -> ParseError: + err: ParseError = {"line": read_int(data), "column": read_int(data), "message": read_str(data)} + tag = read_tag(data) + if tag == LITERAL_TRUE: + err["blocker"] = True + elif tag == LITERAL_FALSE: + err["blocker"] = False + else: + assert tag == LITERAL_NONE + if (code := read_str_opt(data)) is not None: + err["code"] = code + return err + + +class FileRawData: + """Raw (binary) data representing parsed, but not deserialized file.""" + + __slots__ = ( + "defs", + "imports", + "raw_errors", + "ignored_lines", + "is_partial_stub_package", + "uses_template_strings", + "source_hash", + "mypy_comments", + ) + + defs: bytes + imports: bytes + raw_errors: list[ParseError] + ignored_lines: dict[int, list[str]] + is_partial_stub_package: bool + uses_template_strings: bool + source_hash: str + mypy_comments: list[tuple[int, str]] + + def __init__( + self, + defs: bytes, + imports: bytes, + raw_errors: list[ParseError], + ignored_lines: dict[int, list[str]], + is_partial_stub_package: bool, + uses_template_strings: bool, + source_hash: str = "", + mypy_comments: list[tuple[int, str]] | None = None, + ) -> None: + self.defs = defs + self.imports = imports + self.raw_errors = raw_errors + self.ignored_lines = ignored_lines + self.is_partial_stub_package = is_partial_stub_package + self.uses_template_strings = uses_template_strings + self.source_hash = source_hash + self.mypy_comments = mypy_comments if mypy_comments is not None else [] + + def write(self, data: WriteBuffer) -> None: + write_bytes(data, self.defs) + write_bytes(data, self.imports) + write_tag(data, LIST_GEN) + write_int_bare(data, len(self.raw_errors)) + for err in self.raw_errors: + write_parse_error(data, err) + write_tag(data, DICT_INT_GEN) + write_int_bare(data, len(self.ignored_lines)) + for line, codes in self.ignored_lines.items(): + write_int(data, line) + write_str_list(data, codes) + write_bool(data, self.is_partial_stub_package) + write_bool(data, self.uses_template_strings) + write_str(data, self.source_hash) + write_tag(data, LIST_GEN) + write_int_bare(data, len(self.mypy_comments)) + for line, text in self.mypy_comments: + write_int(data, line) + write_str(data, text) + + @classmethod + def read(cls, data: ReadBuffer) -> FileRawData: + defs = read_bytes(data) + imports = read_bytes(data) + assert read_tag(data) == LIST_GEN + raw_errors = [read_parse_error(data) for _ in range(read_int_bare(data))] + assert read_tag(data) == DICT_INT_GEN + ignored_lines = {read_int(data): read_str_list(data) for _ in range(read_int_bare(data))} + is_partial_stub_package = read_bool(data) + uses_template_strings = read_bool(data) + source_hash = read_str(data) + assert read_tag(data) == LIST_GEN + mypy_comments = [(read_int(data), read_str(data)) for _ in range(read_int_bare(data))] + return FileRawData( + defs, + imports, + raw_errors, + ignored_lines, + is_partial_stub_package, + uses_template_strings, + source_hash, + mypy_comments, + ) + + +class MypyFile(SymbolNode): + """The abstract syntax tree of a single source file.""" + + __slots__ = ( + "_fullname", + "path", + "defs", + "alias_deps", + "module_refs", + "is_bom", + "names", + "imports", + "ignored_lines", + "skipped_lines", + "is_stub", + "is_cache_skeleton", + "is_partial_stub_package", + "uses_template_strings", + "plugin_deps", + "future_import_flags", + "_is_typeshed_file", + "raw_data", + ) + + __match_args__ = ("name", "path", "defs") + + # Fully qualified module name + _fullname: str + # Path to the file (empty string if not known) + path: str + # Top-level definitions and statements + defs: list[Statement] + # Type alias dependencies as mapping from target to set of alias full names + alias_deps: defaultdict[str, set[str]] + # The set of all dependencies (suppressed or not) that this module accesses, either + # directly or indirectly. + module_refs: set[str] + # Is there a UTF-8 BOM at the start? + is_bom: bool + names: SymbolTable + # All import nodes within the file (also ones within functions etc.) + imports: list[ImportBase] + # Lines on which to ignore certain errors when checking. + # If the value is empty, ignore all errors; otherwise, the list contains all + # error codes to ignore. + ignored_lines: dict[int, list[str]] + # Lines that were skipped during semantic analysis e.g. due to ALWAYS_FALSE, MYPY_FALSE, + # or platform/version checks. Those lines would not be type-checked. + skipped_lines: set[int] + # Is this file represented by a stub file (.pyi)? + is_stub: bool + # Is this loaded from the cache and thus missing the actual body of the file? + is_cache_skeleton: bool + # Does this represent an __init__.pyi stub with a module __getattr__ + # (i.e. a partial stub package), for such packages we suppress any missing + # module errors in addition to missing attribute errors. + is_partial_stub_package: bool + # True if module contains at least one t-string (PEP 750 TemplateStr). + uses_template_strings: bool + # Plugin-created dependencies + plugin_deps: dict[str, set[str]] + # Future imports defined in this file. Populated during semantic analysis. + future_import_flags: set[str] + _is_typeshed_file: bool | None + # For native parser store actual serialized data here. + raw_data: FileRawData | None + + def __init__( + self, + defs: list[Statement], + imports: list[ImportBase], + is_bom: bool = False, + ignored_lines: dict[int, list[str]] | None = None, + ) -> None: + super().__init__() + self.defs = defs + self.line = 1 # Dummy line number + self.column = 0 # Dummy column + self.imports = imports + self.is_bom = is_bom + self.alias_deps = defaultdict(set) + self.module_refs = set() + self.plugin_deps = {} + if ignored_lines: + self.ignored_lines = ignored_lines + else: + self.ignored_lines = {} + self.skipped_lines = set() + + self.path = "" + self.is_stub = False + self.is_cache_skeleton = False + self.is_partial_stub_package = False + self.uses_template_strings = False + self.future_import_flags = set() + self._is_typeshed_file = None + self.raw_data = None + + def local_definitions(self, *, impl_only: bool = False) -> Iterator[Definition]: + """Return all definitions within the module (including nested). + + This doesn't include imported definitions. + """ + return local_definitions(self.names, self.fullname, impl_only=impl_only) + + @property + def name(self) -> str: + return "" if not self._fullname else self._fullname.split(".")[-1] + + @property + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_mypy_file(self) + + def is_package_init_file(self) -> bool: + return len(self.path) != 0 and os.path.basename(self.path).startswith("__init__.") + + def is_future_flag_set(self, flag: str) -> bool: + return flag in self.future_import_flags + + def is_typeshed_file(self, options: Options) -> bool: + # Cache result since this is called a lot + if self._is_typeshed_file is None: + self._is_typeshed_file = is_typeshed_file(options.abs_custom_typeshed_dir, self.path) + return self._is_typeshed_file + + def serialize(self) -> JsonDict: + return { + ".class": "MypyFile", + "_fullname": self._fullname, + "names": self.names.serialize(self._fullname), + "is_stub": self.is_stub, + "path": self.path, + "is_partial_stub_package": self.is_partial_stub_package, + "future_import_flags": sorted(self.future_import_flags), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> MypyFile: + assert data[".class"] == "MypyFile", data + tree = MypyFile([], []) + tree._fullname = data["_fullname"] + tree.names = SymbolTable.deserialize(data["names"]) + tree.is_stub = data["is_stub"] + tree.path = data["path"] + tree.is_partial_stub_package = data["is_partial_stub_package"] + tree.is_cache_skeleton = True + tree.future_import_flags = set(data["future_import_flags"]) + return tree + + def write(self, data: WriteBuffer) -> None: + write_tag(data, MYPY_FILE) + write_str(data, self._fullname) + self.names.write(data, self._fullname) + write_bool(data, self.is_stub) + write_str(data, self.path) + write_bool(data, self.is_partial_stub_package) + write_str_list(data, sorted(self.future_import_flags)) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> MypyFile: + assert read_tag(data) == MYPY_FILE + tree = MypyFile([], []) + tree._fullname = read_str(data) + tree.names = SymbolTable.read(data) + tree.is_stub = read_bool(data) + tree.path = read_str(data) + tree.is_partial_stub_package = read_bool(data) + tree.future_import_flags = set(read_str_list(data)) + tree.is_cache_skeleton = True + assert read_tag(data) == END_TAG + return tree + + +class ImportBase(Statement): + """Base class for all import statements.""" + + __slots__ = ("is_unreachable", "is_top_level", "is_mypy_only", "assignments") + + is_unreachable: bool # Set by semanal.SemanticAnalyzerPass1 if inside `if False` etc. + is_top_level: bool # Ditto if outside any class or def + is_mypy_only: bool # Ditto if inside `if TYPE_CHECKING` or `if MYPY` + + # If an import replaces existing definitions, we construct dummy assignment + # statements that assign the imported names to the names in the current scope, + # for type checking purposes. Example: + # + # x = 1 + # from m import x <-- add assignment representing "x = m.x" + assignments: list[AssignmentStmt] + + def __init__(self) -> None: + super().__init__() + self.assignments = [] + self.is_unreachable = False + self.is_top_level = False + self.is_mypy_only = False + + +class Import(ImportBase): + """import m [as n]""" + + __slots__ = ("ids",) + + __match_args__ = ("ids",) + + ids: list[tuple[str, str | None]] # (module id, as id) + + def __init__(self, ids: list[tuple[str, str | None]]) -> None: + super().__init__() + self.ids = ids + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_import(self) + + +class ImportFrom(ImportBase): + """from m import x [as y], ...""" + + __slots__ = ("id", "names", "relative") + + __match_args__ = ("id", "names", "relative") + + id: str + relative: int + names: list[tuple[str, str | None]] # Tuples (name, as name) + + def __init__(self, id: str, relative: int, names: list[tuple[str, str | None]]) -> None: + super().__init__() + self.id = id + self.names = names + self.relative = relative + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_import_from(self) + + +class ImportAll(ImportBase): + """from m import *""" + + __slots__ = ("id", "relative") + + __match_args__ = ("id", "relative") + + id: str + relative: int + + def __init__(self, id: str, relative: int) -> None: + super().__init__() + self.id = id + self.relative = relative + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_import_all(self) + + +FUNCBASE_FLAGS: Final = ["is_property", "is_class", "is_static", "is_final"] + + +class FuncBase(Node): + """Abstract base class for function-like nodes. + + N.B: Although this has SymbolNode subclasses (FuncDef, + OverloadedFuncDef), avoid calling isinstance(..., FuncBase) on + something that is typed as SymbolNode. This is to work around + mypy bug #3603, in which mypy doesn't understand multiple + inheritance very well, and will assume that a SymbolNode + cannot be a FuncBase. + + Instead, test against SYMBOL_FUNCBASE_TYPES, which enumerates + SymbolNode subclasses that are also FuncBase subclasses. + """ + + __slots__ = ( + "type", + "unanalyzed_type", + "info", + "is_property", + "is_class", # Uses "@classmethod" (explicit or implicit) + "is_static", # Uses "@staticmethod" (explicit or implicit) + "is_final", # Uses "@final" + "is_explicit_override", # Uses "@override" + "is_type_check_only", # Uses "@type_check_only" + "def_or_infer_vars", + "_fullname", + ) + + def __init__(self) -> None: + super().__init__() + # Type signature. This is usually CallableType or Overloaded, but it can be + # something else for decorated functions. + self.type: mypy.types.ProperType | None = None + # Original, not semantically analyzed type (used for reprocessing) + self.unanalyzed_type: mypy.types.ProperType | None = None + # If method, reference to TypeInfo + self.info = FUNC_NO_INFO + self.is_property = False + self.is_class = False + # Is this a `@staticmethod` (explicit or implicit)? + # Note: use has_self_or_cls_argument to check if there is `self` or `cls` argument + self.is_static = False + self.is_final = False + self.is_explicit_override = False + self.is_type_check_only = False + # Can this function/method define variables or infer variables defined outside? + # In particular, we set this in cases like: + # x = None + # def foo() -> None: + # global x + # x = 1 + # and + # class C: + # x = None + # def foo(self) -> None: + # self.x = 1 + self.def_or_infer_vars = False + # Name with module prefix + self._fullname = "" + + @property + @abstractmethod + def name(self) -> str: + pass + + @property + def fullname(self) -> str: + return self._fullname + + @property + def has_self_or_cls_argument(self) -> bool: + """If used as a method, does it have an argument for method binding (`self`, `cls`)? + + This is true for `__new__` even though `__new__` does not undergo method binding, + because we still usually assume that `cls` corresponds to the enclosing class. + """ + return not self.is_static or self.name == "__new__" + + +OverloadPart: _TypeAlias = Union["FuncDef", "Decorator"] + + +class OverloadedFuncDef(FuncBase, SymbolNode, Statement): + """A logical node representing all the variants of a multi-declaration function. + + A multi-declaration function is often an @overload, but can also be a + @property with a setter and a/or a deleter. + + This node has no explicit representation in the source program. + Overloaded variants must be consecutive in the source file. + """ + + __slots__ = ( + "items", + "unanalyzed_items", + "impl", + "deprecated", + "setter_index", + "_is_trivial_self", + ) + + items: list[OverloadPart] + unanalyzed_items: list[OverloadPart] + impl: OverloadPart | None + deprecated: str | None + setter_index: int | None + + def __init__(self, items: list[OverloadPart]) -> None: + super().__init__() + self.items = items + self.unanalyzed_items = items.copy() + self.impl = None + self.deprecated = None + self.setter_index = None + self._is_trivial_self: bool | None = None + if items: + # TODO: figure out how to reliably set end position (we don't know the impl here). + self.set_line(items[0].line, items[0].column) + + @property + def name(self) -> str: + if self.items: + return self.items[0].name + else: + # This may happen for malformed overload + assert self.impl is not None + return self.impl.name + + @property + def is_trivial_self(self) -> bool: + """Check we can use bind_self() fast path for this overload. + + This will return False if at least one overload: + * Has an explicit self annotation, or Self in signature. + * Has a non-trivial decorator. + """ + if self._is_trivial_self is not None: + return self._is_trivial_self + for i, item in enumerate(self.items): + # Note: bare @property is removed in visit_decorator(). + trivial = 1 if i > 0 or not self.is_property else 0 + if isinstance(item, FuncDef): + if not item.is_trivial_self: + self._is_trivial_self = False + return False + elif len(item.decorators) > trivial or not item.func.is_trivial_self: + self._is_trivial_self = False + return False + self._is_trivial_self = True + return True + + @property + def setter(self) -> Decorator: + # Do some consistency checks first. + first_item = self.items[0] + assert isinstance(first_item, Decorator) + assert first_item.var.is_settable_property + assert self.setter_index is not None + item = self.items[self.setter_index] + assert isinstance(item, Decorator) + return item + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_overloaded_func_def(self) + + def serialize(self) -> JsonDict: + return { + ".class": "OverloadedFuncDef", + "items": [i.serialize() for i in self.items], + "type": None if self.type is None else self.type.serialize(), + "fullname": self._fullname, + "impl": None if self.impl is None else self.impl.serialize(), + "flags": get_flags(self, FUNCBASE_FLAGS), + "deprecated": self.deprecated, + "setter_index": self.setter_index, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: + assert data[".class"] == "OverloadedFuncDef" + res = OverloadedFuncDef( + [cast(OverloadPart, SymbolNode.deserialize(d)) for d in data["items"]] + ) + if data.get("impl") is not None: + res.impl = cast(OverloadPart, SymbolNode.deserialize(data["impl"])) + # set line for empty overload items, as not set in __init__ + if len(res.items) > 0: + res.set_line(res.impl.line) + if data.get("type") is not None: + typ = mypy.types.deserialize_type(data["type"]) + assert isinstance(typ, mypy.types.ProperType) + res.type = typ + res._fullname = data["fullname"] + set_flags(res, data["flags"]) + res.deprecated = data["deprecated"] + res.setter_index = data["setter_index"] + # NOTE: res.info will be set in the fixup phase. + return res + + def write(self, data: WriteBuffer) -> None: + write_tag(data, OVERLOADED_FUNC_DEF) + write_tag(data, LIST_GEN) + write_int_bare(data, len(self.items)) + for item in self.items: + item.write(data) + mypy.types.write_type_opt(data, self.type) + write_str(data, self._fullname) + if self.impl is None: + write_tag(data, LITERAL_NONE) + else: + self.impl.write(data) + write_flags(data, [self.is_property, self.is_class, self.is_static, self.is_final]) + write_str_opt(data, self.deprecated) + write_int_opt(data, self.setter_index) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> OverloadedFuncDef: + assert read_tag(data) == LIST_GEN + res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int_bare(data))]) + typ = mypy.types.read_type_opt(data) + if typ is not None: + assert isinstance(typ, mypy.types.ProperType) + res.type = typ + res._fullname = read_str(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + res.impl = read_overload_part(data, tag) + # set line for empty overload items, as not set in __init__ + if len(res.items) > 0: + res.set_line(res.impl.line) + res.is_property, res.is_class, res.is_static, res.is_final = read_flags(data, num_flags=4) + res.deprecated = read_str_opt(data) + res.setter_index = read_int_opt(data) + # NOTE: res.info will be set in the fixup phase. + assert read_tag(data) == END_TAG + return res + + def is_dynamic(self) -> bool: + return all(item.is_dynamic() for item in self.items) + + +class Argument(Node): + """A single argument in a FuncItem.""" + + __slots__ = ("variable", "type_annotation", "initializer", "kind", "pos_only") + + __match_args__ = ("variable", "type_annotation", "initializer", "kind", "pos_only") + + def __init__( + self, + variable: Var, + type_annotation: mypy.types.Type | None, + initializer: Expression | None, + kind: ArgKind, + pos_only: bool = False, + ) -> None: + super().__init__() + self.variable = variable + self.type_annotation = type_annotation + self.initializer = initializer + self.kind = kind # must be an ARG_* constant + self.pos_only = pos_only + + def set_line( + self, + target: Context | int, + column: int | None = None, + end_line: int | None = None, + end_column: int | None = None, + ) -> None: + super().set_line(target, column, end_line, end_column) + + if self.initializer and self.initializer.line < 0: + self.initializer.set_line(self.line, self.column, self.end_line, self.end_column) + + self.variable.set_line(self.line, self.column, self.end_line, self.end_column) + + +# These specify the kind of a TypeParam +TYPE_VAR_KIND: Final = 0 +PARAM_SPEC_KIND: Final = 1 +TYPE_VAR_TUPLE_KIND: Final = 2 + + +class TypeParam: + __slots__ = ("name", "kind", "upper_bound", "values", "default") + + def __init__( + self, + name: str, + kind: int, + upper_bound: mypy.types.Type | None, + values: list[mypy.types.Type], + default: mypy.types.Type | None, + ) -> None: + self.name = name + self.kind = kind + self.upper_bound = upper_bound + self.values = values + self.default = default + + +FUNCITEM_FLAGS: Final = FUNCBASE_FLAGS + [ + "is_overload", + "is_generator", + "is_coroutine", + "is_async_generator", + "is_awaitable_coroutine", +] + + +class FuncItem(FuncBase): + """Base class for nodes usable as overloaded function items.""" + + __slots__ = ( + "arguments", # Note that can be unset if deserialized (type is a lie!) + "arg_names", # Names of arguments + "arg_kinds", # Kinds of arguments + "min_args", # Minimum number of arguments + "max_pos", # Maximum number of positional arguments, -1 if no explicit + # limit (*args not included) + "type_args", # New-style type parameters (PEP 695) + "body", # Body of the function + "is_overload", # Is this an overload variant of function with more than + # one overload variant? + "is_generator", # Contains a yield statement? + "is_coroutine", # Defined using 'async def' syntax? + "is_async_generator", # Is an async def generator? + "is_awaitable_coroutine", # Decorated with '@{typing,asyncio}.coroutine'? + "expanded", # Variants of function with type variables with values expanded + ) + + __deletable__ = ("arguments", "max_pos", "min_args") + + def __init__( + self, + arguments: list[Argument] | None = None, + body: Block | None = None, + typ: mypy.types.FunctionLike | None = None, + type_args: list[TypeParam] | None = None, + ) -> None: + super().__init__() + self.arguments = arguments or [] + self.arg_names = [None if arg.pos_only else arg.variable.name for arg in self.arguments] + self.arg_kinds: list[ArgKind] = [arg.kind for arg in self.arguments] + self.max_pos: int = self.arg_kinds.count(ARG_POS) + self.arg_kinds.count(ARG_OPT) + self.type_args: list[TypeParam] | None = type_args + self.body: Block = body or Block([]) + self.type = typ + self.unanalyzed_type = typ + self.is_overload: bool = False + self.is_generator: bool = False + self.is_coroutine: bool = False + self.is_async_generator: bool = False + self.is_awaitable_coroutine: bool = False + self.expanded: list[FuncItem] = [] + + self.min_args = 0 + for i in range(len(self.arguments)): + if self.arguments[i] is None and i < self.max_fixed_argc(): + self.min_args = i + 1 + + def max_fixed_argc(self) -> int: + return self.max_pos + + def is_dynamic(self) -> bool: + return ( + self.type is None + or isinstance(self.type, mypy.types.CallableType) + and self.type.implicit + ) + + +FUNCDEF_FLAGS: Final = FUNCITEM_FLAGS + [ + "is_decorated", + "is_conditional", + "is_trivial_body", + "is_trivial_self", + "is_mypy_only", +] + +# Abstract status of a function +NOT_ABSTRACT: Final = 0 +# Explicitly abstract (with @abstractmethod or overload without implementation) +IS_ABSTRACT: Final = 1 +# Implicitly abstract: used for functions with trivial bodies defined in Protocols +IMPLICITLY_ABSTRACT: Final = 2 + + +class FuncDef(FuncItem, SymbolNode, Statement): + """Function definition. + + This is a non-lambda function defined using 'def'. + """ + + __slots__ = ( + "_name", + "is_decorated", + "is_conditional", + "abstract_status", + "original_def", + "is_trivial_body", + "is_trivial_self", + "is_invalid_redefinition", + "is_mypy_only", + # Present only when a function is decorated with @typing.dataclass_transform or similar + "dataclass_transform_spec", + "docstring", + "deprecated", + "original_first_arg", + ) + + __match_args__ = ("name", "arguments", "type", "body") + + # Note that all __init__ args must have default values + def __init__( + self, + name: str = "", # Function name + arguments: list[Argument] | None = None, + body: Block | None = None, + typ: mypy.types.FunctionLike | None = None, + type_args: list[TypeParam] | None = None, + ) -> None: + super().__init__(arguments, body, typ, type_args) + self._name = name + self.is_decorated = False + self.is_conditional = False # Defined conditionally (within block)? + self.abstract_status = NOT_ABSTRACT + # Is this an abstract method with trivial body? + # Such methods can't be called via super(). + self.is_trivial_body = False + # Original conditional definition + self.original_def: None | FuncDef | Var | Decorator = None + # Definitions that appear in if TYPE_CHECKING are marked with this flag. + self.is_mypy_only = False + self.dataclass_transform_spec: DataclassTransformSpec | None = None + self.docstring: str | None = None + self.deprecated: str | None = None + # This is used to simplify bind_self() logic in trivial cases (which are + # the majority). In cases where self is not annotated and there are no Self + # in the signature we can simply drop the first argument. + self.is_trivial_self = False + # This is needed because for positional-only arguments the name is set to None, + # but we sometimes still want to show it in error messages. + if arguments: + self.original_first_arg: str | None = arguments[0].variable.name + else: + self.original_first_arg = None + # Whether this function is an invalid redefinition of variable with the same name? + # We record this status to avoid multiple (similar but different) errors in case + # of partial types etc. + self.is_invalid_redefinition = False + + @property + def name(self) -> str: + return self._name + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_func_def(self) + + def serialize(self) -> JsonDict: + # We're deliberating omitting arguments and storing only arg_names and + # arg_kinds for space-saving reasons (arguments is not used in later + # stages of mypy). + # TODO: After a FuncDef is deserialized, the only time we use `arg_names` + # and `arg_kinds` is when `type` is None and we need to infer a type. Can + # we store the inferred type ahead of time? + return { + ".class": "FuncDef", + "name": self._name, + "fullname": self._fullname, + "arg_names": self.arg_names, + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "type": None if self.type is None else self.type.serialize(), + "flags": get_flags(self, FUNCDEF_FLAGS), + "abstract_status": self.abstract_status, + # TODO: Do we need expanded, original_def? + "dataclass_transform_spec": ( + None + if self.dataclass_transform_spec is None + else self.dataclass_transform_spec.serialize() + ), + "deprecated": self.deprecated, + "original_first_arg": self.original_first_arg, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> FuncDef: + assert data[".class"] == "FuncDef" + body = Block([]) + ret = FuncDef( + data["name"], + [], + body, + ( + None + if data["type"] is None + else cast(mypy.types.FunctionLike, mypy.types.deserialize_type(data["type"])) + ), + ) + ret._fullname = data["fullname"] + set_flags(ret, data["flags"]) + # NOTE: ret.info is set in the fixup phase. + ret.arg_names = data["arg_names"] + ret.original_first_arg = data.get("original_first_arg") + ret.arg_kinds = [ARG_KINDS[x] for x in data["arg_kinds"]] + ret.abstract_status = data["abstract_status"] + ret.dataclass_transform_spec = ( + DataclassTransformSpec.deserialize(data["dataclass_transform_spec"]) + if data["dataclass_transform_spec"] is not None + else None + ) + ret.deprecated = data["deprecated"] + # Leave these uninitialized so that future uses will trigger an error + del ret.arguments + del ret.max_pos + del ret.min_args + return ret + + def write(self, data: WriteBuffer) -> None: + write_tag(data, FUNC_DEF) + write_str(data, self._name) + mypy.types.write_type_opt(data, self.type) + write_str(data, self._fullname) + write_flags( + data, + [ + self.is_property, + self.is_class, + self.is_static, + self.is_final, + self.is_overload, + self.is_generator, + self.is_coroutine, + self.is_async_generator, + self.is_awaitable_coroutine, + self.is_decorated, + self.is_conditional, + self.is_trivial_body, + self.is_trivial_self, + self.is_mypy_only, + ], + ) + write_str_opt_list(data, self.arg_names) + write_int_list(data, [int(ak.value) for ak in self.arg_kinds]) + write_int(data, self.abstract_status) + if self.dataclass_transform_spec is None: + write_tag(data, LITERAL_NONE) + else: + self.dataclass_transform_spec.write(data) + write_str_opt(data, self.deprecated) + write_str_opt(data, self.original_first_arg) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> FuncDef: + name = read_str(data) + typ: mypy.types.FunctionLike | None = None + tag = read_tag(data) + if tag != LITERAL_NONE: + typ = mypy.types.read_function_like(data, tag) + ret = FuncDef(name, [], Block([]), typ) + ret._fullname = read_str(data) + ( + ret.is_property, + ret.is_class, + ret.is_static, + ret.is_final, + ret.is_overload, + ret.is_generator, + ret.is_coroutine, + ret.is_async_generator, + ret.is_awaitable_coroutine, + ret.is_decorated, + ret.is_conditional, + ret.is_trivial_body, + ret.is_trivial_self, + ret.is_mypy_only, + ) = read_flags(data, num_flags=14) + # NOTE: ret.info is set in the fixup phase. + ret.arg_names = read_str_opt_list(data) + ret.arg_kinds = [ARG_KINDS[ak] for ak in read_int_list(data)] + ret.abstract_status = read_int(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC + ret.dataclass_transform_spec = DataclassTransformSpec.read(data) + ret.deprecated = read_str_opt(data) + ret.original_first_arg = read_str_opt(data) + # Leave these uninitialized so that future uses will trigger an error + del ret.arguments + del ret.max_pos + del ret.min_args + assert read_tag(data) == END_TAG + return ret + + +# All types that are both SymbolNodes and FuncBases. See the FuncBase +# docstring for the rationale. +# See https://github.com/python/mypy/pull/13607#issuecomment-1236357236 +# TODO: we want to remove this at some point and just use `FuncBase` ideally. +SYMBOL_FUNCBASE_TYPES: Final = (OverloadedFuncDef, FuncDef) + + +class Decorator(SymbolNode, Statement): + """A decorated function. + + A single Decorator object can include any number of function decorators. + """ + + __slots__ = ("func", "decorators", "original_decorators", "var", "is_overload") + + __match_args__ = ("decorators", "var", "func") + + func: FuncDef # Decorated function + decorators: list[Expression] # Decorators (may be empty) + # Some decorators are removed by semanal, keep the original here. + original_decorators: list[Expression] + # TODO: This is mostly used for the type; consider replacing with a 'type' attribute + var: Var # Represents the decorated function obj + is_overload: bool + + def __init__(self, func: FuncDef, decorators: list[Expression], var: Var) -> None: + super().__init__() + self.func = func + self.decorators = decorators + self.original_decorators = decorators.copy() + self.var = var + self.is_overload = False + + @property + def name(self) -> str: + return self.func.name + + @property + def fullname(self) -> str: + return self.func.fullname + + @property + def is_final(self) -> bool: + return self.func.is_final + + @property + def info(self) -> TypeInfo: + return self.func.info + + @property + def type(self) -> mypy.types.Type | None: + return self.var.type + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_decorator(self) + + def serialize(self) -> JsonDict: + return { + ".class": "Decorator", + "func": self.func.serialize(), + "var": self.var.serialize(), + "is_overload": self.is_overload, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> Decorator: + assert data[".class"] == "Decorator" + dec = Decorator(FuncDef.deserialize(data["func"]), [], Var.deserialize(data["var"])) + dec.is_overload = data["is_overload"] + return dec + + def write(self, data: WriteBuffer) -> None: + write_tag(data, DECORATOR) + self.func.write(data) + self.var.write(data) + write_bool(data, self.is_overload) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> Decorator: + assert read_tag(data) == FUNC_DEF + func = FuncDef.read(data) + assert read_tag(data) == VAR + var = Var.read(data) + dec = Decorator(func, [], var) + dec.is_overload = read_bool(data) + assert read_tag(data) == END_TAG + return dec + + def is_dynamic(self) -> bool: + return self.func.is_dynamic() + + +VAR_FLAGS: Final = [ + "is_self", + "is_cls", + "is_initialized_in_class", + "is_staticmethod", + "is_classmethod", + "is_property", + "is_settable_property", + "is_suppressed_import", + "is_classvar", + "is_abstract_var", + "is_final", + "is_index_var", + "final_unset_in_class", + "final_set_in_init", + "explicit_self_type", + "is_ready", + "is_inferred", + "invalid_partial_type", + "from_module_getattr", + "has_explicit_value", + "allow_incompatible_override", + "is_sentinel", +] + + +class Var(SymbolNode): + """A variable. + + It can refer to global/local variable or a data attribute. + """ + + __slots__ = ( + "_name", + "_fullname", + "info", + "type", + "setter_type", + "final_value", + "is_self", + "is_cls", + "is_ready", + "is_inferred", + "is_initialized_in_class", + "is_staticmethod", + "is_classmethod", + "is_property", + "is_settable_property", + "is_classvar", + "is_abstract_var", + "is_final", + "is_index_var", + "final_unset_in_class", + "final_set_in_init", + "is_suppressed_import", + "explicit_self_type", + "from_module_getattr", + "has_explicit_value", + "allow_incompatible_override", + "invalid_partial_type", + "is_argument", + "is_sentinel", + ) + + __match_args__ = ("name", "type", "final_value") + + def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: + super().__init__() + self._name = name # Name without module prefix + # TODO: Should be Optional[str] + self._fullname = "" # Name with module prefix + # TODO: Should be Optional[TypeInfo] + self.info = VAR_NO_INFO + self.type: mypy.types.Type | None = type # Declared or inferred type, or None + # The setter type for settable properties. + self.setter_type: mypy.types.CallableType | None = None + # Is this the first argument to an ordinary method (usually "self")? + self.is_self = False + # Is this the first argument to a classmethod (typically "cls")? + self.is_cls = False + self.is_ready = True # If inferred, is the inferred type available? + self.is_inferred = self.type is None + # Is this variable declared in class body? The name is confusing, but it + # is a very old attribute, and changing will break some plugins. + self.is_initialized_in_class = False + self.is_staticmethod = False + self.is_classmethod = False + self.is_property = False + self.is_settable_property = False + self.is_classvar = False + self.is_abstract_var = False + self.is_index_var = False + # Set to true when this variable refers to a module we were unable to + # parse for some reason (eg a silenced module) + self.is_suppressed_import = False + # Was this "variable" (rather a constant) defined as Final[...]? + self.is_final = False + # If constant value is a simple literal, + # store the literal value (unboxed) for the benefit of + # tools like mypyc. + self.final_value: int | float | complex | bool | str | None = None + # Where the value was set (only for class attributes) + self.final_unset_in_class = False + self.final_set_in_init = False + # This is True for a variable that was declared on self with an explicit type: + # class C: + # def __init__(self) -> None: + # self.x: int + # This case is important because this defines a new Var, even if there is one + # present in a superclass (without explicit type this doesn't create a new Var). + # See SemanticAnalyzer.analyze_member_lvalue() for details. + self.explicit_self_type = False + # If True, this is an implicit Var created due to module-level __getattr__. + self.from_module_getattr = False + # Var can be created with an explicit value `a = 1` or without one `a: int`, + # we need a way to tell which one is which. + self.has_explicit_value = False + # If True, subclasses can override this with an incompatible type. + self.allow_incompatible_override = False + # If True, this means we didn't manage to infer full type and fall back to + # something like list[Any]. We may decide to not use such types as context. + self.invalid_partial_type = False + # Is it a variable symbol for a function argument? + self.is_argument = False + # Was this variable created by PEP 661 sentinel()/Sentinel() syntax? + self.is_sentinel = False + + @property + def name(self) -> str: + return self._name + + @property + def fullname(self) -> str: + return self._fullname + + def __repr__(self) -> str: + name = self.fullname or self.name + return f"" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_var(self) + + def serialize(self) -> JsonDict: + # TODO: Leave default values out? + # NOTE: Sometimes self.is_ready is False here, but we don't care. + data: JsonDict = { + ".class": "Var", + "name": self._name, + "fullname": self._fullname, + "type": None if self.type is None else self.type.serialize(), + "setter_type": None if self.setter_type is None else self.setter_type.serialize(), + "flags": get_flags(self, VAR_FLAGS), + } + if self.final_value is not None: + data["final_value"] = self.final_value + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> Var: + assert data[".class"] == "Var" + name = data["name"] + type = None if data["type"] is None else mypy.types.deserialize_type(data["type"]) + setter_type = ( + None + if data["setter_type"] is None + else mypy.types.deserialize_type(data["setter_type"]) + ) + v = Var(name, type) + assert ( + setter_type is None + or isinstance(setter_type, mypy.types.ProperType) + and isinstance(setter_type, mypy.types.CallableType) + ) + v.setter_type = setter_type + v.is_ready = False # Override True default set in __init__ + v._fullname = data["fullname"] + set_flags(v, data["flags"]) + v.final_value = data.get("final_value") + return v + + def write(self, data: WriteBuffer) -> None: + write_tag(data, VAR) + write_str(data, self._name) + mypy.types.write_type_opt(data, self.type) + mypy.types.write_type_opt(data, self.setter_type) + write_str(data, self._fullname) + write_flags( + data, + [ + self.is_initialized_in_class, + self.is_staticmethod, + self.is_classmethod, + self.is_property, + self.is_settable_property, + self.is_suppressed_import, + self.is_classvar, + self.is_abstract_var, + self.is_final, + self.is_index_var, + self.final_unset_in_class, + self.final_set_in_init, + self.explicit_self_type, + self.is_ready, + self.is_inferred, + self.invalid_partial_type, + self.from_module_getattr, + self.has_explicit_value, + self.allow_incompatible_override, + self.is_sentinel, + ], + ) + write_literal(data, self.final_value) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> Var: + name = read_str(data) + typ = mypy.types.read_type_opt(data) + v = Var(name, typ) + setter_type: mypy.types.CallableType | None = None + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == mypy.types.CALLABLE_TYPE + setter_type = mypy.types.CallableType.read(data) + v.setter_type = setter_type + v._fullname = read_str(data) + ( + v.is_initialized_in_class, + v.is_staticmethod, + v.is_classmethod, + v.is_property, + v.is_settable_property, + v.is_suppressed_import, + v.is_classvar, + v.is_abstract_var, + v.is_final, + v.is_index_var, + v.final_unset_in_class, + v.final_set_in_init, + v.explicit_self_type, + v.is_ready, + v.is_inferred, + v.invalid_partial_type, + v.from_module_getattr, + v.has_explicit_value, + v.allow_incompatible_override, + v.is_sentinel, + ) = read_flags(data, num_flags=20) + tag = read_tag(data) + if tag == LITERAL_COMPLEX: + v.final_value = complex(read_float_bare(data), read_float_bare(data)) + elif tag != LITERAL_NONE: + val = read_literal(data, tag) + assert not isinstance(val, mypy.types.SentinelValue) + v.final_value = val + assert read_tag(data) == END_TAG + return v + + +class ClassDef(Statement): + """Class definition""" + + __slots__ = ( + "name", + "_fullname", + "defs", + "type_args", + "type_vars", + "base_type_exprs", + "removed_base_type_exprs", + "info", + "metaclass", + "decorators", + "keywords", + "analyzed", + "has_incompatible_baseclass", + "docstring", + "removed_statements", + ) + + __match_args__ = ("name", "defs") + + name: str # Name of the class without module prefix + _fullname: str # Fully qualified name of the class + defs: Block + # New-style type parameters (PEP 695), unanalyzed + type_args: list[TypeParam] | None + # Semantically analyzed type parameters (all syntax variants) + type_vars: list[mypy.types.TypeVarLikeType] + # Base class expressions (not semantically analyzed -- can be arbitrary expressions) + base_type_exprs: list[Expression] + # Special base classes like Generic[...] get moved here during semantic analysis + removed_base_type_exprs: list[Expression] + info: TypeInfo # Related TypeInfo + metaclass: Expression | None + decorators: list[Expression] + keywords: dict[str, Expression] + analyzed: Expression | None + has_incompatible_baseclass: bool + # Used by special forms like NamedTuple and TypedDict to store invalid statements + removed_statements: list[Statement] + + def __init__( + self, + name: str, + defs: Block, + type_vars: list[mypy.types.TypeVarLikeType] | None = None, + base_type_exprs: list[Expression] | None = None, + metaclass: Expression | None = None, + keywords: list[tuple[str, Expression]] | None = None, + type_args: list[TypeParam] | None = None, + ) -> None: + super().__init__() + self.name = name + self._fullname = "" + self.defs = defs + self.type_vars = type_vars or [] + self.type_args = type_args + self.base_type_exprs = base_type_exprs or [] + self.removed_base_type_exprs = [] + self.info = CLASSDEF_NO_INFO + self.metaclass = metaclass + self.decorators = [] + self.keywords = dict(keywords) if keywords else {} + self.analyzed = None + self.has_incompatible_baseclass = False + self.docstring: str | None = None + self.removed_statements = [] + + @property + def fullname(self) -> str: + return self._fullname + + @fullname.setter + def fullname(self, v: str) -> None: + self._fullname = v + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_class_def(self) + + def is_generic(self) -> bool: + return self.info.is_generic() + + def serialize(self) -> JsonDict: + # Not serialized: defs, base_type_exprs, metaclass, decorators, + # analyzed (for named tuples etc.) + return { + ".class": "ClassDef", + "name": self.name, + "fullname": self.fullname, + "type_vars": [v.serialize() for v in self.type_vars], + } + + @classmethod + def deserialize(cls, data: JsonDict) -> ClassDef: + assert data[".class"] == "ClassDef" + res = ClassDef( + data["name"], + Block([]), + # https://github.com/python/mypy/issues/12257 + [ + cast(mypy.types.TypeVarLikeType, mypy.types.deserialize_type(v)) + for v in data["type_vars"] + ], + ) + res.fullname = data["fullname"] + return res + + def write(self, data: WriteBuffer) -> None: + write_tag(data, CLASS_DEF) + write_str(data, self.name) + mypy.types.write_type_list(data, self.type_vars) + write_str(data, self.fullname) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> ClassDef: + res = ClassDef(read_str(data), Block([]), mypy.types.read_type_var_likes(data)) + res.fullname = read_str(data) + assert read_tag(data) == END_TAG + return res + + +class GlobalDecl(Statement): + """Declaration global x, y, ...""" + + __slots__ = ("names",) + + __match_args__ = ("names",) + + names: list[str] + + def __init__(self, names: list[str]) -> None: + super().__init__() + self.names = names + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_global_decl(self) + + +class NonlocalDecl(Statement): + """Declaration nonlocal x, y, ...""" + + __slots__ = ("names",) + + __match_args__ = ("names",) + + names: list[str] + + def __init__(self, names: list[str]) -> None: + super().__init__() + self.names = names + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_nonlocal_decl(self) + + +class Block(Statement): + __slots__ = ("body", "is_unreachable") + + __match_args__ = ("body", "is_unreachable") + + def __init__(self, body: list[Statement], *, is_unreachable: bool = False) -> None: + super().__init__() + self.body = body + # True if we can determine that this block is not executed during semantic + # analysis. For example, this applies to blocks that are protected by + # something like "if PY3:" when using Python 2. However, some code is + # only considered unreachable during type checking and this is not true + # in those cases. + self.is_unreachable = is_unreachable + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_block(self) + + +# Statements + + +class ExpressionStmt(Statement): + """An expression as a statement, such as print(s).""" + + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression + + def __init__(self, expr: Expression) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_expression_stmt(self) + + +class AssignmentStmt(Statement): + """Assignment statement. + + The same node class is used for single assignment, multiple assignment + (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments + that define new names, and assignments with explicit types ("# type: t" + or "x: t [= ...]"). + + An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, or IndexExpr. + """ + + __slots__ = ( + "lvalues", + "rvalue", + "type", + "unanalyzed_type", + "new_syntax", + "is_alias_def", + "is_final_def", + "invalid_recursive_alias", + ) + + __match_args__ = ("lvalues", "rvalues", "type") + + lvalues: list[Lvalue] + # This is a TempNode if and only if no rvalue (x: t). + rvalue: Expression + # Declared type in a comment, may be None. + type: mypy.types.Type | None + # Original, not semantically analyzed type in annotation (used for reprocessing) + unanalyzed_type: mypy.types.Type | None + # This indicates usage of PEP 526 type annotation syntax in assignment. + new_syntax: bool + # Does this assignment define a type alias? + is_alias_def: bool + # Is this a final definition? + # Final attributes can't be re-assigned once set, and can't be overridden + # in a subclass. This flag is not set if an attempted declaration was found to + # be invalid during semantic analysis. It is still set to `True` if + # a final declaration overrides another final declaration (this is checked + # during type checking when MROs are known). + is_final_def: bool + # Stop further processing of this assignment, to prevent flipping back and forth + # during semantic analysis passes. + invalid_recursive_alias: bool + + def __init__( + self, + lvalues: list[Lvalue], + rvalue: Expression, + type: mypy.types.Type | None = None, + new_syntax: bool = False, + ) -> None: + super().__init__() + self.lvalues = lvalues + self.rvalue = rvalue + self.type = type + self.unanalyzed_type = type + self.new_syntax = new_syntax + self.is_alias_def = False + self.is_final_def = False + self.invalid_recursive_alias = False + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_assignment_stmt(self) + + +class OperatorAssignmentStmt(Statement): + """Operator assignment statement such as x += 1""" + + __slots__ = ("op", "lvalue", "rvalue") + + __match_args__ = ("lvalue", "op", "rvalue") + + op: str # TODO: Enum? + lvalue: Lvalue + rvalue: Expression + + def __init__(self, op: str, lvalue: Lvalue, rvalue: Expression) -> None: + super().__init__() + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_operator_assignment_stmt(self) + + +class WhileStmt(Statement): + __slots__ = ("expr", "body", "else_body") + + __match_args__ = ("expr", "body", "else_body") + + expr: Expression + body: Block + else_body: Block | None + + def __init__(self, expr: Expression, body: Block, else_body: Block | None) -> None: + super().__init__() + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_while_stmt(self) + + +class ForStmt(Statement): + __slots__ = ( + "index", + "index_type", + "unanalyzed_index_type", + "inferred_item_type", + "inferred_iterator_type", + "expr", + "body", + "else_body", + "is_async", + ) + + __match_args__ = ("index", "index_type", "expr", "body", "else_body") + + # Index variables + index: Lvalue + # Type given by type comments for index, can be None + index_type: mypy.types.Type | None + # Original, not semantically analyzed type in annotation (used for reprocessing) + unanalyzed_index_type: mypy.types.Type | None + # Inferred iterable item type + inferred_item_type: mypy.types.Type | None + # Inferred iterator type + inferred_iterator_type: mypy.types.Type | None + # Expression to iterate + expr: Expression + body: Block + else_body: Block | None + is_async: bool # True if `async for ...` (PEP 492, Python 3.5) + + def __init__( + self, + index: Lvalue, + expr: Expression, + body: Block, + else_body: Block | None, + index_type: mypy.types.Type | None = None, + ) -> None: + super().__init__() + self.index = index + self.index_type = index_type + self.unanalyzed_index_type = index_type + self.inferred_item_type = None + self.inferred_iterator_type = None + self.expr = expr + self.body = body + self.else_body = else_body + self.is_async = False + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_for_stmt(self) + + +class ReturnStmt(Statement): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression | None + + def __init__(self, expr: Expression | None) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_return_stmt(self) + + +class AssertStmt(Statement): + __slots__ = ("expr", "msg") + + __match_args__ = ("expr", "msg") + + expr: Expression + msg: Expression | None + + def __init__(self, expr: Expression, msg: Expression | None = None) -> None: + super().__init__() + self.expr = expr + self.msg = msg + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_assert_stmt(self) + + +class DelStmt(Statement): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Lvalue + + def __init__(self, expr: Lvalue) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_del_stmt(self) + + +class BreakStmt(Statement): + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_break_stmt(self) + + +class ContinueStmt(Statement): + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_continue_stmt(self) + + +class PassStmt(Statement): + __slots__ = () + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_pass_stmt(self) + + +class IfStmt(Statement): + __slots__ = ("expr", "body", "else_body", "unreachable_else") + + __match_args__ = ("expr", "body", "else_body", "unreachable_else") + + expr: list[Expression] + body: list[Block] + else_body: Block | None + # (If there is actually no else statement, semantic analysis may nevertheless create an + # empty else block and mark it permanently as unreachable to tell that the control flow + # must always go through the if block.) + unreachable_else: bool + # (Type checking may modify this flag repeatedly to indicate whether an actually available + # or unavailable else block is unreachable, considering the current type information.) + + def __init__(self, expr: list[Expression], body: list[Block], else_body: Block | None) -> None: + super().__init__() + self.expr = expr + self.body = body + self.else_body = else_body + self.unreachable_else = False + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_if_stmt(self) + + +class RaiseStmt(Statement): + __slots__ = ("expr", "from_expr") + + __match_args__ = ("expr", "from_expr") + + # Plain 'raise' is a valid statement. + expr: Expression | None + from_expr: Expression | None + + def __init__(self, expr: Expression | None, from_expr: Expression | None) -> None: + super().__init__() + self.expr = expr + self.from_expr = from_expr + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_raise_stmt(self) + + +class TryStmt(Statement): + __slots__ = ("body", "types", "vars", "handlers", "else_body", "finally_body", "is_star") + + __match_args__ = ("body", "types", "vars", "handlers", "else_body", "finally_body", "is_star") + + body: Block # Try body + # Plain 'except:' also possible + types: list[Expression | None] # Except type expressions + vars: list[NameExpr | None] # Except variable names + handlers: list[Block] # Except bodies + else_body: Block | None + finally_body: Block | None + # Whether this is try ... except* (added in Python 3.11) + is_star: bool + + def __init__( + self, + body: Block, + vars: list[NameExpr | None], + types: list[Expression | None], + handlers: list[Block], + else_body: Block | None, + finally_body: Block | None, + ) -> None: + super().__init__() + self.body = body + self.vars = vars + self.types = types + self.handlers = handlers + self.else_body = else_body + self.finally_body = finally_body + self.is_star = False + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_try_stmt(self) + + +class WithStmt(Statement): + __slots__ = ("expr", "target", "unanalyzed_type", "analyzed_types", "body", "is_async") + + __match_args__ = ("expr", "target", "body") + + expr: list[Expression] + target: list[Lvalue | None] + # Type given by type comments for target, can be None + unanalyzed_type: mypy.types.Type | None + # Semantically analyzed types from type comment (TypeList type expanded) + analyzed_types: list[mypy.types.Type] + body: Block + is_async: bool # True if `async with ...` (PEP 492, Python 3.5) + + def __init__( + self, + expr: list[Expression], + target: list[Lvalue | None], + body: Block, + target_type: mypy.types.Type | None = None, + ) -> None: + super().__init__() + self.expr = expr + self.target = target + self.unanalyzed_type = target_type + self.analyzed_types = [] + self.body = body + self.is_async = False + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_with_stmt(self) + + +class MatchStmt(Statement): + __slots__ = ("subject", "subject_dummy", "patterns", "guards", "bodies") + + __match_args__ = ("subject", "patterns", "guards", "bodies") + + subject: Expression + subject_dummy: NameExpr | None + patterns: list[ConcretePattern] + guards: list[Expression | None] + bodies: list[Block] + + def __init__( + self, + subject: Expression, + patterns: list[ConcretePattern], + guards: list[Expression | None], + bodies: list[Block], + ) -> None: + super().__init__() + assert len(patterns) == len(guards) == len(bodies) + self.subject = subject + self.subject_dummy = None + self.patterns = patterns + self.guards = guards + self.bodies = bodies + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_match_stmt(self) + + +class TypeAliasStmt(Statement): + __slots__ = ("name", "type_args", "value", "invalid_recursive_alias", "alias_node") + + __match_args__ = ("name", "type_args", "value") + + name: NameExpr + type_args: list[TypeParam] + value: LambdaExpr # Return value will get translated into a type + invalid_recursive_alias: bool + alias_node: TypeAlias | None + + def __init__(self, name: NameExpr, type_args: list[TypeParam], value: LambdaExpr) -> None: + super().__init__() + self.name = name + self.type_args = type_args + self.value = value + self.invalid_recursive_alias = False + self.alias_node = None + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_type_alias_stmt(self) + + +# Expressions + + +class IntExpr(Expression): + """Integer literal""" + + __slots__ = ("value",) + + __match_args__ = ("value",) + + value: int # 0 by default + + def __init__(self, value: int) -> None: + super().__init__() + self.value = value + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_int_expr(self) + + +# How mypy uses StrExpr and BytesExpr: +# +# b'x' -> BytesExpr +# 'x', u'x' -> StrExpr + + +class StrExpr(Expression): + """String literal""" + + __slots__ = ("value", "as_type", "has_surrogates") + + __match_args__ = ("value",) + + value: str # '' by default + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None + # This indicates whether original string literal contained Unicode surrogate + # codepoints. Those are not supported by Ruff parser and are replaced by + # replacement characters. Thus, we can't support them in mypyc. + has_surrogates: bool + + def __init__(self, value: str) -> None: + super().__init__() + self.value = value + self.as_type = NotParsed.VALUE + self.has_surrogates = False + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_str_expr(self) + + +def is_StrExpr_list(seq: list[Expression]) -> TypeGuard[list[StrExpr]]: # noqa: N802 + return all(isinstance(item, StrExpr) for item in seq) + + +class BytesExpr(Expression): + """Bytes literal""" + + __slots__ = ("value",) + + __match_args__ = ("value",) + + # Note: we deliberately do NOT use bytes here because it ends up + # unnecessarily complicating a lot of the result logic. For example, + # we'd have to worry about converting the bytes into a format we can + # easily serialize/deserialize to and from JSON, would have to worry + # about turning the bytes into a human-readable representation in + # error messages... + # + # It's more convenient to just store the human-readable representation + # from the very start. + value: str + + def __init__(self, value: str) -> None: + super().__init__() + self.value = value + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_bytes_expr(self) + + +class FloatExpr(Expression): + """Float literal""" + + __slots__ = ("value",) + + __match_args__ = ("value",) + + value: float # 0.0 by default + + def __init__(self, value: float) -> None: + super().__init__() + self.value = value + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_float_expr(self) + + +class ComplexExpr(Expression): + """Complex literal""" + + __slots__ = ("value",) + + __match_args__ = ("value",) + + value: complex + + def __init__(self, value: complex) -> None: + super().__init__() + self.value = value + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_complex_expr(self) + + +class EllipsisExpr(Expression): + """Ellipsis (...)""" + + __slots__ = () + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_ellipsis(self) + + +class StarExpr(Expression): + """Star expression""" + + __slots__ = ("expr", "valid") + + __match_args__ = ("expr", "valid") + + expr: Expression + valid: bool + + def __init__(self, expr: Expression) -> None: + super().__init__() + self.expr = expr + + # Whether this starred expression is used in a tuple/list and as lvalue + self.valid = False + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_star_expr(self) + + +class RefExpr(Expression): + """Abstract base class for name-like constructs""" + + __slots__ = ( + "kind", + "node", + "_fullname", + "is_new_def", + "is_inferred_def", + "is_alias_rvalue", + "type_guard", + "type_is", + ) + + def __init__(self) -> None: + super().__init__() + # LDEF/GDEF/MDEF/... (None if not available) + self.kind: int | None = None + # Var, FuncDef or TypeInfo that describes this + self.node: SymbolNode | None = None + # Fully qualified name (or name if not global) + self._fullname = "" + # Does this define a new name? + self.is_new_def = False + # Does this define a new name with inferred type? + # + # For members, after semantic analysis, this does not take base + # classes into consideration at all; the type checker deals with these. + self.is_inferred_def = False + # Is this expression appears as an rvalue of a valid type alias definition? + self.is_alias_rvalue = False + # Cache type guard from callable_type.type_guard + self.type_guard: mypy.types.Type | None = None + # And same for TypeIs + self.type_is: mypy.types.Type | None = None + + @property + def fullname(self) -> str: + return self._fullname + + @fullname.setter + def fullname(self, v: str) -> None: + self._fullname = v + + +class NameExpr(RefExpr): + """Name expression + + This refers to a local name, global name or a module. + """ + + __slots__ = ("name", "is_special_form") + + __match_args__ = ("name", "node") + + def __init__(self, name: str) -> None: + super().__init__() + self.name = name # Name referred to + # Is this a l.h.s. of a special form assignment like typed dict or type variable? + self.is_special_form = False + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_name_expr(self) + + def serialize(self) -> JsonDict: + assert False, f"Serializing NameExpr: {self}" + + +class MemberExpr(RefExpr): + """Member access expression x.y""" + + __slots__ = ("expr", "name", "def_var") + + __match_args__ = ("expr", "name", "node") + + def __init__(self, expr: Expression, name: str) -> None: + super().__init__() + self.expr = expr + self.name = name + # The variable node related to a definition through 'self.x = '. + # The nodes of other kinds of member expressions are resolved during type checking. + self.def_var: Var | None = None + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_member_expr(self) + + +# Kinds of arguments +@unique +class ArgKind(Enum): + # Positional argument + ARG_POS = 0 + # Positional, optional argument (functions only, not calls) + ARG_OPT = 1 + # *arg argument + ARG_STAR = 2 + # Keyword argument x=y in call, or keyword-only function arg + ARG_NAMED = 3 + # **arg argument + ARG_STAR2 = 4 + # In an argument list, keyword-only and also optional + ARG_NAMED_OPT = 5 + + def is_positional(self, star: bool = False) -> bool: + return self == ARG_POS or self == ARG_OPT or (star and self == ARG_STAR) + + def is_named(self, star: bool = False) -> bool: + return self == ARG_NAMED or self == ARG_NAMED_OPT or (star and self == ARG_STAR2) + + def is_required(self) -> bool: + return self == ARG_POS or self == ARG_NAMED + + def is_optional(self) -> bool: + return self == ARG_OPT or self == ARG_NAMED_OPT + + def is_star(self) -> bool: + return self == ARG_STAR or self == ARG_STAR2 + + +ARG_POS: Final = ArgKind.ARG_POS +ARG_OPT: Final = ArgKind.ARG_OPT +ARG_STAR: Final = ArgKind.ARG_STAR +ARG_NAMED: Final = ArgKind.ARG_NAMED +ARG_STAR2: Final = ArgKind.ARG_STAR2 +ARG_NAMED_OPT: Final = ArgKind.ARG_NAMED_OPT + +ARG_KINDS: Final = (ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, ARG_NAMED_OPT) + + +class CallExpr(Expression): + """Call expression. + + This can also represent several special forms that are syntactically calls + such as cast(...) and None # type: .... + """ + + __slots__ = ("callee", "args", "arg_kinds", "arg_names", "analyzed") + + __match_args__ = ("callee", "args", "arg_kinds", "arg_names") + + def __init__( + self, + callee: Expression, + args: list[Expression], + arg_kinds: list[ArgKind], + arg_names: list[str | None], + analyzed: Expression | None = None, + ) -> None: + super().__init__() + if not arg_names: + arg_names = [None] * len(args) + + self.callee = callee + self.args = args + self.arg_kinds = arg_kinds # ARG_ constants + # Each name can be None if not a keyword argument. + self.arg_names: list[str | None] = arg_names + # If not None, the node that represents the meaning of the CallExpr. For + # cast(...) this is a CastExpr. + self.analyzed = analyzed + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_call_expr(self) + + +class YieldFromExpr(Expression): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression + + def __init__(self, expr: Expression) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_yield_from_expr(self) + + +class YieldExpr(Expression): + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression | None + + def __init__(self, expr: Expression | None) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_yield_expr(self) + + +class IndexExpr(Expression): + """Index expression x[y]. + + Also wraps type application such as List[int] as a special form. + """ + + __slots__ = ("base", "index", "method_type", "analyzed", "as_type") + + __match_args__ = ("base", "index") + + base: Expression + index: Expression + # Inferred __getitem__ method type + method_type: mypy.types.Type | None + # If not None, this is actually semantically a type application + # Class[type, ...] or a type alias initializer. + analyzed: TypeApplication | TypeAliasExpr | None + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None + + def __init__(self, base: Expression, index: Expression) -> None: + super().__init__() + self.base = base + self.index = index + self.method_type = None + self.analyzed = None + self.as_type = NotParsed.VALUE + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_index_expr(self) + + +class UnaryExpr(Expression): + """Unary operation""" + + __slots__ = ("op", "expr", "method_type") + + __match_args__ = ("op", "expr") + + op: str # TODO: Enum? + expr: Expression + # Inferred operator method type + method_type: mypy.types.Type | None + + def __init__(self, op: str, expr: Expression) -> None: + super().__init__() + self.op = op + self.expr = expr + self.method_type = None + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_unary_expr(self) + + +class AssignmentExpr(Expression): + """Assignment expressions in Python 3.8+, like "a := 2".""" + + __slots__ = ("target", "value") + + __match_args__ = ("target", "value") + + def __init__(self, target: NameExpr, value: Expression) -> None: + super().__init__() + self.target = target + self.value = value + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_assignment_expr(self) + + +class OpExpr(Expression): + """Binary operation. + + The dot (.), [] and comparison operators have more specific nodes. + """ + + __slots__ = ( + "op", + "left", + "right", + "method_type", + "right_always", + "right_unreachable", + "analyzed", + "as_type", + ) + + __match_args__ = ("left", "op", "right") + + op: str # TODO: Enum? + left: Expression + right: Expression + # Inferred type for the operator method type (when relevant). + method_type: mypy.types.Type | None + # Per static analysis only: Is the right side going to be evaluated every time? + right_always: bool + # Per static analysis only: Is the right side unreachable? + right_unreachable: bool + # Used for expressions that represent a type "X | Y" in some contexts + analyzed: TypeAliasExpr | None + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None + + def __init__( + self, op: str, left: Expression, right: Expression, analyzed: TypeAliasExpr | None = None + ) -> None: + super().__init__() + self.op = op + self.left = left + self.right = right + self.method_type = None + self.right_always = False + self.right_unreachable = False + self.analyzed = analyzed + self.as_type = NotParsed.VALUE + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_op_expr(self) + + +# Expression subtypes that could represent the root of a valid type expression. +# +# May have an "as_type" attribute to hold the type for a type expression parsed +# during the SemanticAnalyzer pass. +MaybeTypeExpression = (IndexExpr, MemberExpr, NameExpr, OpExpr, StrExpr) + + +class ComparisonExpr(Expression): + """Comparison expression (e.g. a < b > c < d).""" + + __slots__ = ("operators", "operands", "method_types") + + __match_args__ = ("operands", "operators") + + operators: list[str] + operands: list[Expression] + # Inferred type for the operator methods (when relevant; None for 'is'). + method_types: list[mypy.types.Type | None] + + def __init__(self, operators: list[str], operands: list[Expression]) -> None: + super().__init__() + self.operators = operators + self.operands = operands + self.method_types = [] + + def pairwise(self) -> Iterator[tuple[str, Expression, Expression]]: + """If this comparison expr is "a < b is c == d", yields the sequence + ("<", a, b), ("is", b, c), ("==", c, d) + """ + for i, operator in enumerate(self.operators): + yield operator, self.operands[i], self.operands[i + 1] + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_comparison_expr(self) + + +class SliceExpr(Expression): + """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). + + This is only valid as index in index expressions. + """ + + __slots__ = ("begin_index", "end_index", "stride") + + __match_args__ = ("begin_index", "end_index", "stride") + + begin_index: Expression | None + end_index: Expression | None + stride: Expression | None + + def __init__( + self, + begin_index: Expression | None, + end_index: Expression | None, + stride: Expression | None, + ) -> None: + super().__init__() + self.begin_index = begin_index + self.end_index = end_index + self.stride = stride + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_slice_expr(self) + + +class CastExpr(Expression): + """Cast expression cast(type, expr).""" + + __slots__ = ("expr", "type") + + __match_args__ = ("expr", "type") + + expr: Expression + type: mypy.types.Type + + def __init__(self, expr: Expression, typ: mypy.types.Type) -> None: + super().__init__() + self.expr = expr + self.type = typ + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_cast_expr(self) + + +class TypeFormExpr(Expression): + """TypeForm(type) expression.""" + + __slots__ = ("type",) + + __match_args__ = ("type",) + + type: mypy.types.Type + + def __init__(self, typ: mypy.types.Type) -> None: + super().__init__() + self.type = typ + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_form_expr(self) + + +class AssertTypeExpr(Expression): + """Represents a typing.assert_type(expr, type) call.""" + + __slots__ = ("expr", "type") + + __match_args__ = ("expr", "type") + + expr: Expression + type: mypy.types.Type + + def __init__(self, expr: Expression, typ: mypy.types.Type) -> None: + super().__init__() + self.expr = expr + self.type = typ + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_assert_type_expr(self) + + +class RevealExpr(Expression): + """Reveal type expression reveal_type(expr) or reveal_locals() expression.""" + + __slots__ = ("expr", "kind", "local_nodes", "is_imported") + + __match_args__ = ("expr", "kind", "local_nodes", "is_imported") + + expr: Expression | None + kind: int + local_nodes: list[Var] | None + + def __init__( + self, + kind: int, + expr: Expression | None = None, + local_nodes: list[Var] | None = None, + is_imported: bool = False, + ) -> None: + super().__init__() + self.expr = expr + self.kind = kind + self.local_nodes = local_nodes + self.is_imported = is_imported + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_reveal_expr(self) + + +class SuperExpr(Expression): + """Expression super().name""" + + __slots__ = ("name", "info", "call") + + __match_args__ = ("name", "call", "info") + + name: str + info: TypeInfo | None # Type that contains this super expression + call: CallExpr # The expression super(...) + + def __init__(self, name: str, call: CallExpr) -> None: + super().__init__() + self.name = name + self.call = call + self.info = None + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_super_expr(self) + + +class LambdaExpr(FuncItem, Expression): + """Lambda expression""" + + __match_args__ = ("arguments", "arg_names", "arg_kinds", "body") + + @property + def name(self) -> str: + return LAMBDA_NAME + + def expr(self) -> Expression: + """Return the expression (the body) of the lambda.""" + ret = self.body.body[-1] + assert isinstance(ret, ReturnStmt) + expr = ret.expr + assert expr is not None # lambda can't have empty body + return expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_lambda_expr(self) + + def is_dynamic(self) -> bool: + return False + + +class ListExpr(Expression): + """List literal expression [...].""" + + __slots__ = ("items",) + + __match_args__ = ("items",) + + items: list[Expression] + + def __init__(self, items: list[Expression]) -> None: + super().__init__() + self.items = items + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_list_expr(self) + + +class DictExpr(Expression): + """Dictionary literal expression {key: value, ...}.""" + + __slots__ = ("items",) + + __match_args__ = ("items",) + + items: list[tuple[Expression | None, Expression]] + + def __init__(self, items: list[tuple[Expression | None, Expression]]) -> None: + super().__init__() + self.items = items + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_dict_expr(self) + + +class TemplateStrExpr(Expression): + """Template string expression t'...'.""" + + __slots__ = ("items", "has_surrogates") + __match_args__ = ("items",) + + # Each item is either: + # - a StrExpr (literal string segment), or + # - a tuple (value_expr, source_text, conversion, format_spec_expr) + # where conversion is str | None ("r", "s", "a", or None) + # and format_spec_expr is Expression | None + items: list[Expression | tuple[Expression, str, str | None, Expression | None]] + has_surrogates: bool + + def __init__( + self, items: list[Expression | tuple[Expression, str, str | None, Expression | None]] + ) -> None: + super().__init__() + self.items = items + self.has_surrogates = False + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_template_str_expr(self) + + +class TupleExpr(Expression): + """Tuple literal expression (..., ...) + + Also lvalue sequences (..., ...) and [..., ...]""" + + __slots__ = ("items",) + + __match_args__ = ("items",) + + items: list[Expression] + + def __init__(self, items: list[Expression]) -> None: + super().__init__() + self.items = items + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_tuple_expr(self) + + +class SetExpr(Expression): + """Set literal expression {value, ...}.""" + + __slots__ = ("items",) + + __match_args__ = ("items",) + + items: list[Expression] + + def __init__(self, items: list[Expression]) -> None: + super().__init__() + self.items = items + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_set_expr(self) + + +class GeneratorExpr(Expression): + """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" + + __slots__ = ("left_expr", "sequences", "condlists", "is_async", "indices") + + __match_args__ = ("left_expr", "indices", "sequences", "condlists") + + left_expr: Expression + sequences: list[Expression] + condlists: list[list[Expression]] + is_async: list[bool] + indices: list[Lvalue] + + def __init__( + self, + left_expr: Expression, + indices: list[Lvalue], + sequences: list[Expression], + condlists: list[list[Expression]], + is_async: list[bool], + ) -> None: + super().__init__() + self.left_expr = left_expr + self.sequences = sequences + self.condlists = condlists + self.indices = indices + self.is_async = is_async + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_generator_expr(self) + + +class ListComprehension(Expression): + """List comprehension (e.g. [x + 1 for x in a])""" + + __slots__ = ("generator",) + + __match_args__ = ("generator",) + + generator: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + super().__init__() + self.generator = generator + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_list_comprehension(self) + + +class SetComprehension(Expression): + """Set comprehension (e.g. {x + 1 for x in a})""" + + __slots__ = ("generator",) + + __match_args__ = ("generator",) + + generator: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + super().__init__() + self.generator = generator + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_set_comprehension(self) + + +class DictionaryComprehension(Expression): + """Dictionary comprehension (e.g. {k: v for k, v in a}""" + + __slots__ = ("key", "value", "sequences", "condlists", "is_async", "indices") + + __match_args__ = ("key", "value", "indices", "sequences", "condlists") + + key: Expression | None + value: Expression + sequences: list[Expression] + condlists: list[list[Expression]] + is_async: list[bool] + indices: list[Lvalue] + + def __init__( + self, + key: Expression | None, + value: Expression, + indices: list[Lvalue], + sequences: list[Expression], + condlists: list[list[Expression]], + is_async: list[bool], + ) -> None: + super().__init__() + self.key = key + self.value = value + self.sequences = sequences + self.condlists = condlists + self.indices = indices + self.is_async = is_async + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_dictionary_comprehension(self) + + +class ConditionalExpr(Expression): + """Conditional expression (e.g. x if y else z)""" + + __slots__ = ("cond", "if_expr", "else_expr") + + __match_args__ = ("if_expr", "cond", "else_expr") + + cond: Expression + if_expr: Expression + else_expr: Expression + + def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: + super().__init__() + self.cond = cond + self.if_expr = if_expr + self.else_expr = else_expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_conditional_expr(self) + + +class TypeApplication(Expression): + """Type application expr[type, ...]""" + + __slots__ = ("expr", "types") + + __match_args__ = ("expr", "types") + + expr: Expression + types: list[mypy.types.Type] + + def __init__(self, expr: Expression, types: list[mypy.types.Type]) -> None: + super().__init__() + self.expr = expr + self.types = types + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_application(self) + + +# Variance of a type variable. For example, T in the definition of +# List[T] is invariant, so List[int] is not a subtype of List[object], +# and also List[object] is not a subtype of List[int]. +# +# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of +# Iterable[object], but not vice versa. +# +# If T is contravariant in Foo[T], Foo[object] is a subtype of +# Foo[int], but not vice versa. +INVARIANT: Final = 0 +COVARIANT: Final = 1 +CONTRAVARIANT: Final = 2 +VARIANCE_NOT_READY: Final = 3 # Variance hasn't been inferred (using Python 3.12 syntax) + + +class TypeVarLikeExpr(SymbolNode, Expression): + """Base class for TypeVarExpr, ParamSpecExpr and TypeVarTupleExpr. + + Note that they are constructed by the semantic analyzer. + """ + + __slots__ = ( + "_name", + "_fullname", + "upper_bound", + "default", + "variance", + "is_new_style", + "default_depends", + ) + + _name: str + _fullname: str + # Upper bound: only subtypes of upper_bound are valid as values. By default + # this is 'object', meaning no restriction. + upper_bound: mypy.types.Type + # Default: used to resolve the TypeVar if the default is not explicitly given. + # By default this is 'AnyType(TypeOfAny.from_omitted_generics)'. See PEP 696. + default: mypy.types.Type + # Variance of the type variable. Invariant is the default. + # TypeVar(..., covariant=True) defines a covariant type variable. + # TypeVar(..., contravariant=True) defines a contravariant type + # variable. + variance: int + # Record instances and type aliases that appear bare/implicit in the default value + # of this type variable. This is needed to detect recursive type variable defaults. + default_depends: set[TypeInfo | TypeAlias] | None + + def __init__( + self, + name: str, + fullname: str, + upper_bound: mypy.types.Type, + default: mypy.types.Type, + variance: int = INVARIANT, + is_new_style: bool = False, + line: int = -1, + ) -> None: + super().__init__(line=line) + self._name = name + self._fullname = fullname + self.upper_bound = upper_bound + self.default = default + self.variance = variance + self.is_new_style = is_new_style + self.default_depends = None + + @property + def name(self) -> str: + return self._name + + @property + def fullname(self) -> str: + return self._fullname + + +# All types that are both SymbolNodes and Expressions. +# Use when common children of them are needed. +SYMBOL_NODE_EXPRESSION_TYPES: Final = (TypeVarLikeExpr,) + + +class TypeVarExpr(TypeVarLikeExpr): + """Type variable expression TypeVar(...). + + This is also used to represent type variables in symbol tables. + + A type variable is not valid as a type unless bound in a TypeVarLikeScope. + That happens within: + + 1. a generic class that uses the type variable as a type argument or + 2. a generic function that refers to the type variable in its signature. + """ + + __slots__ = ("values",) + + __match_args__ = ("name", "values", "upper_bound", "default") + + # Value restriction: only types in the list are valid as values. If the + # list is empty, there is no restriction. + values: list[mypy.types.Type] + + def __init__( + self, + name: str, + fullname: str, + values: list[mypy.types.Type], + upper_bound: mypy.types.Type, + default: mypy.types.Type, + variance: int = INVARIANT, + is_new_style: bool = False, + line: int = -1, + ) -> None: + super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line) + self.values = values + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_var_expr(self) + + def serialize(self) -> JsonDict: + return { + ".class": "TypeVarExpr", + "name": self._name, + "fullname": self._fullname, + "values": [t.serialize() for t in self.values], + "upper_bound": self.upper_bound.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarExpr: + assert data[".class"] == "TypeVarExpr" + return TypeVarExpr( + data["name"], + data["fullname"], + [mypy.types.deserialize_type(v) for v in data["values"]], + mypy.types.deserialize_type(data["upper_bound"]), + mypy.types.deserialize_type(data["default"]), + data["variance"], + ) + + def write(self, data: WriteBuffer) -> None: + write_tag(data, TYPE_VAR_EXPR) + write_str(data, self._name) + write_str(data, self._fullname) + mypy.types.write_type_list(data, self.values) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> TypeVarExpr: + ret = TypeVarExpr( + read_str(data), + read_str(data), + mypy.types.read_type_list(data), + mypy.types.read_type(data), + mypy.types.read_type(data), + read_int(data), + ) + assert read_tag(data) == END_TAG + return ret + + +class ParamSpecExpr(TypeVarLikeExpr): + __slots__ = () + + __match_args__ = ("name", "upper_bound", "default") + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_paramspec_expr(self) + + def serialize(self) -> JsonDict: + return { + ".class": "ParamSpecExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": self.upper_bound.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> ParamSpecExpr: + assert data[".class"] == "ParamSpecExpr" + return ParamSpecExpr( + data["name"], + data["fullname"], + mypy.types.deserialize_type(data["upper_bound"]), + mypy.types.deserialize_type(data["default"]), + data["variance"], + ) + + def write(self, data: WriteBuffer) -> None: + write_tag(data, PARAM_SPEC_EXPR) + write_str(data, self._name) + write_str(data, self._fullname) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> ParamSpecExpr: + ret = ParamSpecExpr( + read_str(data), + read_str(data), + mypy.types.read_type(data), + mypy.types.read_type(data), + read_int(data), + ) + assert read_tag(data) == END_TAG + return ret + + +class TypeVarTupleExpr(TypeVarLikeExpr): + """Type variable tuple expression TypeVarTuple(...).""" + + __slots__ = "tuple_fallback" + + tuple_fallback: mypy.types.Instance + + __match_args__ = ("name", "upper_bound", "default") + + def __init__( + self, + name: str, + fullname: str, + upper_bound: mypy.types.Type, + tuple_fallback: mypy.types.Instance, + default: mypy.types.Type, + variance: int = INVARIANT, + is_new_style: bool = False, + line: int = -1, + ) -> None: + super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line) + self.tuple_fallback = tuple_fallback + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_var_tuple_expr(self) + + def serialize(self) -> JsonDict: + return { + ".class": "TypeVarTupleExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": self.upper_bound.serialize(), + "tuple_fallback": self.tuple_fallback.serialize(), + "default": self.default.serialize(), + "variance": self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: + assert data[".class"] == "TypeVarTupleExpr" + return TypeVarTupleExpr( + data["name"], + data["fullname"], + mypy.types.deserialize_type(data["upper_bound"]), + mypy.types.Instance.deserialize(data["tuple_fallback"]), + mypy.types.deserialize_type(data["default"]), + data["variance"], + ) + + def write(self, data: WriteBuffer) -> None: + write_tag(data, TYPE_VAR_TUPLE_EXPR) + self.tuple_fallback.write(data) + write_str(data, self._name) + write_str(data, self._fullname) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> TypeVarTupleExpr: + assert read_tag(data) == mypy.types.INSTANCE + fallback = mypy.types.Instance.read(data) + ret = TypeVarTupleExpr( + read_str(data), + read_str(data), + mypy.types.read_type(data), + fallback, + mypy.types.read_type(data), + read_int(data), + ) + assert read_tag(data) == END_TAG + return ret + + +class TypeAliasExpr(Expression): + """Type alias expression (rvalue).""" + + __slots__ = ("node",) + + __match_args__ = ("node",) + + node: TypeAlias + + def __init__(self, node: TypeAlias) -> None: + super().__init__() + self.node = node + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_alias_expr(self) + + +class NamedTupleExpr(Expression): + """Named tuple expression namedtuple(...) or NamedTuple(...).""" + + __slots__ = ("info", "is_typed") + + __match_args__ = ("info",) + + # The class representation of this named tuple (its tuple_type attribute contains + # the tuple item types) + info: TypeInfo + is_typed: bool # whether this class was created with typing(_extensions).NamedTuple + + def __init__(self, info: TypeInfo, is_typed: bool = False) -> None: + super().__init__() + self.info = info + self.is_typed = is_typed + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_namedtuple_expr(self) + + +class TypedDictExpr(Expression): + """Typed dict expression TypedDict(...).""" + + __slots__ = ("info",) + + __match_args__ = ("info",) + + # The class representation of this typed dict + info: TypeInfo + + def __init__(self, info: TypeInfo) -> None: + super().__init__() + self.info = info + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_typeddict_expr(self) + + +class EnumCallExpr(Expression): + """Named tuple expression Enum('name', 'val1 val2 ...').""" + + __slots__ = ("info", "items", "values") + + __match_args__ = ("info", "items", "values") + + # The class representation of this enumerated type + info: TypeInfo + # The item names (for debugging) + items: list[str] + values: list[Expression | None] + + def __init__(self, info: TypeInfo, items: list[str], values: list[Expression | None]) -> None: + super().__init__() + self.info = info + self.items = items + self.values = values + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_enum_call_expr(self) + + +class PromoteExpr(Expression): + """Ducktype class decorator expression _promote(...).""" + + __slots__ = ("type",) + + type: mypy.types.ProperType + + def __init__(self, type: mypy.types.ProperType) -> None: + super().__init__() + self.type = type + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit__promote_expr(self) + + +class NewTypeExpr(Expression): + """NewType expression NewType(...).""" + + __slots__ = ("name", "old_type", "info") + + __match_args__ = ("name", "old_type", "info") + + name: str + # The base type (the second argument to NewType) + old_type: mypy.types.Type | None + # The synthesized class representing the new type (inherits old_type) + info: TypeInfo | None + + def __init__( + self, name: str, old_type: mypy.types.Type | None, line: int, column: int + ) -> None: + super().__init__(line=line, column=column) + self.name = name + self.old_type = old_type + self.info = None + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_newtype_expr(self) + + +class AwaitExpr(Expression): + """Await expression (await ...).""" + + __slots__ = ("expr",) + + __match_args__ = ("expr",) + + expr: Expression + + def __init__(self, expr: Expression) -> None: + super().__init__() + self.expr = expr + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_await_expr(self) + + +# Constants + + +class TempNode(Expression): + """Temporary dummy node used during type checking. + + This node is not present in the original program; it is just an artifact + of the type checker implementation. It only represents an opaque node with + some fixed type. + """ + + __slots__ = ("type", "no_rhs") + + type: mypy.types.Type + # Is this TempNode used to indicate absence of a right hand side in an annotated assignment? + # (e.g. for 'x: int' the rvalue is TempNode(AnyType(TypeOfAny.special_form), no_rhs=True)) + no_rhs: bool + + def __init__( + self, typ: mypy.types.Type, no_rhs: bool = False, *, context: Context | None = None + ) -> None: + """Construct a dummy node; optionally borrow line/column from context object.""" + super().__init__() + self.type = typ + self.no_rhs = no_rhs + if context is not None: + self.line = context.line + self.column = context.column + + def __repr__(self) -> str: + return "TempNode:%d(%s)" % (self.line, str(self.type)) + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_temp_node(self) + + +# Special attributes not collected as protocol members by Python 3.12 +# See typing._SPECIAL_NAMES +EXCLUDED_PROTOCOL_ATTRIBUTES: Final = frozenset( + { + "__abstractmethods__", + "__annotations__", + "__dict__", + "__doc__", + "__init__", + "__module__", + "__new__", + "__slots__", + "__subclasshook__", + "__weakref__", + "__class_getitem__", # Since Python 3.9 + } +) + +# Attributes that can optionally be defined in the body of a subclass of +# enum.Enum but are removed from the class __dict__ by EnumMeta. +EXCLUDED_ENUM_ATTRIBUTES: Final = frozenset({"_ignore_", "_order_", "__order__"}) + + +class TypeInfo(SymbolNode): + """The type structure of a single class. + + Each TypeInfo corresponds one-to-one to a ClassDef, which + represents the AST of the class. + + In type-theory terms, this is a "type constructor", and if the + class is generic then it will be a type constructor of higher kind. + Where the class is used in an actual type, it's in the form of an + Instance, which amounts to a type application of the tycon to + the appropriate number of arguments. + """ + + __slots__ = ( + "_fullname", + "module_name", + "defn", + "mro", + "_mro_refs", + "bad_mro", + "is_final", + "is_disjoint_base", + "declared_metaclass", + "metaclass_type", + "names", + "is_abstract", + "is_protocol", + "runtime_protocol", + "abstract_attributes", + "deletable_attributes", + "slots", + "assuming", + "assuming_proper", + "inferring", + "is_enum", + "fallback_to_any", + "meta_fallback_to_any", + "type_vars", + "has_param_spec_type", + "bases", + "_promote", + "tuple_type", + "special_alias", + "is_named_tuple", + "typeddict_type", + "is_newtype", + "is_intersection", + "metadata", + "alt_promote", + "has_type_var_tuple_type", + "type_var_tuple_prefix", + "type_var_tuple_suffix", + "self_type", + "dataclass_transform_spec", + "is_type_check_only", + "deprecated", + "type_object_type", + "default_depends", + "typeddict_data", + ) + + _fullname: str # Fully qualified name + # Fully qualified name for the module this type was defined in. This + # information is also in the fullname, but is harder to extract in the + # case of nested class definitions. + module_name: str + defn: ClassDef # Corresponding ClassDef + # Method Resolution Order: the order of looking up attributes. The first + # value always to refers to this class. + mro: list[TypeInfo] + # Used to stash the names of the mro classes temporarily between + # deserialization and fixup. See deserialize() for why. + _mro_refs: list[str] | None + bad_mro: bool # Could not construct full MRO + is_final: bool + is_disjoint_base: bool + + declared_metaclass: mypy.types.Instance | None + metaclass_type: mypy.types.Instance | None + + names: SymbolTable # Names defined directly in this type + is_abstract: bool # Does the class have any abstract attributes? + is_protocol: bool # Is this a protocol class? + runtime_protocol: bool # Does this protocol support isinstance checks? + # List of names of abstract attributes together with their abstract status. + # The abstract status must be one of `NOT_ABSTRACT`, `IS_ABSTRACT`, `IMPLICITLY_ABSTRACT`. + abstract_attributes: list[tuple[str, int]] + deletable_attributes: list[str] # Used by mypyc only + # Does this type have concrete `__slots__` defined? + # If class does not have `__slots__` defined then it is `None`, + # if it has empty `__slots__` then it is an empty set. + slots: set[str] | None + + # The attributes 'assuming' and 'assuming_proper' represent structural subtype matrices. + # + # In languages with structural subtyping, one can keep a global subtype matrix like this: + # . A B C . + # A 1 0 0 + # B 1 1 1 + # C 1 0 1 + # . + # where 1 indicates that the type in corresponding row is a subtype of the type + # in corresponding column. This matrix typically starts filled with all 1's and + # a typechecker tries to "disprove" every subtyping relation using atomic (or nominal) types. + # However, we don't want to keep this huge global state. Instead, we keep the subtype + # information in the form of list of pairs (subtype, supertype) shared by all Instances + # with given supertype's TypeInfo. When we enter a subtype check we push a pair in this list + # thus assuming that we started with 1 in corresponding matrix element. Such algorithm allows + # to treat recursive and mutually recursive protocols and other kinds of complex situations. + # + # If concurrent/parallel type checking will be added in future, + # then there should be one matrix per thread/process to avoid false negatives + # during the type checking phase. + assuming: list[tuple[mypy.types.Instance, mypy.types.Instance]] + assuming_proper: list[tuple[mypy.types.Instance, mypy.types.Instance]] + # Ditto for temporary 'inferring' stack of recursive constraint inference. + # It contains Instances of protocol types that appeared as an argument to + # constraints.infer_constraints(). We need 'inferring' to avoid infinite recursion for + # recursive and mutually recursive protocols. + # + # We make 'assuming' and 'inferring' attributes here instead of passing they as kwargs, + # since this would require to pass them in many dozens of calls. In particular, + # there is a dependency infer_constraint -> is_subtype -> is_callable_subtype -> + # -> infer_constraints. + inferring: list[mypy.types.Instance] + # 'inferring' and 'assuming' can't be made sets, since we need to use + # is_same_type to correctly treat unions. + + # Classes inheriting from Enum shadow their true members with a __getattr__, so we + # have to treat them as a special case. + is_enum: bool + # If true, any unknown attributes should have type 'Any' instead + # of generating a type error. This would be true if there is a + # base class with type 'Any', but other use cases may be + # possible. This is similar to having __getattr__ that returns Any + # (and __setattr__), but without the __getattr__ method. + fallback_to_any: bool + + # Same as above but for cases where metaclass has type Any. This will suppress + # all attribute errors only for *class object* access. + meta_fallback_to_any: bool + + # Information related to type annotations. + + # Generic type variable names (full names) + type_vars: list[str] + + # Whether this class has a ParamSpec type variable + has_param_spec_type: bool + + # Direct base classes. + bases: list[mypy.types.Instance] + + # Another type which this type will be treated as a subtype of, + # even though it's not a subclass in Python. The non-standard + # `@_promote` decorator introduces this, and there are also + # several builtin examples, in particular `int` -> `float`. + _promote: list[mypy.types.ProperType] + + # This is used for promoting native integer types such as 'i64' to + # 'int'. (_promote is used for the other direction.) This only + # supports one-step promotions (e.g., i64 -> int, not + # i64 -> int -> float, and this isn't used to promote in joins. + # + # This results in some unintuitive results, such as that even + # though i64 is compatible with int and int is compatible with + # float, i64 is *not* compatible with float. + alt_promote: mypy.types.Instance | None + + # Representation of a Tuple[...] base class, if the class has any + # (e.g., for named tuples). If this is not None, the actual Type + # object used for this class is not an Instance but a TupleType; + # the corresponding Instance is set as the fallback type of the + # tuple type. + tuple_type: mypy.types.TupleType | None + + # Is this a named tuple type? + is_named_tuple: bool + + # If this class is defined by the TypedDict type constructor, + # then this is not None. + typeddict_type: mypy.types.TypedDictType | None + + # Is this a newtype type? + is_newtype: bool + + # Is this a synthesized intersection type? + is_intersection: bool + + # This is a dictionary that will be serialized and un-serialized as is. + # It is useful for plugins to add their data to save in the cache. + metadata: dict[str, JsonDict] + + # Store type alias representing this type (for named tuples and TypedDicts). + # Although definitions of these types are stored in symbol tables as TypeInfo, + # when a type analyzer will find them, it should construct a TupleType, or + # a TypedDict type. However, we can't use the plain types, since if the definition + # is recursive, this will create an actual recursive structure of types (i.e. as + # internal Python objects) causing infinite recursions everywhere during type checking. + # To overcome this, we create a TypeAlias node, that will point to these types. + # We store this node in the `special_alias` attribute, because it must be the same node + # in case we are doing multiple semantic analysis passes. + special_alias: TypeAlias | None + + # Shared type variable for typing.Self in this class (if used, otherwise None). + self_type: mypy.types.TypeVarType | None + + # Added if the corresponding class is directly decorated with `typing.dataclass_transform` + dataclass_transform_spec: DataclassTransformSpec | None + + # Is set to `True` when class is decorated with `@typing.type_check_only` + is_type_check_only: bool + + # The type's deprecation message (in case it is deprecated) + deprecated: str | None + + # Cached value of class constructor type, i.e. the type of class object when it + # appears in runtime context. + type_object_type: mypy.types.FunctionLike | None + + # Type variables whose defaults depend on defaults of type variables in other classes + # and type aliases. We keep track of this to safely handle situations like this one: + # class C[T = D]: ... + # class D[S = C]: ... + # x: C + # Since we apply fix_instance() eagerly, inferring a precise type is quite tricky. + # Therefore, we infer the type of `x` as `C[D[Any]]` to avoid infinite recursion. + # Keys are type variable full names. + default_depends: dict[str, set[TypeAlias | TypeInfo]] + + # If defn is TypedDictType, stores information needed for delayed validation of inheritance. + typeddict_data: TypedDictData | None + + FLAGS: Final = [ + "is_abstract", + "is_enum", + "fallback_to_any", + "meta_fallback_to_any", + "is_named_tuple", + "is_newtype", + "is_protocol", + "runtime_protocol", + "is_final", + "is_disjoint_base", + "is_intersection", + ] + + def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None: + """Initialize a TypeInfo.""" + super().__init__() + self._fullname = defn.fullname + self.names = names + self.defn = defn + self.module_name = module_name + self.type_vars = [] + self.has_param_spec_type = False + self.has_type_var_tuple_type = False + self.bases = [] + self.mro = [] + self._mro_refs = None + self.bad_mro = False + self.declared_metaclass = None + self.metaclass_type = None + self.is_abstract = False + self.abstract_attributes = [] + self.deletable_attributes = [] + self.slots = None + self.assuming = [] + self.assuming_proper = [] + self.inferring = [] + self.is_protocol = False + self.runtime_protocol = False + self.type_var_tuple_prefix: int | None = None + self.type_var_tuple_suffix: int | None = None + self.add_type_vars() + self.is_final = False + self.is_disjoint_base = False + self.is_enum = False + self.fallback_to_any = False + self.meta_fallback_to_any = False + self._promote = [] + self.alt_promote = None + self.tuple_type = None + self.special_alias = None + self.is_named_tuple = False + self.typeddict_type = None + self.is_newtype = False + self.is_intersection = False + self.metadata = {} + self.self_type = None + self.dataclass_transform_spec = None + self.is_type_check_only = False + self.deprecated = None + self.type_object_type = None + self.default_depends = {} + self.typeddict_data = None + + def add_type_vars(self) -> None: + self.has_type_var_tuple_type = False + if self.defn.type_vars: + for i, vd in enumerate(self.defn.type_vars): + if isinstance(vd, mypy.types.ParamSpecType): + self.has_param_spec_type = True + if isinstance(vd, mypy.types.TypeVarTupleType): + assert not self.has_type_var_tuple_type + self.has_type_var_tuple_type = True + self.type_var_tuple_prefix = i + self.type_var_tuple_suffix = len(self.defn.type_vars) - i - 1 + self.type_vars.append(vd.name) + + @property + def name(self) -> str: + """Short name.""" + return self.defn.name + + @property + def fullname(self) -> str: + return self._fullname + + def is_generic(self) -> bool: + """Is the type generic (i.e. does it have type variables)?""" + return len(self.type_vars) > 0 + + def get(self, name: str) -> SymbolTableNode | None: + for cls in self.mro: + n = cls.names.get(name) + if n: + return n + return None + + def get_containing_type_info(self, name: str) -> TypeInfo | None: + for cls in self.mro: + if name in cls.names: + return cls + return None + + @property + def protocol_members(self) -> list[str]: + # Protocol members are names of all attributes/methods defined in a protocol + # and in all its supertypes (except for 'object'). + members: set[str] = set() + assert self.mro, "This property can be only accessed after MRO is (re-)calculated" + for base in self.mro[:-1]: # we skip "object" since everyone implements it + if base.is_protocol: + for name, node in base.names.items(): + if isinstance(node.node, (TypeAlias, TypeVarExpr, MypyFile)): + # These are auxiliary definitions (and type aliases are prohibited). + continue + if name in EXCLUDED_PROTOCOL_ATTRIBUTES: + continue + members.add(name) + return sorted(members) + + @property + def enum_members(self) -> list[str]: + # TODO: cache the results? + members = [] + for name, sym in self.names.items(): + # Case 1: + # + # class MyEnum(Enum): + # @member + # def some(self): ... + if isinstance(sym.node, Decorator): + if any( + dec.fullname == "enum.member" + for dec in sym.node.decorators + if isinstance(dec, RefExpr) + ): + members.append(name) + continue + # Case 2: + # + # class MyEnum(Enum): + # x = 1 + # + # Case 3: + # + # class MyEnum(Enum): + # class Other: ... + elif isinstance(sym.node, (Var, TypeInfo)): + if ( + # TODO: properly support ignored names from `_ignore_` + name in EXCLUDED_ENUM_ATTRIBUTES + or is_sunder(name) + or name.startswith("__") # dunder and private + ): + continue # name is excluded + + if isinstance(sym.node, Var): + if not sym.node.has_explicit_value: + continue # unannotated value not a member + + typ = mypy.types.get_proper_type(sym.node.type) + if ( + isinstance(typ, mypy.types.FunctionLike) and not typ.is_type_obj() + ) or ( # explicit `@member` is required + isinstance(typ, mypy.types.Instance) + and typ.type.fullname == "enum.nonmember" + ): + continue # name is not a member + + members.append(name) + return members + + def __getitem__(self, name: str) -> SymbolTableNode: + n = self.get(name) + if n: + return n + else: + raise KeyError(name) + + def __repr__(self) -> str: + return f"" + + def __bool__(self) -> bool: + # We defined this here instead of just overriding it in + # FakeInfo so that mypyc can generate a direct call instead of + # using the generic bool handling. + return not isinstance(self, FakeInfo) + + def has_readable_member(self, name: str) -> bool: + return self.get(name) is not None + + def get_method(self, name: str) -> FuncBase | Decorator | None: + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + elif possible_redefinitions := sorted( + [n for n in cls.names.keys() if n.startswith(f"{name}-redefinition")] + ): + node = cls.names[possible_redefinitions[-1]].node + else: + continue + if isinstance(node, SYMBOL_FUNCBASE_TYPES): + return node + elif isinstance(node, Decorator): # Two `if`s make `mypyc` happy + return node + else: + return None + return None + + def calculate_metaclass_type(self) -> mypy.types.Instance | None: + declared = self.declared_metaclass + if declared is not None and not declared.type.has_base("builtins.type"): + return declared + if self._fullname == "builtins.type": + return mypy.types.Instance(self, []) + + winner = declared + for super_class in self.mro[1:]: + super_meta = super_class.declared_metaclass + if super_meta is None or super_meta.type is None: + continue + if winner is None: + winner = super_meta + continue + if winner.type.has_base(super_meta.type.fullname): + continue + if super_meta.type.has_base(winner.type.fullname): + winner = super_meta + continue + # metaclass conflict + winner = None + break + + return winner + + def explain_metaclass_conflict(self) -> str | None: + # Compare to logic in calculate_metaclass_type + declared = self.declared_metaclass + if declared is not None and not declared.type.has_base("builtins.type"): + return None + if self._fullname == "builtins.type": + return None + + winner = declared + if declared is None: + resolution_steps = [] + else: + resolution_steps = [f'"{declared.type.fullname}" (metaclass of "{self.fullname}")'] + for super_class in self.mro[1:]: + super_meta = super_class.declared_metaclass + if super_meta is None or super_meta.type is None: + continue + if winner is None: + winner = super_meta + resolution_steps.append( + f'"{winner.type.fullname}" (metaclass of "{super_class.fullname}")' + ) + continue + if winner.type.has_base(super_meta.type.fullname): + continue + if super_meta.type.has_base(winner.type.fullname): + winner = super_meta + resolution_steps.append( + f'"{winner.type.fullname}" (metaclass of "{super_class.fullname}")' + ) + continue + # metaclass conflict + conflict = f'"{super_meta.type.fullname}" (metaclass of "{super_class.fullname}")' + return f"{' > '.join(resolution_steps)} conflicts with {conflict}" + + return None + + def is_metaclass(self, *, precise: bool = False) -> bool: + return ( + self.has_base("builtins.type") + or self.fullname == "abc.ABCMeta" + or (self.fallback_to_any and not precise) + ) + + def has_base(self, fullname: str) -> bool: + """Return True if type has a base type with the specified name. + + This can be either via extension or via implementation. + """ + for cls in self.mro: + if cls.fullname == fullname: + return True + return False + + def get_base(self, fullname: str) -> TypeInfo: + for cls in self.mro: + if cls.fullname == fullname: + return cls + assert False, f"Missing base {fullname} for {self.fullname}" + + def direct_base_classes(self) -> list[TypeInfo]: + """Return a direct base classes. + + Omit base classes of other base classes. + """ + return [base.type for base in self.bases] + + def update_tuple_type(self, typ: mypy.types.TupleType) -> None: + """Update tuple_type and special_alias as needed.""" + self.tuple_type = typ + alias = TypeAlias.from_tuple_type(self) + if not self.special_alias: + self.special_alias = alias + else: + self.special_alias.target = alias.target + # Invalidate recursive status cache in case it was previously set. + self.special_alias._is_recursive = None + + def update_typeddict_type(self, typ: mypy.types.TypedDictType) -> None: + """Update typeddict_type and special_alias as needed.""" + self.typeddict_type = typ + alias = TypeAlias.from_typeddict_type(self) + if not self.special_alias: + self.special_alias = alias + else: + self.special_alias.target = alias.target + # Invalidate recursive status cache in case it was previously set. + self.special_alias._is_recursive = None + + def __str__(self) -> str: + """Return a string representation of the type. + + This includes the most important information about the type. + """ + options = Options() + return self.dump( + str_conv=mypy.strconv.StrConv(options=options), + type_str_conv=mypy.types.TypeStrVisitor(options=options), + ) + + def dump( + self, str_conv: mypy.strconv.StrConv, type_str_conv: mypy.types.TypeStrVisitor + ) -> str: + """Return a string dump of the contents of the TypeInfo.""" + + base: str = "" + + def type_str(typ: mypy.types.Type) -> str: + return typ.accept(type_str_conv) + + head = "TypeInfo" + str_conv.format_id(self) + if self.bases: + base = f"Bases({', '.join(type_str(base) for base in self.bases)})" + mro = "Mro({})".format( + ", ".join(item.fullname + str_conv.format_id(item) for item in self.mro) + ) + names = [] + for name in sorted(self.names): + description = name + str_conv.format_id(self.names[name].node) + node = self.names[name].node + if isinstance(node, Var) and node.type: + description += f" ({type_str(node.type)})" + names.append(description) + items = [f"Name({self.fullname})", base, mro, ("Names", names)] + if self.declared_metaclass: + items.append(f"DeclaredMetaclass({type_str(self.declared_metaclass)})") + if self.metaclass_type: + items.append(f"MetaclassType({type_str(self.metaclass_type)})") + return mypy.strconv.dump_tagged(items, head, str_conv=str_conv) + + def serialize(self) -> JsonDict: + # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. + data = { + ".class": "TypeInfo", + "module_name": self.module_name, + "fullname": self.fullname, + "names": self.names.serialize(self.fullname), + "defn": self.defn.serialize(), + "abstract_attributes": self.abstract_attributes, + "type_vars": self.type_vars, + "has_param_spec_type": self.has_param_spec_type, + "bases": [b.serialize() for b in self.bases], + "mro": [c.fullname for c in self.mro], + "_promote": [p.serialize() for p in self._promote], + "alt_promote": None if self.alt_promote is None else self.alt_promote.serialize(), + "declared_metaclass": ( + None if self.declared_metaclass is None else self.declared_metaclass.serialize() + ), + "metaclass_type": ( + None if self.metaclass_type is None else self.metaclass_type.serialize() + ), + "tuple_type": None if self.tuple_type is None else self.tuple_type.serialize(), + "typeddict_type": ( + None if self.typeddict_type is None else self.typeddict_type.serialize() + ), + "flags": get_flags(self, TypeInfo.FLAGS), + "metadata": self.metadata, + "slots": sorted(self.slots) if self.slots is not None else None, + "deletable_attributes": self.deletable_attributes, + "self_type": self.self_type.serialize() if self.self_type is not None else None, + "dataclass_transform_spec": ( + self.dataclass_transform_spec.serialize() + if self.dataclass_transform_spec is not None + else None + ), + "deprecated": self.deprecated, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeInfo: + names = SymbolTable.deserialize(data["names"]) + defn = ClassDef.deserialize(data["defn"]) + module_name = data["module_name"] + ti = TypeInfo(names, defn, module_name) + ti._fullname = data["fullname"] + ti.abstract_attributes = [(attr[0], attr[1]) for attr in data["abstract_attributes"]] + ti.type_vars = data["type_vars"] + ti.has_param_spec_type = data["has_param_spec_type"] + ti.bases = [mypy.types.Instance.deserialize(b) for b in data["bases"]] + _promote = [] + for p in data["_promote"]: + t = mypy.types.deserialize_type(p) + assert isinstance(t, mypy.types.ProperType) + _promote.append(t) + ti._promote = _promote + ti.alt_promote = ( + None + if data["alt_promote"] is None + else mypy.types.Instance.deserialize(data["alt_promote"]) + ) + ti.declared_metaclass = ( + None + if data["declared_metaclass"] is None + else mypy.types.Instance.deserialize(data["declared_metaclass"]) + ) + ti.metaclass_type = ( + None + if data["metaclass_type"] is None + else mypy.types.Instance.deserialize(data["metaclass_type"]) + ) + # NOTE: ti.mro will be set in the fixup phase based on these + # names. The reason we need to store the mro instead of just + # recomputing it from base classes has to do with a subtle + # point about fine-grained incremental: the cache files might + # not be loaded until after a class in the mro has changed its + # bases, which causes the mro to change. If we recomputed our + # mro, we would compute the *new* mro, which leaves us with no + # way to detect that the mro has changed! Thus we need to make + # sure to load the original mro so that once the class is + # rechecked, it can tell that the mro has changed. + ti._mro_refs = data["mro"] + ti.tuple_type = ( + None + if data["tuple_type"] is None + else mypy.types.TupleType.deserialize(data["tuple_type"]) + ) + ti.typeddict_type = ( + None + if data["typeddict_type"] is None + else mypy.types.TypedDictType.deserialize(data["typeddict_type"]) + ) + ti.metadata = data["metadata"] + ti.slots = set(data["slots"]) if data["slots"] is not None else None + ti.deletable_attributes = data["deletable_attributes"] + set_flags(ti, data["flags"]) + st = data["self_type"] + ti.self_type = mypy.types.TypeVarType.deserialize(st) if st is not None else None + if data.get("dataclass_transform_spec") is not None: + ti.dataclass_transform_spec = DataclassTransformSpec.deserialize( + data["dataclass_transform_spec"] + ) + ti.deprecated = data.get("deprecated") + return ti + + def write(self, data: WriteBuffer) -> None: + write_tag(data, TYPE_INFO) + self.names.write(data, self.fullname) + self.defn.write(data) + write_str(data, self.module_name) + write_str(data, self.fullname) + write_str_list(data, [a for a, _ in self.abstract_attributes]) + write_int_list(data, [s for _, s in self.abstract_attributes]) + write_str_list(data, self.type_vars) + write_bool(data, self.has_param_spec_type) + mypy.types.write_type_list(data, self.bases) + write_str_list(data, [c.fullname for c in self.mro]) + mypy.types.write_type_list(data, self._promote) + mypy.types.write_type_opt(data, self.alt_promote) + mypy.types.write_type_opt(data, self.declared_metaclass) + mypy.types.write_type_opt(data, self.metaclass_type) + mypy.types.write_type_opt(data, self.tuple_type) + mypy.types.write_type_opt(data, self.typeddict_type) + write_flags( + data, + [ + self.is_abstract, + self.is_enum, + self.fallback_to_any, + self.meta_fallback_to_any, + self.is_named_tuple, + self.is_newtype, + self.is_protocol, + self.runtime_protocol, + self.is_final, + self.is_disjoint_base, + self.is_intersection, + ], + ) + write_json(data, self.metadata) + if self.slots is None: + write_tag(data, LITERAL_NONE) + else: + write_str_list(data, sorted(self.slots)) + write_str_list(data, self.deletable_attributes) + mypy.types.write_type_opt(data, self.self_type) + if self.dataclass_transform_spec is None: + write_tag(data, LITERAL_NONE) + else: + self.dataclass_transform_spec.write(data) + write_str_opt(data, self.deprecated) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> TypeInfo: + names = SymbolTable.read(data) + assert read_tag(data) == CLASS_DEF + defn = ClassDef.read(data) + module_name = read_str(data) + ti = TypeInfo(names, defn, module_name) + ti._fullname = read_str(data) + attrs = read_str_list(data) + statuses = read_int_list(data) + ti.abstract_attributes = list(zip(attrs, statuses)) + ti.type_vars = read_str_list(data) + ti.has_param_spec_type = read_bool(data) + ti.bases = [] + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): + assert read_tag(data) == mypy.types.INSTANCE + ti.bases.append(mypy.types.Instance.read(data)) + # NOTE: ti.mro will be set in the fixup phase based on these + # names. The reason we need to store the mro instead of just + # recomputing it from base classes has to do with a subtle + # point about fine-grained incremental: the cache files might + # not be loaded until after a class in the mro has changed its + # bases, which causes the mro to change. If we recomputed our + # mro, we would compute the *new* mro, which leaves us with no + # way to detect that the mro has changed! Thus, we need to make + # sure to load the original mro so that once the class is + # rechecked, it can tell that the mro has changed. + ti._mro_refs = read_str_list(data) + ti._promote = cast(list[mypy.types.ProperType], mypy.types.read_type_list(data)) + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE + ti.alt_promote = mypy.types.Instance.read(data) + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE + ti.declared_metaclass = mypy.types.Instance.read(data) + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE + ti.metaclass_type = mypy.types.Instance.read(data) + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TUPLE_TYPE + ti.tuple_type = mypy.types.TupleType.read(data) + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPED_DICT_TYPE + ti.typeddict_type = mypy.types.TypedDictType.read(data) + ( + ti.is_abstract, + ti.is_enum, + ti.fallback_to_any, + ti.meta_fallback_to_any, + ti.is_named_tuple, + ti.is_newtype, + ti.is_protocol, + ti.runtime_protocol, + ti.is_final, + ti.is_disjoint_base, + ti.is_intersection, + ) = read_flags(data, num_flags=11) + ti.metadata = read_json(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LIST_STR + ti.slots = {read_str_bare(data) for _ in range(read_int_bare(data))} + ti.deletable_attributes = read_str_list(data) + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPE_VAR_TYPE + ti.self_type = mypy.types.TypeVarType.read(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC + ti.dataclass_transform_spec = DataclassTransformSpec.read(data) + ti.deprecated = read_str_opt(data) + assert read_tag(data) == END_TAG + return ti + + +class FakeInfo(TypeInfo): + __slots__ = ("msg",) + + # types.py defines a single instance of this class, called types.NOT_READY. + # This instance is used as a temporary placeholder in the process of de-serialization + # of 'Instance' types. The de-serialization happens in two steps: In the first step, + # Instance.type is set to NOT_READY. In the second step (in fixup.py) it is replaced by + # an actual TypeInfo. If you see the assertion error below, then most probably something + # went wrong during the second step and an 'Instance' that raised this error was not fixed. + # Note: + # 'None' is not used as a dummy value for two reasons: + # 1. This will require around 80-100 asserts to make 'mypy --strict-optional mypy' + # pass cleanly. + # 2. If NOT_READY value is accidentally used somewhere, it will be obvious where the value + # is from, whereas a 'None' value could come from anywhere. + # + # Additionally, this serves as a more general-purpose placeholder + # for missing TypeInfos in a number of places where the excuses + # for not being Optional are a little weaker. + # + # TypeInfo defines a __bool__ method that returns False for FakeInfo + # so that it can be conveniently tested against in the same way that it + # would be if things were properly optional. + def __init__(self, msg: str) -> None: + self.msg = msg + + def __getattribute__(self, attr: str) -> type: + # Handle __class__ so that isinstance still works... + if attr == "__class__": + return object.__getattribute__(self, attr) # type: ignore[no-any-return] + raise AssertionError(object.__getattribute__(self, "msg")) + + +VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info") +CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info") +FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info") +MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") + + +class TypeAlias(SymbolNode): + """ + A symbol node representing a type alias. + + Type alias is a static concept, in contrast to variables with types + like Type[...]. Namely: + * type aliases + - can be used in type context (annotations) + - cannot be re-assigned + * variables with type Type[...] + - cannot be used in type context + - but can be re-assigned + + An alias can be defined only by an assignment to a name (not any other lvalues). + + Such assignment defines an alias by default. To define a variable, + an explicit Type[...] annotation is required. As an exception, + at non-global scope non-subscripted rvalue creates a variable even without + an annotation. This exception exists to accommodate the common use case of + class-valued attributes. See SemanticAnalyzerPass2.check_and_set_up_type_alias + for details. + + Aliases can be generic. We use bound type variables for generic aliases, similar + to classes. Essentially, type aliases work as macros that expand textually. + The definition and expansion rules are following: + + 1. An alias targeting a generic class without explicit variables act as + the given class (this doesn't apply to TypedDict, Tuple and Callable, which + are not proper classes but special type constructors): + + A = List + AA = List[Any] + + x: A # same as List[Any] + x: A[int] # same as List[int] + + x: AA # same as List[Any] + x: AA[int] # Error! + + C = Callable # Same as Callable[..., Any] + T = Tuple # Same as Tuple[Any, ...] + + 2. An alias using explicit type variables in its rvalue expects + replacements (type arguments) for these variables. If missing, they + are treated as Any, like for other generics: + + B = List[Tuple[T, T]] + + x: B # same as List[Tuple[Any, Any]] + x: B[int] # same as List[Tuple[int, int]] + + def f(x: B[T]) -> T: ... # without T, Any would be used here + + 3. An alias can be defined using another aliases. In the definition + rvalue the Any substitution doesn't happen for top level unsubscripted + generic classes: + + A = List + B = A # here A is expanded to List, _not_ List[Any], + # to match the Python runtime behaviour + x: B[int] # same as List[int] + C = List[A] # this expands to List[List[Any]] + + AA = List[T] + D = AA # here AA expands to List[Any] + x: D[int] # Error! + + Note: the fact that we support aliases like `A = List` means that the target + type will be initially an instance type with wrong number of type arguments. + Such instances are all fixed either during or after main semantic analysis passes. + We therefore store the difference between `List` and `List[Any]` rvalues (targets) + using the `no_args` flag. + + Meaning of other fields: + + target: The target type. For generic aliases contains bound type variables + as nested types (currently TypeVar and ParamSpec are supported). + _fullname: Qualified name of this type alias. This is used in particular + to track fine-grained dependencies from aliases. + module: Module where the alias was defined. + alias_tvars: Type variables used to define this alias. + normalized: Used to distinguish between `A = List`, and `A = list`. Both + are internally stored using `builtins.list` (because `typing.List` is + itself an alias), while the second cannot be subscripted because of + Python runtime limitation. + line and column: Line and column on the original alias definition. + eager: If True, immediately expand alias when referred to (useful for aliases + within functions that can't be looked up from the symbol table) + """ + + __slots__ = ( + "target", + "_fullname", + "module", + "alias_tvars", + "no_args", + "normalized", + "_is_recursive", + "eager", + "tvar_tuple_index", + "python_3_12_type_alias", + "default_depends", + ) + + __match_args__ = ("name", "target", "alias_tvars", "no_args") + + def __init__( + self, + target: mypy.types.Type, + fullname: str, + module: str, + line: int, + column: int, + *, + alias_tvars: list[mypy.types.TypeVarLikeType] | None = None, + no_args: bool = False, + normalized: bool = False, + eager: bool = False, + python_3_12_type_alias: bool = False, + ) -> None: + self._fullname = fullname + self.module = module + self.target = target + if alias_tvars is None: + alias_tvars = [] + self.alias_tvars = alias_tvars + self.no_args = no_args + self.normalized = normalized + # This attribute is manipulated by TypeAliasType. If non-None, + # it is the cached value. + self._is_recursive: bool | None = None + self.eager = eager + self.python_3_12_type_alias = python_3_12_type_alias + self.tvar_tuple_index = None + # This plays the same role as TypeInfo.default_depends attribute. + self.default_depends: dict[str, set[TypeAlias | TypeInfo]] = {} + for i, t in enumerate(alias_tvars): + if isinstance(t, mypy.types.TypeVarTupleType): + self.tvar_tuple_index = i + super().__init__(line, column) + + @classmethod + def from_tuple_type(cls, info: TypeInfo) -> TypeAlias: + """Generate an alias to the tuple type described by a given TypeInfo. + + NOTE: this doesn't set type alias type variables (for generic tuple types), + they must be set by the caller (when fully analyzed). + """ + assert info.tuple_type + # TODO: is it possible to refactor this to set the correct type vars here? + return TypeAlias( + info.tuple_type.copy_modified( + # Create an Instance similar to fill_typevars(). + fallback=mypy.types.Instance( + info, mypy.types.type_vars_as_args(info.defn.type_vars) + ) + ), + info.fullname, + info.module_name, + info.line, + info.column, + ) + + @classmethod + def from_typeddict_type(cls, info: TypeInfo) -> TypeAlias: + """Generate an alias to the TypedDict type described by a given TypeInfo. + + NOTE: this doesn't set type alias type variables (for generic TypedDicts), + they must be set by the caller (when fully analyzed). + """ + assert info.typeddict_type + # TODO: is it possible to refactor this to set the correct type vars here? + return TypeAlias( + info.typeddict_type.copy_modified( + # Create an Instance similar to fill_typevars(). + fallback=mypy.types.Instance( + info, mypy.types.type_vars_as_args(info.defn.type_vars) + ) + ), + info.fullname, + info.module_name, + info.line, + info.column, + ) + + @property + def name(self) -> str: + return self._fullname.split(".")[-1] + + @property + def fullname(self) -> str: + return self._fullname + + @property + def has_param_spec_type(self) -> bool: + return any(isinstance(v, mypy.types.ParamSpecType) for v in self.alias_tvars) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias(self) + + def serialize(self) -> JsonDict: + data: JsonDict = { + ".class": "TypeAlias", + "fullname": self._fullname, + "module": self.module, + "target": self.target.serialize(), + "alias_tvars": [v.serialize() for v in self.alias_tvars], + "no_args": self.no_args, + "normalized": self.normalized, + "python_3_12_type_alias": self.python_3_12_type_alias, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> TypeAlias: + assert data[".class"] == "TypeAlias" + fullname = data["fullname"] + module = data["module"] + alias_tvars = [mypy.types.deserialize_type(v) for v in data["alias_tvars"]] + assert all(isinstance(t, mypy.types.TypeVarLikeType) for t in alias_tvars) + target = mypy.types.deserialize_type(data["target"]) + no_args = data["no_args"] + normalized = data["normalized"] + python_3_12_type_alias = data["python_3_12_type_alias"] + return cls( + target, + fullname, + module, + -1, + -1, + alias_tvars=cast(list[mypy.types.TypeVarLikeType], alias_tvars), + no_args=no_args, + normalized=normalized, + python_3_12_type_alias=python_3_12_type_alias, + ) + + def write(self, data: WriteBuffer) -> None: + write_tag(data, TYPE_ALIAS) + write_str(data, self._fullname) + write_str(data, self.module) + self.target.write(data) + mypy.types.write_type_list(data, self.alias_tvars) + write_bool(data, self.no_args) + write_bool(data, self.normalized) + write_bool(data, self.python_3_12_type_alias) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> TypeAlias: + fullname = read_str(data) + module = read_str(data) + target = mypy.types.read_type(data) + alias_tvars = mypy.types.read_type_var_likes(data) + ret = TypeAlias( + target, + fullname, + module, + -1, + -1, + alias_tvars=alias_tvars, + no_args=read_bool(data), + normalized=read_bool(data), + python_3_12_type_alias=read_bool(data), + ) + assert read_tag(data) == END_TAG + return ret + + +class PlaceholderNode(SymbolNode): + """Temporary symbol node that will later become a real SymbolNode. + + These are only present during semantic analysis when using the new + semantic analyzer. These are created if some essential dependencies + of a definition are not yet complete. + + A typical use is for names imported from a module which is still + incomplete (within an import cycle): + + from m import f # Initially may create PlaceholderNode + + This is particularly important if the imported shadows a name from + an enclosing scope or builtins: + + from m import int # Placeholder avoids mixups with builtins.int + + Another case where this is useful is when there is another definition + or assignment: + + from m import f + def f() -> None: ... + + In the above example, the presence of PlaceholderNode allows us to + handle the second definition as a redefinition. + + They are also used to create PlaceholderType instances for types + that refer to incomplete types. Example: + + class C(Sequence[C]): ... + + We create a PlaceholderNode (with becomes_typeinfo=True) for C so + that the type C in Sequence[C] can be bound. + + Attributes: + + fullname: Full name of the PlaceholderNode. + node: AST node that contains the definition that caused this to + be created. This is useful for tracking order of incomplete definitions + and for debugging. + becomes_typeinfo: If True, this refers something that could later + become a TypeInfo. It can't be used with type variables, in + particular, as this would cause issues with class type variable + detection. + + The long-term purpose of placeholder nodes/types is to evolve into + something that can support general recursive types. + """ + + __slots__ = ("_fullname", "node", "becomes_typeinfo") + + def __init__( + self, fullname: str, node: Node, line: int, *, becomes_typeinfo: bool = False + ) -> None: + self._fullname = fullname + self.node = node + self.becomes_typeinfo = becomes_typeinfo + self.line = line + + @property + def name(self) -> str: + return self._fullname.split(".")[-1] + + @property + def fullname(self) -> str: + return self._fullname + + def serialize(self) -> JsonDict: + assert False, "PlaceholderNode can't be serialized" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_placeholder_node(self) + + +class SymbolTableNode: + """Description of a name binding in a symbol table. + + These are only used as values in module (global), function (local) + and class symbol tables (see SymbolTable). The name that is bound is + the key in SymbolTable. + + Symbol tables don't contain direct references to AST nodes primarily + because there can be multiple symbol table references to a single + AST node (due to imports and aliases), and different references can + behave differently. This class describes the unique properties of + each reference. + + The most fundamental attribute is 'node', which is the AST node that + the name refers to. + + The kind is usually one of LDEF, GDEF or MDEF, depending on the scope + of the definition. These three kinds can usually be used + interchangeably and the difference between local, global and class + scopes is mostly descriptive, with no semantic significance. + However, some tools that consume mypy ASTs may care about these so + they should be correct. + + Attributes: + _node: AST node of definition. Among others, this can be one of + FuncDef, Var, TypeInfo, TypeVarExpr or MypyFile -- or None + for cross_ref that hasn't been fixed up yet. Should not be accessed + directly, only via the `node` property. + kind: Kind of node. Possible values: + - LDEF: local definition + - GDEF: global (module-level) definition + - MDEF: class member definition + - UNBOUND_IMPORTED: temporary kind for imported names (we + don't know the final kind yet) + module_public: If False, this name won't be imported via + 'from import *'. This has no effect on names within + classes. + module_hidden: If True, the name will never be exported (needed for + stub files) + cross_ref: For deserialized MypyFile nodes, the referenced module + name; for other nodes, optionally the name of the referenced object. + implicit: Was this defined by assignment to self attribute? + plugin_generated: Was this symbol generated by a plugin? + (And therefore needs to be removed in aststrip.) + no_serialize: Do not serialize this node if True. This is used for internal + and/or temporary symbols such as function redefinitions. + unfixed: Indicates that this symbol is fresh after deserialization and + needs fixup, such as resolving cross-references etc. + stored_info: TypeInfo containing this symbol. Normally code accesses this + on the `node` attribute, but it may be not ready during deserialization, + so we temporarily store info on the symbol itself. + + NOTE: No other attributes should be added to this class unless they + are shared by all node kinds. + """ + + __slots__ = ( + "kind", + "_node", + "_node_bytes", + "_node_tag", + "module_public", + "module_hidden", + "cross_ref", + "implicit", + "plugin_generated", + "no_serialize", + "unfixed", + "stored_info", + ) + + def __init__( + self, + kind: int, + node: SymbolNode | None, + module_public: bool = True, + implicit: bool = False, + module_hidden: bool = False, + *, + plugin_generated: bool = False, + no_serialize: bool = False, + ) -> None: + self.kind = kind + self._node = node + self._node_bytes = b"" + self._node_tag: Tag = 0 + self.module_public = module_public + self.implicit = implicit + self.module_hidden = module_hidden + self.cross_ref: str | None = None + self.plugin_generated = plugin_generated + self.no_serialize = no_serialize + self.unfixed = False + self.stored_info: TypeInfo | None = None + + @property + def fullname(self) -> str | None: + if self.node is not None: + return self.node.fullname + else: + return None + + @property + def type(self) -> mypy.types.Type | None: + node = self.node + if isinstance(node, (Var, SYMBOL_FUNCBASE_TYPES)) and node.type is not None: + return node.type + elif isinstance(node, Decorator): + return node.var.type + else: + return None + + @property + def node(self) -> SymbolNode | None: + if self.unfixed: + node_fixer = modules_state.node_fixer + assert node_fixer is not None + if self.cross_ref is not None: + node_fixer.resolve_cross_ref(self) + else: + if self._node is None: + self._node = read_symbol(ReadBuffer(self._node_bytes), self._node_tag) + self._node_bytes = b"" + node = self._node + assert node is not None + if self.stored_info is not None: + set_info(node, self.stored_info) + self.stored_info = None + node.accept(node_fixer) + self.unfixed = False + return self._node + + def read_node_no_fixup(self) -> SymbolNode | None: + """Return the deserialized node without performing cross-reference fixup. + + This is intended for introspection tools (such as mypy.exportjson) that read + cache files in isolation, where no node fixer is available. + """ + if self._node is None and self._node_bytes: + self._node = read_symbol(ReadBuffer(self._node_bytes), self._node_tag) + self._node_bytes = b"" + return self._node + + def copy(self) -> SymbolTableNode: + new = SymbolTableNode( + self.kind, self._node, self.module_public, self.implicit, self.module_hidden + ) + new.cross_ref = self.cross_ref + new.unfixed = self.unfixed + new.stored_info = self.stored_info + return new + + def __str__(self) -> str: + s = f"{node_kinds[self.kind]}/{short_type(self.node)}" + if isinstance(self.node, SymbolNode): + s += f" ({self.node.fullname})" + # Include declared type of variables and functions. + if self.type is not None: + s += f" : {self.type}" + if self.cross_ref: + s += f" cross_ref:{self.cross_ref}" + return s + + def serialize(self, prefix: str, name: str) -> JsonDict: + """Serialize a SymbolTableNode. + + Args: + prefix: full name of the containing module or class; or None + name: name of this object relative to the containing object + """ + data: JsonDict = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} + if self.module_hidden: + data["module_hidden"] = True + if not self.module_public: + data["module_public"] = False + if self.implicit: + data["implicit"] = True + if self.plugin_generated: + data["plugin_generated"] = True + if isinstance(self.node, MypyFile): + data["cross_ref"] = self.node.fullname + else: + assert self.node is not None, f"{prefix}:{name}" + if prefix is not None: + fullname = self.node.fullname + if ( + "." in fullname + and fullname != prefix + "." + name + and not (isinstance(self.node, Var) and self.node.from_module_getattr) + ): + assert not isinstance( + self.node, PlaceholderNode + ), f"Definition of {fullname} is unexpectedly incomplete" + data["cross_ref"] = fullname + return data + data["node"] = self.node.serialize() + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> SymbolTableNode: + assert data[".class"] == "SymbolTableNode" + kind = inverse_node_kinds[data["kind"]] + if "cross_ref" in data: + # This will be fixed up later. + stnode = SymbolTableNode(kind, None) + stnode.cross_ref = data["cross_ref"] + stnode.unfixed = True + else: + assert "node" in data, data + node = SymbolNode.deserialize(data["node"]) + stnode = SymbolTableNode(kind, node) + if not isinstance(node, TypeInfo): + stnode.unfixed = True + if "module_hidden" in data: + stnode.module_hidden = data["module_hidden"] + if "module_public" in data: + stnode.module_public = data["module_public"] + if "implicit" in data: + stnode.implicit = data["implicit"] + if "plugin_generated" in data: + stnode.plugin_generated = data["plugin_generated"] + return stnode + + def write(self, data: WriteBuffer, prefix: str, name: str) -> None: + write_tag(data, SYMBOL_TABLE_NODE) + write_int(data, self.kind) + write_bool(data, self.module_hidden) + write_bool(data, self.module_public) + write_bool(data, self.implicit) + write_bool(data, self.plugin_generated) + + cross_ref = None + if isinstance(self.node, MypyFile): + cross_ref = self.node.fullname + else: + assert self.node is not None, f"{prefix}:{name}" + if prefix is not None: + fullname = self.node.fullname + if ( + "." in fullname + and fullname != prefix + "." + name + and not (isinstance(self.node, Var) and self.node.from_module_getattr) + ): + assert not isinstance( + self.node, PlaceholderNode + ), f"Definition of {fullname} is unexpectedly incomplete" + cross_ref = fullname + + write_str_opt(data, cross_ref) + if cross_ref is None: + assert self.node is not None + self.node.write(data) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> SymbolTableNode: + assert read_tag(data) == SYMBOL_TABLE_NODE + sym = SymbolTableNode(read_int(data), None) + sym.module_hidden = read_bool(data) + sym.module_public = read_bool(data) + sym.implicit = read_bool(data) + sym.plugin_generated = read_bool(data) + cross_ref = read_str_opt(data) + if cross_ref is None: + tag = read_tag(data) + if tag == TYPE_INFO: + sym._node = TypeInfo.read(data) + else: + sym._node_bytes = extract_symbol(data) + sym._node_tag = tag + sym.unfixed = True + else: + sym.cross_ref = cross_ref + sym.unfixed = True + assert read_tag(data) == END_TAG + return sym + + +class SymbolTable(dict[str, SymbolTableNode]): + """Static representation of a namespace dictionary. + + This is used for module, class and function namespaces. + """ + + __slots__ = () + + def __str__(self) -> str: + a: list[str] = [] + for key, value in self.items(): + # Filter out the implicit import of builtins. + if isinstance(value, SymbolTableNode): + if ( + value.fullname != "builtins" + and (value.fullname or "").split(".")[-1] not in implicit_module_attrs + ): + a.append(" " + str(key) + " : " + str(value)) + else: + # Used in debugging: + a.append(" ") # type: ignore[unreachable] + a = sorted(a) + a.insert(0, "SymbolTable(") + a[-1] += ")" + return "\n".join(a) + + def copy(self) -> SymbolTable: + return SymbolTable([(key, node.copy()) for key, node in self.items()]) + + def serialize(self, fullname: str) -> JsonDict: + data: JsonDict = {".class": "SymbolTable"} + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + data[key] = value.serialize(fullname, key) + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> SymbolTable: + assert data[".class"] == "SymbolTable" + st = SymbolTable() + for key, value in data.items(): + if key != ".class": + st[key] = SymbolTableNode.deserialize(value) + return st + + def write(self, data: WriteBuffer, fullname: str) -> None: + size = 0 + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + size += 1 + # We intentionally tag SymbolTable as a simple dictionary str -> SymbolTableNode. + write_tag(data, DICT_STR_GEN) + write_int_bare(data, size) + for key in sorted(self): + value = self[key] + if key == "__builtins__" or value.no_serialize: + continue + write_str_bare(data, key) + value.write(data, fullname, key) + + @classmethod + def read(cls, data: ReadBuffer) -> SymbolTable: + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return SymbolTable( + [(read_str_bare(data), SymbolTableNode.read(data)) for _ in range(size)] + ) + + +class DataclassTransformSpec: + """Specifies how a dataclass-like transform should be applied. The fields here are based on the + parameters accepted by `typing.dataclass_transform`.""" + + __slots__ = ( + "eq_default", + "order_default", + "kw_only_default", + "frozen_default", + "field_specifiers", + ) + + def __init__( + self, + *, + eq_default: bool | None = None, + order_default: bool | None = None, + kw_only_default: bool | None = None, + field_specifiers: tuple[str, ...] | None = None, + # Specified outside of PEP 681: + # frozen_default was added to CPythonin https://github.com/python/cpython/pull/99958 citing + # positive discussion in typing-sig + frozen_default: bool | None = None, + ) -> None: + self.eq_default = eq_default if eq_default is not None else True + self.order_default = order_default if order_default is not None else False + self.kw_only_default = kw_only_default if kw_only_default is not None else False + self.frozen_default = frozen_default if frozen_default is not None else False + self.field_specifiers = field_specifiers if field_specifiers is not None else () + + def serialize(self) -> JsonDict: + return { + "eq_default": self.eq_default, + "order_default": self.order_default, + "kw_only_default": self.kw_only_default, + "frozen_default": self.frozen_default, + "field_specifiers": list(self.field_specifiers), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: + return DataclassTransformSpec( + eq_default=data.get("eq_default"), + order_default=data.get("order_default"), + kw_only_default=data.get("kw_only_default"), + frozen_default=data.get("frozen_default"), + field_specifiers=tuple(data.get("field_specifiers", [])), + ) + + def write(self, data: WriteBuffer) -> None: + write_tag(data, DT_SPEC) + write_bool(data, self.eq_default) + write_bool(data, self.order_default) + write_bool(data, self.kw_only_default) + write_bool(data, self.frozen_default) + write_str_list(data, self.field_specifiers) + write_tag(data, END_TAG) + + @classmethod + def read(cls, data: ReadBuffer) -> DataclassTransformSpec: + ret = DataclassTransformSpec( + eq_default=read_bool(data), + order_default=read_bool(data), + kw_only_default=read_bool(data), + frozen_default=read_bool(data), + field_specifiers=tuple(read_str_list(data)), + ) + assert read_tag(data) == END_TAG + return ret + + +class TypedDictFieldSource: + """Source of a TypedDict field definition, used for forming error messages. + + May be defined directly on the type, or on a base class. + """ + + __slots__ = ("base", "ctx") + + base: TypeInfo | None + ctx: Context + + def __init__(self, base: TypeInfo | None, ctx: Context) -> None: + self.base = base + self.ctx = ctx + + +class TypedDictData: + """Stores information needed for delayed validation of TypedDict inheritance.""" + + __slots__ = ("ready", "bases", "field_sources") + + # If False, the type definition referenced a placeholder + ready: bool + + bases: list[tuple[TypeInfo, dict[str, mypy.types.Type]]] + field_sources: dict[str, TypedDictFieldSource] + + def __init__( + self, + ready: bool, + bases: list[tuple[TypeInfo, dict[str, mypy.types.Type]]], + field_sources: dict[str, TypedDictFieldSource], + ) -> None: + self.ready = ready + self.bases = bases + self.field_sources = field_sources + + +@trait +class SplittingVisitor: + # If True, process function definitions. If False, don't. This is used + # for processing module top levels in fine-grained incremental mode. + recurse_into_functions: bool + + @contextmanager + def set_recurse_into_functions(self) -> Iterator[None]: + """Temporarily set recurse_into_functions to True. + + This is used to process top-level functions/methods as a whole. + """ + old_recurse_into_functions = self.recurse_into_functions + self.recurse_into_functions = True + try: + yield + finally: + self.recurse_into_functions = old_recurse_into_functions + + +def get_flags(node: Node, names: list[str]) -> list[str]: + return [name for name in names if getattr(node, name)] + + +def set_flags(node: Node, flags: list[str]) -> None: + for name in flags: + setattr(node, name, True) + + +def get_member_expr_fullname(expr: MemberExpr) -> str | None: + """Return the qualified name representation of a member expression. + + Return a string of form foo.bar, foo.bar.baz, or similar, or None if the + argument cannot be represented in this form. + """ + initial: str | None = None + if isinstance(expr.expr, NameExpr): + initial = expr.expr.name + elif isinstance(expr.expr, MemberExpr): + initial = get_member_expr_fullname(expr.expr) + if initial is None: + return None + return f"{initial}.{expr.name}" + + +deserialize_map: Final = { + key: obj.deserialize + for key, obj in globals().items() + if type(obj) is not FakeInfo + and isinstance(obj, type) + and issubclass(obj, SymbolNode) + and obj is not SymbolNode +} + + +def check_arg_kinds( + arg_kinds: list[ArgKind], nodes: list[T], fail: Callable[[str, T], None] +) -> None: + is_var_arg = False + is_kw_arg = False + seen_named = False + seen_opt = False + for kind, node in zip(arg_kinds, nodes): + if kind == ARG_POS: + if is_var_arg or is_kw_arg or seen_named or seen_opt: + fail( + "Required positional args may not appear after default, named or var args", + node, + ) + break + elif kind == ARG_OPT: + if is_var_arg or is_kw_arg or seen_named: + fail("Positional default args may not appear after named or var args", node) + break + seen_opt = True + elif kind == ARG_STAR: + if is_var_arg or is_kw_arg or seen_named: + fail("Var args may not appear after named or var args", node) + break + is_var_arg = True + elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: + seen_named = True + if is_kw_arg: + fail("A **kwargs argument must be the last argument", node) + break + elif kind == ARG_STAR2: + if is_kw_arg: + fail("You may only have one **kwargs argument", node) + break + is_kw_arg = True + + +def check_param_names( + names: Sequence[str | None], + nodes: list[T], + fail: Callable[[str, T], None], + description: str = "function definition", +) -> None: + seen_names: set[str | None] = set() + for name, node in zip(names, nodes): + if name is not None and name in seen_names: + fail(f'Duplicate parameter "{name}" in {description}', node) + break + seen_names.add(name) + + +def is_class_var(expr: NameExpr) -> bool: + """Return whether the expression is ClassVar[...]""" + if isinstance(expr.node, Var): + return expr.node.is_classvar + return False + + +def is_final_node(node: SymbolNode | None) -> bool: + """Check whether `node` corresponds to a final attribute.""" + return isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)) and node.is_final + + +def get_func_def(typ: mypy.types.CallableType) -> SymbolNode | None: + definition = typ.definition + if isinstance(definition, Decorator): + definition = definition.func + return definition + + +def local_definitions( + names: SymbolTable, name_prefix: str, info: TypeInfo | None = None, impl_only: bool = False +) -> Iterator[Definition]: + """Iterate over local definitions (not imported) in a symbol table. + + Recursively iterate over class members and nested classes. If impl_only is True, do + not yield the classes themselves, only methods. + """ + # TODO: What should the name be? Or maybe remove it? + for name, symnode in names.items(): + shortname = name + if "-redef" in name: + # Restore original name from mangled name of multiply defined function + shortname = name.split("-redef")[0] + fullname = name_prefix + "." + shortname + node = symnode.node + if node and node.fullname == fullname: + yield_node = True + if impl_only: + if not isinstance(node, (FuncDef, OverloadedFuncDef, Decorator)): + yield_node = False + else: + impl = node.func if isinstance(node, Decorator) else node + yield_node = not impl.def_or_infer_vars + # This logic for plugin generated nodes preserves historical behavior. + # On one hand, we generally type-check plugin generated nodes, since + # some 3rd party plugins may rely on this behavior. On the other hand + # we skip nodes generated by mypy itself, these nodes are not added to + # the class AST (only to symbol table) as they often cut corners. + if symnode.plugin_generated and info and node not in info.defn.defs.body: + yield_node = False + if isinstance(node, (FuncDef, OverloadedFuncDef, Decorator)) and "@" in fullname: + yield_node = False + if yield_node: + yield fullname, symnode, info + if isinstance(node, TypeInfo): + yield from local_definitions(node.names, fullname, node, impl_only) + + +def set_info(node: SymbolNode, info: TypeInfo) -> None: + """Add `info` attribute to all relevant components of the node.""" + if isinstance(node, (FuncDef, Var)): + node.info = info + elif isinstance(node, Decorator): + node.var.info = info + node.func.info = info + elif isinstance(node, OverloadedFuncDef): + node.info = info + for item in node.items: + set_info(item, info) + if node.impl: + set_info(node.impl, info) + + +def func_scoped_name(name: str, line: int) -> str: + """Mangled name to use when storing function-scoped symbols in global symbol tables.""" + return f"{name}@{line}" + + +def inline_base(name: str, index: int) -> str: + """Synthetic name to use when storing inlined base classes in symbol tables.""" + return f"{name}@base{index + 1}" + + +# See docstring for mypy/cache.py for reserved tag ranges. +MYPY_FILE: Final[Tag] = 50 +OVERLOADED_FUNC_DEF: Final[Tag] = 51 +FUNC_DEF: Final[Tag] = 52 +DECORATOR: Final[Tag] = 53 +VAR: Final[Tag] = 54 +TYPE_VAR_EXPR: Final[Tag] = 55 +PARAM_SPEC_EXPR: Final[Tag] = 56 +TYPE_VAR_TUPLE_EXPR: Final[Tag] = 57 +TYPE_INFO: Final[Tag] = 58 +TYPE_ALIAS: Final[Tag] = 59 +CLASS_DEF: Final[Tag] = 60 +SYMBOL_TABLE_NODE: Final[Tag] = 61 + +# Tags 160+ are shared with the ast_serialize Rust extension and must be kept in sync. +EXPR_STMT: Final[Tag] = 160 +CALL_EXPR: Final[Tag] = 161 +NAME_EXPR: Final[Tag] = 162 +STR_EXPR: Final[Tag] = 163 +IMPORT: Final[Tag] = 164 +MEMBER_EXPR: Final[Tag] = 165 +OP_EXPR: Final[Tag] = 166 +INT_EXPR: Final[Tag] = 167 +IF_STMT: Final[Tag] = 168 +ASSIGNMENT_STMT: Final[Tag] = 169 +TUPLE_EXPR: Final[Tag] = 170 +BLOCK: Final[Tag] = 171 +INDEX_EXPR: Final[Tag] = 172 +LIST_EXPR: Final[Tag] = 173 +SET_EXPR: Final[Tag] = 174 +RETURN_STMT: Final[Tag] = 175 +WHILE_STMT: Final[Tag] = 176 +COMPARISON_EXPR: Final[Tag] = 177 +BOOL_OP_EXPR: Final[Tag] = 178 +FUNC_DEF_STMT: Final[Tag] = 179 +PASS_STMT: Final[Tag] = 180 +FLOAT_EXPR: Final[Tag] = 181 +UNARY_EXPR: Final[Tag] = 182 +DICT_EXPR: Final[Tag] = 183 +COMPLEX_EXPR: Final[Tag] = 184 +SLICE_EXPR: Final[Tag] = 185 +TEMP_NODE: Final[Tag] = 186 +RAISE_STMT: Final[Tag] = 187 +BREAK_STMT: Final[Tag] = 188 +CONTINUE_STMT: Final[Tag] = 189 +GENERATOR_EXPR: Final[Tag] = 190 +YIELD_EXPR: Final[Tag] = 191 +YIELD_FROM_EXPR: Final[Tag] = 192 +LIST_COMPREHENSION: Final[Tag] = 193 +SET_COMPREHENSION: Final[Tag] = 194 +DICT_COMPREHENSION: Final[Tag] = 195 +IMPORT_FROM: Final[Tag] = 196 +ASSERT_STMT: Final[Tag] = 197 +FOR_STMT: Final[Tag] = 198 +WITH_STMT: Final[Tag] = 199 +OPERATOR_ASSIGNMENT_STMT: Final[Tag] = 200 +TRY_STMT: Final[Tag] = 201 +ELLIPSIS_EXPR: Final[Tag] = 202 +CONDITIONAL_EXPR: Final[Tag] = 203 +DEL_STMT: Final[Tag] = 204 +FSTRING_EXPR: Final[Tag] = 205 +FSTRING_INTERPOLATION: Final[Tag] = 206 +LAMBDA_EXPR: Final[Tag] = 207 +ASSIGNMENT_EXPR: Final[Tag] = 208 +STAR_EXPR: Final[Tag] = 209 +BYTES_EXPR: Final[Tag] = 210 +GLOBAL_DECL: Final[Tag] = 211 +NONLOCAL_DECL: Final[Tag] = 212 +AWAIT_EXPR: Final[Tag] = 213 +BIG_INT_EXPR: Final[Tag] = 214 +IMPORT_ALL: Final[Tag] = 215 +MATCH_STMT: Final[Tag] = 216 +AS_PATTERN: Final[Tag] = 217 +OR_PATTERN: Final[Tag] = 218 +VALUE_PATTERN: Final[Tag] = 219 +SINGLETON_PATTERN: Final[Tag] = 220 +SEQUENCE_PATTERN: Final[Tag] = 221 +STARRED_PATTERN: Final[Tag] = 222 +MAPPING_PATTERN: Final[Tag] = 223 +CLASS_PATTERN: Final[Tag] = 224 +TYPE_ALIAS_STMT: Final[Tag] = 225 +IMPORT_METADATA: Final[Tag] = 226 +IMPORTFROM_METADATA: Final[Tag] = 227 +IMPORTALL_METADATA: Final[Tag] = 228 +TSTRING_EXPR: Final[Tag] = 229 + + +def read_symbol(data: ReadBuffer, tag: Tag) -> SymbolNode: + # The branches here are ordered manually by type "popularity". + if tag == VAR: + return Var.read(data) + if tag == FUNC_DEF: + return FuncDef.read(data) + if tag == DECORATOR: + return Decorator.read(data) + if tag == OVERLOADED_FUNC_DEF: + return OverloadedFuncDef.read(data) + if tag == TYPE_VAR_EXPR: + return TypeVarExpr.read(data) + if tag == TYPE_ALIAS: + return TypeAlias.read(data) + if tag == PARAM_SPEC_EXPR: + return ParamSpecExpr.read(data) + if tag == TYPE_VAR_TUPLE_EXPR: + return TypeVarTupleExpr.read(data) + assert False, f"Unknown symbol tag {tag}" + + +def read_overload_part(data: ReadBuffer, tag: Tag | None = None) -> OverloadPart: + if tag is None: + tag = read_tag(data) + if tag == DECORATOR: + return Decorator.read(data) + if tag == FUNC_DEF: + return FuncDef.read(data) + assert False, f"Invalid tag for an OverloadPart {tag}" diff --git a/mypy/semanal.py b/mypy/semanal.py new file mode 100644 index 000000000000..5ef5223c2346 --- /dev/null +++ b/mypy/semanal.py @@ -0,0 +1,8662 @@ +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. Populate symbol tables. The semantic analyzer also detects +special forms which reuse generic syntax such as NamedTuple and +cast(). Multiple analysis iterations may be needed to analyze forward +references and import cycles. Each iteration "fills in" additional +bindings and references until everything has been bound. + +For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable (Var) node. The second assignment would also +be analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis of types is implemented in typeanal.py. + +See semanal_main.py for the top-level logic. + +Some important properties: + +* After semantic analysis is complete, no PlaceholderNode and + PlaceholderType instances should remain. During semantic analysis, + if we encounter one of these, the current target should be deferred. + +* A TypeInfo is only created once we know certain basic information about + a type, such as the MRO, existence of a Tuple base class (e.g., for named + tuples), and whether we have a TypedDict. We use a temporary + PlaceholderNode node in the symbol table if some such information is + missing. + +* For assignments, we only add a non-placeholder symbol table entry once + we know the sort of thing being defined (variable, NamedTuple, type alias, + etc.). + +* Every part of the analysis step must support multiple iterations over + the same AST nodes, and each iteration must be able to fill in arbitrary + things that were missing or incomplete in previous iterations. + +* Changes performed by the analysis need to be reversible, since mypy + daemon strips and reuses existing ASTs (to improve performance and/or + reduce memory use). +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Collection, Iterable, Iterator +from contextlib import contextmanager +from typing import Any, Final, TypeAlias as _TypeAlias, TypeGuard, TypeVar, cast +from typing_extensions import assert_never + +from mypy import errorcodes as codes, message_registry +from mypy.constant_fold import constant_fold_expr +from mypy.errorcodes import PROPERTY_DECORATOR, ErrorCode +from mypy.errors import Errors, report_internal_error +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import ErrorMessage +from mypy.messages import ( + SUGGESTED_TEST_FIXTURES, + TYPES_FOR_UNIMPORTED_HINTS, + MessageBuilder, + best_matches, + pretty_seq, +) +from mypy.mro import MroError, calculate_mro +from mypy.nodes import ( + ARG_NAMED, + ARG_POS, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + GDEF, + IMPLICITLY_ABSTRACT, + INVARIANT, + IS_ABSTRACT, + LDEF, + MDEF, + NOT_ABSTRACT, + PARAM_SPEC_KIND, + REVEAL_LOCALS, + REVEAL_TYPE, + RUNTIME_PROTOCOL_DECOS, + SYMBOL_FUNCBASE_TYPES, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, + VARIANCE_NOT_READY, + ArgKind, + AssertStmt, + AssertTypeExpr, + AssignmentExpr, + AssignmentStmt, + AwaitExpr, + Block, + BreakStmt, + BytesExpr, + CallExpr, + CastExpr, + ClassDef, + ComparisonExpr, + ComplexExpr, + ConditionalExpr, + Context, + ContinueStmt, + DataclassTransformSpec, + Decorator, + DelStmt, + DictExpr, + DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, + Expression, + ExpressionStmt, + FakeExpression, + FloatExpr, + ForStmt, + FuncBase, + FuncDef, + FuncItem, + GeneratorExpr, + GlobalDecl, + IfStmt, + Import, + ImportAll, + ImportBase, + ImportFrom, + IndexExpr, + IntExpr, + LambdaExpr, + ListComprehension, + ListExpr, + Lvalue, + MatchStmt, + MaybeTypeExpression, + MemberExpr, + MypyFile, + NamedTupleExpr, + NameExpr, + Node, + NonlocalDecl, + OperatorAssignmentStmt, + OpExpr, + OverloadedFuncDef, + OverloadPart, + ParamSpecExpr, + PassStmt, + PlaceholderNode, + PromoteExpr, + RaiseStmt, + RefExpr, + ReturnStmt, + RevealExpr, + SetComprehension, + SetExpr, + SliceExpr, + SplittingVisitor, + StarExpr, + Statement, + StrExpr, + SuperExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TemplateStrExpr, + TempNode, + TryStmt, + TupleExpr, + TypeAlias, + TypeAliasExpr, + TypeAliasStmt, + TypeApplication, + TypedDictExpr, + TypeFormExpr, + TypeInfo, + TypeParam, + TypeVarExpr, + TypeVarLikeExpr, + TypeVarTupleExpr, + UnaryExpr, + Var, + WhileStmt, + WithStmt, + YieldExpr, + YieldFromExpr, + func_scoped_name, + get_member_expr_fullname, + implicit_module_attrs, + inline_base, + is_final_node, + type_aliases, + type_aliases_source_versions, + typing_extensions_aliases, +) +from mypy.options import Options +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) +from mypy.plugin import ( + ClassDefContext, + DynamicClassDefContext, + Plugin, + SemanticAnalyzerPluginInterface, +) +from mypy.plugins import dataclasses as dataclasses_plugin +from mypy.reachability import ( + ALWAYS_FALSE, + ALWAYS_TRUE, + MYPY_FALSE, + MYPY_TRUE, + infer_condition_value, + infer_reachability_of_if_statement, + infer_reachability_of_match_statement, +) +from mypy.scope import Scope +from mypy.semanal_enum import EnumCallAnalyzer +from mypy.semanal_namedtuple import NamedTupleAnalyzer +from mypy.semanal_newtype import NewTypeAnalyzer +from mypy.semanal_shared import ( + ALLOW_INCOMPATIBLE_OVERRIDE, + PRIORITY_FALLBACKS, + SemanticAnalyzerInterface, + calculate_tuple_fallback, + find_dataclass_transform_spec, + has_placeholder, + parse_bool, + require_bool_literal_argument, + set_callable_name as set_callable_name, +) +from mypy.semanal_typeddict import TypedDictAnalyzer +from mypy.tvar_scope import TypeVarLikeScope +from mypy.typeanal import ( + SELF_TYPE_NAMES, + FindTypeVarVisitor, + TypeAnalyser, + TypeVarDefaultTranslator, + TypeVarLikeList, + analyze_type_alias, + check_for_explicit_any, + check_vec_type_args, + detect_diverging_alias, + find_self_type, + fix_instance, + has_any_from_unimported_type, + type_constructors, + validate_instance, +) +from mypy.typeops import function_type, get_type_vars, try_getting_str_literals_from_type +from mypy.types import ( + ANNOTATED_TYPE_NAMES, + ASSERT_TYPE_NAMES, + DATACLASS_TRANSFORM_NAMES, + DEPRECATED_TYPE_NAMES, + DISJOINT_BASE_DECORATOR_NAMES, + FINAL_DECORATOR_NAMES, + FINAL_TYPE_NAMES, + IMPORTED_REVEAL_TYPE_NAMES, + NEVER_NAMES, + OVERLOAD_NAMES, + OVERRIDE_DECORATOR_NAMES, + PROTOCOL_NAMES, + REVEAL_TYPE_NAMES, + SENTINEL_TYPE_NAMES, + TPDICT_NAMES, + TYPE_ALIAS_NAMES, + TYPE_CHECK_ONLY_NAMES, + TYPE_NAMES, + TYPE_VAR_LIKE_NAMES, + TYPED_NAMEDTUPLE_NAMES, + UNPACK_TYPE_NAMES, + AnyType, + CallableType, + FunctionLike, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + PlaceholderType, + ProperType, + SentinelValue, + TrivialSyntheticTypeTranslator, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeOfAny, + TypeStrVisitor, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UnionType, + UnpackType, + flatten_nested_tuples, + get_proper_type, + get_proper_types, + has_type_vars, + is_named_instance, + remove_dups, + type_vars_as_args, +) +from mypy.types_utils import is_invalid_recursive_alias, store_argument_type +from mypy.typevars import fill_typevars +from mypy.util import correct_relative_import, is_dunder, module_prefix, unmangle, unnamed_function +from mypy.visitor import NodeVisitor + +T = TypeVar("T") + + +# Whether to print diagnostic information for failed full parses +# in SemanticAnalyzer.try_parse_as_type_expression(). +# +# See also: misc/analyze_typeform_stats.py +DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES: Final = False + + +FUTURE_IMPORTS: Final = { + "__future__.nested_scopes": "nested_scopes", + "__future__.generators": "generators", + "__future__.division": "division", + "__future__.absolute_import": "absolute_import", + "__future__.with_statement": "with_statement", + "__future__.print_function": "print_function", + "__future__.unicode_literals": "unicode_literals", + "__future__.barry_as_FLUFL": "barry_as_FLUFL", + "__future__.generator_stop": "generator_stop", + "__future__.annotations": "annotations", +} + + +# Special cased built-in classes that are needed for basic functionality and need to be +# available very early on. +CORE_BUILTIN_CLASSES: Final = ["object", "bool", "function"] + + +# Python has several different scope/namespace kinds with subtly different semantics. +SCOPE_GLOBAL: Final = 0 # Module top level +SCOPE_CLASS: Final = 1 # Class body +SCOPE_FUNC: Final = 2 # Function or lambda +SCOPE_COMPREHENSION: Final = 3 # Comprehension or generator expression +SCOPE_ANNOTATION: Final = 4 # Annotation scopes for type parameters and aliases (PEP 695) + + +# Used for tracking incomplete references +Tag: _TypeAlias = int + + +# Matches two words separated by whitespace, where each word lacks +# any symbols which have special meaning in a type expression. +# +# Any string literal matching this common pattern cannot be a valid +# type expression and can be ignored quickly when attempting to parse a +# string literal as a type expression. +_MULTIPLE_WORDS_NONTYPE_RE = re.compile(r'\s*[^\s.\'"|\[]+\s+[^\s.\'"|\[]') + +# Characters which never appear in a valid type expression. +# NOTE: Allows '*' for (PEP 646 Unpack) and '+' for (Literal[+N]) +# NOTE: Stored as a tuple of single-character strings (rather than a str) +# so that iterating it does not allocate a new string per character +# when compiled with mypyc. +_NONTYPE_CHARS: Final = tuple("!:/<>@%$^?;&~`\\") + + +class SemanticAnalyzer( + NodeVisitor[None], SemanticAnalyzerInterface, SemanticAnalyzerPluginInterface, SplittingVisitor +): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for an + AST. Note that type checking is performed as a separate pass. + """ + + __deletable__ = ["patches", "options", "cur_mod_node"] + + # Module name space + modules: dict[str, MypyFile] + # Global name space for current module + globals: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls: list[set[str]] + # Names declared using "nonlocal" (separate set for each scope) + nonlocal_decls: list[set[str]] + # Local names of function scopes; None for non-function scopes. + locals: list[SymbolTable | None] + # Type of each scope (SCOPE_*, indexes match locals) + scope_stack: list[int] + # Nested block depths of scopes + block_depth: list[int] + # TypeInfo of directly enclosing class (or None) + _type: TypeInfo | None = None + # Stack of outer classes (the second tuple item contains tvars). + type_stack: list[TypeInfo | None] + # Type variables bound by the current scope, be it class or function + tvar_scope: TypeVarLikeScope + # Per-module options + options: Options + + # Stack of functions being analyzed + function_stack: list[FuncItem] + + # Set to True if semantic analysis defines a name, or replaces a + # placeholder definition. If some iteration makes no progress, + # there can be at most one additional final iteration (see below). + progress = False + deferred = False # Set to true if another analysis pass is needed + incomplete = False # Set to true if current module namespace is missing things + # Is this the final iteration of semantic analysis (where we report + # unbound names due to cyclic definitions and should not defer)? + _final_iteration = False + # These names couldn't be added to the symbol table due to incomplete deps. + # Note that missing names are per module, _not_ per namespace. This means that e.g. + # a missing name at global scope will block adding same name at a class scope. + # This should not affect correctness and is purely a performance issue, + # since it can cause unnecessary deferrals. These are represented as + # PlaceholderNodes in the symbol table. We use this to ensure that the first + # definition takes precedence even if it's incomplete. + # + # Note that a star import adds a special name '*' to the set, this blocks + # adding _any_ names in the current file. + missing_names: list[set[str]] + # Callbacks that will be called after semantic analysis to tweak things. + patches: list[tuple[int, Callable[[], None]]] + loop_depth: list[int] # Depth of breakable loops + cur_mod_id = "" # Current module id (or None) (phase 2) + _is_stub_file = False # Are we analyzing a stub file? + _is_typeshed_stub_file = False # Are we analyzing a typeshed stub file? + imports: set[str] # Imported modules (during phase 2 analysis) + # Note: some imports (and therefore dependencies) might + # not be found in phase 1, for example due to * imports. + errors: Errors # Keeps track of generated errors + plugin: Plugin # Mypy plugin for special casing of library features + statement: Statement | None = None # Statement/definition being analyzed + + # Mapping from 'async def' function definitions to their return type wrapped as a + # 'Coroutine[Any, Any, T]'. Used to keep track of whether a function definition's + # return type has already been wrapped, by checking if the function definition's + # type is stored in this mapping and that it still matches. + wrapped_coro_return_types: dict[FuncDef, Type] = {} + + def __init__( + self, + modules: dict[str, MypyFile], + missing_modules: dict[str, int], + incomplete_namespaces: set[str], + errors: Errors, + plugin: Plugin, + import_map: dict[str, set[str]], + ) -> None: + """Construct semantic analyzer. + + We reuse the same semantic analyzer instance across multiple modules. + + Args: + modules: Global modules dictionary + missing_modules: Modules that could not be imported encountered so far + incomplete_namespaces: Namespaces that are being populated during semantic analysis + (can contain modules and classes within the current SCC; mutated by the caller) + errors: Report analysis errors using this instance + """ + self.locals = [None] + self.scope_stack = [SCOPE_GLOBAL] + # Saved namespaces from previous iteration. Every top-level function/method body is + # analyzed in several iterations until all names are resolved. We need to save + # the local namespaces for the top level function and all nested functions between + # these iterations. See also semanal_main.process_top_level_function(). + self.saved_locals: dict[ + FuncItem | GeneratorExpr | DictionaryComprehension, SymbolTable + ] = {} + self.imports = set() + self._type = None + self.type_stack = [] + # Are the namespaces of classes being processed complete? + self.incomplete_type_stack: list[bool] = [] + self.tvar_scope = TypeVarLikeScope() + self.function_stack = [] + self.block_depth = [0] + self.loop_depth = [0] + self.errors = errors + self.modules = modules + self.import_map = import_map + self.msg = MessageBuilder(errors, modules) + self.missing_modules = missing_modules + self.missing_names = [set()] + # These namespaces are still in process of being populated. If we encounter a + # missing name in these namespaces, we need to defer the current analysis target, + # since it's possible that the name will be there once the namespace is complete. + self.incomplete_namespaces = incomplete_namespaces + self.all_exports: list[str] = [] + self.plugin = plugin + self.recurse_into_functions = True + self.scope = Scope() + + # Trace line numbers for every file where deferral happened during analysis of + # current SCC or top-level function. + self.deferral_debug_context: list[tuple[str, int]] = [] + + # This is needed to properly support recursive type aliases. The problem is that + # Foo[Bar] could mean three things depending on context: a target for type alias, + # a normal index expression (including enum index), or a type application. + # The latter is particularly problematic as it can falsely create incomplete + # refs while analysing rvalues of type aliases. To avoid this we first analyse + # rvalues while temporarily setting this to True. + self.basic_type_applications = False + + # Used to temporarily enable unbound type variables in some contexts. Namely, + # in base class expressions, and in right hand sides of type aliases. Do not add + # new uses of this, as this may cause leaking `UnboundType`s to type checking. + self.allow_unbound_tvars = False + + # Used to pass information about current overload index to visit_func_def(). + self.current_overload_item: int | None = None + + # Used to track whether currently inside an except* block. This helps + # to invoke errors when continue/break/return is used inside except* block. + self.inside_except_star_block: bool = False + # Used to track edge case when return is still inside except* if it enters a loop + self.return_stmt_inside_except_star_block: bool = False + + self._str_type: Instance | None = None + self._function_type: Instance | None = None + self._object_type: Instance | None = None + + # TypeForm profiling counters + self.type_expression_parse_count: int = 0 # Total try_parse_as_type_expression calls + self.type_expression_full_parse_success_count: int = 0 # Successful full parses + self.type_expression_full_parse_failure_count: int = 0 # Failed full parses + + # Imports of submodules transitively visible from given module. + # This is needed to support patterns like this + # [a.py] + # import b + # import foo + # foo.bar # <- this should work even if bar is not re-exported in foo + # [b.py] + # import foo.bar + self.transitive_submodule_imports: dict[str, set[str]] = {} + + # Instances and type aliases that were fixed using default values of type + # variables. This can be used on-demand by type analyzer. Use record_fixed_type() + # to create the set lazily. + self.types_fixed: set[TypeInfo | TypeAlias] | None = None + + # Stack of type variables that have been removed from current class because they + # cannot be bound unambiguously. This can happen if a (regular) type variable + # with a default follows a type variable tuple. + self.removed_type_vars: list[list[TypeVarType]] = [[]] + + # mypyc doesn't properly handle implementing an abstractproperty + # with a regular attribute, so we make them properties + @property + def type(self) -> TypeInfo | None: + return self._type + + @property + def is_stub_file(self) -> bool: + return self._is_stub_file + + @property + def is_typeshed_stub_file(self) -> bool: + return self._is_typeshed_stub_file + + @property + def final_iteration(self) -> bool: + return self._final_iteration + + @contextmanager + def allow_unbound_tvars_set(self) -> Iterator[None]: + old = self.allow_unbound_tvars + self.allow_unbound_tvars = True + try: + yield + finally: + self.allow_unbound_tvars = old + + @contextmanager + def inside_except_star_block_set( + self, value: bool, entering_loop: bool = False + ) -> Iterator[None]: + old = self.inside_except_star_block + self.inside_except_star_block = value + + # Return statement would still be in except* scope if entering loops + if not entering_loop: + old_return_stmt_flag = self.return_stmt_inside_except_star_block + self.return_stmt_inside_except_star_block = value + + try: + yield + finally: + self.inside_except_star_block = old + if not entering_loop: + self.return_stmt_inside_except_star_block = old_return_stmt_flag + + # + # Preparing module (performed before semantic analysis) + # + + def prepare_file(self, file_node: MypyFile) -> None: + """Prepare a freshly parsed file for semantic analysis.""" + if "builtins" in self.modules: + file_node.names["__builtins__"] = SymbolTableNode(GDEF, self.modules["builtins"]) + if file_node.fullname == "builtins": + self.prepare_builtins_namespace(file_node) + if file_node.fullname == "typing": + self.prepare_typing_namespace(file_node, type_aliases) + if file_node.fullname == "typing_extensions": + self.prepare_typing_namespace(file_node, typing_extensions_aliases) + + def prepare_typing_namespace(self, file_node: MypyFile, aliases: dict[str, str]) -> None: + """Remove dummy alias definitions such as List = TypeAlias(object) from typing. + + They will be replaced with real aliases when corresponding targets are ready. + """ + + # This is all pretty unfortunate. typeshed now has a + # sys.version_info check for OrderedDict, and we shouldn't + # take it out, because it is correct and a typechecker should + # use that as a source of truth. But instead we rummage + # through IfStmts to remove the info first. (I tried to + # remove this whole machinery and ran into issues with the + # builtins/typing import cycle.) + def helper(defs: list[Statement]) -> None: + for stmt in defs.copy(): + if isinstance(stmt, IfStmt): + for body in stmt.body: + helper(body.body) + if stmt.else_body: + helper(stmt.else_body.body) + if ( + isinstance(stmt, AssignmentStmt) + and len(stmt.lvalues) == 1 + and isinstance(stmt.lvalues[0], NameExpr) + ): + # Assignment to a simple name, remove it if it is a dummy alias. + if f"{file_node.fullname}.{stmt.lvalues[0].name}" in aliases: + defs.remove(stmt) + + helper(file_node.defs) + + def prepare_builtins_namespace(self, file_node: MypyFile) -> None: + """Add certain special-cased definitions to the builtins module. + + Some definitions are too special or fundamental to be processed + normally from the AST. + """ + names = file_node.names + + # Add empty definition for core built-in classes, since they are required for basic + # operation. These will be completed later on. + for name in CORE_BUILTIN_CLASSES: + cdef = ClassDef(name, Block([])) # Dummy ClassDef, will be replaced later + info = TypeInfo(SymbolTable(), cdef, "builtins") + info._fullname = f"builtins.{name}" + names[name] = SymbolTableNode(GDEF, info) + + bool_info = names["bool"].node + assert isinstance(bool_info, TypeInfo) + bool_type = Instance(bool_info, []) + + special_var_types: list[tuple[str, Type]] = [ + ("None", NoneType()), + # reveal_type is a mypy-only function that gives an error with + # the type of its arg. + ("reveal_type", AnyType(TypeOfAny.special_form)), + # reveal_locals is a mypy-only function that gives an error with the types of + # locals + ("reveal_locals", AnyType(TypeOfAny.special_form)), + ("True", bool_type), + ("False", bool_type), + ("__debug__", bool_type), + ] + + for name, typ in special_var_types: + v = Var(name, typ) + v._fullname = f"builtins.{name}" + file_node.names[name] = SymbolTableNode(GDEF, v) + + # + # Analyzing a target + # + + def refresh_partial( + self, + node: MypyFile | FuncDef | OverloadedFuncDef, + patches: list[tuple[int, Callable[[], None]]], + final_iteration: bool, + file_node: MypyFile, + options: Options, + active_type: TypeInfo | None = None, + ) -> None: + """Refresh a stale target in fine-grained incremental mode.""" + self.patches = patches + self.deferred = False + self.incomplete = False + self._final_iteration = final_iteration + self.missing_names[-1] = set() + + with self.file_context(file_node, options, active_type): + if isinstance(node, MypyFile): + self.refresh_top_level(node) + else: + self.recurse_into_functions = True + self.accept(node) + del self.patches + + def ad_hoc_error(self, msg: str) -> None: + n = TempNode(AnyType(TypeOfAny.special_form)) + n.line = 1 + n.column = 0 + n.end_line = 1 + n.end_column = 0 + self.fail(msg, n) + + def refresh_top_level(self, file_node: MypyFile) -> None: + """Reanalyze a stale module top-level in fine-grained incremental mode.""" + if self.options.allow_redefinition and not self.options.local_partial_types: + self.ad_hoc_error( + "--local-partial-types must be enabled if using --allow-redefinition" + ) + if self.options.allow_redefinition and self.options.allow_redefinition_old: + self.ad_hoc_error( + "--allow-redefinition-old and --allow-redefinition should not be used together" + ) + if not self.options.local_partial_types and self.options.num_workers > 0: + self.ad_hoc_error("--local-partial-types must be enabled in parallel mode") + self.recurse_into_functions = False + self.add_implicit_module_attrs(file_node) + for d in file_node.defs: + self.accept(d) + if file_node.fullname == "typing": + self.add_builtin_aliases(file_node) + if file_node.fullname == "typing_extensions": + self.add_typing_extension_aliases(file_node) + self.adjust_public_exports() + self.all_exports = [] + + def add_implicit_module_attrs(self, file_node: MypyFile) -> None: + """Manually add implicit definitions of module '__name__' etc.""" + str_type: Type | None = self.named_type_or_none("builtins.str") + if str_type is None: + str_type = UnboundType("builtins.str") + inst: Type | None + for name, t in implicit_module_attrs.items(): + if name == "__doc__": + typ: Type = str_type + elif name == "__path__": + if not file_node.is_package_init_file(): + continue + # Need to construct the type ourselves, to avoid issues with __builtins__.list + # not being subscriptable or typing.List not getting bound + inst = self.named_type_or_none("builtins.list", [str_type]) + if inst is None: + assert not self.final_iteration, "Cannot find builtins.list to add __path__" + self.defer() + return + typ = inst + elif name == "__annotations__": + inst = self.named_type_or_none( + "builtins.dict", [str_type, AnyType(TypeOfAny.special_form)] + ) + if inst is None: + assert ( + not self.final_iteration + ), "Cannot find builtins.dict to add __annotations__" + self.defer() + return + typ = inst + elif name == "__spec__": + if self.options.use_builtins_fixtures: + inst = self.named_type_or_none("builtins.object") + else: + inst = self.named_type_or_none("importlib.machinery.ModuleSpec") + if inst is None: + if ( + self.final_iteration + or self.options.clone_for_module("importlib.machinery").follow_imports + == "skip" + ): + # If we are not allowed to resolve imports from `importlib.machinery`, + # ModuleSpec will not be available at any iteration. + # Use the fallback earlier. + # (see https://github.com/python/mypy/issues/18237) + inst = self.named_type_or_none("builtins.object") + assert inst is not None, "Cannot find builtins.object" + else: + self.defer() + return + if file_node.name == "__main__": + # https://docs.python.org/3/reference/import.html#main-spec + inst = UnionType.make_union([inst, NoneType()]) + typ = inst + else: + assert t is not None, f"type should be specified for {name}" + typ = UnboundType(t) + + existing = file_node.names.get(name) + if existing is not None and not isinstance(existing.node, PlaceholderNode): + # Already exists. + continue + + an_type = self.anal_type(typ) + if an_type: + var = Var(name, an_type) + var._fullname = self.qualified_name(name) + var.is_ready = True + self.add_symbol(name, var, dummy_context()) + else: + self.add_symbol( + name, + PlaceholderNode(self.qualified_name(name), file_node, -1), + dummy_context(), + ) + + def add_builtin_aliases(self, tree: MypyFile) -> None: + """Add builtin type aliases to typing module. + + For historical reasons, the aliases like `List = list` are not defined + in typeshed stubs for typing module. Instead we need to manually add the + corresponding nodes on the fly. We explicitly mark these aliases as normalized, + so that a user can write `typing.List[int]`. + """ + assert tree.fullname == "typing" + for alias, target_name in type_aliases.items(): + if ( + alias in type_aliases_source_versions + and type_aliases_source_versions[alias] > self.options.python_version + ): + # This alias is not available on this Python version. + continue + name = alias.split(".")[-1] + if name in tree.names and not isinstance(tree.names[name].node, PlaceholderNode): + continue + self.create_alias(tree, target_name, alias, name) + + def add_typing_extension_aliases(self, tree: MypyFile) -> None: + """Typing extensions module does contain some type aliases. + + We need to analyze them as such, because in typeshed + they are just defined as `_Alias()` call. + Which is not supported natively. + """ + assert tree.fullname == "typing_extensions" + + for alias, target_name in typing_extensions_aliases.items(): + name = alias.split(".")[-1] + if name in tree.names and isinstance(tree.names[name].node, TypeAlias): + continue # Do not reset TypeAliases on the second pass. + + # We need to remove any node that is there at the moment. It is invalid. + tree.names.pop(name, None) + + # Now, create a new alias. + self.create_alias(tree, target_name, alias, name) + + def create_alias(self, tree: MypyFile, target_name: str, alias: str, name: str) -> None: + tag = self.track_incomplete_refs() + n = self.lookup_fully_qualified_or_none(target_name) + if n: + if isinstance(n.node, PlaceholderNode): + self.mark_incomplete(name, tree) + else: + # Found built-in class target. Create alias. + target = self.named_type_or_none(target_name, []) + assert target is not None + # Transform List to List[Any], etc. + fix_instance( + target, self.fail, self.note, disallow_any=False, options=self.options + ) + alias_node = TypeAlias( + target, + alias, + tree.fullname, + line=-1, + column=-1, # there is no context + no_args=True, + normalized=True, + ) + self.add_symbol(name, alias_node, tree) + elif self.found_incomplete_ref(tag): + # Built-in class target may not ready yet -- defer. + self.mark_incomplete(name, tree) + else: + # Test fixtures may be missing some builtin classes, which is okay. + # Kill the placeholder if there is one. + if name in tree.names: + assert isinstance(tree.names[name].node, PlaceholderNode) + del tree.names[name] + + def adjust_public_exports(self) -> None: + """Adjust the module visibility of globals due to __all__.""" + if "__all__" in self.globals: + for name, g in self.globals.items(): + # Being included in __all__ explicitly exports and makes public. + if name in self.all_exports: + g.module_public = True + g.module_hidden = False + # But when __all__ is defined, and a symbol is not included in it, + # it cannot be public. + else: + g.module_public = False + + @contextmanager + def file_context( + self, file_node: MypyFile, options: Options, active_type: TypeInfo | None = None + ) -> Iterator[None]: + """Configure analyzer for analyzing targets within a file/class. + + Args: + file_node: target file + options: options specific to the file + active_type: must be the surrounding class to analyze method targets + """ + scope = self.scope + self.options = options + self.errors.set_file(file_node.path, file_node.fullname, scope=scope, options=options) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname + with scope.module_scope(self.cur_mod_id): + self._is_stub_file = file_node.path.lower().endswith(".pyi") + self._is_typeshed_stub_file = file_node.is_typeshed_file(options) + self.globals = file_node.names + self.tvar_scope = TypeVarLikeScope() + + self.named_tuple_analyzer = NamedTupleAnalyzer(options, self, self.msg) + self.typed_dict_analyzer = TypedDictAnalyzer(options, self, self.msg) + self.enum_call_analyzer = EnumCallAnalyzer(options, self) + self.newtype_analyzer = NewTypeAnalyzer(options, self, self.msg) + + # Counter that keeps track of references to undefined things potentially caused by + # incomplete namespaces. + self.num_incomplete_refs = 0 + + if active_type: + enclosing_fullname = active_type.fullname.rsplit(".", 1)[0] + if "." in enclosing_fullname: + enclosing_node = self.lookup_fully_qualified_or_none(enclosing_fullname) + if enclosing_node and isinstance(enclosing_node.node, TypeInfo): + self._type = enclosing_node.node + self.push_type_args(active_type.defn.type_args, active_type.defn) + self.incomplete_type_stack.append(False) + scope.enter_class(active_type) + self.enter_class(active_type.defn.info) + for tvar in active_type.defn.type_vars: + self.tvar_scope.bind_existing(tvar) + + yield + + if active_type: + scope.leave_class() + self.leave_class() + self._type = None + self.incomplete_type_stack.pop() + self.pop_type_args(active_type.defn.type_args) + del self.options + + # + # Functions + # + + def visit_func_def(self, defn: FuncDef) -> None: + self.statement = defn + + # Visit default values because they may contain assignment expressions. + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + + defn.is_conditional = self.block_depth[-1] > 0 + + # Set full names even for those definitions that aren't added + # to a symbol table. For example, for overload items. + defn._fullname = self.qualified_name(defn.name) + + # We don't add module top-level functions to symbol tables + # when we analyze their bodies in the second phase on analysis, + # since they were added in the first phase. Nested functions + # get always added, since they aren't separate targets. + if not self.recurse_into_functions or len(self.function_stack) > 0: + if not defn.is_decorated and not defn.is_overload: + self.add_function_to_symbol_table(defn) + + if not self.recurse_into_functions and not defn.def_or_infer_vars: + return + + with self.scope.function_scope(defn), self.set_recurse_into_functions(): + with self.inside_except_star_block_set(value=False): + self.analyze_func_def(defn) + + def function_fullname(self, fullname: str) -> str: + if self.current_overload_item is None: + return fullname + return f"{fullname}#{self.current_overload_item}" + + def analyze_func_def(self, defn: FuncDef) -> None: + if self.push_type_args(defn.type_args, defn) is None: + self.defer(defn) + return + + self.function_stack.append(defn) + + if defn.type: + assert isinstance(defn.type, CallableType) + has_self_type = self.update_function_type_variables(defn.type, defn) + else: + has_self_type = False + + self.function_stack.pop() + + if self.is_class_scope(): + # Method definition + assert self.type is not None + defn.info = self.type + if defn.type is not None and defn.name in ("__init__", "__init_subclass__"): + assert isinstance(defn.type, CallableType) + if isinstance(get_proper_type(defn.type.ret_type), AnyType): + defn.type = defn.type.copy_modified(ret_type=NoneType()) + self.prepare_method_signature(defn, self.type, has_self_type) + + # Analyze function signature + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + if defn.type: + self.check_classvar_in_signature(defn.type) + assert isinstance(defn.type, CallableType) + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + analyzer = self.type_analyzer() + tag = self.track_incomplete_refs() + result = analyzer.visit_callable_type(defn.type, nested=False, namespace=fullname) + # Don't store not ready types (including placeholders). + if self.found_incomplete_ref(tag) or has_placeholder(result): + self.defer(defn) + self.pop_type_args(defn.type_args) + return + assert isinstance(result, ProperType) + if isinstance(result, CallableType): + # type guards need to have a positional argument, to spec + skip_self = self.is_class_scope() and not defn.is_static + if result.type_guard and ARG_POS not in result.arg_kinds[skip_self:]: + self.fail( + "TypeGuard functions must have a positional argument", + result, + code=codes.VALID_TYPE, + ) + # in this case, we just kind of just ... remove the type guard. + result = result.copy_modified(type_guard=None) + if result.type_is and ARG_POS not in result.arg_kinds[skip_self:]: + self.fail( + '"TypeIs" functions must have a positional argument', + result, + code=codes.VALID_TYPE, + ) + result = result.copy_modified(type_is=None) + + result = self.remove_unpack_kwargs(defn, result) + if has_self_type and self.type is not None: + info = self.type + if info.self_type is not None: + result.variables = (info.self_type,) + result.variables + defn.type = result + self.add_type_alias_deps(analyzer.aliases_used) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + assert isinstance(defn.type, CallableType) + defn.type = set_callable_name(defn.type, defn) + + self.analyze_arg_initializers(defn) + self.analyze_function_body(defn) + + if self.is_class_scope(): + assert self.type is not None + # Mark protocol methods with empty bodies as implicitly abstract. + # This makes explicit protocol subclassing type-safe. + if ( + self.type.is_protocol + and not self.is_stub_file # Bodies in stub files are always empty. + and (not isinstance(self.scope.function, OverloadedFuncDef) or defn.is_property) + and defn.abstract_status != IS_ABSTRACT + and is_trivial_body(defn.body) + ): + defn.abstract_status = IMPLICITLY_ABSTRACT + if ( + is_trivial_body(defn.body) + and not self.is_stub_file + and defn.abstract_status != NOT_ABSTRACT + ): + defn.is_trivial_body = True + + if ( + defn.is_coroutine + and isinstance(defn.type, CallableType) + and self.wrapped_coro_return_types.get(defn) != defn.type + ): + if defn.is_async_generator: + # Async generator types are handled elsewhere + pass + else: + # A coroutine defined as `async def foo(...) -> T: ...` + # has external return type `Coroutine[Any, Any, T]`. + any_type = AnyType(TypeOfAny.special_form) + ret_type = self.named_type_or_none( + "typing.Coroutine", [any_type, any_type, defn.type.ret_type] + ) + assert ret_type is not None, "Internal error: typing.Coroutine not found" + defn.type = defn.type.copy_modified(ret_type=ret_type) + self.wrapped_coro_return_types[defn] = defn.type + + self.pop_type_args(defn.type_args) + + def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType: + if not typ.arg_kinds or typ.arg_kinds[-1] is not ArgKind.ARG_STAR2: + return typ + last_type = typ.arg_types[-1] + if not isinstance(last_type, UnpackType): + return typ + p_last_type = get_proper_type(last_type.type) + if not isinstance(p_last_type, TypedDictType): + self.fail("Unpack item in ** parameter must be a TypedDict", last_type) + new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] + return typ.copy_modified(arg_types=new_arg_types) + overlap = set(typ.arg_names) & set(p_last_type.items) + # It is OK for TypedDict to have a key named 'kwargs'. + overlap.discard(typ.arg_names[-1]) + if overlap: + overlapped = ", ".join([f'"{name}"' for name in sorted(filter(None, overlap))]) + self.fail( + f"Overlap between parameter names and ** TypedDict items: {overlapped}", defn + ) + new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] + return typ.copy_modified(arg_types=new_arg_types) + # OK, everything looks right now, mark the callable type as using unpack. + new_arg_types = typ.arg_types[:-1] + [p_last_type] + return typ.copy_modified(arg_types=new_arg_types, unpack_kwargs=True) + + def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: bool) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special, as well as __new__. + functype = func.type + if func.name == "__new__": + func.is_static = True + if func.has_self_or_cls_argument: + if func.name in ["__init_subclass__", "__class_getitem__"]: + func.is_class = True + if func.arguments and isinstance(functype, CallableType): + self_type = get_proper_type(functype.arg_types[0]) + if isinstance(self_type, AnyType): + if has_self_type: + assert self.type is not None and self.type.self_type is not None + leading_type: Type = self.type.self_type + else: + func.is_trivial_self = True + leading_type = fill_typevars(info) + if func.is_class or func.name == "__new__": + leading_type = self.class_type(leading_type) + if not has_placeholder(leading_type): + func.type = replace_implicit_first_type(functype, leading_type) + elif has_self_type and isinstance(func.unanalyzed_type, CallableType): + if not isinstance(get_proper_type(func.unanalyzed_type.arg_types[0]), AnyType): + if self.is_expected_self_type( + self_type, func.is_class or func.name == "__new__" + ): + # This error is off by default, since it is explicitly allowed + # by the PEP 673. + self.fail( + 'Redundant "Self" annotation for the first method argument', + func, + code=codes.REDUNDANT_SELF_TYPE, + ) + else: + self.fail( + "Method cannot have explicit self annotation and Self type", func + ) + elif has_self_type: + self.fail("Static methods cannot use Self type", func) + + def is_expected_self_type(self, typ: Type, is_classmethod: bool) -> bool: + """Does this (analyzed or not) type represent the expected Self type for a method?""" + assert self.type is not None + typ = get_proper_type(typ) + if is_classmethod: + if isinstance(typ, TypeType): + return self.is_expected_self_type(typ.item, is_classmethod=False) + if isinstance(typ, UnboundType): + sym = self.lookup_qualified(typ.name, typ, suppress_errors=True) + if sym is not None and sym.fullname in TYPE_NAMES and typ.args: + return self.is_expected_self_type(typ.args[0], is_classmethod=False) + if sym is not None and sym.fullname in ANNOTATED_TYPE_NAMES and typ.args: + return self.is_expected_self_type(typ.args[0], is_classmethod=True) + return False + if isinstance(typ, TypeVarType): + return typ == self.type.self_type + if isinstance(typ, UnboundType): + sym = self.lookup_qualified(typ.name, typ, suppress_errors=True) + if sym is not None and sym.fullname in ANNOTATED_TYPE_NAMES and typ.args: + return self.is_expected_self_type(typ.args[0], is_classmethod=False) + return sym is not None and sym.fullname in SELF_TYPE_NAMES + return False + + def set_original_def(self, previous: Node | None, new: FuncDef | Decorator) -> bool: + """If 'new' conditionally redefine 'previous', set 'previous' as original + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + def f(): ... + def f(): ... # Error: 'f' redefined + """ + if isinstance(new, Decorator): + new = new.func + if ( + isinstance(previous, (FuncDef, Decorator)) + and unnamed_function(new.name) + and unnamed_function(previous.name) + ): + return True + if isinstance(previous, (FuncDef, Var, Decorator)) and new.is_conditional: + new.original_def = previous + return True + else: + return False + + def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) -> bool: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. Return True, if the signature contains typing.Self + type, or False otherwise. + """ + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + a = self.type_analyzer() + fun_type.variables, has_self_type = a.bind_function_type_variables(fun_type, defn) + if has_self_type and self.type is not None: + self.setup_self_type() + if defn.type_args: + bound_fullnames = {v.fullname for v in fun_type.variables} + declared_fullnames = {self.qualified_name(p.name) for p in defn.type_args} + extra = sorted(bound_fullnames - declared_fullnames) + if extra: + self.msg.type_parameters_should_be_declared( + [n.split(".")[-1] for n in extra], defn + ) + return has_self_type + + def setup_self_type(self) -> None: + """Setup a (shared) Self type variable for current class. + + We intentionally don't add it to the class symbol table, + so it can be accessed only by mypy and will not cause + clashes with user defined names. + """ + assert self.type is not None + info = self.type + if info.self_type is not None: + if has_placeholder(info.self_type.upper_bound): + # Similar to regular (user defined) type variables. + self.process_placeholder( + None, + "Self upper bound", + info, + force_progress=info.self_type.upper_bound != fill_typevars(info), + ) + else: + return + info.self_type = TypeVarType( + "Self", + f"{info.fullname}.Self", + id=TypeVarId(0), # 0 is a special value for self-types. + values=[], + upper_bound=fill_typevars(info), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + self.statement = defn + self.add_function_to_symbol_table(defn) + + if not self.recurse_into_functions and not defn.def_or_infer_vars: + return + + # NB: Since _visit_overloaded_func_def will call accept on the + # underlying FuncDefs, the function might get entered twice. + # This is fine, though, because only the outermost function is + # used to compute targets. + with self.scope.function_scope(defn), self.set_recurse_into_functions(): + self.analyze_overloaded_func_def(defn) + + @contextmanager + def overload_item_set(self, item: int | None) -> Iterator[None]: + self.current_overload_item = item + try: + yield + finally: + self.current_overload_item = None + + def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + # OverloadedFuncDef refers to any legitimate situation where you have + # more than one declaration for the same function in a row. This occurs + # with a @property with a setter or a deleter, and for a classic + # @overload. + + defn._fullname = self.qualified_name(defn.name) + # TODO: avoid modifying items. + defn.items = defn.unanalyzed_items.copy() + + first_item = defn.items[0] + first_item.is_overload = True + with self.overload_item_set(0): + first_item.accept(self) + + bare_setter_type = None + is_property = False + if isinstance(first_item, Decorator) and first_item.func.is_property: + is_property = True + # This is a property. + first_item.func.is_overload = True + bare_setter_type = self.analyze_property_with_multi_part_definition(defn) + typ = function_type(first_item.func, self.function_type()) + assert isinstance(typ, CallableType) + typ.definition = first_item + types = [typ] + else: + # This is a normal overload. Find the item signatures, the + # implementation (if outside a stub), and any missing @overload + # decorators. + types, impl, non_overload_indexes = self.analyze_overload_sigs_and_impl(defn) + defn.impl = impl + if non_overload_indexes: + self.handle_missing_overload_decorators( + defn, non_overload_indexes, some_overload_decorators=len(types) > 0 + ) + # If we found an implementation, remove it from the overload item list, + # as it's special. + if impl is not None: + assert impl is defn.items[-1] + defn.items = defn.items[:-1] + elif not non_overload_indexes: + self.handle_missing_overload_implementation(defn) + + if types and not any( + # If some overload items are decorated with other decorators, then + # the overload type will be determined during type checking. + # Note: bare @property is removed in visit_decorator(). + isinstance(it, Decorator) + and len(it.decorators) > (1 if i > 0 or not is_property else 0) + for i, it in enumerate(defn.items) + ): + # TODO: should we enforce decorated overloads consistency somehow? + # Some existing code uses both styles: + # * Put decorator only on implementation, use "effective" types in overloads + # * Put decorator everywhere, use "bare" types in overloads. + defn.type = Overloaded(types) + defn.type.line = defn.line + # In addition, we can set the getter/setter type for valid properties as some + # code paths may either use the above type, or var.type etc. of the first item. + if isinstance(first_item, Decorator) and bare_setter_type: + first_item.var.type = types[0] + first_item.var.setter_type = bare_setter_type + + if not defn.items: + # It was not a real overload after all, but function redefinition. We've + # visited the redefinition(s) already. + if not defn.impl: + # For really broken overloads with no items and no implementation we need to keep + # at least one item to hold basic information like function name. + defn.impl = defn.unanalyzed_items[-1] + return + + # We know this is an overload def. Infer properties and perform some checks. + self.process_deprecated_overload(defn) + self.process_final_in_overload(defn) + self.process_static_or_class_method_in_overload(defn) + self.process_overload_impl(defn) + + def process_deprecated_overload(self, defn: OverloadedFuncDef) -> None: + if defn.is_property: + return + + if isinstance(impl := defn.impl, Decorator) and ( + (deprecated := impl.func.deprecated) is not None + ): + defn.deprecated = deprecated + for item in defn.items: + if isinstance(item, Decorator): + item.func.deprecated = deprecated + + for item in defn.items: + deprecation = False + if isinstance(item, Decorator): + for d in item.decorators: + if deprecation and refers_to_fullname(d, OVERLOAD_NAMES): + self.msg.note("@overload should be placed before @deprecated", d) + elif (deprecated := self.get_deprecated(d)) is not None: + deprecation = True + if isinstance(typ := item.func.type, CallableType): + typestr = f" {typ.accept(TypeStrVisitor(options=self.options))} " + else: + typestr = " " + item.func.deprecated = ( + f"overload{typestr}of function {defn.fullname} is deprecated: " + f"{deprecated}" + ) + + @staticmethod + def get_deprecated(expression: Expression) -> str | None: + if ( + isinstance(expression, CallExpr) + and refers_to_fullname(expression.callee, DEPRECATED_TYPE_NAMES) + and (len(args := expression.args) >= 1) + and isinstance(deprecated := args[0], StrExpr) + ): + return deprecated.value + return None + + def process_overload_impl(self, defn: OverloadedFuncDef) -> None: + """Set flags for an overload implementation. + + Currently, this checks for a trivial body in protocols classes, + where it makes the method implicitly abstract. + """ + if defn.impl is None: + return + impl = defn.impl if isinstance(defn.impl, FuncDef) else defn.impl.func + if is_trivial_body(impl.body) and self.is_class_scope() and not self.is_stub_file: + assert self.type is not None + if self.type.is_protocol: + impl.abstract_status = IMPLICITLY_ABSTRACT + if impl.abstract_status != NOT_ABSTRACT: + impl.is_trivial_body = True + + def analyze_overload_sigs_and_impl( + self, defn: OverloadedFuncDef + ) -> tuple[list[CallableType], OverloadPart | None, list[int]]: + """Find overload signatures, the implementation, and items with missing @overload. + + Assume that the first was already analyzed. As a side effect: + analyzes remaining items and updates 'is_overload' flags. + """ + types = [] + non_overload_indexes = [] + impl: OverloadPart | None = None + for i, item in enumerate(defn.items): + if i != 0: + # Assume that the first item was already visited + item.is_overload = True + with self.overload_item_set(i if i < len(defn.items) - 1 else None): + item.accept(self) + # TODO: support decorated overloaded functions properly + if isinstance(item, Decorator): + callable = function_type(item.func, self.function_type()) + assert isinstance(callable, CallableType) + callable.definition = item + if not any(refers_to_fullname(dec, OVERLOAD_NAMES) for dec in item.decorators): + if i == len(defn.items) - 1 and not self.is_stub_file: + # Last item outside a stub is impl + impl = item + else: + # Oops it wasn't an overload after all. A clear error + # will vary based on where in the list it is, record + # that. + non_overload_indexes.append(i) + else: + item.func.is_overload = True + types.append(callable) + if item.var.is_property: + self.fail("An overload can not be a property", item) + # If any item was decorated with `@override`, the whole overload + # becomes an explicit override. + defn.is_explicit_override |= item.func.is_explicit_override + elif isinstance(item, FuncDef): + if i == len(defn.items) - 1 and not self.is_stub_file: + impl = item + else: + non_overload_indexes.append(i) + return types, impl, non_overload_indexes + + def handle_missing_overload_decorators( + self, + defn: OverloadedFuncDef, + non_overload_indexes: list[int], + some_overload_decorators: bool, + ) -> None: + """Generate errors for overload items without @overload. + + Side effect: remote non-overload items. + """ + if some_overload_decorators: + # Some of them were overloads, but not all. + for idx in non_overload_indexes: + if self.is_stub_file: + self.fail( + "An implementation for an overloaded function " + "is not allowed in a stub file", + defn.items[idx], + ) + else: + self.fail( + "The implementation for an overloaded function must come last", + defn.items[idx], + ) + else: + for idx in non_overload_indexes[1:]: + self.name_already_defined(defn.name, defn.items[idx], defn.items[0]) + if defn.impl: + self.name_already_defined(defn.name, defn.impl, defn.items[0]) + # Remove the non-overloads + for idx in reversed(non_overload_indexes): + del defn.items[idx] + + def handle_missing_overload_implementation(self, defn: OverloadedFuncDef) -> None: + """Generate error about missing overload implementation (only if needed).""" + if not self.is_stub_file: + if self.type and self.type.is_protocol and not self.is_func_scope(): + # An overloaded protocol method doesn't need an implementation, + # but if it doesn't have one, then it is considered abstract. + for item in defn.items: + if isinstance(item, Decorator): + item.func.abstract_status = IS_ABSTRACT + else: + item.abstract_status = IS_ABSTRACT + elif all( + isinstance(item, Decorator) and item.func.abstract_status == IS_ABSTRACT + for item in defn.items + ): + # Since there is no implementation, it can't be called via super(). + if defn.items: + assert isinstance(defn.items[0], Decorator) + defn.items[0].func.is_trivial_body = True + else: + self.fail( + "An overloaded function outside a stub file must have an implementation", + defn, + code=codes.NO_OVERLOAD_IMPL, + ) + + def process_final_in_overload(self, defn: OverloadedFuncDef) -> None: + """Detect the @final status of an overloaded function (and perform checks).""" + # If the implementation is marked as @final (or the first overload in + # stubs), then the whole overloaded definition if @final. + if any(item.is_final for item in defn.items): + # We anyway mark it as final because it was probably the intention. + defn.is_final = True + # Only show the error once per overload + bad_final = next(ov for ov in defn.items if ov.is_final) + if not self.is_stub_file: + self.fail("@final should be applied only to overload implementation", bad_final) + elif any(item.is_final for item in defn.items[1:]): + bad_final = next(ov for ov in defn.items[1:] if ov.is_final) + self.fail( + "In a stub file @final must be applied only to the first overload", bad_final + ) + if defn.impl is not None and defn.impl.is_final: + defn.is_final = True + + def process_static_or_class_method_in_overload(self, defn: OverloadedFuncDef) -> None: + class_status = [] + static_status = [] + for item in defn.items: + if isinstance(item, Decorator): + inner = item.func + elif isinstance(item, FuncDef): + inner = item + else: + assert False, f"The 'item' variable is an unexpected type: {type(item)}" + class_status.append(inner.is_class) + static_status.append(inner.is_static) + + if defn.impl is not None: + if isinstance(defn.impl, Decorator): + inner = defn.impl.func + elif isinstance(defn.impl, FuncDef): + inner = defn.impl + else: + assert False, f"Unexpected impl type: {type(defn.impl)}" + class_status.append(inner.is_class) + static_status.append(inner.is_static) + + if len(set(class_status)) != 1: + self.msg.overload_inconsistently_applies_decorator("classmethod", defn) + elif len(set(static_status)) != 1: + self.msg.overload_inconsistently_applies_decorator("staticmethod", defn) + else: + defn.is_class = class_status[0] + defn.is_static = static_status[0] + + def analyze_property_with_multi_part_definition( + self, defn: OverloadedFuncDef + ) -> CallableType | None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + Return bare setter type (without any other decorators applied), this may be used + by the caller for performance optimizations. + """ + defn.is_property = True + items = defn.items + first_item = defn.items[0] + assert isinstance(first_item, Decorator) + deleted_items = [] + bare_setter_type = None + func_name = first_item.func.name + for i, item in enumerate(items[1:]): + if isinstance(item, Decorator): + item.func.accept(self) + if item.decorators: + first_node = item.decorators[0] + if self._is_valid_property_decorator(first_node, func_name): + # Get abstractness from the original definition. + item.func.abstract_status = first_item.func.abstract_status + if first_node.name == "setter": + # The first item represents the entire property. + first_item.var.is_settable_property = True + setter_func_type = function_type(item.func, self.function_type()) + assert isinstance(setter_func_type, CallableType) + bare_setter_type = setter_func_type + defn.setter_index = i + 1 + for other_node in item.decorators[1:]: + other_node.accept(self) + else: + self.fail( + f'Only supported top decorators are "@{func_name}.setter" and "@{func_name}.deleter"', + first_node, + ) + else: + self.fail(f'Unexpected definition for property "{func_name}"', item) + deleted_items.append(i + 1) + for i in reversed(deleted_items): + del items[i] + + for item in items[1:]: + if isinstance(item, Decorator): + for d in item.decorators: + if (deprecated := self.get_deprecated(d)) is not None: + item.func.deprecated = ( + f"function {item.fullname} is deprecated: {deprecated}" + ) + return bare_setter_type + + def _is_valid_property_decorator( + self, deco: Expression, property_name: str + ) -> TypeGuard[MemberExpr]: + if not isinstance(deco, MemberExpr): + return False + if not isinstance(deco.expr, NameExpr) or deco.expr.name != property_name: + return False + if deco.name not in {"setter", "deleter"}: + # This intentionally excludes getter. While `@prop.getter` is valid at + # runtime, that would mean replacing the already processed getter type. + # Such usage is almost definitely a mistake (except for overrides in + # subclasses but we don't support them anyway) and might be a typo + # (only one letter away from `setter`), it's likely almost never used, + # so supporting it properly won't pay off. + return False + return True + + def add_function_to_symbol_table(self, func: FuncDef | OverloadedFuncDef) -> None: + if self.is_class_scope(): + assert self.type is not None + func.info = self.type + func._fullname = self.qualified_name(func.name) + self.add_symbol(func.name, func, func) + + def analyze_arg_initializers(self, defn: FuncItem) -> None: + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + # Analyze default arguments + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + + def analyze_function_body(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): + # Bind the type variables again to visit the body. + if defn.type: + a = self.type_analyzer() + typ = defn.type + assert isinstance(typ, CallableType) + a.bind_function_type_variables(typ, defn) + for i in range(len(typ.arg_types)): + store_argument_type(defn, i, typ, self.named_type) + self.function_stack.append(defn) + with self.enter(defn): + for arg in defn.arguments: + self.add_local(arg.variable, defn) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and defn.has_self_or_cls_argument and defn.arguments: + if not defn.is_class: + defn.arguments[0].variable.is_self = True + else: + defn.arguments[0].variable.is_cls = True + + defn.body.accept(self) + self.function_stack.pop() + + def check_classvar_in_signature(self, typ: ProperType) -> None: + t: ProperType + if isinstance(typ, Overloaded): + for t in typ.items: + self.check_classvar_in_signature(t) + return + if not isinstance(typ, CallableType): + return + for t in get_proper_types(typ.arg_types) + [get_proper_type(typ.ret_type)]: + if self.is_classvar(t): + self.fail_invalid_classvar(t) + # Show only one error per signature + break + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = fdef.type + assert isinstance(sig, CallableType) + if len(sig.arg_types) < len(fdef.arguments): + self.fail("Type signature has too few arguments", fdef) + # Add dummy Any arguments to prevent crashes later. + num_extra_anys = len(fdef.arguments) - len(sig.arg_types) + extra_anys = [AnyType(TypeOfAny.from_error)] * num_extra_anys + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail("Type signature has too many arguments", fdef, blocker=True) + + def visit_decorator(self, dec: Decorator) -> None: + self.statement = dec + # TODO: better don't modify them at all. + dec.decorators = dec.original_decorators.copy() + dec.func.is_conditional = self.block_depth[-1] > 0 + if not dec.is_overload: + self.add_symbol(dec.name, dec, dec) + dec.func._fullname = self.qualified_name(dec.name) + dec.var._fullname = self.qualified_name(dec.name) + for d in dec.decorators: + d.accept(self) + removed: list[int] = [] + no_type_check = False + could_be_decorated_property = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, "abc.abstractmethod"): + removed.append(i) + dec.func.abstract_status = IS_ABSTRACT + self.check_decorated_function_is_method("abstractmethod", dec) + elif refers_to_fullname(d, ("asyncio.coroutines.coroutine", "types.coroutine")): + removed.append(i) + dec.func.is_awaitable_coroutine = True + elif refers_to_fullname(d, "builtins.staticmethod"): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method("staticmethod", dec) + elif refers_to_fullname(d, "builtins.classmethod"): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method("classmethod", dec) + elif refers_to_fullname(d, OVERRIDE_DECORATOR_NAMES): + removed.append(i) + dec.func.is_explicit_override = True + self.check_decorated_function_is_method("override", dec) + elif refers_to_fullname( + d, + ( + "builtins.property", + "abc.abstractproperty", + "functools.cached_property", + "enum.property", + "types.DynamicClassAttribute", + ), + ): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, "abc.abstractproperty"): + dec.func.abstract_status = IS_ABSTRACT + elif refers_to_fullname(d, "functools.cached_property"): + dec.var.is_settable_property = True + self.check_decorated_function_is_method("property", dec) + elif refers_to_fullname(d, "typing.no_type_check"): + dec.var.type = AnyType(TypeOfAny.special_form) + no_type_check = True + elif refers_to_fullname(d, FINAL_DECORATOR_NAMES): + if self.is_class_scope(): + assert self.type is not None, "No type set at class scope" + if self.type.is_protocol: + self.msg.protocol_members_cant_be_final(d) + else: + dec.func.is_final = True + dec.var.is_final = True + removed.append(i) + else: + self.fail("@final cannot be used with non-method functions", d) + elif refers_to_fullname(d, TYPE_CHECK_ONLY_NAMES): + # TODO: support `@overload` funcs. + dec.func.is_type_check_only = True + elif isinstance(d, CallExpr) and refers_to_fullname( + d.callee, DATACLASS_TRANSFORM_NAMES + ): + dec.func.dataclass_transform_spec = self.parse_dataclass_transform_spec(d) + elif (deprecated := self.get_deprecated(d)) is not None: + dec.func.deprecated = f"function {dec.fullname} is deprecated: {deprecated}" + elif not dec.var.is_property: + # We have seen a "non-trivial" decorator before seeing @property, if + # we will see a @property later, give an error, as we don't support this. + could_be_decorated_property = True + for i in reversed(removed): + del dec.decorators[i] + if (not dec.is_overload or dec.var.is_property) and self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + if no_type_check: + erase_func_annotations(dec.func) + if not no_type_check and (self.recurse_into_functions or dec.func.def_or_infer_vars): + with self.set_recurse_into_functions(): + dec.func.accept(self) + if could_be_decorated_property and dec.decorators and dec.var.is_property: + self.fail( + "Decorators on top of @property are not supported", dec, code=PROPERTY_DECORATOR + ) + if (dec.func.is_static or dec.func.is_class) and dec.var.is_property: + self.fail("Only instance methods can be decorated with @property", dec) + if dec.func.abstract_status == IS_ABSTRACT and dec.func.is_final: + self.fail(f"Method {dec.func.name} is both abstract and final", dec) + if dec.func.is_static and dec.func.is_class: + self.fail(message_registry.CLASS_PATTERN_CLASS_OR_STATIC_METHOD, dec) + + def check_decorated_function_is_method(self, decorator: str, context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail(f'"{decorator}" used with a non-method', context) + + # + # Classes + # + + def visit_class_def(self, defn: ClassDef) -> None: + self.statement = defn + self.incomplete_type_stack.append(not defn.info) + namespace = self.qualified_name(defn.name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + if self.push_type_args(defn.type_args, defn) is None: + self.mark_incomplete(defn.name, defn) + return + self.removed_type_vars.append([]) + self.analyze_class(defn) + self.removed_type_vars.pop() + self.pop_type_args(defn.type_args) + self.incomplete_type_stack.pop() + + def push_type_args( + self, type_args: list[TypeParam] | None, context: Context + ) -> list[tuple[str, TypeVarLikeExpr]] | None: + if not type_args: + return [] + self.locals.append(SymbolTable()) + self.scope_stack.append(SCOPE_ANNOTATION) + tvs: list[tuple[str, TypeVarLikeExpr]] = [] + for p in type_args: + tv = self.analyze_type_param(p, context) + if tv is None: + return None + tvs.append((p.name, tv)) + + if self.is_defined_type_param(p.name): + self.fail(f'"{p.name}" already defined as a type parameter', context) + else: + assert self.add_symbol( + p.name, tv, context, no_progress=True, type_param=True + ), "Type parameter should not be discarded" + + return tvs + + def is_defined_type_param(self, name: str) -> bool: + for names in self.locals: + if names is None: + continue + if name in names: + node = names[name].node + if isinstance(node, TypeVarLikeExpr): + return True + return False + + def analyze_type_param( + self, type_param: TypeParam, context: Context + ) -> TypeVarLikeExpr | None: + fullname = self.qualified_name(type_param.name) + if type_param.upper_bound: + upper_bound = self.anal_type(type_param.upper_bound, allow_placeholder=True) + # TODO: we should validate the upper bound is valid for a given kind. + if upper_bound is None: + # This and below copies special-casing for old-style type variables, that + # is equally necessary for new-style classes to break a vicious circle. + upper_bound = PlaceholderType(None, [], context.line) + else: + if type_param.kind == TYPE_VAR_TUPLE_KIND: + upper_bound = self.named_type("builtins.tuple", [self.object_type()]) + else: + upper_bound = self.object_type() + # Reset fixed types both before and after each collection just in case. + if self.types_fixed is not None: + self.types_fixed.clear() + if type_param.default: + default = self.anal_type( + type_param.default, + allow_placeholder=True, + allow_unbound_tvars=True, + report_invalid_types=False, + allow_param_spec_literals=type_param.kind == PARAM_SPEC_KIND, + allow_tuple_literal=type_param.kind == PARAM_SPEC_KIND, + allow_unpack=type_param.kind == TYPE_VAR_TUPLE_KIND, + analyzing_tvar_def=True, + ) + if default is None: + default = PlaceholderType(None, [], context.line) + elif type_param.kind == TYPE_VAR_KIND: + default = self.check_typevar_default(default, type_param.default) + elif type_param.kind == PARAM_SPEC_KIND: + default = self.check_paramspec_default(default, type_param.default) + elif type_param.kind == TYPE_VAR_TUPLE_KIND: + default = self.check_typevartuple_default(default, type_param.default) + else: + default = AnyType(TypeOfAny.from_omitted_generics) + default_depends = self.types_fixed + self.types_fixed = None + if type_param.kind == TYPE_VAR_KIND: + values: list[Type] = [] + if type_param.values: + for value in type_param.values: + analyzed = self.anal_type(value, allow_placeholder=True) + if analyzed is None: + analyzed = PlaceholderType(None, [], context.line) + if has_type_vars(analyzed): + self.fail(message_registry.TYPE_VAR_GENERIC_CONSTRAINT_TYPE, context) + values.append(AnyType(TypeOfAny.from_error)) + else: + values.append(analyzed) + tv = TypeVarExpr( + name=type_param.name, + fullname=fullname, + values=values, + upper_bound=upper_bound, + default=default, + variance=VARIANCE_NOT_READY, + is_new_style=True, + line=context.line, + ) + elif type_param.kind == PARAM_SPEC_KIND: + tv = ParamSpecExpr( + name=type_param.name, + fullname=fullname, + upper_bound=upper_bound, + default=default, + is_new_style=True, + line=context.line, + ) + else: + assert type_param.kind == TYPE_VAR_TUPLE_KIND + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + tv = TypeVarTupleExpr( + name=type_param.name, + fullname=fullname, + upper_bound=upper_bound, + tuple_fallback=tuple_fallback, + default=default, + is_new_style=True, + line=context.line, + ) + tv.default_depends = default_depends + return tv + + def pop_type_args(self, type_args: list[TypeParam] | None) -> None: + if not type_args: + return + self.locals.pop() + self.scope_stack.pop() + + def analyze_class(self, defn: ClassDef) -> None: + fullname = self.qualified_name(defn.name) + if not defn.info and not self.is_core_builtin_class(defn): + # Add placeholder so that self-references in base classes can be + # resolved. We don't want this to cause a deferral, since if there + # are no incomplete references, we'll replace this with a TypeInfo + # before returning. + placeholder = PlaceholderNode(fullname, defn, defn.line, becomes_typeinfo=True) + self.add_symbol(defn.name, placeholder, defn, can_defer=False) + + tag = self.track_incomplete_refs() + + # Restore base classes after previous iteration (things like Generic[T] might be removed). + defn.base_type_exprs.extend(defn.removed_base_type_exprs) + defn.removed_base_type_exprs.clear() + + self.infer_metaclass_and_bases_from_compat_helpers(defn) + + bases = defn.base_type_exprs + bases, tvar_defs, is_protocol, declared_tvars = ( + self.clean_up_bases_and_infer_type_variables(defn, bases, context=defn) + ) + + self.check_type_alias_bases(bases) + default_depends: dict[str, set[TypeAlias | TypeInfo]] = {} + for _, tv in declared_tvars: + if tv.default_depends is not None: + default_depends[tv.fullname] = tv.default_depends + + if any(has_placeholder(tvd) for tvd in tvar_defs): + # Some type variable bounds or values are not ready, we need to + # re-analyze this class. Note we force progress to handle cases like + # class C[T = C], this matches logic in process_typevar_parameters() + # for "old style" type variables. + self.defer(force_progress=tvar_defs != defn.type_vars) + + self.analyze_class_keywords(defn) + bases_result = self.analyze_base_classes(defn.name, bases) + if bases_result is None or self.found_incomplete_ref(tag): + # Something was incomplete. Defer current target. + self.mark_incomplete(defn.name, defn) + return + + base_types, base_error = bases_result + if any(isinstance(base, PlaceholderType) for base, _ in base_types): + # We need to know the TypeInfo of each base to construct the MRO. Placeholder types + # are okay in nested positions, since they can't affect the MRO. + self.mark_incomplete(defn.name, defn) + return + if any(has_placeholder(base) for base, _ in base_types): + # We need to manually call defer() in case a placeholder was brought by a + # type variable default, so that type analyzer didn't call it. + self.defer() + + declared_metaclass, should_defer, any_meta = self.get_declared_metaclass( + defn.name, defn.metaclass + ) + if should_defer or self.found_incomplete_ref(tag): + # Metaclass was not ready. Defer current target. + self.mark_incomplete(defn.name, defn) + return + + if self.analyze_typeddict_classdef(defn): + if defn.info: + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + defn.info.default_depends = default_depends + return + + if self.analyze_namedtuple_classdef(defn, tvar_defs): + if defn.info: + defn.info.default_depends = default_depends + return + + # Create TypeInfo for class now that base classes and the MRO can be calculated. + self.prepare_class_def(defn) + self.setup_type_vars(defn, tvar_defs) + defn.info.default_depends = default_depends + if base_error: + defn.info.fallback_to_any = True + if any_meta: + defn.info.meta_fallback_to_any = True + + with self.scope.class_scope(defn.info): + self.configure_base_classes(defn, base_types) + defn.info.is_protocol = is_protocol + self.recalculate_metaclass(defn, declared_metaclass) + defn.info.runtime_protocol = False + + if defn.type_args: + # PEP 695 type parameters are not in scope in class decorators, so + # temporarily disable type parameter namespace. + type_params_names = self.locals.pop() + self.scope_stack.pop() + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + if defn.type_args: + self.locals.append(type_params_names) + self.scope_stack.append(SCOPE_ANNOTATION) + + self.analyze_class_body_common(defn) + + def check_type_alias_bases(self, bases: list[Expression]) -> None: + for base in bases: + if isinstance(base, IndexExpr): + base = base.base + if ( + isinstance(base, RefExpr) + and isinstance(base.node, TypeAlias) + and base.node.python_3_12_type_alias + ): + self.fail( + 'Type alias defined using "type" statement not valid as base class', base + ) + + def setup_type_vars(self, defn: ClassDef, tvar_defs: list[TypeVarLikeType]) -> None: + seen_tvt = False + valid_tvar_defs = [] + for tv in tvar_defs: + if seen_tvt and isinstance(tv, TypeVarType) and tv.has_default(): + self.fail( + message_registry.NO_DEFAULT_AFTER_TYPEVAR_TUPLE, defn, code=codes.TYPE_VAR + ) + # Remove the ambiguous type variable, and record it, so that we can replace + # all its uses with Any. + self.removed_type_vars[-1].append(tv) + continue + if isinstance(tv, TypeVarTupleType): + seen_tvt = True + valid_tvar_defs.append(tv) + defn.type_vars = valid_tvar_defs + defn.info.type_vars = [] + # we want to make sure any additional logic in add_type_vars gets run + defn.info.add_type_vars() + + def setup_alias_type_vars(self, defn: ClassDef) -> None: + assert defn.info.special_alias is not None + defn.info.special_alias.alias_tvars = list(defn.type_vars) + # It is a bit unfortunate that we need to inline some logic from TypeAlias constructor, + # but it is required, since type variables may change during semantic analyzer passes. + for i, t in enumerate(defn.type_vars): + if isinstance(t, TypeVarTupleType): + defn.info.special_alias.tvar_tuple_index = i + target = defn.info.special_alias.target + assert isinstance(target, ProperType) + if isinstance(target, TypedDictType): + target.fallback.args = type_vars_as_args(defn.type_vars) + elif isinstance(target, TupleType): + target.partial_fallback.args = type_vars_as_args(defn.type_vars) + else: + assert False, f"Unexpected special alias type: {type(target)}" + + def is_core_builtin_class(self, defn: ClassDef) -> bool: + return self.cur_mod_id == "builtins" and defn.name in CORE_BUILTIN_CLASSES + + def analyze_class_body_common(self, defn: ClassDef) -> None: + """Parts of class body analysis that are common to all kinds of class defs.""" + self.enter_class(defn.info) + if any(b.self_type is not None for b in defn.info.mro): + self.setup_self_type() + defn.defs.accept(self) + self.apply_class_plugin_hooks(defn) + self.leave_class() + + def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: + if ( + defn.info + and defn.info.typeddict_type + and (defn.info.typeddict_data is None or defn.info.typeddict_data.ready) + ): + # Don't reprocess everything + is_typeddict = True + info = defn.info + else: + is_typeddict, info = self.typed_dict_analyzer.analyze_typeddict_classdef(defn) + if is_typeddict: + if info is None: + self.mark_incomplete(defn.name, defn) + else: + self.prepare_class_def(defn, info) + for decorator in defn.decorators: + decorator.accept(self) + if defn.info: + self.analyze_class_decorator_common(defn, decorator) + return True + return False + + def analyze_namedtuple_classdef( + self, defn: ClassDef, tvar_defs: list[TypeVarLikeType] + ) -> bool: + """Check if this class can define a named tuple.""" + if ( + defn.info + and defn.info.is_named_tuple + and defn.info.tuple_type + and not has_placeholder(defn.info.tuple_type) + ): + # Don't reprocess everything. We just need to process methods defined + # in the named tuple class body. + is_named_tuple = True + info: TypeInfo | None = defn.info + else: + is_named_tuple, info = self.named_tuple_analyzer.analyze_namedtuple_classdef( + defn, self.is_stub_file + ) + if is_named_tuple: + if info is None: + self.mark_incomplete(defn.name, defn) + else: + self.prepare_class_def(defn, info) + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + with self.scope.class_scope(defn.info): + for deco in defn.decorators: + deco.accept(self) + self.analyze_class_decorator_common(defn, deco) + with self.named_tuple_analyzer.save_namedtuple_body(info): + self.analyze_class_body_common(defn) + return True + return False + + def apply_class_plugin_hooks(self, defn: ClassDef) -> None: + """Apply a plugin hook that may infer a more precise definition for a class.""" + + for decorator in defn.decorators: + decorator_name = self.get_fullname_for_hook(decorator) + if decorator_name: + hook = self.plugin.get_class_decorator_hook(decorator_name) + # Special case: if the decorator is itself decorated with + # typing.dataclass_transform, apply the hook for the dataclasses plugin + # TODO: remove special casing here + if hook is None and find_dataclass_transform_spec(decorator): + hook = dataclasses_plugin.dataclass_tag_callback + if hook: + hook(ClassDefContext(defn, decorator, self)) + + if defn.metaclass: + metaclass_name = self.get_fullname_for_hook(defn.metaclass) + if metaclass_name: + hook = self.plugin.get_metaclass_hook(metaclass_name) + if hook: + hook(ClassDefContext(defn, defn.metaclass, self)) + + for base_expr in defn.base_type_exprs: + base_name = self.get_fullname_for_hook(base_expr) + if base_name: + hook = self.plugin.get_base_class_hook(base_name) + if hook: + hook(ClassDefContext(defn, base_expr, self)) + + # Check if the class definition itself triggers a dataclass transform (via a parent class/ + # metaclass) + spec = find_dataclass_transform_spec(defn) + if spec is not None: + dataclasses_plugin.add_dataclass_tag(defn.info) + + def get_fullname_for_hook(self, expr: Expression) -> str | None: + if isinstance(expr, CallExpr): + return self.get_fullname_for_hook(expr.callee) + elif isinstance(expr, IndexExpr): + return self.get_fullname_for_hook(expr.base) + elif isinstance(expr, RefExpr): + if expr.fullname: + return expr.fullname + # If we don't have a fullname look it up. This happens because base classes are + # analyzed in a different manner (see exprtotype.py) and therefore those AST + # nodes will not have full names. + sym = self.lookup_type_node(expr) + if sym: + return sym.fullname + return None + + def analyze_class_keywords(self, defn: ClassDef) -> None: + for value in defn.keywords.values(): + value.accept(self) + + def enter_class(self, info: TypeInfo) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.scope_stack.append(SCOPE_CLASS) + self.block_depth.append(-1) # The class body increments this to 0 + self.loop_depth.append(0) + self._type = info + self.missing_names.append(set()) + + def leave_class(self) -> None: + """Restore analyzer state.""" + self.block_depth.pop() + self.loop_depth.pop() + self.locals.pop() + self.scope_stack.pop() + self._type = self.type_stack.pop() + self.missing_names.pop() + + def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None: + decorator.accept(self) + self.analyze_class_decorator_common(defn, decorator) + if isinstance(decorator, RefExpr): + if decorator.fullname in RUNTIME_PROTOCOL_DECOS: + if defn.info.is_protocol: + defn.info.runtime_protocol = True + else: + self.fail("@runtime_checkable can only be used with protocol classes", defn) + elif isinstance(decorator, CallExpr) and refers_to_fullname( + decorator.callee, DATACLASS_TRANSFORM_NAMES + ): + defn.info.dataclass_transform_spec = self.parse_dataclass_transform_spec(decorator) + + def analyze_class_decorator_common(self, defn: ClassDef, decorator: Expression) -> None: + """Common method for applying class decorators. + + Called on regular classes, typeddicts, and namedtuples. + """ + info = defn.info + if refers_to_fullname(decorator, FINAL_DECORATOR_NAMES): + info.is_final = True + elif refers_to_fullname(decorator, DISJOINT_BASE_DECORATOR_NAMES): + if info.is_protocol: + self.fail("@disjoint_base cannot be used with protocol class", decorator) + elif info.typeddict_type is not None: + self.fail("@disjoint_base cannot be used with TypedDict", decorator) + else: + info.is_disjoint_base = True + elif refers_to_fullname(decorator, TYPE_CHECK_ONLY_NAMES): + info.is_type_check_only = True + elif (deprecated := self.get_deprecated(decorator)) is not None: + info.deprecated = f"class {defn.fullname} is deprecated: {deprecated}" + + def clean_up_bases_and_infer_type_variables( + self, defn: ClassDef, base_type_exprs: list[Expression], context: Context + ) -> tuple[list[Expression], list[TypeVarLikeType], bool, list[tuple[str, TypeVarLikeExpr]]]: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + + Returns a tuple: + (remaining base expressions, type variables, is protocol, type variable expressions). + """ + removed: list[int] = [] + declared_tvars: TypeVarLikeList = [] + is_protocol = False + has_type_var_tuple = False + if defn.type_args is not None: + for p in defn.type_args: + node = self.lookup(p.name, context) + assert node is not None + assert isinstance(node.node, TypeVarLikeExpr) + if isinstance(node.node, TypeVarTupleExpr): + if has_type_var_tuple: + self.fail("Can only use one type var tuple in a class def", context) + continue + has_type_var_tuple = True + declared_tvars.append((p.name, node.node)) + + for i, base_expr in enumerate(base_type_exprs): + if isinstance(base_expr, StarExpr): + base_expr.valid = True + self.analyze_type_expr(base_expr) + + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + result = self.analyze_class_typevar_declaration(base, has_type_var_tuple) + if result is not None: + tvars = result[0] + is_protocol |= result[1] + if declared_tvars: + if defn.type_args: + if is_protocol: + self.fail('No arguments expected for "Protocol" base class', context) + else: + self.fail("Generic[...] base class is redundant", context) + else: + self.fail( + "Only single Generic[...] or Protocol[...] can be in bases", context + ) + removed.append(i) + declared_tvars.extend(tvars) + if isinstance(base, UnboundType): + sym = self.lookup_qualified(base.name, base) + if sym is not None and sym.node is not None: + if sym.node.fullname in PROTOCOL_NAMES and i not in removed: + # also remove bare 'Protocol' bases + removed.append(i) + is_protocol = True + + all_tvars = self.get_all_bases_tvars(base_type_exprs, removed) + if declared_tvars: + if len(remove_dups(declared_tvars)) < len(declared_tvars) and not defn.type_args: + self.fail("Duplicate type variables in Generic[...] or Protocol[...]", context) + declared_tvars = remove_dups(declared_tvars) + if not set(all_tvars).issubset(set(declared_tvars)): + if defn.type_args: + undeclared = sorted(set(all_tvars) - set(declared_tvars)) + self.msg.type_parameters_should_be_declared( + [tv[0] for tv in undeclared], context + ) + else: + self.fail( + "If Generic[...] or Protocol[...] is present" + " it should list all type variables", + context, + ) + # In case of error, Generic tvars will go first + declared_tvars = remove_dups(declared_tvars + all_tvars) + else: + declared_tvars = all_tvars + for i in reversed(removed): + # We need to actually remove the base class expressions like Generic[T], + # mostly because otherwise they will create spurious dependencies in fine + # grained incremental mode. + defn.removed_base_type_exprs.append(defn.base_type_exprs[i]) + del base_type_exprs[i] + tvar_defs = self.tvar_defs_from_tvars(declared_tvars, context) + return base_type_exprs, tvar_defs, is_protocol, declared_tvars + + def analyze_class_typevar_declaration( + self, base: Type, has_type_var_tuple: bool + ) -> tuple[TypeVarLikeList, bool] | None: + """Analyze type variables declared using Generic[...] or Protocol[...]. + + Args: + base: Non-analyzed base class + + Return None if the base class does not declare type variables. Otherwise, + return the type variables. + """ + if not isinstance(base, UnboundType): + return None + unbound = base + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if ( + sym.node.fullname == "typing.Generic" + or sym.node.fullname in PROTOCOL_NAMES + and base.args + ): + is_proto = sym.node.fullname != "typing.Generic" + tvars: TypeVarLikeList = [] + for arg in unbound.args: + tag = self.track_incomplete_refs() + tvar = self.analyze_unbound_tvar(arg) + if tvar: + if isinstance(tvar[1], TypeVarTupleExpr): + if has_type_var_tuple: + self.fail("Can only use one type var tuple in a class def", base) + continue + has_type_var_tuple = True + tvars.append(tvar) + elif not self.found_incomplete_ref(tag): + self.fail("Free type variable expected in %s[...]" % sym.node.name, base) + return tvars, is_proto + return None + + def analyze_unbound_tvar(self, t: Type) -> tuple[str, TypeVarLikeExpr] | None: + if isinstance(t, UnpackType) and isinstance(t.type, UnboundType): + return self.analyze_unbound_tvar_impl(t.type, is_unpacked=True) + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym and sym.fullname in UNPACK_TYPE_NAMES: + if not t.args: + # Unpack used without arguments, e.g. `Protocol[Unpack]` + return None + inner_t = t.args[0] + if isinstance(inner_t, UnboundType): + return self.analyze_unbound_tvar_impl(inner_t, is_unpacked=True) + return None + return self.analyze_unbound_tvar_impl(t) + return None + + def analyze_unbound_tvar_impl( + self, t: UnboundType, is_unpacked: bool = False, is_typealias_param: bool = False + ) -> tuple[str, TypeVarLikeExpr] | None: + assert not is_unpacked or not is_typealias_param, "Mutually exclusive conditions" + sym = self.lookup_qualified(t.name, t) + if sym and isinstance(sym.node, PlaceholderNode): + self.record_incomplete_ref() + if not is_unpacked and sym and isinstance(sym.node, ParamSpecExpr): + if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): + # It's bound by our type variable scope + return None + return t.name, sym.node + if (is_unpacked or is_typealias_param) and sym and isinstance(sym.node, TypeVarTupleExpr): + if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): + # It's bound by our type variable scope + return None + return t.name, sym.node + if sym is None or not isinstance(sym.node, TypeVarExpr) or is_unpacked: + return None + elif sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): + # It's bound by our type variable scope + return None + else: + assert isinstance(sym.node, TypeVarExpr) + return t.name, sym.node + + def find_type_var_likes(self, t: Type) -> TypeVarLikeList: + visitor = FindTypeVarVisitor(self, self.tvar_scope) + t.accept(visitor) + return visitor.type_var_likes + + def get_all_bases_tvars( + self, base_type_exprs: list[Expression], removed: list[int] + ) -> TypeVarLikeList: + """Return all type variable references in bases.""" + tvars: TypeVarLikeList = [] + for i, base_expr in enumerate(base_type_exprs): + if i not in removed: + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + base_tvars = self.find_type_var_likes(base) + tvars.extend(base_tvars) + return remove_dups(tvars) + + def tvar_defs_from_tvars( + self, tvars: TypeVarLikeList, context: Context + ) -> list[TypeVarLikeType]: + tvar_defs: list[TypeVarLikeType] = [] + last_tvar_name_with_default: str | None = None + for name, tvar_expr in tvars: + # PEP-695 type variables that are redeclared in an inner scope are warned + # about elsewhere. + if not tvar_expr.is_new_style and not self.tvar_scope.allow_binding( + tvar_expr.fullname + ): + self.fail( + message_registry.TYPE_VAR_REDECLARED_IN_NESTED_CLASS.format(name), context + ) + tvar_def = self.tvar_scope.bind_new(name, tvar_expr, self.fail, context) + # Fix any residual UnboundTypes in the generated TypeVarLike, keep + # TypeVarLikeExpr untouched as it may be shared by multiple classes. + tvar_def.default = tvar_def.default.accept( + TypeVarDefaultTranslator(self, tvar_expr.name, context) + ) + if last_tvar_name_with_default is not None and not tvar_def.has_default(): + self.msg.tvar_without_default_type( + tvar_def.name, last_tvar_name_with_default, context + ) + tvar_def.default = AnyType(TypeOfAny.from_error) + elif tvar_def.has_default(): + last_tvar_name_with_default = tvar_def.name + tvar_defs.append(tvar_def) + return tvar_defs + + def get_and_bind_all_tvars(self, type_exprs: list[Expression]) -> list[TypeVarLikeType]: + """Return all type variable references in item type expressions. + + This is a helper for generic TypedDicts and NamedTuples. Essentially it is + a simplified version of the logic we use for ClassDef bases. We duplicate + some amount of code, because it is hard to refactor common pieces. + """ + tvars: dict[str, tuple[TypeVarLikeExpr, Expression]] = {} + for base_expr in type_exprs: + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + for name, expr in self.find_type_var_likes(base): + tvars.setdefault(name, (expr, base_expr)) + tvar_defs = [] + for name, (tvar_expr, context) in tvars.items(): + tvar_def = self.tvar_scope.bind_new(name, tvar_expr, self.fail, context) + tvar_defs.append(tvar_def) + return tvar_defs + + def class_fullname(self, name: str, line: int) -> str: + if not self.is_nested_within_func_scope(): + return self.qualified_name(name) + name = func_scoped_name(name, line) + return f"{self.cur_mod_id}.{name}" + + def prepare_class_def(self, defn: ClassDef, info: TypeInfo | None = None) -> None: + """Prepare for the analysis of a class definition. + + Create an empty TypeInfo and store it in a symbol table, or if the 'info' + argument is provided, store it instead (used for magic type definitions). + """ + if not defn.info: + defn.fullname = self.class_fullname(defn.name, defn.line) + info = info or self.make_empty_type_info(defn) + defn.info = info + info.defn = defn + self.add_symbol(defn.name, defn.info, defn) + if self.is_nested_within_func_scope(): + self.add_global_symbol(defn.name, defn, defn.info) + + def make_empty_type_info(self, defn: ClassDef) -> TypeInfo: + if ( + self.is_module_scope() + and self.cur_mod_id == "builtins" + and defn.name in CORE_BUILTIN_CLASSES + ): + # Special case core built-in classes. A TypeInfo was already + # created for it before semantic analysis, but with a dummy + # ClassDef. Patch the real ClassDef object. + info = self.globals[defn.name].node + assert isinstance(info, TypeInfo) + else: + info = TypeInfo(SymbolTable(), defn, self.cur_mod_id) + info.set_line(defn) + return info + + def get_name_repr_of_expr(self, expr: Expression) -> str | None: + """Try finding a short simplified textual representation of a base class expression.""" + if isinstance(expr, NameExpr): + return expr.name + if isinstance(expr, MemberExpr): + return get_member_expr_fullname(expr) + if isinstance(expr, IndexExpr): + return self.get_name_repr_of_expr(expr.base) + if isinstance(expr, CallExpr): + return self.get_name_repr_of_expr(expr.callee) + return None + + def analyze_base_classes( + self, cls_name: str, base_type_exprs: list[Expression] + ) -> tuple[list[tuple[ProperType, Expression]], bool] | None: + """Analyze base class types. + + Return None if some definition was incomplete. Otherwise, return a tuple + with these items: + + * List of (analyzed type, original expression) tuples + * Boolean indicating whether one of the bases had a semantic analysis error + """ + is_error = False + bases = [] + for i, base_expr in enumerate(base_type_exprs): + if ( + isinstance(base_expr, RefExpr) + and base_expr.fullname in TYPED_NAMEDTUPLE_NAMES + TPDICT_NAMES + ) or ( + isinstance(base_expr, CallExpr) + and isinstance(base_expr.callee, RefExpr) + and base_expr.callee.fullname in TPDICT_NAMES + ): + # Ignore magic bases for now. + # For example: + # class Foo(TypedDict): ... # RefExpr + # class Foo(NamedTuple): ... # RefExpr + # class Foo(TypedDict("Foo", {"a": int})): ... # CallExpr + continue + + try: + base = self.expr_to_analyzed_type( + base_expr, + allow_placeholder=True, + allow_type_any=True, + unique_name=inline_base(cls_name, i), + ) + except TypeTranslationError: + name = self.get_name_repr_of_expr(base_expr) + if isinstance(base_expr, CallExpr): + msg = "Unsupported dynamic base class" + else: + msg = "Invalid base class" + if name: + msg += f' "{name}"' + self.fail(msg, base_expr) + is_error = True + continue + if base is None: + return None + base = get_proper_type(base) + bases.append((base, base_expr)) + return bases, is_error + + def configure_base_classes( + self, defn: ClassDef, bases: list[tuple[ProperType, Expression]] + ) -> None: + """Set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + base_types: list[Instance] = [] + info = defn.info + + for base, base_expr in bases: + if isinstance(base, TupleType): + actual_base = self.configure_tuple_base_class(defn, base) + base_types.append(actual_base) + elif isinstance(base, Instance): + if base.type.is_newtype: + self.fail('Cannot subclass "NewType"', defn) + base_types.append(base) + elif isinstance(base, AnyType): + if self.options.disallow_subclassing_any: + if isinstance(base_expr, (NameExpr, MemberExpr)): + msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' + else: + msg = 'Class cannot subclass value of type "Any"' + self.fail(msg, base_expr) + info.fallback_to_any = True + elif isinstance(base, TypedDictType): + base_types.append(base.fallback) + else: + msg = "Invalid base class" + name = self.get_name_repr_of_expr(base_expr) + if name: + msg += f' "{name}"' + self.fail(msg, base_expr) + info.fallback_to_any = True + if self.options.disallow_any_unimported and has_any_from_unimported_type(base): + if isinstance(base_expr, (NameExpr, MemberExpr)): + prefix = f"Base type {base_expr.name}" + else: + prefix = "Base type" + self.msg.unimported_type_becomes_any(prefix, base, base_expr) + check_for_explicit_any( + base, self.options, self.is_typeshed_stub_file, self.msg, context=base_expr + ) + + # Add 'object' as implicit base if there is no other base class. + if not base_types and defn.fullname != "builtins.object": + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. + if not self.verify_base_classes(defn): + self.set_dummy_mro(defn.info) + return + if not self.verify_duplicate_base_classes(defn): + # We don't want to block the typechecking process, + # so, we just insert `Any` as the base class and show an error. + self.set_any_mro(defn.info) + self.calculate_class_mro(defn, self.object_type) + + def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instance: + info = defn.info + + # There may be an existing valid tuple type from previous semanal iterations. + # Use equality to check if it is the case. + if info.tuple_type and info.tuple_type != base and not has_placeholder(info.tuple_type): + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.has_incompatible_baseclass = True + if has_placeholder(base): + self.process_placeholder( + None, "tuple base", defn, force_progress=base != info.tuple_type + ) + info.update_tuple_type(base) + self.setup_alias_type_vars(defn) + + if base.partial_fallback.type.fullname == "builtins.tuple" and not has_placeholder(base): + # Fallback can only be safely calculated after semantic analysis, since base + # classes may be incomplete. Postpone the calculation. + self.schedule_patch(PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(base)) + + return base.partial_fallback + + def set_dummy_mro(self, info: TypeInfo) -> None: + # Give it an MRO consisting of just the class itself and object. + info.mro = [info, self.object_type().type] + info.bad_mro = True + + def set_any_mro(self, info: TypeInfo) -> None: + # Give it an MRO consisting direct `Any` subclass. + info.fallback_to_any = True + info.mro = [info, self.object_type().type] + + def calculate_class_mro( + self, defn: ClassDef, obj_type: Callable[[], Instance] | None = None + ) -> None: + """Calculate method resolution order for a class. + + `obj_type` exists just to fill in empty base class list in case of an error. + """ + try: + calculate_mro(defn.info, obj_type) + except MroError: + self.fail( + f'Cannot determine consistent method resolution order (MRO) for "{defn.name}"', + defn, + ) + self.set_dummy_mro(defn.info) + # Allow plugins to alter the MRO to handle the fact that `def mro()` + # on metaclasses permits MRO rewriting. + if defn.fullname: + hook = self.plugin.get_customize_class_mro_hook(defn.fullname) + if hook: + hook(ClassDefContext(defn, FakeExpression(), self)) + + def infer_metaclass_and_bases_from_compat_helpers(self, defn: ClassDef) -> None: + """Lookup for special metaclass declarations, and update defn fields accordingly. + + * six.with_metaclass(M, B1, B2, ...) + * @six.add_metaclass(M) + * future.utils.with_metaclass(M, B1, B2, ...) + * past.utils.with_metaclass(M, B1, B2, ...) + """ + + # Look for six.with_metaclass(M, B1, B2, ...) + with_meta_expr: Expression | None = None + if len(defn.base_type_exprs) == 1: + base_expr = defn.base_type_exprs[0] + if isinstance(base_expr, CallExpr) and isinstance(base_expr.callee, RefExpr): + self.analyze_type_expr(base_expr) + if ( + base_expr.callee.fullname + in { + "six.with_metaclass", + "future.utils.with_metaclass", + "past.utils.with_metaclass", + } + and len(base_expr.args) >= 1 + and all(kind == ARG_POS for kind in base_expr.arg_kinds) + ): + with_meta_expr = base_expr.args[0] + defn.base_type_exprs = base_expr.args[1:] + + # Look for @six.add_metaclass(M) + add_meta_expr: Expression | None = None + for dec_expr in defn.decorators: + if isinstance(dec_expr, CallExpr) and isinstance(dec_expr.callee, RefExpr): + dec_expr.callee.accept(self) + if ( + dec_expr.callee.fullname == "six.add_metaclass" + and len(dec_expr.args) == 1 + and dec_expr.arg_kinds[0] == ARG_POS + ): + add_meta_expr = dec_expr.args[0] + break + + metas = {defn.metaclass, with_meta_expr, add_meta_expr} - {None} + if len(metas) == 0: + return + if len(metas) > 1: + self.fail("Multiple metaclass definitions", defn, code=codes.METACLASS) + return + defn.metaclass = metas.pop() + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + cycle = False + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail("Cycle in inheritance hierarchy", defn) + cycle = True + return not cycle + + def verify_duplicate_base_classes(self, defn: ClassDef) -> bool: + dup = find_duplicate(defn.info.direct_base_classes()) + if dup: + self.fail(f'Duplicate base class "{dup.name}"', defn) + return not dup + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def get_declared_metaclass( + self, name: str, metaclass_expr: Expression | None + ) -> tuple[Instance | None, bool, bool]: + """Get declared metaclass from metaclass expression. + + Returns a tuple of three values: + * A metaclass instance or None + * A boolean indicating whether we should defer + * A boolean indicating whether we should set metaclass Any fallback + (either for Any metaclass or invalid/dynamic metaclass). + + The two boolean flags can only be True if instance is None. + """ + declared_metaclass = None + if metaclass_expr: + metaclass_name = None + if isinstance(metaclass_expr, NameExpr): + metaclass_name = metaclass_expr.name + elif isinstance(metaclass_expr, MemberExpr): + metaclass_name = get_member_expr_fullname(metaclass_expr) + if metaclass_name is None: + self.fail( + f'Dynamic metaclass not supported for "{name}"', + metaclass_expr, + code=codes.METACLASS, + ) + return None, False, True + sym = self.lookup_qualified(metaclass_name, metaclass_expr) + if sym is None: + # Probably a name error - it is already handled elsewhere + return None, False, True + if isinstance(sym.node, Var) and isinstance(get_proper_type(sym.node.type), AnyType): + if self.options.disallow_subclassing_any: + self.fail( + f'Class cannot use "{sym.node.name}" as a metaclass (has type "Any")', + metaclass_expr, + code=codes.METACLASS, + ) + return None, False, True + if isinstance(sym.node, PlaceholderNode): + return None, True, False # defer later in the caller + + # Support type aliases, like `_Meta: TypeAlias = type` + metaclass_info: Node | None = sym.node + if ( + isinstance(sym.node, TypeAlias) + and not sym.node.python_3_12_type_alias + and not sym.node.alias_tvars + ): + target = get_proper_type(sym.node.target) + if isinstance(target, Instance): + metaclass_info = target.type + + if not isinstance(metaclass_info, TypeInfo) or metaclass_info.tuple_type is not None: + self.fail( + f'Invalid metaclass "{metaclass_name}"', metaclass_expr, code=codes.METACLASS + ) + return None, False, False + if not metaclass_info.is_metaclass(): + self.fail( + 'Metaclasses not inheriting from "type" are not supported', + metaclass_expr, + code=codes.METACLASS, + ) + return None, False, False + inst = fill_typevars(metaclass_info) + assert isinstance(inst, Instance) + declared_metaclass = inst + return declared_metaclass, False, False + + def recalculate_metaclass(self, defn: ClassDef, declared_metaclass: Instance | None) -> None: + defn.info.declared_metaclass = declared_metaclass + defn.info.metaclass_type = defn.info.calculate_metaclass_type() + if any(info.is_protocol for info in defn.info.mro): + if ( + not defn.info.metaclass_type + or defn.info.metaclass_type.type.fullname == "builtins.type" + ): + # All protocols and their subclasses have ABCMeta metaclass by default. + # TODO: add a metaclass conflict check if there is another metaclass. + abc_meta = self.named_type_or_none("abc.ABCMeta", []) + if abc_meta is not None: # May be None in tests with incomplete lib-stub. + defn.info.metaclass_type = abc_meta + if defn.info.metaclass_type and defn.info.metaclass_type.type.has_base("enum.EnumMeta"): + defn.info.is_enum = True + if defn.type_vars: + self.fail("Enum class cannot be generic", defn) + + # + # Imports + # + + def visit_import(self, i: Import) -> None: + self.statement = i + for id, as_id in i.ids: + # Modules imported in a stub file without using 'import X as X' won't get exported + # When implicit re-exporting is disabled, we have the same behavior as stubs. + use_implicit_reexport = not self.is_stub_file and self.options.implicit_reexport + if as_id is not None: + base_id = id + imported_id = as_id + module_public = use_implicit_reexport or id == as_id + else: + base_id = id.split(".")[0] + imported_id = base_id + module_public = use_implicit_reexport + + if base_id in self.modules: + node = self.modules[base_id] + if self.is_func_scope(): + kind = LDEF + elif self.type is not None: + kind = MDEF + else: + kind = GDEF + symbol = SymbolTableNode( + kind, node, module_public=module_public, module_hidden=not module_public + ) + self.add_imported_symbol( + imported_id, + symbol, + context=i, + module_public=module_public, + module_hidden=not module_public, + ) + else: + self.add_unknown_imported_symbol( + imported_id, + context=i, + target_name=base_id, + module_public=module_public, + module_hidden=not module_public, + ) + + def visit_import_from(self, imp: ImportFrom) -> None: + self.statement = imp + module_id = self.correct_relative_import(imp) + module = self.modules.get(module_id) + for id, as_id in imp.names: + fullname = module_id + "." + id + self.set_future_import_flags(fullname) + if module is None: + node = None + elif module_id == self.cur_mod_id and fullname in self.modules: + # Submodule takes precedence over definition in surround package, for + # compatibility with runtime semantics in typical use cases. This + # could more precisely model runtime semantics by taking into account + # the line number beyond which the local definition should take + # precedence, but doesn't seem to be important in most use cases. + node = SymbolTableNode(GDEF, self.modules[fullname]) + else: + if id == as_id == "__all__": + # For modules with __all__ public status of symbols is determined uniquely + # by contents of __all__, so we can recover the latter here, and avoid + # serializing this (redundant) information in MypyFile. + self.all_exports[:] = [ + name for name, sym in module.names.items() if sym.module_public + ] + node = module.names.get(id) + + missing_submodule = False + imported_id = as_id or id + + # Modules imported in a stub file without using 'from Y import X as X' will + # not get exported. + # When implicit re-exporting is disabled, we have the same behavior as stubs. + use_implicit_reexport = not self.is_stub_file and self.options.implicit_reexport + module_public = use_implicit_reexport or (as_id is not None and id == as_id) + + # If the module does not contain a symbol with the name 'id', + # try checking if it's a module instead. + if not node: + mod = self.modules.get(fullname) + if mod is not None: + kind = self.current_symbol_kind() + node = SymbolTableNode(kind, mod) + elif fullname in self.missing_modules: + missing_submodule = True + # If it is still not resolved, check for a module level __getattr__ + if module and not node and "__getattr__" in module.names: + # We store the fullname of the original definition so that we can + # detect whether two imported names refer to the same thing. + fullname = module_id + "." + id + gvar = self.create_getattr_var(module.names["__getattr__"], imported_id, fullname) + if gvar: + self.add_symbol( + imported_id, + gvar, + imp, + module_public=module_public, + module_hidden=not module_public, + ) + continue + + if node: + self.process_imported_symbol( + node, module_id, id, imported_id, fullname, module_public, context=imp + ) + if node.module_hidden: + self.report_missing_module_attribute( + module_id, + id, + imported_id, + module_public=module_public, + module_hidden=not module_public, + context=imp, + add_unknown_imported_symbol=False, + ) + elif module and not missing_submodule: + # Target module exists but the imported name is missing or hidden. + self.report_missing_module_attribute( + module_id, + id, + imported_id, + module_public=module_public, + module_hidden=not module_public, + context=imp, + ) + else: + # Import of a missing (sub)module. + self.add_unknown_imported_symbol( + imported_id, + imp, + target_name=fullname, + module_public=module_public, + module_hidden=not module_public, + ) + + def process_imported_symbol( + self, + node: SymbolTableNode, + module_id: str, + id: str, + imported_id: str, + fullname: str, + module_public: bool, + context: ImportBase, + ) -> None: + module_hidden = not module_public and ( + # `from package import submodule` should work regardless of whether package + # re-exports submodule, so we shouldn't hide it + not isinstance(node.node, MypyFile) + or fullname not in self.modules + # but given `from somewhere import random_unrelated_module` we should hide + # random_unrelated_module + or not fullname.startswith(self.cur_mod_id + ".") + ) + + if isinstance(node.node, PlaceholderNode): + if self.final_iteration: + self.report_missing_module_attribute( + module_id, + id, + imported_id, + module_public=module_public, + module_hidden=module_hidden, + context=context, + ) + return + else: + # This might become a type. + self.mark_incomplete( + imported_id, + node.node, + module_public=module_public, + module_hidden=module_hidden, + becomes_typeinfo=True, + ) + # NOTE: we take the original node even for final `Var`s. This is to support + # a common pattern when constants are re-exported (same applies to import *). + self.add_imported_symbol( + imported_id, node, context, module_public=module_public, module_hidden=module_hidden + ) + + def report_missing_module_attribute( + self, + import_id: str, + source_id: str, + imported_id: str, + module_public: bool, + module_hidden: bool, + context: Node, + add_unknown_imported_symbol: bool = True, + ) -> None: + # Missing attribute. + if self.is_incomplete_namespace(import_id): + # We don't know whether the name will be there, since the namespace + # is incomplete. Defer the current target. + self.mark_incomplete( + imported_id, context, module_public=module_public, module_hidden=module_hidden + ) + return + message = f'Module "{import_id}" has no attribute "{source_id}"' + # Suggest alternatives, if any match is found. + module = self.modules.get(import_id) + if module: + if source_id in module.names.keys() and not module.names[source_id].module_public: + message = ( + f'Module "{import_id}" does not explicitly export attribute "{source_id}"' + ) + elif not ( + self.options.ignore_errors + or self.cur_mod_node.path in self.errors.ignored_files + or self.errors.prefer_simple_messages() + ): + alternatives = set(module.names.keys()).difference({source_id}) + matches = best_matches(source_id, alternatives, n=3) + if matches: + suggestion = f"; maybe {pretty_seq(matches, 'or')}?" + message += f"{suggestion}" + self.fail(message, context, code=codes.ATTR_DEFINED) + if add_unknown_imported_symbol: + self.add_unknown_imported_symbol( + imported_id, + context, + target_name=None, + module_public=module_public, + module_hidden=not module_public, + ) + + if import_id == "typing": + # The user probably has a missing definition in a test fixture. Let's verify. + fullname = f"builtins.{source_id.lower()}" + if ( + self.lookup_fully_qualified_or_none(fullname) is None + and fullname in SUGGESTED_TEST_FIXTURES + ): + # Yes. Generate a helpful note. + self.msg.add_fixture_note(fullname, context) + else: + typing_extensions = self.modules.get("typing_extensions") + if typing_extensions and source_id in typing_extensions.names: + self.msg.note( + f"Use `from typing_extensions import {source_id}` instead", + context, + code=codes.ATTR_DEFINED, + ) + self.msg.note( + "See https://mypy.readthedocs.io/en/stable/runtime_troubles.html#using-new-additions-to-the-typing-module", + context, + code=codes.ATTR_DEFINED, + ) + + def process_import_over_existing_name( + self, + imported_id: str, + existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase, + ) -> bool: + if existing_symbol.node is module_symbol.node: + # We added this symbol on previous iteration. + return False + if existing_symbol.kind in (LDEF, GDEF, MDEF) and isinstance( + existing_symbol.node, (Var, FuncDef, TypeInfo, Decorator, TypeAlias) + ): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + if isinstance(rvalue.node, TypeAlias): + # Suppress bogus errors from the dummy assignment if rvalue is an alias. + # Otherwise mypy may complain that alias is invalid in runtime context. + rvalue.is_alias_rvalue = True + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def correct_relative_import(self, node: ImportFrom | ImportAll) -> str: + import_id, ok = correct_relative_import( + self.cur_mod_id, node.relative, node.id, self.cur_mod_node.is_package_init_file() + ) + if not ok: + self.fail("Relative import climbs too many namespaces", node) + return import_id + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + if self.is_incomplete_namespace(i_id): + # Any names could be missing from the current namespace if the target module + # namespace is incomplete. + self.mark_incomplete("*", i) + for name, node in m.names.items(): + if node.no_serialize: + # This is either internal or generated symbol, skip it to avoid problems + # like accidental name conflicts or invalid cross-references. + continue + fullname = i_id + "." + name + self.set_future_import_flags(fullname) + # if '__all__' exists, all nodes not included have had module_public set to + # False, and we can skip checking '_' because it's been explicitly included. + if node.module_public and (not name.startswith("_") or "__all__" in m.names): + if isinstance(node.node, MypyFile): + # Star import of submodule from a package, add it as a dependency. + self.imports.add(node.node.fullname) + # `from x import *` always reexports symbols + self.add_imported_symbol( + name, node, context=i, module_public=True, module_hidden=False + ) + # This is a (minimalist) copy of the logic in visit_import_from(), we need + # to clean-up any remaining placeholders by replacing them with Var(Any). + if isinstance(node.node, PlaceholderNode) and self.final_iteration: + self.add_unknown_imported_symbol( + name, i, target_name=None, module_public=True, module_hidden=False + ) + + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + # + # Assignment + # + + def visit_assignment_expr(self, s: AssignmentExpr) -> None: + s.value.accept(self) + if self.is_func_scope(): + if not self.check_valid_comprehension(s): + return + self.analyze_lvalue(s.target, escape_comprehensions=True, has_explicit_value=True) + + def check_valid_comprehension(self, s: AssignmentExpr) -> bool: + """Check that assignment expression is not nested within comprehension at class scope. + + class C: + [(j := i) for i in [1, 2, 3]] + is a syntax error that is not enforced by Python parser, but at later steps. + """ + for i, scope_type in enumerate(reversed(self.scope_stack)): + if scope_type != SCOPE_COMPREHENSION and i < len(self.locals) - 1: + if self.locals[-1 - i] is None: + self.fail( + "Assignment expression within a comprehension" + " cannot be used in a class body", + s, + code=codes.SYNTAX, + serious=True, + blocker=True, + ) + return False + break + return True + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.statement = s + + # Special case assignment like X = X. + if self.analyze_identity_global_assignment(s): + return + + tag = self.track_incomplete_refs() + + # Here we have a chicken and egg problem: at this stage we can't call + # can_be_type_alias(), because we have not enough information about rvalue. + # But we can't use a full visit because it may emit extra incomplete refs (namely + # when analysing any type applications there) thus preventing the further analysis. + # To break the tie, we first analyse rvalue partially, if it can be a type alias. + if self.can_possibly_be_type_form(s): + old_basic_type_applications = self.basic_type_applications + self.basic_type_applications = True + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) + self.basic_type_applications = old_basic_type_applications + elif self.can_possibly_be_typevarlike_declaration(s): + # Allow unbound tvars inside TypeVarLike defaults to be evaluated later + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) + else: + s.rvalue.accept(self) + + if self.found_incomplete_ref(tag) or self.should_wait_rhs(s.rvalue): + # Initializer couldn't be fully analyzed. Defer the current node and give up. + # Make sure that if we skip the definition of some local names, they can't be + # added later in this scope, since an earlier definition should take precedence. + for expr in names_modified_by_assignment(s): + self.mark_incomplete(expr.name, expr) + return + if self.can_possibly_be_type_form(s): + # Now re-visit those rvalues that were we skipped type applications above. + # This should be safe as generally semantic analyzer is idempotent. + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) + + # The r.h.s. is now ready to be classified, first check if it is a special form: + special_form = False + # * type alias + if self.check_and_set_up_type_alias(s): + s.is_alias_def = True + special_form = True + elif isinstance(s.rvalue, CallExpr): + # * type variable definition + if self.process_typevar_declaration(s): + special_form = True + elif self.process_paramspec_declaration(s): + special_form = True + elif self.process_typevartuple_declaration(s): + special_form = True + # * type constructors + elif self.analyze_namedtuple_assign(s): + special_form = True + elif self.analyze_typeddict_assign(s): + special_form = True + elif self.newtype_analyzer.process_newtype_declaration(s): + special_form = True + elif self.analyze_enum_assign(s): + special_form = True + + if special_form: + self.record_special_form_lvalue(s) + return + # Clear the alias flag if assignment turns out not a special form after all. It + # may be set to True while there were still placeholders due to forward refs. + s.is_alias_def = False + + sentinel_definition = self.is_sentinel_declaration(s) + + # OK, this is a regular assignment, perform the necessary analysis steps. + s.is_final_def = self.unwrap_final(s) + if sentinel_definition: + s.is_final_def = True + self.analyze_lvalues(s) + if sentinel_definition: + self.setup_sentinel_var(s) + self.check_final_implicit_def(s) + self.store_final_status(s) + self.check_classvar(s) + self.process_type_annotation(s) + self.analyze_rvalue_as_type_form(s) + self.apply_dynamic_class_hook(s) + if not s.type: + self.process_module_assignment(s.lvalues, s.rvalue, s) + self.process__all__(s) + self.process__deletable__(s) + self.process__slots__(s) + + def is_sentinel_declaration(self, s: AssignmentStmt) -> bool: + """Does this assignment define a PEP 661 sentinel singleton? + + This includes both the original `NAME = Sentinel("NAME")` call and a + plain alias of an existing sentinel, e.g. `ALIAS = NAME` or + `ALIAS = mod.NAME`, so that re-exports of a sentinel keep working as + the same sentinel type. + """ + if self.is_nested_within_func_scope() or s.unanalyzed_type is not None: + return False + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return False + if isinstance(s.rvalue, RefExpr): + return isinstance(s.rvalue.node, Var) and s.rvalue.node.is_sentinel + if not isinstance(s.rvalue, CallExpr): + return False + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return False + if call.callee.fullname not in SENTINEL_TYPE_NAMES: + return False + if not call.args or call.arg_kinds[0] != ARG_POS or not isinstance(call.args[0], StrExpr): + return False + return True + + def setup_sentinel_var(self, s: AssignmentStmt) -> None: + lvalue = s.lvalues[0] + assert isinstance(lvalue, NameExpr) + if not isinstance(lvalue.node, Var): + return + lvalue.is_special_form = True + var = lvalue.node + var.is_sentinel = True + if isinstance(s.rvalue, RefExpr): + # An alias of an existing sentinel: reuse its already-computed type. + assert isinstance(s.rvalue.node, Var) + typ = get_proper_type(s.rvalue.node.type) + assert isinstance(typ, LiteralType) + else: + typ = self.sentinel_type_for_var(var, s.rvalue) + if typ is not None: + s.type = typ + + def sentinel_type_for_var(self, var: Var, rvalue: Expression) -> LiteralType | None: + assert isinstance(rvalue, CallExpr) + callee = rvalue.callee + assert isinstance(callee, RefExpr) + typ = self.named_type_or_none(callee.fullname) + if typ is None: + return None + name = f"{self.type.name}.{var.name}" if self.type is not None else var.name + return LiteralType( + SentinelValue(var.fullname, name), fallback=typ, line=rvalue.line, column=rvalue.column + ) + + def analyze_identity_global_assignment(self, s: AssignmentStmt) -> bool: + """Special case 'X = X' in global scope. + + This allows supporting some important use cases. + + Return true if special casing was applied. + """ + if not isinstance(s.rvalue, NameExpr) or len(s.lvalues) != 1: + # Not of form 'X = X' + return False + lvalue = s.lvalues[0] + if not isinstance(lvalue, NameExpr) or s.rvalue.name != lvalue.name: + # Not of form 'X = X' + return False + if self.type is not None or self.is_func_scope(): + # Not in global scope + return False + # It's an assignment like 'X = X' in the global scope. + name = lvalue.name + sym = self.lookup(name, s) + if sym is None: + if self.final_iteration: + # Fall back to normal assignment analysis. + return False + else: + self.defer() + return True + else: + if sym.node is None: + # Something special -- fall back to normal assignment analysis. + return False + if name not in self.globals: + # The name is from builtins. Add an alias to the current module. + self.add_symbol(name, sym.node, s) + if not isinstance(sym.node, PlaceholderNode): + for node in s.rvalue, lvalue: + node.node = sym.node + node.kind = GDEF + node.fullname = sym.node.fullname + return True + + def should_wait_rhs(self, rv: Expression) -> bool: + """Can we already classify this r.h.s. of an assignment or should we wait? + + This returns True if we don't have enough information to decide whether + an assignment is just a normal variable definition or a special form. + Always return False if this is a final iteration. This will typically cause + the lvalue to be classified as a variable plus emit an error. + """ + if self.final_iteration: + # No chance, nothing has changed. + return False + if isinstance(rv, NameExpr): + n = self.lookup(rv.name, rv) + if n and isinstance(n.node, PlaceholderNode) and not n.node.becomes_typeinfo: + return True + elif isinstance(rv, MemberExpr): + fname = get_member_expr_fullname(rv) + if fname: + n = self.lookup_qualified(fname, rv, suppress_errors=True) + if n and isinstance(n.node, PlaceholderNode) and not n.node.becomes_typeinfo: + return True + elif isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr): + return self.should_wait_rhs(rv.base) + elif isinstance(rv, CallExpr) and isinstance(rv.callee, RefExpr): + # This is only relevant for builtin SCC where things like 'TypeVar' + # may be not ready. + return self.should_wait_rhs(rv.callee) + return False + + def can_be_type_alias(self, rv: Expression, allow_none: bool = False) -> bool: + """Is this a valid r.h.s. for an alias definition? + + Note: this function should be only called for expressions where self.should_wait_rhs() + returns False. + """ + if isinstance(rv, RefExpr) and self.is_type_ref(rv, bare=True): + return True + if isinstance(rv, IndexExpr) and self.is_type_ref(rv.base, bare=False): + return True + if self.is_none_alias(rv): + return True + if allow_none and isinstance(rv, NameExpr) and rv.fullname == "builtins.None": + return True + if isinstance(rv, OpExpr) and rv.op == "|": + if self.is_stub_file: + return True + if self.can_be_type_alias(rv.left, allow_none=True) and self.can_be_type_alias( + rv.right, allow_none=True + ): + return True + return False + + def can_possibly_be_type_form(self, s: AssignmentStmt) -> bool: + """Like can_be_type_alias(), but simpler and doesn't require fully analyzed rvalue. + + Instead, use lvalues/annotations structure to figure out whether this can potentially be + a type alias definition, NamedTuple, or TypedDict. Another difference from above function + is that we are only interested IndexExpr, CallExpr and OpExpr rvalues, since only those + can be potentially recursive (things like `A = A` are never valid). + """ + if len(s.lvalues) > 1: + return False + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.callee, RefExpr): + ref = s.rvalue.callee.fullname + return ref in TPDICT_NAMES or ref in TYPED_NAMEDTUPLE_NAMES + if not isinstance(s.lvalues[0], NameExpr): + return False + if s.unanalyzed_type is not None and not self.is_pep_613(s): + return False + if not isinstance(s.rvalue, (IndexExpr, OpExpr)): + return False + # Something that looks like Foo = Bar[Baz, ...] + return True + + def can_possibly_be_typevarlike_declaration(self, s: AssignmentStmt) -> bool: + """Check if r.h.s. can be a TypeVarLike declaration.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return False + if not isinstance(s.rvalue, CallExpr) or not isinstance(s.rvalue.callee, NameExpr): + return False + ref = s.rvalue.callee + ref.accept(self) + return ref.fullname in TYPE_VAR_LIKE_NAMES + + def is_type_ref(self, rv: Expression, bare: bool = False) -> bool: + """Does this expression refer to a type? + + This includes: + * Special forms, like Any or Union + * Classes (except subscripted enums) + * Other type aliases + * PlaceholderNodes with becomes_typeinfo=True (these can be not ready class + definitions, and not ready aliases). + + If bare is True, this is not a base of an index expression, so some special + forms are not valid (like a bare Union). + + Note: This method should be only used in context of a type alias definition. + This method can only return True for RefExprs, to check if C[int] is a valid + target for type alias call this method on expr.base (i.e. on C in C[int]). + See also can_be_type_alias(). + """ + if not isinstance(rv, RefExpr): + return False + if isinstance(rv.node, TypeVarLikeExpr): + self.fail(f'Type variable "{rv.fullname}" is invalid as target for type alias', rv) + return False + + if bare: + # These three are valid even if bare, for example + # A = Tuple is just equivalent to A = Tuple[Any, ...]. + valid_refs = {"typing.Any", "typing.Tuple", "typing.Callable"} + else: + valid_refs = type_constructors + + if isinstance(rv.node, TypeAlias) or rv.fullname in valid_refs: + return True + if isinstance(rv.node, TypeInfo): + if bare: + return True + # Assignment color = Color['RED'] defines a variable, not an alias. + return not rv.node.is_enum + if isinstance(rv.node, Var): + return rv.node.fullname in NEVER_NAMES + + if isinstance(rv, NameExpr): + n = self.lookup(rv.name, rv) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True + elif isinstance(rv, MemberExpr): + fname = get_member_expr_fullname(rv) + if fname: + # The r.h.s. for variable definitions may not be a type reference but just + # an instance attribute, so suppress the errors. + n = self.lookup_qualified(fname, rv, suppress_errors=True) + if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo: + return True + return False + + def is_none_alias(self, node: Expression) -> bool: + """Is this a r.h.s. for a None alias? + + We special case the assignments like Void = type(None), to allow using + Void in type annotations. + """ + if isinstance(node, CallExpr): + if ( + isinstance(node.callee, NameExpr) + and len(node.args) == 1 + and isinstance(node.args[0], NameExpr) + ): + call = self.lookup_qualified(node.callee.name, node.callee) + arg = self.lookup_qualified(node.args[0].name, node.args[0]) + if ( + call is not None + and call.node + and call.node.fullname == "builtins.type" + and arg is not None + and arg.node + and arg.node.fullname == "builtins.None" + ): + return True + return False + + def record_special_form_lvalue(self, s: AssignmentStmt) -> None: + """Record minimal necessary information about l.h.s. of a special form. + + This exists mostly for compatibility with the old semantic analyzer. + """ + lvalue = s.lvalues[0] + assert isinstance(lvalue, NameExpr) + lvalue.is_special_form = True + if self.current_symbol_kind() == GDEF: + lvalue.fullname = self.qualified_name(lvalue.name) + lvalue.kind = self.current_symbol_kind() + + def analyze_enum_assign(self, s: AssignmentStmt) -> bool: + """Check if s defines an Enum.""" + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, EnumCallExpr): + # This is an analyzed enum definition. + # It is valid iff it can be stored correctly, failures were already reported. + return self._is_single_name_assignment(s) + return self.enum_call_analyzer.process_enum_call(s) + + def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: + """Check if s defines a namedtuple.""" + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, NamedTupleExpr): + if s.rvalue.analyzed.info.tuple_type and not has_placeholder( + s.rvalue.analyzed.info.tuple_type + ): + # This is an analyzed named tuple definition. + # It is valid iff it can be stored correctly, failures were already reported. + return self._is_single_name_assignment(s) + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): + return False + lvalue = s.lvalues[0] + if isinstance(lvalue, MemberExpr): + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.callee, RefExpr): + fullname = s.rvalue.callee.fullname + if fullname == "collections.namedtuple" or fullname in TYPED_NAMEDTUPLE_NAMES: + self.fail("NamedTuple type as an attribute is not supported", lvalue) + return False + name = lvalue.name + namespace = self.qualified_name(name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( + s.rvalue, name + ) + if internal_name is None: + return False + if internal_name != name: + self.fail( + 'First argument to namedtuple() should be "{}", not "{}"'.format( + name, internal_name + ), + s.rvalue, + code=codes.NAME_MATCH, + ) + return True + # Yes, it's a valid namedtuple, but defer if it is not ready. + if not info: + self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + self.setup_type_vars(info.defn, tvar_defs) + self.setup_alias_type_vars(info.defn) + return True + + def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: + """Check if s defines a typed dict.""" + if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, TypedDictExpr): + if s.rvalue.analyzed.info.typeddict_type and not has_placeholder( + s.rvalue.analyzed.info.typeddict_type + ): + # This is an analyzed typed dict definition. + # It is valid iff it can be stored correctly, failures were already reported. + return self._is_single_name_assignment(s) + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): + return False + lvalue = s.lvalues[0] + name = lvalue.name + namespace = self.qualified_name(name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict( + s.rvalue, name + ) + if not is_typed_dict: + return False + if isinstance(lvalue, MemberExpr): + self.fail("TypedDict type as attribute is not supported", lvalue) + return False + # Yes, it's a valid typed dict, but defer if it is not ready. + if not info: + self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + defn = info.defn + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + return True + + def _is_single_name_assignment(self, s: AssignmentStmt) -> bool: + return len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) + + def analyze_lvalues(self, s: AssignmentStmt) -> None: + # We cannot use s.type, because analyze_simple_literal_type() will set it. + explicit = s.unanalyzed_type is not None + if self.is_final_type(s.unanalyzed_type): + # We need to exclude bare Final. + assert isinstance(s.unanalyzed_type, UnboundType) + if not s.unanalyzed_type.args: + explicit = False + + if s.rvalue: + if isinstance(s.rvalue, TempNode): + has_explicit_value = not s.rvalue.no_rhs + else: + has_explicit_value = True + else: + has_explicit_value = False + + for lval in s.lvalues: + self.analyze_lvalue( + lval, + explicit_type=explicit, + is_final=s.is_final_def, + has_explicit_value=has_explicit_value, + ) + + def analyze_rvalue_as_type_form(self, s: AssignmentStmt) -> None: + self.try_parse_as_type_expression(s.rvalue) + + def apply_dynamic_class_hook(self, s: AssignmentStmt) -> None: + if not isinstance(s.rvalue, CallExpr): + return + fname = "" + call = s.rvalue + while True: + if isinstance(call.callee, RefExpr): + fname = call.callee.fullname + # check if method call + if not fname and isinstance(call.callee, MemberExpr): + callee_expr = call.callee.expr + if isinstance(callee_expr, RefExpr) and callee_expr.fullname: + method_name = call.callee.name + fname = callee_expr.fullname + "." + method_name + elif ( + isinstance(callee_expr, IndexExpr) + and isinstance(callee_expr.base, RefExpr) + and isinstance(callee_expr.analyzed, TypeApplication) + ): + method_name = call.callee.name + fname = callee_expr.base.fullname + "." + method_name + elif isinstance(callee_expr, CallExpr): + # check if chain call + call = callee_expr + continue + break + if not fname: + return + hook = self.plugin.get_dynamic_class_hook(fname) + if not hook: + return + for lval in s.lvalues: + if not isinstance(lval, NameExpr): + continue + hook(DynamicClassDefContext(call, lval.name, self)) + + def unwrap_final(self, s: AssignmentStmt) -> bool: + """Strip Final[...] if present in an assignment. + + This is done to invoke type inference during type checking phase for this + assignment. Also, Final[...] doesn't affect type in any way -- it is rather an + access qualifier for given `Var`. + + Also perform various consistency checks. + + Returns True if Final[...] was present. + """ + if not s.unanalyzed_type or not self.is_final_type(s.unanalyzed_type): + return False + assert isinstance(s.unanalyzed_type, UnboundType) + if len(s.unanalyzed_type.args) > 1: + self.fail("Final[...] takes at most one type argument", s.unanalyzed_type) + invalid_bare_final = False + if not s.unanalyzed_type.args: + s.type = None + if ( + isinstance(s.rvalue, TempNode) + and s.rvalue.no_rhs + # Filter duplicate errors, we already reported this: + and not (self.type and self.type.is_named_tuple) + ): + invalid_bare_final = True + self.fail("Type in Final[...] can only be omitted if there is an initializer", s) + else: + s.type = s.unanalyzed_type.args[0] + + if ( + s.type is not None + and self.options.python_version < (3, 13) + and self.is_classvar(s.type) + ): + self.fail("Variable should not be annotated with both ClassVar and Final", s) + return False + + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], RefExpr): + self.fail("Invalid final declaration", s) + return False + lval = s.lvalues[0] + assert isinstance(lval, RefExpr) + + # Reset inferred status if it was set due to simple literal rvalue on previous iteration. + # TODO: this is a best-effort quick fix, we should avoid the need to manually sync this, + # see https://github.com/python/mypy/issues/6458. + if lval.is_new_def: + lval.is_inferred_def = s.type is None + + if self.loop_depth[-1] > 0: + self.fail("Cannot use Final inside a loop", s) + if self.type and self.type.is_protocol: + if self.is_class_scope(): + self.msg.protocol_members_cant_be_final(s) + if ( + isinstance(s.rvalue, TempNode) + and s.rvalue.no_rhs + and not self.is_stub_file + and not self.is_class_scope() + ): + if not invalid_bare_final: # Skip extra error messages. + self.msg.final_without_value(s) + return True + + def check_final_implicit_def(self, s: AssignmentStmt) -> None: + """Do basic checks for final declaration on self in __init__. + + Additional re-definition checks are performed by `analyze_lvalue`. + """ + if not s.is_final_def: + return + lval = s.lvalues[0] + assert isinstance(lval, RefExpr) + if isinstance(lval, MemberExpr): + if not self.is_self_member_ref(lval): + self.fail("Final can be only applied to a name or an attribute on self", s) + s.is_final_def = False + return + else: + assert self.function_stack + if self.function_stack[-1].name != "__init__": + self.fail("Can only declare a final attribute in class body or __init__", s) + s.is_final_def = False + return + + def store_final_status(self, s: AssignmentStmt) -> None: + """If this is a locally valid final declaration, set the corresponding flag on `Var`.""" + if s.is_final_def: + if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr): + node = s.lvalues[0].node + if isinstance(node, Var): + node.is_final = True + if s.type: + node.final_value = constant_fold_expr(s.rvalue, self.cur_mod_id) + if self.is_class_scope() and ( + isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs + ): + node.final_unset_in_class = True + else: + for lval in self.flatten_lvalues(s.lvalues): + # Special case: we are working with an `Enum`: + # + # class MyEnum(Enum): + # key = 'some value' + # + # Here `key` is implicitly final. In runtime, code like + # + # MyEnum.key = 'modified' + # + # will fail with `AttributeError: Cannot reassign members.` + # That's why we need to replicate this. + if ( + isinstance(lval, NameExpr) + and isinstance(self.type, TypeInfo) + and self.type.is_enum + ): + cur_node = self.type.names.get(lval.name, None) + if ( + cur_node + and isinstance(cur_node.node, Var) + and not (isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs) + ): + # Double underscored members are writable on an `Enum`. + # (Except read-only `__members__` but that is handled in type checker) + cur_node.node.is_final = s.is_final_def = not is_dunder(cur_node.node.name) + + # Special case: deferred initialization of a final attribute in __init__. + # In this case we just pretend this is a valid final definition to suppress + # errors about assigning to final attribute. + if isinstance(lval, MemberExpr) and self.is_self_member_ref(lval): + assert self.type, "Self member outside a class" + cur_node = self.type.names.get(lval.name, None) + if cur_node and isinstance(cur_node.node, Var) and cur_node.node.is_final: + assert self.function_stack + current_function = self.function_stack[-1] + if ( + current_function.name == "__init__" + and cur_node.node.final_unset_in_class + and not cur_node.node.final_set_in_init + and not (isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs) + ): + cur_node.node.final_set_in_init = True + s.is_final_def = True + + def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: + res: list[Expression] = [] + for lv in lvalues: + if isinstance(lv, (TupleExpr, ListExpr)): + res.extend(self.flatten_lvalues(lv.items)) + else: + res.append(lv) + return res + + def process_type_annotation(self, s: AssignmentStmt) -> None: + """Analyze type annotation or infer simple literal type.""" + if s.type: + lvalue = s.lvalues[-1] + allow_tuple_literal = isinstance(lvalue, TupleExpr) + analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) + # Don't store not ready types (including placeholders). + if analyzed is None or has_placeholder(analyzed): + self.defer(s) + return + s.type = analyzed + if ( + self.type + and self.type.is_protocol + and isinstance(lvalue, NameExpr) + and isinstance(s.rvalue, TempNode) + and s.rvalue.no_rhs + ): + if isinstance(lvalue.node, Var): + lvalue.node.is_abstract_var = True + else: + if ( + self.type + and self.type.is_protocol + and self.is_annotated_protocol_member(s) + and not self.is_func_scope() + ): + self.fail("All protocol members must have explicitly declared types", s) + # Set the type if the rvalue is a simple literal (even if the above error occurred). + if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr): + ref_expr = s.lvalues[0] + safe_literal_inference = True + if self.type and isinstance(ref_expr, NameExpr) and len(self.type.mro) > 1: + # Check if there is a definition in supertype. If yes, we can't safely + # decide here what to infer: int or Literal[42]. + safe_literal_inference = self.type.mro[1].get(ref_expr.name) is None + if safe_literal_inference and ref_expr.is_inferred_def: + s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + + def is_annotated_protocol_member(self, s: AssignmentStmt) -> bool: + """Check whether a protocol member is annotated. + + There are some exceptions that can be left unannotated, like ``__slots__``.""" + return any( + (isinstance(lv, NameExpr) and lv.name != "__slots__" and lv.is_inferred_def) + for lv in s.lvalues + ) + + def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: + """Return builtins.int if rvalue is an int literal, etc. + + If this is a 'Final' context, we return "Literal[...]" instead. + """ + if self.function_stack: + # Skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + + value = constant_fold_expr(rvalue, self.cur_mod_id) + if value is None or isinstance(value, complex): + return None + + if isinstance(value, bool): + type_name = "builtins.bool" + elif isinstance(value, int): + type_name = "builtins.int" + elif isinstance(value, str): + type_name = "builtins.str" + elif isinstance(value, float): + type_name = "builtins.float" + + typ = self.named_type_or_none(type_name) + if typ and is_final: + return typ.copy_modified(last_known_value=LiteralType(value=value, fallback=typ)) + return typ + + def analyze_alias( + self, + name: str, + rvalue: Expression, + allow_placeholder: bool = False, + declared_type_vars: TypeVarLikeList | None = None, + all_declared_type_params_names: list[str] | None = None, + python_3_12_type_alias: bool = False, + ) -> tuple[ + Type | None, list[TypeVarLikeType], set[str], bool, dict[str, set[TypeAlias | TypeInfo]] + ]: + """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable). + + If yes, return the corresponding type, a list of type variables for generic aliases, + a set of names the alias depends on, whether the original type has empty tuple index, + and any type variables whose defaults depend on other classes or type aliases. + An example for the dependencies: + A = int + B = str + analyze_alias(dict[A, B])[2] == {'__main__.A', '__main__.B'} + """ + dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) + global_scope = not self.type and not self.function_stack + try: + typ = expr_to_unanalyzed_type( + rvalue, self.options, self.is_stub_file, lookup_qualified=self.lookup_qualified + ) + except TypeTranslationError: + self.fail( + "Invalid type alias: expression is not a valid type", rvalue, code=codes.VALID_TYPE + ) + return None, [], set(), False, {} + + found_type_vars = self.find_type_var_likes(typ) + namespace = self.qualified_name(name) + alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + tvar_defs = self.tvar_defs_from_tvars(alias_type_vars, typ) + + if python_3_12_type_alias: + with self.allow_unbound_tvars_set(): + rvalue.accept(self) + + new_tvar_defs = [] + erase_tvar_defs = [] + variadic = False + for td in tvar_defs: + if variadic and isinstance(td, TypeVarType) and td.has_default(): + self.fail( + message_registry.NO_DEFAULT_AFTER_TYPEVAR_TUPLE, + rvalue, + code=codes.TYPE_VAR, + ) + # Remove the ambiguous type variable, and record it, so that we can + # replace all its uses with Any. + erase_tvar_defs.append(td) + continue + if isinstance(td, TypeVarTupleType): + # There can be only one variadic variable at most, + # the error is reported elsewhere. + if variadic: + continue + variadic = True + new_tvar_defs.append(td) + + analyzed, depends_on = analyze_type_alias( + typ, + self, + self.tvar_scope, + self.plugin, + self.options, + self.cur_mod_node, + self.is_typeshed_stub_file, + allow_placeholder=allow_placeholder, + in_dynamic_func=dynamic, + global_scope=global_scope, + allowed_alias_tvars=tvar_defs, + erase_tvar_defs=erase_tvar_defs, + alias_type_params_names=all_declared_type_params_names, + python_3_12_type_alias=python_3_12_type_alias, + ) + + indexed = bool(isinstance(typ, UnboundType) and (typ.args or typ.empty_tuple_index)) + default_depends = {} + for _, tv in alias_type_vars: + if tv.default_depends is not None: + default_depends[tv.fullname] = tv.default_depends + return analyzed, new_tvar_defs, depends_on, indexed, default_depends + + def is_pep_613(self, s: AssignmentStmt) -> bool: + if s.unanalyzed_type is not None and isinstance(s.unanalyzed_type, UnboundType): + lookup = self.lookup_qualified(s.unanalyzed_type.name, s, suppress_errors=True) + if lookup and lookup.fullname in TYPE_ALIAS_NAMES: + return True + return False + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: + """Check if assignment creates a type alias and set it up as needed. + + Return True if it is a type alias (even if the target is not ready), + or False otherwise. + + Note: the resulting types for subscripted (including generic) aliases + are also stored in rvalue.analyzed. + """ + if s.invalid_recursive_alias: + return True + lvalue = s.lvalues[0] + if len(s.lvalues) > 1 or not isinstance(lvalue, NameExpr): + # First rule: Only simple assignments like Alias = ... create aliases. + return False + + pep_613 = self.is_pep_613(s) + if not pep_613 and s.unanalyzed_type is not None: + # Second rule: Explicit type (cls: Type[A] = A) always creates variable, not alias. + # unless using PEP 613 `cls: TypeAlias = A` + return False + + # It can be `A = TypeAliasType('A', ...)` call, in this case, + # we just take the second argument and analyze it: + type_params: TypeVarLikeList | None + all_type_params_names: list[str] | None + if self.check_type_alias_type_call(s.rvalue, name=lvalue.name): + rvalue = ( + s.rvalue.args[1] + if s.rvalue.arg_kinds[1] == ARG_POS + else s.rvalue.args[s.rvalue.arg_names.index("value")] + ) + pep_695 = True + type_params, all_type_params_names = self.analyze_type_alias_type_params(s.rvalue) + else: + rvalue = s.rvalue + pep_695 = False + type_params = None + all_type_params_names = None + + if isinstance(rvalue, CallExpr) and rvalue.analyzed: + return False + + existing = self.current_symbol_table().get(lvalue.name) + # Third rule: type aliases can't be re-defined. For example: + # A: Type[float] = int + # A = float # OK, but this doesn't define an alias + # B = int + # B = float # Error! + # Don't create an alias in these cases: + if existing and ( + isinstance(existing.node, Var) # existing variable + or (isinstance(existing.node, TypeAlias) and not s.is_alias_def) # existing alias + or (isinstance(existing.node, PlaceholderNode) and existing.node.node.line < s.line) + ): # previous incomplete definition + # TODO: find a more robust way to track the order of definitions. + # Note: if is_alias_def=True, this is just a node from previous iteration. + if isinstance(existing.node, TypeAlias) and not s.is_alias_def: + self.fail( + 'Cannot assign multiple types to name "{}"' + ' without an explicit "type[...]" annotation'.format(lvalue.name), + lvalue, + ) + return False + + non_global_scope = self.type or self.is_func_scope() + if not pep_613 and not pep_695 and isinstance(rvalue, RefExpr) and non_global_scope: + # Fourth rule (special case): Non-subscripted right hand side creates a variable + # at class and function scopes. For example: + # + # class Model: + # ... + # class C: + # model = Model # this is automatically a variable with type 'Type[Model]' + # + # without this rule, this typical use case will require a lot of explicit + # annotations (see the second rule). + return False + if not pep_613 and not pep_695 and not self.can_be_type_alias(rvalue): + return False + + if existing and not isinstance(existing.node, (PlaceholderNode, TypeAlias)): + # Cannot redefine existing node as type alias. + return False + + res: Type | None = None + if self.is_none_alias(rvalue): + res = NoneType() + alias_tvars: list[TypeVarLikeType] = [] + depends_on: set[str] = set() + indexed = False + default_depends: dict[str, set[TypeAlias | TypeInfo]] = {} + else: + tag = self.track_incomplete_refs() + res, alias_tvars, depends_on, indexed, default_depends = self.analyze_alias( + lvalue.name, + rvalue, + allow_placeholder=True, + declared_type_vars=type_params, + all_declared_type_params_names=all_type_params_names, + ) + if not res: + return False + if not self.is_func_scope(): + # Only marking incomplete for top-level placeholders makes recursive aliases like + # `A = Sequence[str | A]` valid here, similar to how we treat base classes in class + # definitions, allowing `class str(Sequence[str]): ...` + incomplete_target = isinstance(res, ProperType) and isinstance( + res, PlaceholderType + ) + else: + incomplete_target = has_placeholder(res) + if self.found_incomplete_ref(tag) or incomplete_target: + # Since we have got here, we know this must be a type alias (incomplete refs + # may appear in nested positions), therefore use becomes_typeinfo=True. + self.mark_incomplete(lvalue.name, rvalue, becomes_typeinfo=True) + return True + + self.add_type_alias_deps(depends_on) + check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s) + # When this type alias gets "inlined", the Any is not explicit anymore, + # so we need to replace it with non-explicit Anys. + res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + # Only show error message once, when the type is fully analyzed. + if not has_placeholder(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) + # Note: with the new (lazy) type alias representation we only need to set no_args to True + # if the expected number of arguments is non-zero, so that aliases like `A = List` work + # but not aliases like `A = TypeAliasType("A", List)` as these need explicit type params. + # However, eagerly expanding aliases like Text = str is a nice performance optimization. + no_args = ( + isinstance(res, ProperType) + and isinstance(res, Instance) + and not pep_695 + and not indexed + ) + if isinstance(res, ProperType) and isinstance(res, Instance): + if not validate_instance(res, self.fail, indexed): + fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) + # Aliases defined within functions can't be accessed outside + # the function, since the symbol table will no longer + # exist. Work around by expanding them eagerly when used. + eager = self.is_func_scope() + alias_node = TypeAlias( + res, + self.qualified_name(lvalue.name), + self.cur_mod_id, + s.line, + s.column, + alias_tvars=alias_tvars, + no_args=no_args, + eager=eager, + python_3_12_type_alias=pep_695, + ) + alias_node.default_depends = default_depends + if isinstance(s.rvalue, (IndexExpr, CallExpr, OpExpr)): + # Note: CallExpr is for "void = type(None)" and OpExpr is for "X | Y" union syntax. + if not isinstance(s.rvalue.analyzed, TypeAliasExpr): + # Any existing node will be updated in-place below. + s.rvalue.analyzed = TypeAliasExpr(alias_node) + s.rvalue.analyzed.line = s.line + # we use the column from resulting target, to get better location for errors + s.rvalue.analyzed.column = res.column + elif isinstance(s.rvalue, RefExpr): + s.rvalue.is_alias_rvalue = True + + updated = False + if existing: + if isinstance(existing.node, TypeAlias): + # Invalidate recursive status cache in case it was previously set. + existing.node._is_recursive = None + if existing.node.target != res: + # Copy expansion to the existing alias, this matches how we update base classes + # for a TypeInfo _in place_ if there are nested placeholders. + existing.node.target = res + existing.node.default_depends = default_depends + existing.node.alias_tvars = alias_tvars + existing.node.no_args = no_args + updated = True + else: + # Otherwise just replace existing placeholder with type alias *in place*. + existing._node = alias_node + updated = True + else: + self.add_symbol(lvalue.name, alias_node, s) + if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias): + alias_node.normalized = rvalue.node.normalized + current_node = existing.node if existing else alias_node + assert isinstance(current_node, TypeAlias) + self.disable_invalid_recursive_aliases(s, current_node, s.rvalue) + # Check for placeholders only after we disable invalid recursive aliases. + # Otherwise, we may get an infinite recursion while visiting the target. + if updated or has_placeholder(res): + self.process_placeholder(lvalue.name, "name", s, force_progress=updated) + if self.is_class_scope(): + assert self.type is not None + if self.type.is_protocol: + self.fail("Type aliases are prohibited in protocol bodies", s) + if not lvalue.name[0].isupper(): + self.note("Use variable annotation syntax to define protocol members", s) + return True + + def check_type_alias_type_call(self, rvalue: Expression, *, name: str) -> TypeGuard[CallExpr]: + if not isinstance(rvalue, CallExpr): + return False + + names = ["typing_extensions.TypeAliasType"] + if self.options.python_version >= (3, 12): + names.append("typing.TypeAliasType") + if not refers_to_fullname(rvalue.callee, tuple(names)): + return False + if not self.check_typevarlike_name(rvalue, name, rvalue): + return False + if rvalue.arg_kinds.count(ARG_POS) != ( + 2 - ("value" in rvalue.arg_names) - ("name" in rvalue.arg_names) + ): + return False + + return True + + def analyze_type_alias_type_params( + self, rvalue: CallExpr + ) -> tuple[TypeVarLikeList, list[str]]: + """Analyze type_params of TypeAliasType. + + Returns declared unbound type variable expressions and a list of all declared type + variable names for error reporting. + """ + if "type_params" in rvalue.arg_names: + type_params_arg = rvalue.args[rvalue.arg_names.index("type_params")] + if not isinstance(type_params_arg, TupleExpr): + self.fail( + "Tuple literal expected as the type_params argument to TypeAliasType", + type_params_arg, + ) + return [], [] + type_params = type_params_arg.items + else: + return [], [] + + declared_tvars: TypeVarLikeList = [] + all_declared_tvar_names: list[str] = [] # includes bound type variables + have_type_var_tuple = False + for tp_expr in type_params: + if isinstance(tp_expr, StarExpr): + tp_expr.valid = False + self.analyze_type_expr(tp_expr) + try: + base = self.expr_to_unanalyzed_type(tp_expr) + except TypeTranslationError: + continue + if not isinstance(base, UnboundType): + continue + + tag = self.track_incomplete_refs() + tvar = self.analyze_unbound_tvar_impl(base, is_typealias_param=True) + if tvar: + if isinstance(tvar[1], TypeVarTupleExpr): + if have_type_var_tuple: + self.fail( + "Can only use one TypeVarTuple in type_params argument to TypeAliasType", + base, + code=codes.TYPE_VAR, + ) + have_type_var_tuple = True + continue + have_type_var_tuple = True + elif not self.found_incomplete_ref(tag): + sym = self.lookup_qualified(base.name, base) + if sym and isinstance(sym.node, TypeVarLikeExpr): + all_declared_tvar_names.append(sym.node.name) # Error will be reported later + else: + self.fail( + "Free type variable expected in type_params argument to TypeAliasType", + base, + code=codes.TYPE_VAR, + ) + if sym and sym.fullname in UNPACK_TYPE_NAMES: + self.note( + "Don't Unpack type variables in type_params", base, code=codes.TYPE_VAR + ) + continue + if tvar in declared_tvars: + self.fail( + f'Duplicate type variable "{tvar[0]}" in type_params argument to TypeAliasType', + base, + code=codes.TYPE_VAR, + ) + continue + if tvar: + all_declared_tvar_names.append(tvar[0]) + declared_tvars.append(tvar) + return declared_tvars, all_declared_tvar_names + + def disable_invalid_recursive_aliases( + self, s: AssignmentStmt | TypeAliasStmt, current_node: TypeAlias, ctx: Context + ) -> None: + """Prohibit and fix recursive type aliases that are invalid/unsupported.""" + messages = [] + if ( + isinstance(current_node.target, TypeAliasType) + and current_node.target.alias is current_node + ): + # We want to have consistent error messages, but not calling name_not_defined(), + # since it will do a bunch of unrelated things we don't want here. + messages.append( + f'Cannot resolve name "{current_node.name}" (possible cyclic definition)' + ) + elif is_invalid_recursive_alias({current_node}, current_node.target): + target = ( + "tuple" if isinstance(get_proper_type(current_node.target), TupleType) else "union" + ) + messages.append(f"Invalid recursive alias: a {target} item of itself") + if detect_diverging_alias(current_node, current_node.target): + messages.append("Invalid recursive alias: type variable nesting on right hand side") + if messages: + current_node.target = AnyType(TypeOfAny.from_error) + s.invalid_recursive_alias = True + for msg in messages: + self.fail(msg, ctx) + + def analyze_lvalue( + self, + lval: Lvalue, + nested: bool = False, + explicit_type: bool = False, + is_final: bool = False, + escape_comprehensions: bool = False, + has_explicit_value: bool = False, + is_index_var: bool = False, + ) -> None: + """Analyze an lvalue or assignment target. + + Args: + lval: The target lvalue + nested: If true, the lvalue is within a tuple or list lvalue expression + explicit_type: Assignment has type annotation + escape_comprehensions: If we are inside a comprehension, set the variable + in the enclosing scope instead. This implements + https://www.python.org/dev/peps/pep-0572/#scope-of-the-target + is_index_var: If lval is the index variable in a for loop + """ + if escape_comprehensions: + assert isinstance(lval, NameExpr), "assignment expression target must be NameExpr" + if isinstance(lval, NameExpr): + self.analyze_name_lvalue( + lval, + explicit_type, + is_final, + escape_comprehensions, + has_explicit_value=has_explicit_value, + is_index_var=is_index_var, + ) + elif isinstance(lval, MemberExpr): + self.analyze_member_lvalue(lval, explicit_type, is_final, has_explicit_value) + if explicit_type and not self.is_self_member_ref(lval): + self.fail("Type cannot be declared in assignment to non-self attribute", lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail("Unexpected type declaration", lval) + lval.accept(self) + elif isinstance(lval, TupleExpr): + self.analyze_tuple_or_list_lvalue(lval, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, explicit_type) + else: + self.fail("Starred assignment target must be in a list or tuple", lval) + else: + self.fail("Invalid assignment target", lval) + + def analyze_name_lvalue( + self, + lvalue: NameExpr, + explicit_type: bool, + is_final: bool, + escape_comprehensions: bool, + has_explicit_value: bool, + is_index_var: bool, + ) -> None: + """Analyze an lvalue that targets a name expression. + + Arguments are similar to "analyze_lvalue". + """ + if lvalue.node: + # This has been bound already in a previous iteration. + return + + name = lvalue.name + if self.is_alias_for_final_name(name): + if is_final: + self.fail("Cannot redefine an existing name as final", lvalue) + else: + self.msg.cant_assign_to_final(name, self.type is not None, lvalue) + + kind = self.current_symbol_kind() + names = self.current_symbol_table(escape_comprehensions=escape_comprehensions) + existing = names.get(name) + + outer = self.is_global_or_nonlocal(name) + if ( + kind == MDEF + and isinstance(self.type, TypeInfo) + and self.type.is_enum + and not name.startswith("__") + ): + # Special case: we need to be sure that `Enum` keys are unique. + if existing is not None and not isinstance(existing.node, PlaceholderNode): + self.fail( + 'Attempted to reuse member name "{}" in Enum definition "{}"'.format( + name, self.type.name + ), + lvalue, + ) + + if explicit_type and has_explicit_value: + self.fail("Enum members must be left unannotated", lvalue) + self.note( + "See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members", + lvalue, + ) + + if (not existing or isinstance(existing.node, PlaceholderNode)) and not outer: + # Define new variable. + var = self.make_name_lvalue_var( + lvalue, kind, not explicit_type, has_explicit_value, is_index_var + ) + added = self.add_symbol(name, var, lvalue, escape_comprehensions=escape_comprehensions) + # Only bind expression if we successfully added name to symbol table. + if added: + lvalue.is_new_def = True + lvalue.is_inferred_def = True + lvalue.kind = kind + lvalue.node = var + if kind == GDEF: + lvalue.fullname = var._fullname + else: + lvalue.fullname = lvalue.name + if self.is_func_scope(): + if unmangle(name) == "_" and not self.options.allow_redefinition: + # Special case for assignment to local named '_': always infer 'Any'. + # This isn't needed with --allow-redefinition, since arbitrary + # types can be assigned to '_' anyway. + typ = AnyType(TypeOfAny.special_form) + self.store_declared_types(lvalue, typ) + if is_final and self.is_final_redefinition(kind, name): + self.fail("Cannot redefine an existing name as final", lvalue) + else: + self.make_name_lvalue_point_to_existing_def(lvalue, explicit_type, is_final) + + def is_final_redefinition(self, kind: int, name: str) -> bool: + if kind == GDEF: + return self.is_mangled_global(name) and not self.is_initial_mangled_global(name) + elif kind == MDEF and self.type: + return unmangle(name) + "'" in self.type.names + return False + + def is_alias_for_final_name(self, name: str) -> bool: + if self.is_func_scope(): + if not name.endswith("'"): + # Not a mangled name -- can't be an alias + return False + name = unmangle(name) + assert self.locals[-1] is not None, "No locals at function scope" + existing = self.locals[-1].get(name) + return existing is not None and is_final_node(existing.node) + elif self.type is not None: + orig_name = unmangle(name) + "'" + if name == orig_name: + return False + existing = self.type.names.get(orig_name) + return existing is not None and is_final_node(existing.node) + else: + orig_name = unmangle(name) + "'" + if name == orig_name: + return False + existing = self.globals.get(orig_name) + return existing is not None and is_final_node(existing.node) + + def make_name_lvalue_var( + self, + lvalue: NameExpr, + kind: int, + inferred: bool, + has_explicit_value: bool, + is_index_var: bool, + ) -> Var: + """Return a Var node for an lvalue that is a name expression.""" + name = lvalue.name + v = Var(name) + v.set_line(lvalue) + v.is_inferred = inferred + if kind == MDEF: + assert self.type is not None + v.info = self.type + v.is_initialized_in_class = True + v.allow_incompatible_override = name in ALLOW_INCOMPATIBLE_OVERRIDE + if kind != LDEF: + v._fullname = self.qualified_name(name) + else: + # fullname should never stay None + v._fullname = name + v.is_ready = False # Type not inferred yet + v.has_explicit_value = has_explicit_value + v.is_index_var = is_index_var + return v + + def make_name_lvalue_point_to_existing_def( + self, lval: NameExpr, explicit_type: bool, is_final: bool + ) -> None: + """Update an lvalue to point to existing definition in the same scope. + + Arguments are similar to "analyze_lvalue". + + Assume that an existing name exists. + """ + if is_final: + # Redefining an existing name with final is always an error. + self.fail("Cannot redefine an existing name as final", lval) + original_def = self.lookup(lval.name, lval, suppress_errors=True) + if original_def is None and self.type and not self.is_func_scope(): + # Workaround to allow "x, x = ..." in class body. + original_def = self.type.get(lval.name) + if explicit_type: + # Don't re-bind if there is a type annotation. + self.name_already_defined(lval.name, lval, original_def) + else: + # Bind to an existing name. + if original_def: + self.bind_name_expr(lval, original_def) + else: + self.name_not_defined(lval.name, lval) + self.check_lvalue_validity(lval.node, lval) + + if self.scope.functions and lval.name in self.global_decls[-1]: + # Technically, we only need to set this if original r.h.s. would be inferred + # as None, but it is tricky to detect reliably during semantic analysis. + if ( + original_def + and isinstance(original_def.node, Var) + and original_def.node.is_inferred + ): + for func in self.scope.functions: + func.def_or_infer_vars = True + + def analyze_tuple_or_list_lvalue(self, lval: TupleExpr, explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [item for item in items if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail("Two starred expressions in assignment", lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue( + lval=i, + nested=True, + explicit_type=explicit_type, + # Lists and tuples always have explicit values defined: + # `a, b, c = value` + has_explicit_value=True, + ) + + def analyze_member_lvalue( + self, lval: MemberExpr, explicit_type: bool, is_final: bool, has_explicit_value: bool + ) -> None: + """Analyze lvalue that is a member expression. + + Arguments: + lval: The target lvalue + explicit_type: Assignment has type annotation + is_final: Is the target final + """ + if lval.node: + # This has been bound already in a previous iteration. + return + lval.accept(self) + if self.is_self_member_ref(lval): + assert self.type, "Self member outside a class" + cur_node = self.type.names.get(lval.name) + node = self.type.get(lval.name) + if cur_node and is_final: + # Overrides will be checked in type checker. + self.fail("Cannot redefine an existing name as final", lval) + # On first encounter with this definition, if this attribute was defined before + # with an inferred type and it's marked with an explicit type now, give an error. + if ( + not lval.node + and cur_node + and isinstance(cur_node.node, Var) + and cur_node.node.is_inferred + and explicit_type + ): + self.attribute_already_defined(lval.name, lval, cur_node) + if self.type.is_protocol and has_explicit_value and cur_node is not None: + # Make this variable non-abstract, it would be safer to do this only if we + # are inside __init__, but we do this always to preserve historical behaviour. + if isinstance(cur_node.node, Var): + cur_node.node.is_abstract_var = False + if ( + # If the attribute of self is not defined, create a new Var, ... + node is None + # ... or if it is defined as abstract in a *superclass*. + or (cur_node is None and isinstance(node.node, Var) and node.node.is_abstract_var) + # ... also an explicit declaration on self also creates a new Var. + # Note that `explicit_type` might have been erased for bare `Final`, + # so we also check if `is_final` is passed. + or (cur_node is None and (explicit_type or is_final)) + ): + if self.type.is_protocol and node is None: + self.fail("Protocol members cannot be defined via assignment to self", lval) + else: + # Implicit attribute definition in __init__. + lval.is_new_def = True + lval.is_inferred_def = True + v = Var(lval.name) + v.set_line(lval) + v._fullname = self.qualified_name(lval.name) + v.info = self.type + v.is_ready = False + v.explicit_self_type = explicit_type or is_final + lval.def_var = v + lval.node = v + # TODO: should we also set lval.kind = MDEF? + self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True) + for func in self.scope.functions: + func.def_or_infer_vars = True + + if self.is_self_member_ref(lval) or self.is_cls_member_ref(lval): + assert self.type, "Self or cls member outside a class" + cur_node = self.type.names.get(lval.name) + if ( + cur_node + and isinstance(cur_node.node, Var) + and cur_node.node.is_inferred + and cur_node.node.is_initialized_in_class + ): + for func in self.scope.functions: + func.def_or_infer_vars = True + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def is_cls_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of cls?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_cls + + def check_lvalue_validity(self, node: Expression | SymbolNode | None, ctx: Context) -> None: + if isinstance(node, TypeVarExpr): + self.fail("Invalid assignment target", ctx) + elif isinstance(node, TypeInfo): + self.fail(message_registry.CANNOT_ASSIGN_TO_TYPE, ctx) + + def store_declared_types(self, lvalue: Lvalue, typ: Type) -> None: + if isinstance(lvalue, RefExpr): + lvalue.is_inferred_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + typ = get_proper_type(typ) + if ( + var.is_final + and isinstance(typ, Instance) + and typ.last_known_value + and not isinstance(typ.last_known_value.value, SentinelValue) + and (not self.type or not self.type.is_enum) + ): + var.final_value = typ.last_known_value.value + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + typ = get_proper_type(typ) + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail("Incompatible number of tuple items", lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail("Tuple type expected for multiple variables", lvalue) + elif isinstance(lvalue, StarExpr): + # Historical behavior for the old parser + self.store_declared_types(lvalue.expr, typ) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> bool: + """Check if s declares a TypeVar; it yes, store it in symbol table. + + Return True if this looks like a type variable declaration (but maybe + with errors), otherwise return False. + """ + call = self.get_typevarlike_declaration(s, ("typing.TypeVar", "typing_extensions.TypeVar")) + if not call: + return False + + name = self.extract_typevarlike_name(s, call) + if name is None: + return False + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_value_types(call.args[1 : 1 + n_values]) + + if self.types_fixed is not None: + self.types_fixed.clear() + res = self.process_typevar_parameters( + call.args[1 + n_values :], + call.arg_names[1 + n_values :], + call.arg_kinds[1 + n_values :], + n_values, + s, + ) + default_depends = self.types_fixed + self.types_fixed = None + if res is None: + return False + variance, upper_bound, default = res + + existing = self.current_symbol_table().get(name) + if existing and not ( + isinstance(existing.node, PlaceholderNode) + or + # Also give error for another type variable with the same name. + (isinstance(existing.node, TypeVarExpr) and existing.node is call.analyzed) + ): + self.fail(f'Cannot redefine "{name}" as a type variable', s) + return False + + if self.options.disallow_any_unimported: + for idx, constraint in enumerate(values, start=1): + if has_any_from_unimported_type(constraint): + prefix = f"Constraint {idx}" + self.msg.unimported_type_becomes_any(prefix, constraint, s) + + if has_any_from_unimported_type(upper_bound): + prefix = "Upper bound of type variable" + self.msg.unimported_type_becomes_any(prefix, upper_bound, s) + + for t in values + [upper_bound, default]: + check_for_explicit_any( + t, self.options, self.is_typeshed_stub_file, self.msg, context=s + ) + + # mypyc suppresses making copies of a function to check each + # possible type, so set the upper bound to Any to prevent that + # from causing errors. + if values and self.options.mypyc: + upper_bound = AnyType(TypeOfAny.implementation_artifact) + + # Yes, it's a valid type variable definition! Add it to the symbol table. + if not call.analyzed: + type_var = TypeVarExpr( + name, self.qualified_name(name), values, upper_bound, default, variance + ) + type_var.default_depends = default_depends + type_var.line = call.line + call.analyzed = type_var + updated = True + else: + assert isinstance(call.analyzed, TypeVarExpr) + updated = ( + values != call.analyzed.values + or upper_bound != call.analyzed.upper_bound + or default != call.analyzed.default + ) + call.analyzed.upper_bound = upper_bound + call.analyzed.values = values + call.analyzed.default = default + call.analyzed.default_depends = default_depends + if any(has_placeholder(v) for v in values): + self.process_placeholder(None, "TypeVar values", s, force_progress=updated) + elif has_placeholder(upper_bound): + self.process_placeholder(None, "TypeVar upper bound", s, force_progress=updated) + elif has_placeholder(default): + self.process_placeholder(None, "TypeVar default", s, force_progress=updated) + + self.add_symbol(name, call.analyzed, s) + return True + + def check_typevar_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if isinstance(typ, AnyType) and typ.is_from_error: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format("TypeVar", "default"), context + ) + return default + + def check_paramspec_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if isinstance(typ, Parameters): + for i, arg_type in enumerate(typ.arg_types): + arg_ptype = get_proper_type(arg_type) + if isinstance(arg_ptype, AnyType) and arg_ptype.is_from_error: + self.fail(f"Argument {i} of ParamSpec default must be a type", context) + elif ( + isinstance(typ, AnyType) + and typ.is_from_error + or not isinstance(typ, (AnyType, UnboundType)) + ): + self.fail( + "The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec", + context, + ) + default = AnyType(TypeOfAny.from_error) + return default + + def check_typevartuple_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if not isinstance(typ, UnpackType): + self.fail("The default argument to TypeVarTuple must be an Unpacked tuple", context) + default = AnyType(TypeOfAny.from_error) + return default + + def check_typevarlike_name(self, call: CallExpr, name: str, context: Context) -> bool: + """Checks that the name of a TypeVar or ParamSpec matches its variable.""" + name = unmangle(name) + assert isinstance(call.callee, RefExpr) + typevarlike_type = ( + call.callee.name if isinstance(call.callee, NameExpr) else call.callee.fullname + ) + if len(call.args) < 1: + self.fail(f"Too few arguments for {typevarlike_type}()", context) + return False + if not isinstance(call.args[0], StrExpr) or call.arg_kinds[0] != ARG_POS: + self.fail(f"{typevarlike_type}() expects a string literal as first argument", context) + return False + elif call.args[0].value != name: + msg = 'String argument 1 "{}" to {}(...) does not match variable name "{}"' + self.fail(msg.format(call.args[0].value, typevarlike_type, name), context) + return False + return True + + def get_typevarlike_declaration( + self, s: AssignmentStmt, typevarlike_types: tuple[str, ...] + ) -> CallExpr | None: + """Returns the call expression if `s` is a declaration of `typevarlike_type` + (TypeVar or ParamSpec), or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + callee = call.callee + if not isinstance(callee, RefExpr): + return None + if callee.fullname not in typevarlike_types: + return None + return call + + def process_typevar_parameters( + self, + args: list[Expression], + names: list[str | None], + kinds: list[ArgKind], + num_values: int, + context: Context, + ) -> tuple[int, Type, Type] | None: + has_values = num_values > 0 + covariant = False + contravariant = False + upper_bound: Type = self.object_type() + default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind.is_named(): + self.fail(message_registry.TYPEVAR_UNEXPECTED_ARGUMENT, context) + return None + if param_name == "covariant": + if isinstance(param_value, NameExpr) and param_value.name in ("True", "False"): + covariant = param_value.name == "True" + else: + self.fail(message_registry.TYPEVAR_VARIANCE_DEF.format("covariant"), context) + return None + elif param_name == "contravariant": + if isinstance(param_value, NameExpr) and param_value.name in ("True", "False"): + contravariant = param_value.name == "True" + else: + self.fail( + message_registry.TYPEVAR_VARIANCE_DEF.format("contravariant"), context + ) + return None + elif param_name == "bound": + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + tv_arg = self.get_typevarlike_argument("TypeVar", param_name, param_value, context) + if tv_arg is None: + return None + upper_bound = tv_arg + elif param_name == "default": + tv_arg = self.get_typevarlike_argument( + "TypeVar", param_name, param_value, context, allow_unbound_tvars=True + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + elif param_name == "values": + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail('TypeVar "values" argument not supported', context) + self.fail( + "Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", context + ) + return None + else: + self.fail( + f'{message_registry.TYPEVAR_UNEXPECTED_ARGUMENT}: "{param_name}"', context + ) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif num_values == 1: + self.fail(message_registry.TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES, context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return variance, upper_bound, default + + def record_fixed_type(self, fixed: TypeInfo | TypeAlias) -> None: + if self.types_fixed is None: + self.types_fixed = set() + self.types_fixed.add(fixed) + + def get_typevarlike_argument( + self, + typevarlike_name: str, + param_name: str, + param_value: Expression, + context: Context, + *, + allow_unbound_tvars: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_typevar_arg: bool = True, + ) -> Type | None: + try: + # We want to use our custom error message below, so we suppress + # the default error message for invalid types here. + analyzed = self.expr_to_analyzed_type( + param_value, + allow_placeholder=True, + report_invalid_types=False, + allow_unbound_tvars=allow_unbound_tvars, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + analyzing_tvar_def=param_name == "default", + ) + if analyzed is None: + # Type variables are special: we need to place them in the symbol table + # soon, even if upper bound is not ready yet. Otherwise, avoiding + # a "deadlock" in this common pattern would be tricky: + # T = TypeVar('T', bound=Custom[Any]) + # class Custom(Generic[T]): + # ... + analyzed = PlaceholderType(None, [], context.line) + if ( + report_invalid_typevar_arg + and isinstance(analyzed, ProperType) + and isinstance(analyzed, AnyType) + and analyzed.is_from_error + ): + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name), + param_value, + ) + # Note: we do not return 'None' here -- we want to continue + # using the AnyType. + return analyzed + except TypeTranslationError: + if report_invalid_typevar_arg: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name), + param_value, + ) + return None + + def extract_typevarlike_name(self, s: AssignmentStmt, call: CallExpr) -> str | None: + if not call: + return None + + lvalue = s.lvalues[0] + assert isinstance(lvalue, NameExpr) + if s.type: + self.fail("Cannot declare the type of a TypeVar or similar construct", s) + return None + + if not self.check_typevarlike_name(call, lvalue.name, s): + return None + return lvalue.name + + def process_paramspec_declaration(self, s: AssignmentStmt) -> bool: + """Checks if s declares a ParamSpec; if yes, store it in symbol table. + + Return True if this looks like a ParamSpec (maybe with errors), otherwise return False. + + In the future, ParamSpec may accept bounds and variance arguments, in which + case more aggressive sharing of code with process_typevar_declaration should be pursued. + """ + call = self.get_typevarlike_declaration( + s, ("typing_extensions.ParamSpec", "typing.ParamSpec") + ) + if not call: + return False + + name = self.extract_typevarlike_name(s, call) + if name is None: + return False + + n_values = call.arg_kinds[1:].count(ARG_POS) + if n_values != 0: + self.fail('Too many positional arguments for "ParamSpec"', s) + + if self.types_fixed is not None: + self.types_fixed.clear() + default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name in zip( + call.args[1 + n_values :], call.arg_names[1 + n_values :] + ): + if param_name == "default": + tv_arg = self.get_typevarlike_argument( + "ParamSpec", + param_name, + param_value, + s, + allow_unbound_tvars=True, + allow_param_spec_literals=True, + report_invalid_typevar_arg=False, + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + default = self.check_paramspec_default(default, param_value) + else: + # ParamSpec is different from a regular TypeVar: + # arguments are not semantically valid. But, allowed in runtime. + # So, we need to warn users about possible invalid usage. + self.fail( + "The variance and bound arguments to ParamSpec do not have defined semantics yet", + s, + ) + default_depends = self.types_fixed + self.types_fixed = None + + # PEP 612 reserves the right to define bound, covariant and contravariant arguments to + # ParamSpec in a later PEP. If and when that happens, we should do something + # on the lines of process_typevar_parameters + + if not call.analyzed: + paramspec_var = ParamSpecExpr( + name, self.qualified_name(name), self.object_type(), default, INVARIANT + ) + paramspec_var.line = call.line + paramspec_var.default_depends = default_depends + call.analyzed = paramspec_var + updated = True + else: + assert isinstance(call.analyzed, ParamSpecExpr) + updated = default != call.analyzed.default + call.analyzed.default = default + call.analyzed.default_depends = default_depends + if has_placeholder(default): + self.process_placeholder(None, "ParamSpec default", s, force_progress=updated) + + self.add_symbol(name, call.analyzed, s) + return True + + def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: + """Checks if s declares a TypeVarTuple; if yes, store it in symbol table. + + Return True if this looks like a TypeVarTuple (maybe with errors), otherwise return False. + """ + call = self.get_typevarlike_declaration( + s, ("typing_extensions.TypeVarTuple", "typing.TypeVarTuple") + ) + if not call: + return False + + n_values = call.arg_kinds[1:].count(ARG_POS) + if n_values != 0: + self.fail('Too many positional arguments for "TypeVarTuple"', s) + + default: Type = AnyType(TypeOfAny.from_omitted_generics) + if self.types_fixed is not None: + self.types_fixed.clear() + for param_value, param_name in zip( + call.args[1 + n_values :], call.arg_names[1 + n_values :] + ): + if param_name == "default": + tv_arg = self.get_typevarlike_argument( + "TypeVarTuple", + param_name, + param_value, + s, + allow_unbound_tvars=True, + report_invalid_typevar_arg=False, + allow_unpack=True, + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + default = self.check_typevartuple_default(default, param_value) + else: + self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s) + + default_depends = self.types_fixed + self.types_fixed = None + + name = self.extract_typevarlike_name(s, call) + if name is None: + return False + + # PEP 646 does not specify the behavior of variance, constraints, or bounds. + if not call.analyzed: + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + typevartuple_var = TypeVarTupleExpr( + name, + self.qualified_name(name), + # Upper bound for *Ts is *tuple[object, ...], it can never be object. + tuple_fallback.copy_modified(), + tuple_fallback, + default, + INVARIANT, + ) + typevartuple_var.line = call.line + typevartuple_var.default_depends = default_depends + call.analyzed = typevartuple_var + updated = True + else: + assert isinstance(call.analyzed, TypeVarTupleExpr) + updated = default != call.analyzed.default + call.analyzed.default = default + call.analyzed.default_depends = default_depends + if has_placeholder(default): + self.process_placeholder(None, "TypeVarTuple default", s, force_progress=updated) + + self.add_symbol(name, call.analyzed, s) + return True + + def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance, line: int) -> TypeInfo: + class_def = ClassDef(name, Block([])) + # Ground rules for classes nested in functions: + # * Use is_nested_within_func_scope(), not is_func_scope(), to determine whether + # to use any special logic, because nothing inside top-level functions is serialized. + # * ClassDef.name is not mangled (i.e. @line suffix is not appended). + # * ClassDef.fullname, and thus TypeInfo.fullname are always pkg.mod.Name@line, any + # "intermediate" classes are not included in the fullname. + # * The caller is responsible for storing the generated TypeInfo twice: once as usual + # with add_symbol(), and once using add_global_symbol() using the mangled name. + # The second one is needed to properly serialize any classes nested in functions. + # TODO: make sure the daemon works well with these rules. + class_def.fullname = self.class_fullname(name, line) + + info = TypeInfo(SymbolTable(), class_def, self.cur_mod_id) + class_def.info = info + mro = basetype_or_fallback.type.mro + if not mro: + # Probably an error, we should not crash so generate something meaningful. + mro = [basetype_or_fallback.type, self.object_type().type] + info.mro = [info] + mro + info.bases = [basetype_or_fallback] + return info + + def analyze_value_types(self, items: list[Expression]) -> list[Type]: + """Analyze types from values expressions in type variable definition.""" + result: list[Type] = [] + for node in items: + try: + analyzed = self.anal_type( + self.expr_to_unanalyzed_type(node), allow_placeholder=True + ) + if analyzed is None: + # Type variables are special: we need to place them in the symbol table + # soon, even if some value is not ready yet, see process_typevar_parameters() + # for an example. + analyzed = PlaceholderType(None, [], node.line) + if has_type_vars(analyzed): + self.fail(message_registry.TYPE_VAR_GENERIC_CONSTRAINT_TYPE, node) + result.append(AnyType(TypeOfAny.from_error)) + else: + result.append(analyzed) + except TypeTranslationError: + self.fail("Type expected", node) + result.append(AnyType(TypeOfAny.from_error)) + return result + + def check_classvar(self, s: AssignmentStmt) -> None: + """Check if assignment defines a class variable.""" + lvalue = s.lvalues[0] + if len(s.lvalues) != 1 or not isinstance(lvalue, RefExpr): + return + if not s.type or not self.is_classvar(s.type): + return + assert isinstance(s.type, UnboundType) + if self.is_class_scope() and isinstance(lvalue, NameExpr): + node = lvalue.node + if isinstance(node, Var): + node.is_classvar = True + analyzed = self.anal_type(s.type) + assert self.type is not None + if ( + analyzed is not None + and self.type.self_type in get_type_vars(analyzed) + and self.type.defn.type_vars + ): + self.fail(message_registry.CLASS_VAR_WITH_GENERIC_SELF, s) + elif not isinstance(lvalue, MemberExpr) or self.is_self_member_ref(lvalue): + # In case of member access, report error only when assigning to self + # Other kinds of member assignments should be already reported + self.fail_invalid_classvar(lvalue) + if not s.type.args: + if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: + if self.options.disallow_any_generics: + self.fail("ClassVar without type argument becomes Any", s, code=codes.TYPE_ARG) + return + s.type = None + + def is_classvar(self, typ: Type) -> bool: + if not isinstance(typ, UnboundType): + return False + sym = self.lookup_qualified(typ.name, typ) + if not sym or not sym.node: + return False + return sym.node.fullname == "typing.ClassVar" + + def is_final_type(self, typ: Type | None) -> bool: + if not isinstance(typ, UnboundType): + return False + sym = self.lookup_qualified(typ.name, typ) + if not sym or not sym.node: + return False + return sym.node.fullname in FINAL_TYPE_NAMES + + def fail_invalid_classvar(self, context: Context) -> None: + self.fail(message_registry.CLASS_VAR_OUTSIDE_OF_CLASS, context) + + def process_module_assignment( + self, lvals: list[Lvalue], rval: Expression, ctx: AssignmentStmt + ) -> None: + """Propagate module references across assignments. + + Recursively handles the simple form of iterable unpacking; doesn't + handle advanced unpacking with *rest, dictionary unpacking, etc. + + In an expression like x = y = z, z is the rval and lvals will be [x, + y]. + + """ + if isinstance(rval, (TupleExpr, ListExpr)) and all( + isinstance(v, TupleExpr) for v in lvals + ): + # rval and all lvals are either list or tuple, so we are dealing + # with unpacking assignment like `x, y = a, b`. Mypy didn't + # understand our all(isinstance(...)), so cast them as TupleExpr + # so mypy knows it is safe to access their .items attribute. + seq_lvals = cast(list[TupleExpr], lvals) + # given an assignment like: + # (x, y) = (m, n) = (a, b) + # we now have: + # seq_lvals = [(x, y), (m, n)] + # seq_rval = (a, b) + # We now zip this into: + # elementwise_assignments = [(a, x, m), (b, y, n)] + # where each elementwise assignment includes one element of rval and the + # corresponding element of each lval. Basically we unpack + # (x, y) = (m, n) = (a, b) + # into elementwise assignments + # x = m = a + # y = n = b + # and then we recursively call this method for each of those assignments. + # If the rval and all lvals are not all of the same length, zip will just ignore + # extra elements, so no error will be raised here; mypy will later complain + # about the length mismatch in type-checking. + elementwise_assignments = zip(rval.items, *[v.items for v in seq_lvals]) + for rv, *lvs in elementwise_assignments: + self.process_module_assignment(lvs, rv, ctx) + elif isinstance(rval, RefExpr): + rnode = self.lookup_type_node(rval) + if rnode and isinstance(rnode.node, MypyFile): + for lval in lvals: + if not isinstance(lval, RefExpr): + continue + # respect explicitly annotated type + if isinstance(lval.node, Var) and lval.node.type is not None: + continue + + # We can handle these assignments to locals and to self + if isinstance(lval, NameExpr): + lnode = self.current_symbol_table().get(lval.name) + elif isinstance(lval, MemberExpr) and self.is_self_member_ref(lval): + assert self.type is not None + lnode = self.type.names.get(lval.name) + else: + continue + + if lnode: + if isinstance(lnode.node, MypyFile) and lnode.node is not rnode.node: + assert isinstance(lval, (NameExpr, MemberExpr)) + self.fail( + 'Cannot assign multiple modules to name "{}" ' + 'without explicit "types.ModuleType" annotation'.format(lval.name), + ctx, + ) + # never create module alias except on initial var definition + elif lval.is_inferred_def: + assert rnode.node is not None + lnode._node = rnode.node + + def process__all__(self, s: AssignmentStmt) -> None: + """Export names if argument is a __all__ assignment.""" + if ( + len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and s.lvalues[0].name == "__all__" + and s.lvalues[0].kind == GDEF + and isinstance(s.rvalue, (ListExpr, TupleExpr)) + ): + self.add_exports(s.rvalue.items) + + def process__deletable__(self, s: AssignmentStmt) -> None: + if not self.options.mypyc: + return + if ( + len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and s.lvalues[0].name == "__deletable__" + and s.lvalues[0].kind == MDEF + ): + rvalue = s.rvalue + if not isinstance(rvalue, (ListExpr, TupleExpr)): + self.fail('"__deletable__" must be initialized with a list or tuple expression', s) + return + items = rvalue.items + attrs = [] + for item in items: + if not isinstance(item, StrExpr): + self.fail('Invalid "__deletable__" item; string literal expected', item) + else: + attrs.append(item.value) + assert self.type + self.type.deletable_attributes = attrs + + def process__slots__(self, s: AssignmentStmt) -> None: + """ + Processing ``__slots__`` if defined in type. + + See: https://docs.python.org/3/reference/datamodel.html#slots + """ + # Later we can support `__slots__` defined as `__slots__ = other = ('a', 'b')` + if ( + isinstance(self.type, TypeInfo) + and len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and s.lvalues[0].name == "__slots__" + and s.lvalues[0].kind == MDEF + ): + # We understand `__slots__` defined as string, tuple, list, set, and dict: + if not isinstance(s.rvalue, (StrExpr, ListExpr, TupleExpr, SetExpr, DictExpr)): + # For example, `__slots__` can be defined as a variable, + # we don't support it for now. + return + + if any(p.slots is None for p in self.type.mro[1:-1]): + # At least one type in mro (excluding `self` and `object`) + # does not have concrete `__slots__` defined. Ignoring. + return + + concrete_slots = True + rvalue: list[Expression] = [] + if isinstance(s.rvalue, StrExpr): + rvalue.append(s.rvalue) + elif isinstance(s.rvalue, (ListExpr, TupleExpr, SetExpr)): + rvalue.extend(s.rvalue.items) + else: + # We have a special treatment of `dict` with possible `{**kwargs}` usage. + # In this case we consider all `__slots__` to be non-concrete. + for key, _ in s.rvalue.items: + if concrete_slots and key is not None: + rvalue.append(key) + else: + concrete_slots = False + + slots = [] + for item in rvalue: + # Special case for `'__dict__'` value: + # when specified it will still allow any attribute assignment. + if isinstance(item, StrExpr) and item.value != "__dict__": + slots.append(item.value) + else: + concrete_slots = False + if not concrete_slots: + # Some slot items are dynamic, we don't want any false positives, + # so, we just pretend that this type does not have any slots at all. + return + + # We need to copy all slots from super types: + for super_type in self.type.mro[1:-1]: + assert super_type.slots is not None + slots.extend(super_type.slots) + self.type.slots = set(slots) + + # + # Misc statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block | None) -> None: + if b: + self.visit_block(b) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + self.statement = s + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + old = self.statement + self.statement = s + if not self.is_func_scope(): + self.fail('"return" outside function', s) + if self.return_stmt_inside_except_star_block: + self.fail('"return" not allowed in except* block', s, serious=True) + if s.expr: + s.expr.accept(self) + self.try_parse_as_type_expression(s.expr) + self.statement = old + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + self.statement = s + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + self.statement = s + if s.expr: + s.expr.accept(self) + if s.msg: + s.msg.accept(self) + + def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: + self.statement = s + s.lvalue.accept(self) + s.rvalue.accept(self) + if ( + isinstance(s.lvalue, NameExpr) + and s.lvalue.name == "__all__" + and s.lvalue.kind == GDEF + and isinstance(s.rvalue, (ListExpr, TupleExpr)) + ): + self.add_exports(s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + self.statement = s + s.expr.accept(self) + self.loop_depth[-1] += 1 + with self.inside_except_star_block_set(value=False, entering_loop=True): + s.body.accept(self) + self.loop_depth[-1] -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + if s.is_async: + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, s, code=codes.SYNTAX) + + self.statement = s + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index, explicit_type=s.index_type is not None, is_index_var=True) + if s.index_type: + if self.is_classvar(s.index_type): + self.fail_invalid_classvar(s.index) + allow_tuple_literal = isinstance(s.index, TupleExpr) + analyzed = self.anal_type(s.index_type, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + self.store_declared_types(s.index, analyzed) + s.index_type = analyzed + + self.loop_depth[-1] += 1 + with self.inside_except_star_block_set(value=False, entering_loop=True): + self.visit_block(s.body) + self.loop_depth[-1] -= 1 + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + self.statement = s + if self.loop_depth[-1] == 0: + self.fail('"break" outside loop', s, serious=True, blocker=True) + if self.inside_except_star_block: + self.fail('"break" not allowed in except* block', s, serious=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + self.statement = s + if self.loop_depth[-1] == 0: + self.fail('"continue" outside loop', s, serious=True, blocker=True) + if self.inside_except_star_block: + self.fail('"continue" not allowed in except* block', s, serious=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + self.statement = s + infer_reachability_of_if_statement(s, self.options) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.statement = s + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor[None]) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var) + with self.inside_except_star_block_set(self.inside_except_star_block or s.is_star): + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + self.statement = s + types: list[Type] = [] + + if s.is_async: + if not self.is_func_scope() or not self.function_stack[-1].is_coroutine: + self.fail(message_registry.ASYNC_WITH_OUTSIDE_COROUTINE, s, code=codes.SYNTAX) + + if s.unanalyzed_type: + assert isinstance(s.unanalyzed_type, ProperType) + actual_targets = [t for t in s.target if t is not None] + if len(actual_targets) == 0: + # We have a type for no targets + self.fail('Invalid type comment: "with" statement has no targets', s) + elif len(actual_targets) == 1: + # We have one target and one type + types = [s.unanalyzed_type] + elif isinstance(s.unanalyzed_type, TupleType): + # We have multiple targets and multiple types + if len(actual_targets) == len(s.unanalyzed_type.items): + types = s.unanalyzed_type.items.copy() + else: + # But it's the wrong number of items + self.fail('Incompatible number of types for "with" targets', s) + else: + # We have multiple targets and one type + self.fail('Multiple types expected for multiple "with" targets', s) + + new_types: list[Type] = [] + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n, explicit_type=s.unanalyzed_type is not None) + + # Since we have a target, pop the next type from types + if types: + t = types.pop(0) + if self.is_classvar(t): + self.fail_invalid_classvar(n) + allow_tuple_literal = isinstance(n, TupleExpr) + analyzed = self.anal_type(t, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + # TODO: Deal with this better + new_types.append(analyzed) + self.store_declared_types(n, analyzed) + + s.analyzed_types = new_types + + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + self.statement = s + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail("Invalid delete target", s) + + def is_valid_del_target(self, s: Expression) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, (TupleExpr, ListExpr)): + return all(self.is_valid_del_target(item) for item in s.items) + else: + return False + + def visit_global_decl(self, g: GlobalDecl) -> None: + self.statement = g + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail(f'Name "{name}" is nonlocal and global', g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + self.statement = d + if self.is_module_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table, scope_type in zip( + reversed(self.locals[:-1]), reversed(self.scope_stack[:-1]) + ): + if table is not None and name in table: + if scope_type == SCOPE_ANNOTATION: + self.fail( + f'nonlocal binding not allowed for type parameter "{name}"', d + ) + break + else: + self.fail(f'No binding for nonlocal "{name}" found', d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail( + 'Name "{}" is already defined in local ' + "scope before nonlocal declaration".format(name), + d, + ) + + if name in self.global_decls[-1]: + self.fail(f'Name "{name}" is nonlocal and global', d) + self.nonlocal_decls[-1].add(name) + + def visit_match_stmt(self, s: MatchStmt) -> None: + self.statement = s + infer_reachability_of_match_statement(s, self.options) + s.subject.accept(self) + for i in range(len(s.patterns)): + s.patterns[i].accept(self) + guard = s.guards[i] + if guard is not None: + guard.accept(self) + self.visit_block(s.bodies[i]) + + def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: + if s.invalid_recursive_alias: + return + self.statement = s + type_params = self.push_type_args(s.type_args, s) + if type_params is None: + self.defer(s) + return + all_type_params_names = [p.name for p in s.type_args] + + try: + existing = self.current_symbol_table().get(s.name.name) + if existing and not ( + isinstance(existing.node, TypeAlias) + or (isinstance(existing.node, PlaceholderNode) and existing.node.line == s.line) + ): + self.already_defined(s.name.name, s, existing, "Name") + return + + tag = self.track_incomplete_refs() + res, alias_tvars, depends_on, indexed, default_depends = self.analyze_alias( + s.name.name, + s.value.expr(), + allow_placeholder=True, + declared_type_vars=type_params, + all_declared_type_params_names=all_type_params_names, + python_3_12_type_alias=True, + ) + if not res: + res = AnyType(TypeOfAny.from_error) + + if not self.is_func_scope(): + # Only marking incomplete for top-level placeholders makes recursive aliases like + # `A = Sequence[str | A]` valid here, similar to how we treat base classes in class + # definitions, allowing `class str(Sequence[str]): ...` + incomplete_target = isinstance(res, ProperType) and isinstance( + res, PlaceholderType + ) + else: + incomplete_target = has_placeholder(res) + + if self.found_incomplete_ref(tag) or incomplete_target: + # Since we have got here, we know this must be a type alias (incomplete refs + # may appear in nested positions), therefore use becomes_typeinfo=True. + self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) + return + + if any(has_placeholder(tv) for tv in alias_tvars): + # Defer the alias if some type variables are not ready, same as for classes. + # Note: progress is forced below (if needed). + self.defer() + + self.add_type_alias_deps(depends_on) + check_for_explicit_any( + res, self.options, self.is_typeshed_stub_file, self.msg, context=s + ) + # When this type alias gets "inlined", the Any is not explicit anymore, + # so we need to replace it with non-explicit Anys. + res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) + eager = self.is_func_scope() + if isinstance(res, ProperType) and isinstance(res, Instance): + if not validate_instance(res, self.fail, indexed): + fix_instance( + res, self.fail, self.note, disallow_any=False, options=self.options + ) + alias_node = TypeAlias( + res, + self.qualified_name(s.name.name), + self.cur_mod_id, + s.line, + s.column, + alias_tvars=alias_tvars, + no_args=False, + eager=eager, + python_3_12_type_alias=True, + ) + alias_node.default_depends = default_depends + s.alias_node = alias_node + + updated = False + if ( + existing + and isinstance(existing.node, (PlaceholderNode, TypeAlias)) + and existing.node.line == s.line + ): + if isinstance(existing.node, TypeAlias): + # Invalidate recursive status cache in case it was previously set. + existing.node._is_recursive = None + if ( + existing.node.target != res + or existing.node.alias_tvars != alias_node.alias_tvars + ): + # Copy expansion to the existing alias, this matches how we update base classes + # for a TypeInfo _in place_ if there are nested placeholders. + existing.node.target = res + existing.node.default_depends = default_depends + existing.node.alias_tvars = alias_tvars + updated = True + else: + # Otherwise just replace existing placeholder with type alias *in place*. + existing._node = alias_node + updated = True + else: + self.add_symbol(s.name.name, alias_node, s) + + current_node = existing.node if existing else alias_node + assert isinstance(current_node, TypeAlias) + self.disable_invalid_recursive_aliases(s, current_node, s.value) + if updated or has_placeholder(res): + self.process_placeholder(s.name.name, "name", s, force_progress=updated) + s.name.accept(self) + finally: + self.pop_type_args(s.type_args) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + self.bind_name_expr(expr, n) + + def bind_name_expr(self, expr: NameExpr, sym: SymbolTableNode) -> None: + """Bind name expression to a symbol table node.""" + if ( + isinstance(sym.node, TypeVarExpr) + and self.tvar_scope.get_binding(sym) + and not self.allow_unbound_tvars + ): + self.fail(f'"{expr.name}" is a type variable and only valid in type context', expr) + elif isinstance(sym.node, PlaceholderNode): + self.process_placeholder(expr.name, "name", expr) + else: + expr.kind = sym.kind + expr.node = sym.node + expr.fullname = sym.fullname or "" + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type and not expr.call.args: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + for arg in expr.call.args: + arg.accept(self) + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + if isinstance(item, StarExpr): + item.valid = True + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + if isinstance(item, StarExpr): + item.valid = True + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + if isinstance(item, StarExpr): + item.valid = True + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + if key is not None: + key.accept(self) + value.accept(self) + + def visit_template_str_expr(self, expr: TemplateStrExpr) -> None: + for item in expr.items: + if isinstance(item, tuple): + item[0].accept(self) + if item[3] is not None: + item[3].accept(self) + else: + item.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail("can't use starred expression here", expr, blocker=True) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): + self.fail('"yield from" outside function', e, serious=True, blocker=True) + elif self.scope_stack[-1] == SCOPE_COMPREHENSION: + self.fail( + '"yield from" inside comprehension or generator expression', + e, + serious=True, + blocker=True, + ) + elif self.function_stack[-1].is_coroutine: + self.fail('"yield from" in async function', e, serious=True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, "typing.cast"): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, "cast"): + return + # Translate first argument to an unanalyzed type. + try: + target = self.expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail("Cast target is not a type", expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, ASSERT_TYPE_NAMES): + if not self.check_fixed_args(expr, 2, "assert_type"): + return + # Translate second argument to an unanalyzed type. + try: + target = self.expr_to_unanalyzed_type(expr.args[1]) + except TypeTranslationError: + self.fail("assert_type() type is not a type", expr) + return + expr.analyzed = AssertTypeExpr(expr.args[0], target) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, REVEAL_TYPE_NAMES): + if not self.check_fixed_args(expr, 1, "reveal_type"): + return + reveal_imported = False + reveal_type_node = self.lookup("reveal_type", expr, suppress_errors=True) + if ( + reveal_type_node + and isinstance(reveal_type_node.node, SYMBOL_FUNCBASE_TYPES) + and reveal_type_node.fullname in IMPORTED_REVEAL_TYPE_NAMES + ): + reveal_imported = True + expr.analyzed = RevealExpr( + kind=REVEAL_TYPE, expr=expr.args[0], is_imported=reveal_imported + ) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, "builtins.reveal_locals"): + # Store the local variable names into the RevealExpr for use in the + # type checking pass + local_nodes: list[Var] = [] + if self.is_module_scope(): + # try to determine just the variable declarations in module scope + # self.globals.values() contains SymbolTableNode's + # Each SymbolTableNode has an attribute node that is nodes.Var + # look for variable nodes that marked as is_inferred + # Each symboltable node has a Var node as .node + local_nodes = [ + n.node + for name, n in self.globals.items() + if getattr(n.node, "is_inferred", False) and isinstance(n.node, Var) + ] + elif self.is_class_scope(): + # type = None # type: Optional[TypeInfo] + if self.type is not None: + local_nodes = [ + st.node for st in self.type.names.values() if isinstance(st.node, Var) + ] + elif self.is_func_scope(): + # locals = None # type: List[Optional[SymbolTable]] + if self.locals is not None: + symbol_table = self.locals[-1] + if symbol_table is not None: + local_nodes = [ + st.node for st in symbol_table.values() if isinstance(st.node, Var) + ] + expr.analyzed = RevealExpr(kind=REVEAL_LOCALS, local_nodes=local_nodes) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, "typing.Any"): + # Special form Any(...) no longer supported. + self.fail("Any(...) is no longer supported. Use cast(Any, ...) instead", expr) + elif refers_to_fullname(expr.callee, "typing._promote"): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, "_promote"): + return + # Translate first argument to an unanalyzed type. + try: + target = self.expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail("Argument 1 to _promote is not a type", expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, "builtins.dict") and not ( + isinstance(expr.callee, RefExpr) + and isinstance(expr.callee.node, TypeAlias) + and not expr.callee.node.no_args + ): + expr.analyzed = self.translate_dict_call(expr) + elif refers_to_fullname(expr.callee, "builtins.divmod"): + if not self.check_fixed_args(expr, 2, "divmod"): + return + expr.analyzed = OpExpr("divmod", expr.args[0], expr.args[1]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname( + expr.callee, ("typing.TypeAliasType", "typing_extensions.TypeAliasType") + ): + with self.allow_unbound_tvars_set(): + for a in expr.args: + a.accept(self) + elif refers_to_fullname(expr.callee, ("typing.TypeForm", "typing_extensions.TypeForm")): + # Special form TypeForm(...). + if not self.check_fixed_args(expr, 1, "TypeForm"): + return + # Translate first argument to an unanalyzed type. + try: + typ = self.expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail("TypeForm argument is not a type", expr) + # Suppress future error: "" not callable + expr.analyzed = CastExpr(expr.args[0], AnyType(TypeOfAny.from_error)) + return + # Piggyback TypeFormExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = TypeFormExpr(typ) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + self.try_parse_as_type_expression(a) + + if ( + isinstance(expr.callee, MemberExpr) + and isinstance(expr.callee.expr, NameExpr) + and expr.callee.expr.name == "__all__" + and expr.callee.expr.kind == GDEF + and expr.callee.name in ("append", "extend", "remove") + ): + if expr.callee.name == "append" and expr.args: + self.add_exports(expr.args[0]) + elif ( + expr.callee.name == "extend" + and expr.args + and isinstance(expr.args[0], (ListExpr, TupleExpr)) + ): + self.add_exports(expr.args[0].items) + elif ( + expr.callee.name == "remove" + and expr.args + and isinstance(expr.args[0], StrExpr) + ): + self.all_exports = [n for n in self.all_exports if n != expr.args[0].value] + + def translate_dict_call(self, call: CallExpr) -> DictExpr | None: + """Translate 'dict(x=y, ...)' to {'x': y, ...} and 'dict()' to {}. + + For other variants of dict(...), return None. + """ + if not all(kind in (ARG_NAMED, ARG_STAR2) for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr( + [ + (StrExpr(key) if key is not None else None, value) + for key, value in zip(call.arg_names, call.args) + ] + ) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = "s" + if numargs == 1: + s = "" + if len(expr.args) != numargs: + self.fail('"%s" expects %d argument%s' % (name, numargs, s), expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail(f'"{name}" must be called with {numargs} positional argument{s}', expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + if isinstance(base, RefExpr) and isinstance(base.node, MypyFile): + # Handle module attribute. + sym = self.get_module_symbol(base.node, expr.name) + if sym: + if isinstance(sym.node, PlaceholderNode): + self.process_placeholder(expr.name, "attribute", expr) + return + self.record_imported_symbol(sym) + expr.kind = sym.kind + expr.fullname = sym.fullname or "" + expr.node = sym.node + elif isinstance(base, RefExpr): + # This branch handles the case C.bar (or cls.bar or self.bar inside + # a classmethod/method), where C is a class and bar is a type + # definition or a module resulting from `import bar` (or a module + # assignment) inside class C. We look up bar in the class' TypeInfo + # namespace. This is done only when bar is a module or a type; + # other things (e.g. methods) are handled by other code in + # checkmember. + type_info = None + if isinstance(base.node, TypeInfo): + # C.bar where C is a class + type_info = base.node + elif isinstance(base.node, Var) and self.type and self.function_stack: + # check for self.bar or cls.bar in method/classmethod + func_def = self.function_stack[-1] + if ( + func_def.has_self_or_cls_argument + and func_def.info is self.type + and isinstance(func_def.type, CallableType) + and func_def.arguments + and base.node is func_def.arguments[0].variable + ): + type_info = self.type + elif isinstance(base.node, TypeAlias) and base.node.no_args: + assert isinstance(base.node.target, ProperType) + if isinstance(base.node.target, Instance): + type_info = base.node.target.type + + if type_info: + n = type_info.names.get(expr.name) + if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)): + self.record_imported_symbol(n) + expr.kind = n.kind + expr.fullname = n.fullname or "" + expr.node = n.node + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + + if expr.op in ("and", "or"): + inferred = infer_condition_value(expr.left, self.options) + if (inferred in (ALWAYS_FALSE, MYPY_FALSE) and expr.op == "and") or ( + inferred in (ALWAYS_TRUE, MYPY_TRUE) and expr.op == "or" + ): + expr.right_unreachable = True + return + elif (inferred in (ALWAYS_TRUE, MYPY_TRUE) and expr.op == "and") or ( + inferred in (ALWAYS_FALSE, MYPY_FALSE) and expr.op == "or" + ): + expr.right_always = True + + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + base = expr.base + base.accept(self) + if ( + isinstance(base, RefExpr) + and isinstance(base.node, TypeInfo) + and not base.node.is_generic() + ): + expr.index.accept(self) + elif ( + isinstance(base, RefExpr) and isinstance(base.node, TypeAlias) + ) or refers_to_class_or_function(base): + # We need to do full processing on every iteration, since some type + # arguments may contain placeholder types. + self.analyze_type_application(expr) + else: + expr.index.accept(self) + + def analyze_type_application(self, expr: IndexExpr) -> None: + """Analyze special form -- type application (either direct or via type aliasing).""" + types = self.analyze_type_application_args(expr) + if types is None: + return + base = expr.base + expr.analyzed = TypeApplication(base, types) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + + if isinstance(base, RefExpr) and base.fullname == "librt.vecs.vec": + # Apply restrictions specific to vec + check_vec_type_args(types, expr, self) + + def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: + """Analyze type arguments (index) in a type application. + + Return None if anything was incomplete. + """ + index = expr.index + tag = self.track_incomplete_refs() + self.analyze_type_expr(index) + if self.found_incomplete_ref(tag): + return None + if self.basic_type_applications: + # Postpone the rest until we have more information (for r.h.s. of an assignment) + return None + types: list[Type] = [] + if isinstance(index, TupleExpr): + items = index.items + is_tuple = isinstance(expr.base, RefExpr) and expr.base.fullname == "builtins.tuple" + if is_tuple and len(items) == 2 and isinstance(items[-1], EllipsisExpr): + items = items[:-1] + else: + items = [index] + + # TODO: this needs a clean-up. + # Probably always allow Parameters literals, and validate in semanal_typeargs.py + base = expr.base + if isinstance(base, RefExpr) and isinstance(base.node, TypeAlias): + allow_unpack = base.node.tvar_tuple_index is not None + alias = base.node + if any(isinstance(t, ParamSpecType) for t in alias.alias_tvars): + has_param_spec = True + num_args = len(alias.alias_tvars) + else: + has_param_spec = False + num_args = -1 + elif isinstance(base, RefExpr) and isinstance(base.node, TypeInfo): + allow_unpack = ( + base.node.has_type_var_tuple_type or base.node.fullname == "builtins.tuple" + ) + has_param_spec = base.node.has_param_spec_type + num_args = len(base.node.type_vars) + else: + allow_unpack = False + has_param_spec = False + num_args = -1 + + for item in items: + try: + typearg = self.expr_to_unanalyzed_type(item, allow_unpack=True) + except TypeTranslationError: + self.fail("Type expected within [...]", expr) + return None + analyzed = self.anal_type( + typearg, + # The type application may appear in base class expression, + # where type variables are not bound yet. Or when accepting + # r.h.s. of type alias before we figured out it is a type alias. + allow_unbound_tvars=self.allow_unbound_tvars, + allow_placeholder=True, + allow_param_spec_literals=has_param_spec, + allow_unpack=allow_unpack, + ) + if analyzed is None: + return None + types.append(analyzed) + + if allow_unpack: + # need to flatten away harmless unpacks like Unpack[tuple[int]] + flattened_items = flatten_nested_tuples(types) + types = self.type_analyzer().check_unpacks_in_list(flattened_items) + if has_param_spec and num_args == 1 and types: + first_arg = get_proper_type(types[0]) + single_any = len(types) == 1 and isinstance(first_arg, AnyType) + if not (single_any or any(isinstance(t, (Parameters, ParamSpecType)) for t in types)): + types = [Parameters(types, [ARG_POS] * len(types), [None] * len(types))] + + return types + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed + + def visit_type_form_expr(self, expr: TypeFormExpr) -> None: + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed + + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> None: + expr.expr.accept(self) + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed + + def visit_reveal_expr(self, expr: RevealExpr) -> None: + if expr.kind == REVEAL_TYPE: + if expr.expr is not None: + expr.expr.accept(self) + else: + # Reveal locals doesn't have an inner expression, there's no + # need to traverse inside it + pass + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + analyzed = self.anal_type(expr.types[i]) + if analyzed is not None: + expr.types[i] = analyzed + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + if any(expr.generator.is_async): + if ( + not self.is_func_scope() + or not self.function_stack + or not self.function_stack[-1].is_coroutine + ): + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, expr, code=codes.SYNTAX) + + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + if any(expr.generator.is_async): + if ( + not self.is_func_scope() + or not self.function_stack + or not self.function_stack[-1].is_coroutine + ): + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, expr, code=codes.SYNTAX) + + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + if any(expr.is_async): + if ( + not self.is_func_scope() + or not self.function_stack + or not self.function_stack[-1].is_coroutine + ): + self.fail(message_registry.ASYNC_FOR_OUTSIDE_COROUTINE, expr, code=codes.SYNTAX) + + with self.enter(expr): + self.analyze_comp_for(expr) + if expr.key is not None: + expr.key.accept(self) + expr.value.accept(self) + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + if isinstance(expr.left_expr, StarExpr): + expr.left_expr.valid = True + with self.enter(expr): + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: GeneratorExpr | DictionaryComprehension) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate( + zip(expr.indices, expr.sequences, expr.condlists) + ): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index, is_index_var=True) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: GeneratorExpr | DictionaryComprehension) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_lambda_expr(self, expr: LambdaExpr) -> None: + self.analyze_arg_initializers(expr) + with self.inside_except_star_block_set(False, entering_loop=False): + self.analyze_function_body(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + analyzed = self.anal_type(expr.type) + if analyzed is not None: + assert isinstance(analyzed, ProperType), "Cannot use type aliases for promotions" + expr.type = analyzed + + def visit_yield_expr(self, e: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail('"yield" outside function', e, serious=True, blocker=True) + elif self.scope_stack[-1] == SCOPE_COMPREHENSION: + self.fail( + '"yield" inside comprehension or generator expression', + e, + serious=True, + blocker=True, + ) + elif self.function_stack[-1].is_coroutine: + self.function_stack[-1].is_generator = True + self.function_stack[-1].is_async_generator = True + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_await_expr(self, expr: AwaitExpr) -> None: + if not self.is_func_scope() or not self.function_stack: + # We check both because is_function_scope() returns True inside comprehensions. + # This is not a blocker, because some environments (like ipython) + # support top level awaits. + self.fail('"await" outside function', expr, serious=True, code=codes.TOP_LEVEL_AWAIT) + elif not self.function_stack[-1].is_coroutine: + self.fail( + '"await" outside coroutine ("async def")', + expr, + serious=True, + code=codes.AWAIT_NOT_ASYNC, + ) + expr.expr.accept(self) + + # + # Patterns + # + + def visit_as_pattern(self, p: AsPattern) -> None: + if p.pattern is not None: + p.pattern.accept(self) + if p.name is not None: + self.analyze_lvalue(p.name) + + def visit_or_pattern(self, p: OrPattern) -> None: + for pattern in p.patterns: + pattern.accept(self) + + def visit_value_pattern(self, p: ValuePattern) -> None: + p.expr.accept(self) + + def visit_sequence_pattern(self, p: SequencePattern) -> None: + for pattern in p.patterns: + pattern.accept(self) + + def visit_starred_pattern(self, p: StarredPattern) -> None: + if p.capture is not None: + self.analyze_lvalue(p.capture) + + def visit_mapping_pattern(self, p: MappingPattern) -> None: + for key in p.keys: + key.accept(self) + for value in p.values: + value.accept(self) + if p.rest is not None: + self.analyze_lvalue(p.rest) + + def visit_class_pattern(self, p: ClassPattern) -> None: + p.class_ref.accept(self) + for pos in p.positionals: + pos.accept(self) + for v in p.keyword_values: + v.accept(self) + + # + # Lookup functions + # + + def lookup( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + node = self._lookup(name, ctx, suppress_errors) + if node is not None: + # This call is unfortunate from performance point of view, but + # needed for rare cases like e.g. testIncrementalChangingAlias. + self.record_imported_symbol(node) + return node + + def record_imported_symbol(self, sym: SymbolTableNode) -> None: + """If the symbol was not defined in current module, add its module to module_refs.""" + if sym.kind == LDEF or sym.node is None: + return + node = sym.node + if isinstance(node, PlaceholderNode) or not node.fullname: + # This node is not ready yet. + return + if node.fullname.startswith(("builtins.", "typing.")): + # Skip dependencies on builtins/typing. + return + # Modules, classes, and type aliases store defining module directly. + if isinstance(node, MypyFile): + fullname = node.fullname + elif isinstance(node, TypeInfo): + fullname = node.module_name + elif isinstance(node, TypeAlias): + fullname = node.module + elif isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)): + # For functions/variables infer defining module from enclosing class. + info = node.var.info if isinstance(node, Decorator) else node.info + if info: + fullname = info.module_name + else: + # global function/variable + fullname = node.fullname.rsplit(".", maxsplit=1)[0] + else: + # Some nodes (currently only TypeVarLikeExpr subclasses) don't store + # module fullname explicitly, infer it from the node fullname iteratively. + # TODO: this is not 100% robust for type variables nested within a class + # with a name that matches name of a submodule. + fullname = node.fullname.rsplit(".", maxsplit=1)[0] + if fullname == self.cur_mod_id: + return + while "." in fullname and fullname not in self.modules: + fullname = fullname.rsplit(".", maxsplit=1)[0] + if fullname != self.cur_mod_id: + self.cur_mod_node.module_refs.add(fullname) + + def _lookup( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + """Look up an unqualified (no dots) name in all active namespaces. + + Note that the result may contain a PlaceholderNode. The caller may + want to defer in that case. + + Generate an error if the name is not defined unless suppress_errors + is true or the current namespace is incomplete. In the latter case + defer. + """ + implicit_name = False + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + if not suppress_errors: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + if not suppress_errors: + self.name_not_defined(name, ctx) + return None + # 2a. Class attributes (if within class definition) + if self.type and not self.is_func_scope() and name in self.type.names: + node = self.type.names[name] + if not node.implicit: + if self.is_active_symbol_in_class_body(node.node): + return node + else: + # Defined through self.x assignment + implicit_name = True + implicit_node = node + # 2b. Class attributes __qualname__ and __module__ + if self.type and not self.is_func_scope() and name in {"__qualname__", "__module__"}: + v = Var(name, self.str_type()) + v._fullname = self.qualified_name(name) + return SymbolTableNode(MDEF, v) + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get("__builtins__", None) + if b: + assert isinstance(b.node, MypyFile) + table = b.node.names + if name in table: + if len(name) > 1 and name[0] == "_" and name[1] != "_": + if not suppress_errors: + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + if not implicit_name and not suppress_errors: + self.name_not_defined(name, ctx) + else: + if implicit_name: + return implicit_node + return None + + def is_active_symbol_in_class_body(self, node: SymbolNode | None) -> bool: + """Can a symbol defined in class body accessed at current statement? + + Only allow access to class attributes textually after + the definition, so that it's possible to fall back to the + outer scope. Example: + + class X: ... + + class C: + X = X # Initializer refers to outer scope + + Nested classes are an exception, since we want to support + arbitrary forward references in type annotations. Also, we + allow forward references to type aliases to support recursive + types. + """ + # TODO: Forward reference to name imported in class body is not + # caught. + if self.statement is None: + # Assume it's fine -- don't have enough context to check + return True + if ( + node is None + or self.is_textually_before_statement(node) + or not self.is_defined_in_current_module(node.fullname) + ): + return True + if self.is_type_like(node): + # Allow forward references to classes/type aliases (see docstring), but + # a forward reference should never shadow an existing regular reference. + if node.name not in self.globals: + return True + global_node = self.globals[node.name] + if not self.is_textually_before_class(global_node.node): + return True + return not self.is_type_like(global_node.node) + return False + + def is_type_like(self, node: SymbolNode | None) -> bool: + return isinstance(node, (TypeInfo, TypeAlias)) or ( + isinstance(node, PlaceholderNode) and node.becomes_typeinfo + ) + + def is_textually_before_statement(self, node: SymbolNode) -> bool: + """Check if a node is defined textually before the current statement + + Note that decorated functions' line number are the same as + the top decorator. + """ + assert self.statement + line_diff = self.statement.line - node.line + + # The first branch handles reference an overloaded function variant inside itself, + # this is a corner case where mypy technically deviates from runtime name resolution, + # but it is fine because we want an overloaded function to be treated as a single unit. + if self.is_overloaded_item(node, self.statement): + return False + elif isinstance(node, Decorator) and not node.is_overload: + return line_diff > len(node.original_decorators) + else: + return line_diff > 0 + + def is_textually_before_class(self, node: SymbolNode | None) -> bool: + """Similar to above, but check if a node is defined before current class.""" + assert self.type is not None + if node is None: + return False + return node.line < self.type.defn.line + + def is_overloaded_item(self, node: SymbolNode, statement: Statement) -> bool: + """Check whether the function belongs to the overloaded variants""" + if isinstance(node, OverloadedFuncDef) and isinstance(statement, FuncDef): + in_items = statement in { + item.func if isinstance(item, Decorator) else item for item in node.items + } + in_impl = node.impl is not None and ( + (isinstance(node.impl, Decorator) and statement is node.impl.func) + or statement is node.impl + ) + return in_items or in_impl + return False + + def is_defined_in_current_module(self, fullname: str | None) -> bool: + if not fullname: + return False + return module_prefix(self.modules, fullname) == self.cur_mod_id + + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + """Lookup a qualified name in all activate namespaces. + + Note that the result may contain a PlaceholderNode. The caller may + want to defer in that case. + + Generate an error if the name is not defined unless suppress_errors + is true or the current namespace is incomplete. In the latter case + defer. + """ + if "." not in name: + # Simple case: look up a short name. + return self.lookup(name, ctx, suppress_errors=suppress_errors) + parts = name.split(".") + namespace = self.cur_mod_id + sym = self.lookup(parts[0], ctx, suppress_errors=suppress_errors) + if sym: + for i in range(1, len(parts)): + node = sym.node + part = parts[i] + if isinstance(node, TypeInfo): + nextsym = node.get(part) + elif isinstance(node, MypyFile): + nextsym = self.get_module_symbol(node, part) + namespace = node.fullname + elif isinstance(node, PlaceholderNode): + return sym + elif isinstance(node, TypeAlias) and node.no_args: + assert isinstance(node.target, ProperType) + if isinstance(node.target, Instance): + nextsym = node.target.type.get(part) + else: + nextsym = None + else: + if isinstance(node, Var): + typ = get_proper_type(node.type) + if isinstance(typ, AnyType): + # Allow access through Var with Any type without error. + return self.implicit_symbol(sym, name, parts[i:], typ) + # This might be something like valid `P.args` or invalid `P.__bound__` access. + # Important note that `ParamSpecExpr` is also ignored in other places. + # See https://github.com/python/mypy/pull/13468 + if isinstance(node, ParamSpecExpr) and part in ("args", "kwargs"): + return None + # Lookup through invalid node, such as variable or function + nextsym = None + if not nextsym or nextsym.module_hidden: + if not suppress_errors: + self.name_not_defined(name, ctx, namespace=namespace) + return None + sym = nextsym + if sym is not None: + self.record_imported_symbol(sym) + return sym + + def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None: + try: + t = self.expr_to_unanalyzed_type(expr) + except TypeTranslationError: + return None + if isinstance(t, UnboundType): + n = self.lookup_qualified(t.name, expr, suppress_errors=True) + return n + return None + + def get_module_symbol(self, node: MypyFile, name: str) -> SymbolTableNode | None: + """Look up a symbol from a module. + + Return None if no matching symbol could be bound. + """ + module = node.fullname + names = node.names + sym = names.get(name) + if not sym: + fullname = module + "." + name + if fullname in self.modules and self.is_visible_import(module, fullname): + sym = SymbolTableNode(GDEF, self.modules[fullname]) + elif self.is_incomplete_namespace(module): + self.record_incomplete_ref() + elif "__getattr__" in names: + gvar = self.create_getattr_var(names["__getattr__"], name, fullname) + if gvar: + sym = SymbolTableNode(GDEF, gvar) + elif self.is_missing_module(fullname): + # We use the fullname of the original definition so that we can + # detect whether two names refer to the same thing. + var_type = AnyType(TypeOfAny.from_unimported_type) + v = Var(name, type=var_type) + v._fullname = fullname + sym = SymbolTableNode(GDEF, v) + elif sym.module_hidden: + sym = None + return sym + + def is_visible_import(self, base_id: str, id: str) -> bool: + # TODO: can we reuse SCC-level tracking from build.py instead? + if id in self.import_map[self.cur_mod_id]: + # Fast path: module is imported locally. + return True + if base_id not in self.transitive_submodule_imports: + # This is a performance optimization for a common pattern. If one module + # in a codebase uses import numpy as np; np.foo.bar, then it is likely that + # other modules use similar pattern as well. So we pre-compute transitive + # dependencies for np, to avoid possible duplicate work in the future. + self.add_transitive_submodule_imports(base_id) + if self.cur_mod_id not in self.transitive_submodule_imports: + self.add_transitive_submodule_imports(self.cur_mod_id) + return id in self.transitive_submodule_imports[self.cur_mod_id] + + def add_transitive_submodule_imports(self, mod_id: str) -> None: + if mod_id not in self.import_map: + return + todo = self.import_map[mod_id] + seen = {mod_id} + result = {mod_id} + while todo: + dep = todo.pop() + if dep in seen: + continue + seen.add(dep) + if "." in dep: + result.add(dep) + if dep in self.transitive_submodule_imports: + result |= self.transitive_submodule_imports[dep] + continue + if dep in self.import_map: + todo |= self.import_map[dep] + self.transitive_submodule_imports[mod_id] = result + + def is_missing_module(self, module: str) -> bool: + return module in self.missing_modules + + def implicit_symbol( + self, sym: SymbolTableNode, name: str, parts: list[str], source_type: AnyType + ) -> SymbolTableNode: + """Create symbol for a qualified name reference through Any type.""" + if sym.node is None: + basename = None + else: + basename = sym.node.fullname + if basename is None: + fullname = name + else: + fullname = basename + "." + ".".join(parts) + var_type = AnyType(TypeOfAny.from_another_any, source_type) + var = Var(parts[-1], var_type) + var._fullname = fullname + return SymbolTableNode(GDEF, var) + + def create_getattr_var( + self, getattr_defn: SymbolTableNode, name: str, fullname: str + ) -> Var | None: + """Create a dummy variable using module-level __getattr__ return type. + + If not possible, return None. + + Note that multiple Var nodes can be created for a single name. We + can use the from_module_getattr and the fullname attributes to + check if two dummy Var nodes refer to the same thing. Reusing Var + nodes would require non-local mutable state, which we prefer to + avoid. + """ + if isinstance(getattr_defn.node, (FuncDef, Var)): + node_type = get_proper_type(getattr_defn.node.type) + if isinstance(node_type, CallableType): + typ = node_type.ret_type + else: + typ = AnyType(TypeOfAny.from_error) + v = Var(name, type=typ) + v._fullname = fullname + v.from_module_getattr = True + return v + return None + + def lookup_fully_qualified(self, fullname: str) -> SymbolTableNode: + ret = self.lookup_fully_qualified_or_none(fullname) + assert ret is not None, fullname + return ret + + def lookup_fully_qualified_or_none(self, fullname: str) -> SymbolTableNode | None: + """Lookup a fully qualified name that refers to a module-level definition. + + Don't assume that the name is defined. This happens in the global namespace -- + the local module namespace is ignored. This does not dereference indirect + refs. + + Note that this can't be used for names nested in class namespaces. + """ + # TODO: unify/clean-up/simplify lookup methods, see #4157. + module, name = fullname.rsplit(".", maxsplit=1) + + if module in self.modules: + # If the module exists, look up the name in the module. + # This is the common case. + filenode = self.modules[module] + result = filenode.names.get(name) + if result is None and self.is_incomplete_namespace(module): + # TODO: More explicit handling of incomplete refs? + self.record_incomplete_ref() + return result + else: + # Else, try to find the longest prefix of the module name that is in the modules dictionary. + splitted_modules = fullname.split(".") + names = [] + + while splitted_modules and ".".join(splitted_modules) not in self.modules: + names.append(splitted_modules.pop()) + + if not splitted_modules or not names: + # If no module or name is found, return None. + return None + + # Reverse the names list to get the correct order of names. + names.reverse() + + module = ".".join(splitted_modules) + filenode = self.modules[module] + result = filenode.names.get(names[0]) + + if result is None and self.is_incomplete_namespace(module): + # TODO: More explicit handling of incomplete refs? + self.record_incomplete_ref() + + for part in names[1:]: + if result is not None and isinstance(result.node, TypeInfo): + result = result.node.names.get(part) + else: + return None + return result + + def object_type(self) -> Instance: + if self._object_type is None: + self._object_type = self.named_type("builtins.object") + return self._object_type + + def str_type(self) -> Instance: + if self._str_type is None: + self._str_type = self.named_type("builtins.str") + return self._str_type + + def function_type(self) -> Instance: + if self._function_type is None: + self._function_type = self.named_type("builtins.function") + return self._function_type + + def named_type(self, fullname: str, args: list[Type] | None = None) -> Instance: + sym = self.lookup_fully_qualified(fullname) + assert sym, "Internal error: attempted to construct unknown type" + node = sym.node + assert isinstance(node, TypeInfo), node + if args: + # TODO: assert len(args) == len(node.defn.type_vars) + return Instance(node, args) + return Instance(node, [AnyType(TypeOfAny.special_form)] * len(node.defn.type_vars)) + + def named_type_or_none(self, fullname: str, args: list[Type] | None = None) -> Instance | None: + sym = self.lookup_fully_qualified_or_none(fullname) + if not sym or isinstance(sym.node, PlaceholderNode): + return None + node = sym.node + if isinstance(node, TypeAlias): + assert isinstance(node.target, Instance) # type: ignore[misc] + node = node.target.type + assert isinstance(node, TypeInfo), node + if args is not None: + # TODO: assert len(args) == len(node.defn.type_vars) + return Instance(node, args) + return Instance(node, [AnyType(TypeOfAny.unannotated)] * len(node.defn.type_vars)) + + def builtin_type(self, fully_qualified_name: str) -> Instance: + """Legacy function -- use named_type() instead.""" + return self.named_type(fully_qualified_name) + + def lookup_current_scope(self, name: str) -> SymbolTableNode | None: + if self.locals[-1] is not None: + return self.locals[-1].get(name) + elif self.type is not None: + return self.type.names.get(name) + else: + return self.globals.get(name) + + # + # Adding symbols + # + + def add_symbol( + self, + name: str, + node: SymbolNode, + context: Context, + module_public: bool = True, + module_hidden: bool = False, + can_defer: bool = True, + escape_comprehensions: bool = False, + no_progress: bool = False, + type_param: bool = False, + ) -> bool: + """Add symbol to the currently active symbol table. + + Generally additions to symbol table should go through this method or + one of the methods below so that kinds, redefinitions, conditional + definitions, and skipped names are handled consistently. + + Return True if we actually added the symbol, or False if we refused to do so + (because something is not ready). + + If can_defer is True, defer current target if adding a placeholder. + """ + if self.is_func_scope(): + kind = LDEF + elif self.type is not None: + kind = MDEF + else: + kind = GDEF + symbol = SymbolTableNode( + kind, node, module_public=module_public, module_hidden=module_hidden + ) + return self.add_symbol_table_node( + name, symbol, context, can_defer, escape_comprehensions, no_progress, type_param + ) + + def add_global_symbol(self, name: str, ctx: Context, node: SymbolNode) -> None: + """Add symbol to a global namespace. + + This doesn't check for previous definition and is only used + for serialization of classes nested in functions/methods. + + Classes defined within methods can be exposed through an + attribute type, but method-level symbol tables aren't serialized. + This method can be used to add such classes to an enclosing, + serialized symbol table. + """ + self.globals[func_scoped_name(name, ctx.line)] = SymbolTableNode(GDEF, node) + + def add_symbol_table_node( + self, + name: str, + symbol: SymbolTableNode, + context: Context | None = None, + can_defer: bool = True, + escape_comprehensions: bool = False, + no_progress: bool = False, + type_param: bool = False, + ) -> bool: + """Add symbol table node to the currently active symbol table. + + Return True if we actually added the symbol, or False if we refused + to do so (because something is not ready or it was a no-op). + + Generate an error if there is an invalid redefinition. + + If context is None, unconditionally add node, since we can't report + an error. Note that this is used by plugins to forcibly replace nodes! + + TODO: Prevent plugins from replacing nodes, as it could cause problems? + + Args: + name: short name of symbol + symbol: Node to add + can_defer: if True, defer current target if adding a placeholder + context: error context (see above about None value) + """ + names = self.current_symbol_table( + escape_comprehensions=escape_comprehensions, type_param=type_param + ) + existing = names.get(name) + if isinstance(symbol.node, PlaceholderNode) and can_defer: + if context is not None: + self.process_placeholder(name, "name", context) + else: + # see note in docstring describing None contexts + self.defer() + + if ( + existing is not None + and context is not None + and not is_valid_replacement(existing, symbol) + ): + # There is an existing node, so this may be a redefinition. + # If the new node points to the same node as the old one, + # or if both old and new nodes are placeholders, we don't + # need to do anything. + old = existing.node + new = symbol.node + if isinstance(new, PlaceholderNode): + # We don't know whether this is okay. Let's wait until the next iteration. + return False + if not is_same_symbol(old, new): + if isinstance(new, (FuncDef, Decorator, OverloadedFuncDef, TypeInfo)): + self.add_redefinition(names, name, symbol) + if isinstance(old, Var) and is_init_only(old): + if old.has_explicit_value: + self.fail("InitVar with default value cannot be redefined", context) + elif ( + not (isinstance(new, (FuncDef, Decorator)) and self.set_original_def(old, new)) + # Avoid (additional) errors for internal symbols. + and "@" not in name + ): + self.name_already_defined(name, context, existing) + elif type_param or ( + name not in self.missing_names[-1] and "*" not in self.missing_names[-1] + ): + names[name] = symbol + if not no_progress: + self.progress = True + return True + return False + + def add_redefinition(self, names: SymbolTable, name: str, symbol: SymbolTableNode) -> None: + """Add a symbol table node that reflects a redefinition as a function or a class. + + Redefinitions need to be added to the symbol table so that they can be found + through AST traversal, but they have dummy names of form 'name-redefinition[N]', + where N ranges over 2, 3, ... (omitted for the first redefinition). + + Note: we always store redefinitions independently of whether they are valid or not + (so they will be semantically analyzed), the caller should give an error for invalid + redefinitions (such as e.g. variable redefined as a class). + """ + i = 1 + # Don't serialize redefined nodes. They are likely to have + # busted internal references which can cause problems with + # serialization, and they can't have any external references to + # them. + symbol.no_serialize = True + while True: + if i == 1: + new_name = f"{name}-redefinition" + else: + new_name = f"{name}-redefinition{i}" + existing = names.get(new_name) + if existing is None: + names[new_name] = symbol + return + elif existing.node is symbol.node: + # Already there + return + i += 1 + + def add_local(self, node: Var | FuncDef | OverloadedFuncDef, context: Context) -> None: + """Add local variable or function.""" + assert self.is_func_scope() + name = node.name + node._fullname = name + self.add_symbol(name, node, context) + + def _get_node_for_class_scoped_import( + self, name: str, symbol_node: SymbolNode | None, context: Context + ) -> SymbolNode | None: + if symbol_node is None: + return None + # I promise this type checks; I'm just making mypyc issues go away. + # mypyc is absolutely convinced that `symbol_node` narrows to a Var in the following, + # when it can also be a FuncBase. Once fixed, `f` in the following can be removed. + # See also https://github.com/mypyc/mypyc/issues/892 + f: Callable[[object], Any] = lambda x: x + if isinstance(f(symbol_node), (Decorator, FuncBase, Var)): + # For imports in class scope, we construct a new node to represent the symbol and + # set its `info` attribute to `self.type`. + existing = self.current_symbol_table().get(name) + if ( + # The redefinition checks in `add_symbol_table_node` don't work for our + # constructed Var / FuncBase, so check for possible redefinitions here. + existing is not None + and isinstance(f(existing.node), (Decorator, FuncBase, Var)) + and ( + isinstance(f(existing.type), f(AnyType)) + or f(existing.type) == f(symbol_node).type + ) + ): + return existing.node + + # Construct the new node + if isinstance(f(symbol_node), (FuncBase, Decorator)): + # In theory we could construct a new node here as well, but in practice + # it doesn't work well, see #12197 + typ: Type | None = AnyType(TypeOfAny.from_error) + self.fail("Unsupported class scoped import", context) + else: + typ = f(symbol_node).type + symbol_node = Var(name, typ) + symbol_node._fullname = self.qualified_name(name) + assert self.type is not None # guaranteed by is_class_scope + symbol_node.info = self.type + symbol_node.line = context.line + symbol_node.column = context.column + return symbol_node + + def add_imported_symbol( + self, + name: str, + node: SymbolTableNode, + context: ImportBase, + module_public: bool, + module_hidden: bool, + ) -> None: + """Add an alias to an existing symbol through import.""" + assert not module_hidden or not module_public + + existing_symbol = self.lookup_current_scope(name) + if ( + existing_symbol + and not isinstance(existing_symbol.node, PlaceholderNode) + and not isinstance(node.node, PlaceholderNode) + ): + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name(name, existing_symbol, node, context): + return + + symbol_node: SymbolNode | None = node.node + + if self.is_class_scope(): + symbol_node = self._get_node_for_class_scoped_import(name, symbol_node, context) + + symbol = SymbolTableNode( + node.kind, symbol_node, module_public=module_public, module_hidden=module_hidden + ) + self.add_symbol_table_node(name, symbol, context) + + def add_unknown_imported_symbol( + self, + name: str, + context: Context, + target_name: str | None, + module_public: bool, + module_hidden: bool, + ) -> None: + """Add symbol that we don't know what it points to because resolving an import failed. + + This can happen if a module is missing, or it is present, but doesn't have + the imported attribute. The `target_name` is the name of symbol in the namespace + it is imported from. For example, for 'from mod import x as y' the target_name is + 'mod.x'. This is currently used only to track logical dependencies. + """ + existing = self.current_symbol_table().get(name) + if existing and isinstance(existing.node, Var) and existing.node.is_suppressed_import: + # This missing import was already added -- nothing to do here. + return + var = Var(name) + if self.options.logical_deps and target_name is not None: + # This makes it possible to add logical fine-grained dependencies + # from a missing module. We can't use this by default, since in a + # few places we assume that the full name points to a real + # definition, but this name may point to nothing. + var._fullname = target_name + elif self.type: + var._fullname = self.type.fullname + "." + name + var.info = self.type + else: + var._fullname = self.qualified_name(name) + var.is_ready = True + any_type = AnyType(TypeOfAny.from_unimported_type, missing_import_name=var._fullname) + var.type = any_type + var.is_suppressed_import = True + self.add_symbol( + name, var, context, module_public=module_public, module_hidden=module_hidden + ) + + # + # Other helpers + # + + @contextmanager + def tvar_scope_frame(self, frame: TypeVarLikeScope) -> Iterator[None]: + old_scope = self.tvar_scope + self.tvar_scope = frame + yield + self.tvar_scope = old_scope + + def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None: + """Defer current analysis target to be analyzed again. + + This must be called if something in the current target is + incomplete or has a placeholder node. However, this must *not* + be called during the final analysis iteration! Instead, an error + should be generated. Often 'process_placeholder' is a good + way to either defer or generate an error. + + NOTE: Some methods, such as 'anal_type', 'mark_incomplete' and + 'record_incomplete_ref', call this implicitly, or when needed. + They are usually preferable to a direct defer() call. + """ + assert not self.final_iteration, "Must not defer during final iteration" + if force_progress: + # Usually, we report progress if we have replaced a placeholder node + # with an actual valid node. However, sometimes we need to update an + # existing node *in-place*. For example, this is used by type aliases + # in context of forward references and/or recursive aliases, and in + # similar situations (recursive named tuples etc). + self.progress = True + self.deferred = True + # Store debug info for this deferral. + line = ( + debug_context.line if debug_context else self.statement.line if self.statement else -1 + ) + self.deferral_debug_context.append((self.cur_mod_id, line)) + + def track_incomplete_refs(self) -> Tag: + """Return tag that can be used for tracking references to incomplete names.""" + return self.num_incomplete_refs + + def found_incomplete_ref(self, tag: Tag) -> bool: + """Have we encountered an incomplete reference since starting tracking?""" + return self.num_incomplete_refs != tag + + def record_incomplete_ref(self) -> None: + """Record the encounter of an incomplete reference and defer current analysis target.""" + self.defer() + self.num_incomplete_refs += 1 + + def mark_incomplete( + self, + name: str, + node: Node, + becomes_typeinfo: bool = False, + module_public: bool = True, + module_hidden: bool = False, + ) -> None: + """Mark a definition as incomplete (and defer current analysis target). + + Also potentially mark the current namespace as incomplete. + + Args: + name: The name that we weren't able to define (or '*' if the name is unknown) + node: The node that refers to the name (definition or lvalue) + becomes_typeinfo: Pass this to PlaceholderNode (used by special forms like + named tuples that will create TypeInfos). + """ + self.defer(node) + if name == "*": + self.incomplete = True + elif not self.is_global_or_nonlocal(name): + fullname = self.qualified_name(name) + assert self.statement + placeholder = PlaceholderNode( + fullname, node, self.statement.line, becomes_typeinfo=becomes_typeinfo + ) + self.add_symbol( + name, + placeholder, + module_public=module_public, + module_hidden=module_hidden, + context=dummy_context(), + ) + self.missing_names[-1].add(name) + + def is_incomplete_namespace(self, fullname: str) -> bool: + """Is a module or class namespace potentially missing some definitions? + + If a name is missing from an incomplete namespace, we'll need to defer the + current analysis target. + """ + return fullname in self.incomplete_namespaces + + def process_placeholder( + self, name: str | None, kind: str, ctx: Context, force_progress: bool = False + ) -> None: + """Process a reference targeting placeholder node. + + If this is not a final iteration, defer current node, + otherwise report an error. + + The 'kind' argument indicates if this a name or attribute expression + (used for better error message). + """ + if self.final_iteration: + self.cannot_resolve_name(name, kind, ctx) + else: + self.defer(ctx, force_progress=force_progress) + + def cannot_resolve_name(self, name: str | None, kind: str, ctx: Context) -> None: + name_format = f' "{name}"' if name else "" + self.fail(f"Cannot resolve {kind}{name_format} (possible cyclic definition)", ctx) + if self.is_func_scope(): + self.note("Recursive types are not allowed at function scope", ctx) + + def qualified_name(self, name: str) -> str: + if self.type is not None: + return self.type._fullname + "." + name + elif self.is_func_scope(): + return name + else: + return self.cur_mod_id + "." + name + + @contextmanager + def enter( + self, function: FuncItem | GeneratorExpr | DictionaryComprehension + ) -> Iterator[None]: + """Enter a function, generator or comprehension scope.""" + names = self.saved_locals.setdefault(function, SymbolTable()) + self.locals.append(names) + is_comprehension = isinstance(function, (GeneratorExpr, DictionaryComprehension)) + self.scope_stack.append(SCOPE_FUNC if not is_comprehension else SCOPE_COMPREHENSION) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + self.loop_depth.append(0) + self.missing_names.append(set()) + try: + yield + finally: + self.locals.pop() + self.scope_stack.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + self.loop_depth.pop() + self.missing_names.pop() + + def is_func_scope(self) -> bool: + scope_type = self.scope_stack[-1] + if scope_type == SCOPE_ANNOTATION: + scope_type = self.scope_stack[-2] + return scope_type in (SCOPE_FUNC, SCOPE_COMPREHENSION) + + def is_nested_within_func_scope(self) -> bool: + """Are we underneath a function scope, even if we are in a nested class also?""" + return any(s in (SCOPE_FUNC, SCOPE_COMPREHENSION) for s in self.scope_stack) + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def is_module_scope(self) -> bool: + return not (self.is_class_scope() or self.is_func_scope()) + + def current_symbol_kind(self) -> int: + if self.is_class_scope(): + kind = MDEF + elif self.is_func_scope(): + kind = LDEF + else: + kind = GDEF + return kind + + def current_symbol_table( + self, escape_comprehensions: bool = False, type_param: bool = False + ) -> SymbolTable: + if type_param and self.scope_stack[-1] == SCOPE_ANNOTATION: + n = self.locals[-1] + assert n is not None + return n + elif self.is_func_scope(): + if self.scope_stack[-1] == SCOPE_ANNOTATION: + n = self.locals[-2] + else: + n = self.locals[-1] + assert n is not None + if escape_comprehensions: + assert len(self.locals) == len(self.scope_stack) + # Retrieve the symbol table from the enclosing non-comprehension scope. + for i, scope_type in enumerate(reversed(self.scope_stack)): + if scope_type != SCOPE_COMPREHENSION: + if i == len(self.locals) - 1: # The last iteration. + # The caller of the comprehension is in the global space. + names = self.globals + else: + names_candidate = self.locals[-1 - i] + assert ( + names_candidate is not None + ), "Escaping comprehension from invalid scope" + names = names_candidate + break + else: + assert False, "Should have at least one non-comprehension scope" + else: + names = n + assert names is not None + elif self.type is not None: + names = self.type.names + else: + names = self.globals + return names + + def is_global_or_nonlocal(self, name: str) -> bool: + return self.is_func_scope() and ( + name in self.global_decls[-1] or name in self.nonlocal_decls[-1] + ) + + def add_exports(self, exp_or_exps: Iterable[Expression] | Expression) -> None: + exps = [exp_or_exps] if isinstance(exp_or_exps, Expression) else exp_or_exps + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.append(exp.value) + + def name_not_defined(self, name: str, ctx: Context, namespace: str | None = None) -> None: + incomplete = self.is_incomplete_namespace(namespace or self.cur_mod_id) + if ( + namespace is None + and self.type + and not self.is_func_scope() + and self.incomplete_type_stack + and self.incomplete_type_stack[-1] + and not self.final_iteration + ): + # We are processing a class body for the first time, so it is incomplete. + incomplete = True + if incomplete: + # Target namespace is incomplete, so it's possible that the name will be defined + # later on. Defer current target. + self.record_incomplete_ref() + return + message = f'Name "{name}" is not defined' + if ( + not self.msg.prefer_simple_messages() + and "." not in name + and not (name.startswith("__") and name.endswith("__")) + and f"builtins.{name}" not in SUGGESTED_TEST_FIXTURES + and ctx.line not in self.errors.ignored_lines.get(self.errors.file, {}) + ): + alternatives = self._get_names_in_scope() + alternatives.discard(name) + matches = best_matches(name, alternatives, n=3) + if matches: + message += f"; did you mean {pretty_seq(matches, 'or')}?" + self.fail(message, ctx, code=codes.NAME_DEFINED) + + if f"builtins.{name}" in SUGGESTED_TEST_FIXTURES: + # The user probably has a missing definition in a test fixture. Let's verify. + fullname = f"builtins.{name}" + if self.lookup_fully_qualified_or_none(fullname) is None: + # Yes. Generate a helpful note. + self.msg.add_fixture_note(fullname, ctx) + + modules_with_unimported_hints = { + name.split(".", 1)[0] for name in TYPES_FOR_UNIMPORTED_HINTS + } + lowercased = {name.lower(): name for name in TYPES_FOR_UNIMPORTED_HINTS} + for module in modules_with_unimported_hints: + fullname = f"{module}.{name}".lower() + if fullname not in lowercased: + continue + # User probably forgot to import these types. + hint = ( + 'Did you forget to import it from "{module}"?' + ' (Suggestion: "from {module} import {name}")' + ).format(module=module, name=lowercased[fullname].rsplit(".", 1)[-1]) + self.note(hint, ctx, code=codes.NAME_DEFINED) + + def _get_names_in_scope(self) -> set[str]: + """Collect all names visible in the current scope for fuzzy matching suggestions. + + This includes: + - Local variables (from function scopes) + - Class attributes (only when directly in class body, not in methods) + - Global/module-level names + - Builtins + """ + names: set[str] = set() + + for table in self.locals: + if table is not None: + names.update(table.keys()) + + if self.is_class_scope(): + assert self.type is not None + names.update(self.type.names.keys()) + + names.update(self.globals.keys()) + + b = self.globals.get("__builtins__", None) + if b: + assert isinstance(b.node, MypyFile) + for builtin_name in b.node.names.keys(): + if not ( + len(builtin_name) > 1 and builtin_name[0] == "_" and builtin_name[1] != "_" + ): + names.add(builtin_name) + + # Filter out internal/dunder names that aren't useful as suggestions + return {n for n in names if not n.startswith("__")} + + def already_defined( + self, name: str, ctx: Context, original_ctx: SymbolTableNode | SymbolNode | None, noun: str + ) -> None: + if isinstance(original_ctx, SymbolTableNode): + node: SymbolNode | None = original_ctx.node + elif isinstance(original_ctx, SymbolNode): + node = original_ctx + else: + node = None + + if isinstance(original_ctx, SymbolTableNode) and isinstance(original_ctx.node, MypyFile): + # Since this is an import, original_ctx.node points to the module definition. + # Therefore, its line number is always 1, which is not useful for this + # error message. + extra_msg = " (by an import)" + elif node and node.line != -1 and self.is_local_name(node.fullname): + # TODO: Using previous symbol node may give wrong line. We should use + # the line number where the binding was established instead. + extra_msg = f" on line {node.line}" + else: + extra_msg = " (possibly by an import)" + self.fail( + f'{noun} "{unmangle(name)}" already defined{extra_msg}', ctx, code=codes.NO_REDEF + ) + + def name_already_defined( + self, name: str, ctx: Context, original_ctx: SymbolTableNode | SymbolNode | None = None + ) -> None: + self.already_defined(name, ctx, original_ctx, noun="Name") + + def attribute_already_defined( + self, name: str, ctx: Context, original_ctx: SymbolTableNode | SymbolNode | None = None + ) -> None: + self.already_defined(name, ctx, original_ctx, noun="Attribute") + + def is_local_name(self, name: str) -> bool: + """Does name look like reference to a definition in the current module?""" + return self.is_defined_in_current_module(name) or "." not in name + + def in_checked_function(self) -> bool: + """Should we type-check the current function? + + - Yes if --check-untyped-defs is set. + - Yes outside functions. + - Yes in annotated functions. + - No otherwise. + """ + if self.options.check_untyped_defs or not self.function_stack: + return True + + current_index = len(self.function_stack) - 1 + while current_index >= 0: + current_func = self.function_stack[current_index] + if not isinstance(current_func, LambdaExpr): + return not current_func.is_dynamic() + + # Special case, `lambda` inherits the "checked" state from its parent. + # Because `lambda` itself cannot be annotated. + # `lambdas` can be deeply nested, so we try to find at least one other parent. + current_index -= 1 + + # This means that we only have a stack of `lambda` functions, + # no regular functions. + return True + + def fail( + self, + msg: str | ErrorMessage, + ctx: Context, + serious: bool = False, + *, + code: ErrorCode | None = None, + blocker: bool = False, + ) -> None: + if not serious and not self.in_checked_function(): + return + # In case it's a bug and we don't really have context + assert ctx is not None, msg + if isinstance(msg, ErrorMessage): + if code is None: + code = msg.code + msg = msg.value + self.errors.report( + ctx.line, + ctx.column, + msg, + blocker=blocker, + code=code, + end_line=ctx.end_line, + end_column=ctx.end_column, + ) + + def note(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None: + if not self.in_checked_function(): + return + self.errors.report(ctx.line, ctx.column, msg, severity="note", code=code) + + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + if feature not in self.options.enable_incomplete_feature: + self.fail( + f'"{feature}" support is experimental,' + f" use --enable-incomplete-feature={feature} to enable", + ctx, + ) + return False + return True + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line, self.errors, self.options) + + def expr_to_analyzed_type( + self, + expr: Expression, + report_invalid_types: bool = True, + allow_placeholder: bool = False, + allow_type_any: bool = False, + allow_unbound_tvars: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + unique_name: str | None = None, + analyzing_tvar_def: bool = False, + ) -> Type | None: + if unique_name is not None and isinstance(expr, CallExpr): + # This is a legacy syntax intended mostly for Python 2, we keep it for + # backwards compatibility, but new features like generic named tuples + # and recursive named tuples will be not supported. + expr.accept(self) + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( + expr, unique_name + ) + if tvar_defs: + self.fail("Generic named tuples are not supported for legacy class syntax", expr) + self.note("Use either Python 3 class syntax, or the assignment syntax", expr) + if internal_name is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + elif not info: + self.defer(expr) + return None + assert info.tuple_type, "NamedTuple without tuple type" + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = self.expr_to_unanalyzed_type(expr) + return self.anal_type( + typ, + report_invalid_types=report_invalid_types, + allow_placeholder=allow_placeholder, + allow_type_any=allow_type_any, + allow_unbound_tvars=allow_unbound_tvars, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + analyzing_tvar_def=analyzing_tvar_def, + ) + + def analyze_type_expr(self, expr: Expression) -> None: + # There are certain expressions that mypy does not need to semantically analyze, + # since they analyzed solely as type. (For example, indexes in type alias definitions + # and base classes in class defs). External consumers of the mypy AST may need + # them semantically analyzed, however, if they need to treat it as an expression + # and not a type. (Which is to say, mypyc needs to do this.) Do the analysis + # in a fresh tvar scope in order to suppress any errors about using type variables. + with self.tvar_scope_frame(TypeVarLikeScope()), self.allow_unbound_tvars_set(): + expr.accept(self) + + def type_analyzer( + self, + *, + tvar_scope: TypeVarLikeScope | None = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_placeholder: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_final: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + prohibit_special_class_field_types: str | None = None, + allow_type_any: bool = False, + analyzing_tvar_def: bool = False, + ) -> TypeAnalyser: + if tvar_scope is None: + tvar_scope = self.tvar_scope + tpan = TypeAnalyser( + self, + tvar_scope, + self.plugin, + self.options, + self.cur_mod_node, + self.is_typeshed_stub_file, + allow_unbound_tvars=allow_unbound_tvars, + allow_tuple_literal=allow_tuple_literal, + report_invalid_types=report_invalid_types, + allow_placeholder=allow_placeholder, + allow_typed_dict_special_forms=allow_typed_dict_special_forms, + allow_final=allow_final, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + prohibit_self_type=prohibit_self_type, + prohibit_special_class_field_types=prohibit_special_class_field_types, + allow_type_any=allow_type_any, + analyzing_tvar_def=analyzing_tvar_def, + erase_tvar_defs=self.removed_type_vars[-1], + ) + tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) + tpan.global_scope = not self.type and not self.function_stack + return tpan + + def expr_to_unanalyzed_type(self, node: Expression, allow_unpack: bool = False) -> ProperType: + return expr_to_unanalyzed_type( + node, self.options, self.is_stub_file, allow_unpack=allow_unpack + ) + + def anal_type( + self, + typ: Type, + *, + tvar_scope: TypeVarLikeScope | None = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_placeholder: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_final: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + prohibit_special_class_field_types: str | None = None, + allow_type_any: bool = False, + analyzing_tvar_def: bool = False, + ) -> Type | None: + """Semantically analyze a type. + + Args: + typ: Type to analyze (if already analyzed, this is a no-op) + allow_placeholder: If True, may return PlaceholderType if + encountering an incomplete definition + + Return None only if some part of the type couldn't be bound *and* it + referred to an incomplete namespace or definition. In this case also + defer as needed. During a final iteration this won't return None; + instead report an error if the type can't be analyzed and return + AnyType. + + In case of other errors, report an error message and return AnyType. + + NOTE: The caller shouldn't defer even if this returns None or a + placeholder type. + """ + has_self_type = find_self_type( + typ, lambda name: self.lookup_qualified(name, typ, suppress_errors=True) + ) + if has_self_type and self.type and prohibit_self_type is None: + self.setup_self_type() + a = self.type_analyzer( + tvar_scope=tvar_scope, + allow_unbound_tvars=allow_unbound_tvars, + allow_tuple_literal=allow_tuple_literal, + allow_placeholder=allow_placeholder, + allow_typed_dict_special_forms=allow_typed_dict_special_forms, + allow_final=allow_final, + allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, + report_invalid_types=report_invalid_types, + prohibit_self_type=prohibit_self_type, + prohibit_special_class_field_types=prohibit_special_class_field_types, + allow_type_any=allow_type_any, + analyzing_tvar_def=analyzing_tvar_def, + ) + tag = self.track_incomplete_refs() + typ = typ.accept(a) + if self.found_incomplete_ref(tag): + # Something could not be bound yet. + return None + self.add_type_alias_deps(a.aliases_used) + return typ + + def class_type(self, self_type: Type) -> Type: + return TypeType.make_normalized(self_type) + + def schedule_patch(self, priority: int, patch: Callable[[], None]) -> None: + self.patches.append((priority, patch)) + + def report_hang(self) -> None: + print("Deferral trace:") + for mod, line in self.deferral_debug_context: + print(f" {mod}:{line}") + self.errors.report( + -1, + -1, + "INTERNAL ERROR: maximum semantic analysis iteration count reached", + blocker=True, + ) + + def add_plugin_dependency(self, trigger: str, target: str | None = None) -> None: + """Add dependency from trigger to a target. + + If the target is not given explicitly, use the current target. + """ + if target is None: + target = self.scope.current_target() + self.cur_mod_node.plugin_deps.setdefault(trigger, set()).add(target) + + def add_type_alias_deps( + self, aliases_used: Collection[str], target: str | None = None + ) -> None: + """Add full names of type aliases on which the current node depends. + + This is used by fine-grained incremental mode to re-check the corresponding nodes. + If `target` is None, then the target node used will be the current scope. + """ + if not aliases_used: + return + if target is None: + target = self.scope.current_target() + self.cur_mod_node.alias_deps[target].update(aliases_used) + + def is_mangled_global(self, name: str) -> bool: + # A global is mangled if there exists at least one renamed variant. + return unmangle(name) + "'" in self.globals + + def is_initial_mangled_global(self, name: str) -> bool: + # If there are renamed definitions for a global, the first one has exactly one prime. + return name == unmangle(name) + "'" + + def parse_bool(self, expr: Expression) -> bool | None: + # This wrapper is preserved for plugins. + return parse_bool(expr) + + def parse_str_literal(self, expr: Expression) -> str | None: + """Attempt to find the string literal value of the given expression. Returns `None` if no + literal value can be found.""" + if isinstance(expr, StrExpr): + return expr.value + if isinstance(expr, RefExpr) and isinstance(expr.node, Var) and expr.node.type is not None: + values = try_getting_str_literals_from_type(expr.node.type) + if values is not None and len(values) == 1: + return values[0] + return None + + def set_future_import_flags(self, module_name: str) -> None: + if module_name in FUTURE_IMPORTS: + self.modules[self.cur_mod_id].future_import_flags.add(FUTURE_IMPORTS[module_name]) + + def is_future_flag_set(self, flag: str) -> bool: + return self.modules[self.cur_mod_id].is_future_flag_set(flag) + + def parse_dataclass_transform_spec(self, call: CallExpr) -> DataclassTransformSpec: + """Build a DataclassTransformSpec from the arguments passed to the given call to + typing.dataclass_transform.""" + parameters = DataclassTransformSpec() + for name, value in zip(call.arg_names, call.args): + # Skip any positional args. Note that any such args are invalid, but we can rely on + # typeshed to enforce this and don't need an additional error here. + if name is None: + continue + + # field_specifiers is currently the only non-boolean argument; check for it first so + # so the rest of the block can fail through to handling booleans + if name == "field_specifiers": + parameters.field_specifiers = self.parse_dataclass_transform_field_specifiers( + value + ) + continue + + boolean = require_bool_literal_argument(self, value, name) + if boolean is None: + continue + + if name == "eq_default": + parameters.eq_default = boolean + elif name == "order_default": + parameters.order_default = boolean + elif name == "kw_only_default": + parameters.kw_only_default = boolean + elif name == "frozen_default": + parameters.frozen_default = boolean + else: + self.fail(f'Unrecognized dataclass_transform parameter "{name}"', call) + + return parameters + + def parse_dataclass_transform_field_specifiers(self, arg: Expression) -> tuple[str, ...]: + if not isinstance(arg, TupleExpr): + self.fail('"field_specifiers" argument must be a tuple literal', arg) + return () + + names = [] + for specifier in arg.items: + if not isinstance(specifier, RefExpr): + self.fail('"field_specifiers" must only contain identifiers', specifier) + return () + names.append(specifier.fullname) + return tuple(names) + + # leafs + def visit_int_expr(self, o: IntExpr, /) -> None: + return None + + def visit_str_expr(self, o: StrExpr, /) -> None: + return None + + def visit_bytes_expr(self, o: BytesExpr, /) -> None: + return None + + def visit_float_expr(self, o: FloatExpr, /) -> None: + return None + + def visit_complex_expr(self, o: ComplexExpr, /) -> None: + return None + + def visit_ellipsis(self, o: EllipsisExpr, /) -> None: + return None + + def visit_temp_node(self, o: TempNode, /) -> None: + return None + + def visit_pass_stmt(self, o: PassStmt, /) -> None: + return None + + def visit_singleton_pattern(self, o: SingletonPattern, /) -> None: + return None + + def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> None: + """Try to parse a value Expression as a type expression. + If success then annotate the Expression with the type that it spells. + If fails then emit no errors and take no further action. + + A value expression that is parsable as a type expression may be used + where a TypeForm is expected to represent the spelled type. + + Unlike ExpressionChecker.try_parse_as_type_expression() + (used in the later TypeChecker pass), this function can recognize + ALL kinds of type expressions, including type expressions containing + string annotations. + + If the provided Expression will be parsable later in + ExpressionChecker.try_parse_as_type_expression(), this function will + skip parsing the Expression to improve performance, because the later + function is called many fewer times (i.e. only lazily in a rare TypeForm + type context) than this function is called (i.e. eagerly for EVERY + expression in certain syntactic positions). + """ + # Count every call to this method for profiling + self.type_expression_parse_count += 1 + + # Bail ASAP if the Expression matches a common pattern that cannot possibly + # be a valid type expression, because this function is called very frequently + if not isinstance(maybe_type_expr, MaybeTypeExpression): + return + # Check types in order from most common to least common, for best performance + if isinstance(maybe_type_expr, (NameExpr, MemberExpr)): + # Defer parsing to the later TypeChecker pass, + # and only lazily in contexts where a TypeForm is expected + return + elif isinstance(maybe_type_expr, StrExpr): + if not self.string_could_be_type_expression(maybe_type_expr): + # Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr, IndexExpr): + if isinstance(maybe_type_expr.base, NameExpr): + if isinstance( + maybe_type_expr.base.node, Var + ) and not self.var_could_be_typing_special_form(maybe_type_expr.base.node): + # Leftmost part of IndexExpr refers to a Var. Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr.base, MemberExpr): + next_leftmost = maybe_type_expr.base + while True: + leftmost = next_leftmost.expr + if not isinstance(leftmost, MemberExpr): + break + next_leftmost = leftmost + if isinstance(leftmost, NameExpr): + if isinstance(leftmost.node, Var): + # Leftmost part of IndexExpr refers to a Var. Not a valid type. + maybe_type_expr.as_type = None + return + else: + # Leftmost part of IndexExpr is not a NameExpr. Not a valid type. + maybe_type_expr.as_type = None + return + else: + # IndexExpr base is neither a NameExpr nor MemberExpr. Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr, OpExpr): + if maybe_type_expr.op != "|": + # Binary operators other than '|' never spell a valid type + maybe_type_expr.as_type = None + return + else: + assert_never(maybe_type_expr) + + with self.isolated_error_analysis(): + try: + t = self.expr_to_analyzed_type(maybe_type_expr) + if self.errors.is_errors(): + t = None + except TypeTranslationError: + # Not a type expression + t = None + + if DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES and t is None: + original_flushed_files = set(self.errors.flushed_files) # save + try: + errors = self.errors.new_messages() # capture + finally: + self.errors.flushed_files = original_flushed_files # restore + + print( + f"SA.try_parse_as_type_expression: Full parse failure: {maybe_type_expr}, errors={errors!r}" + ) + + # Count full parse attempts for profiling + if t is not None: + self.type_expression_full_parse_success_count += 1 + else: + self.type_expression_full_parse_failure_count += 1 + + maybe_type_expr.as_type = t + + def string_could_be_type_expression(self, maybe_type_expr: StrExpr) -> bool: + str_value = maybe_type_expr.value # cache + if str_value.isidentifier(): + # Filter out string literals which look like an identifier but + # cannot be a type expression, for a few common reasons + sym = self.lookup(str_value, UnboundType(str_value), suppress_errors=True) + if sym is None: + # Does not refer to anything in the local symbol table + return False + else: # sym is not None + node = sym.node # cache + # The following early-reject checks are mutually exclusive, + # ordered by decreasing rejection frequency (measured on + # mypy's self-check) so the commonest rejections exit first. + # - TypeVarExpr, TypeVarTupleExpr, ParamSpecExpr (~951) + # - Var (~157) + # - FuncDef, OverloadedFuncDef, MypyFile (~48) + # - PlaceholderNode (~23) + unbound_tvar_or_paramspec = ( + isinstance(node, (TypeVarExpr, TypeVarTupleExpr, ParamSpecExpr)) + and self.tvar_scope.get_binding(sym) is None + ) + if unbound_tvar_or_paramspec: + # Either: + # 1. unbound_tvar: 'Type variable "{}" is unbound' [codes.VALID_TYPE] + # 2. unbound_paramspec: f'ParamSpec "{name}" is unbound' [codes.VALID_TYPE] + return False + if ( + isinstance(node, Var) + and self.var_type_is_known(node) + and not self.var_could_be_typing_special_form(node) + ): + # Var whose type is known and is not a special form. + # It is a value, not a type expression. + return False + if isinstance(node, (FuncDef, OverloadedFuncDef, MypyFile)): + # Functions and modules are never type expressions. + return False + if isinstance(node, PlaceholderNode) and not node.becomes_typeinfo: + # Either: + # 1. f'Cannot resolve name "{t.name}" (possible cyclic definition)' + # 2. Reference to an unknown placeholder node. + return False + elif (leftmost_name := dotted_identifier_leftmost(str_value)) is not None: + # Dotted-name string (e.g. "builtins.tuple", "typing.Mapping"). + # Look up the leftmost component; if it cannot be a type prefix + # then the whole dotted name cannot spell a type. Mirrors the + # IndexExpr-with-MemberExpr-base filter logic below. + sym = self.lookup(leftmost_name, UnboundType(leftmost_name), suppress_errors=True) + if sym is None: + # Leftmost component does not refer to anything in scope + return False + node = sym.node # cache + if isinstance(node, PlaceholderNode) and not node.becomes_typeinfo: + # Either: + # 1. f'Cannot resolve name "{t.name}" (possible cyclic definition)' + # 2. Reference to an unknown placeholder node. + return False + if isinstance(node, Var): + # Leftmost component is a Var: it is a value, so it cannot be + # the module or class prefix of a dotted type name. + return False + else: # does not look like an identifier or dotted identifier + if '"' in str_value or "'" in str_value: + # Only valid inside a Literal[...] or Annotated[..., ...] type + if "[" not in str_value: + # Cannot be a Literal[...] or Annotated[..., ...] type + return False + elif len(str_value) < 2 or str_value.isspace(): + # Whitespace-only strings cannot be valid types. Very short strings can + # only be valid if they are identifiers, but we already checked for those. + return False + # Filter out string literals with common patterns that could not + # possibly be in a type expression + if _MULTIPLE_WORDS_NONTYPE_RE.match(str_value): + # A common pattern in string literals containing a sentence. + # But cannot be a type expression. + return False + # Skip some checks when a non-zero even number of single or double quotes + # signals a possible Literal[...] component, whose quoted content + # could contain anything: symbols or identifiers that would be + # incorrectly processed by some checks. + sq = str_value.count("'") + dq = str_value.count('"') + if not ((sq > 0 and sq % 2 == 0) or (dq > 0 and dq % 2 == 0)): + # Filter out string literals containing characters or boundary + # patterns that never appear in valid type expressions: + # - Leading '.' (incomplete dotted name, file extension, etc) + # - Trailing '.' (incomplete dotted name, file extension, etc) + # - Characters never valid in a type expression (e.g. '/', ':', '<', '>', '@') + # - '-' not directly preceded by '[' (which can occur in Literal[-N]) + # NOTE: str_value is never empty here. Branches above return + # for every string shorter than 2 characters. + if str_value[0] == "." or str_value[-1] == "." or has_nontype_char(str_value): + return False + return True + + @staticmethod + def var_could_be_typing_special_form(var: Var) -> bool: + return ( + var.fullname.startswith("typing.") + or var.fullname.startswith("typing_extensions.") + or var.fullname.startswith("mypy_extensions.") + ) + + @staticmethod + def var_type_is_known(var: Var) -> bool: + return not ( + (var_typ_p := get_proper_type(var.type)) is None + or isinstance(var_typ_p, (AnyType, PlaceholderType, UnboundType)) + ) + + @contextmanager + def isolated_error_analysis(self) -> Iterator[None]: + """ + Context manager for performing error analysis that should not + affect the main SemanticAnalyzer state. + + Upon entering this context, `self.errors` will start empty. + Within this context, you can analyze expressions for errors. + Upon exiting this context, the original `self.errors` will be restored, + and any errors collected during the analysis will be discarded. + """ + # Save state + original_errors = self.errors + original_num_incomplete_refs = self.num_incomplete_refs + original_progress = self.progress + original_deferred = self.deferred + original_deferral_debug_context_len = len(self.deferral_debug_context) + + self.errors = Errors(Options()) + try: + yield + finally: + # Restore state + self.errors = original_errors + self.num_incomplete_refs = original_num_incomplete_refs + self.progress = original_progress + self.deferred = original_deferred + del self.deferral_debug_context[original_deferral_debug_context_len:] + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + if len(sig.arg_types) == 0: + return sig + return sig.copy_modified(arg_types=[new] + sig.arg_types[1:]) + elif isinstance(sig, Overloaded): + return Overloaded( + [cast(CallableType, replace_implicit_first_type(i, new)) for i in sig.items] + ) + else: + assert False + + +def refers_to_fullname(node: Expression, fullnames: str | tuple[str, ...]) -> bool: + """Is node a name or member expression with the given full name?""" + if not isinstance(fullnames, tuple): + fullnames = (fullnames,) + + if not isinstance(node, RefExpr): + return False + if node.fullname in fullnames: + return True + if isinstance(node.node, TypeAlias) and not node.node.python_3_12_type_alias: + return is_named_instance(node.node.target, fullnames) + return False + + +def refers_to_class_or_function(node: Expression) -> bool: + """Does semantically analyzed node refer to a class?""" + return isinstance(node, RefExpr) and isinstance( + node.node, (TypeInfo, FuncDef, OverloadedFuncDef) + ) + + +def find_duplicate(list: list[T]) -> T | None: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def remove_imported_names_from_symtable(names: SymbolTable, module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed: list[str] = [] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname + prefix = fullname[: fullname.rfind(".")] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def make_any_non_explicit(t: Type) -> Type: + """Replace all Any types within in with Any that has attribute 'explicit' set to False""" + return t.accept(MakeAnyNonExplicit()) + + +class MakeAnyNonExplicit(TrivialSyntheticTypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if t.type_of_any == TypeOfAny.explicit: + return t.copy_modified(TypeOfAny.special_form) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + +def make_any_non_unimported(t: Type) -> Type: + """Replace all Any types that come from unimported types with special form Any.""" + return t.accept(MakeAnyNonUnimported()) + + +class MakeAnyNonUnimported(TrivialSyntheticTypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if t.type_of_any == TypeOfAny.from_unimported_type: + return t.copy_modified(TypeOfAny.special_form, missing_import_name=None) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + +def apply_semantic_analyzer_patches(patches: list[tuple[int, Callable[[], None]]]) -> None: + """Call patch callbacks in the right order. + + This should happen after semantic analyzer pass 3. + """ + patches_by_priority = sorted(patches, key=lambda x: x[0]) + for priority, patch_func in patches_by_priority: + patch_func() + + +def names_modified_by_assignment(s: AssignmentStmt) -> list[NameExpr]: + """Return all unqualified (short) names assigned to in an assignment statement.""" + result: list[NameExpr] = [] + for lvalue in s.lvalues: + result += names_modified_in_lvalue(lvalue) + return result + + +def names_modified_in_lvalue(lvalue: Lvalue) -> list[NameExpr]: + """Return all NameExpr assignment targets in an Lvalue.""" + if isinstance(lvalue, NameExpr): + return [lvalue] + elif isinstance(lvalue, StarExpr): + return names_modified_in_lvalue(lvalue.expr) + elif isinstance(lvalue, (ListExpr, TupleExpr)): + result: list[NameExpr] = [] + for item in lvalue.items: + result += names_modified_in_lvalue(item) + return result + return [] + + +def is_same_var_from_getattr(n1: SymbolNode | None, n2: SymbolNode | None) -> bool: + """Do n1 and n2 refer to the same Var derived from module-level __getattr__?""" + return ( + isinstance(n1, Var) + and n1.from_module_getattr + and isinstance(n2, Var) + and n2.from_module_getattr + and n1.fullname == n2.fullname + ) + + +def dummy_context() -> Context: + return TempNode(AnyType(TypeOfAny.special_form)) + + +def is_valid_replacement(old: SymbolTableNode, new: SymbolTableNode) -> bool: + """Can symbol table node replace an existing one? + + These are the only valid cases: + + 1. Placeholder gets replaced with a non-placeholder + 2. Placeholder that isn't known to become type replaced with a + placeholder that can become a type + """ + if isinstance(old.node, PlaceholderNode): + if isinstance(new.node, PlaceholderNode): + return not old.node.becomes_typeinfo and new.node.becomes_typeinfo + else: + return True + return False + + +def is_same_symbol(a: SymbolNode | None, b: SymbolNode | None) -> bool: + return ( + a == b + or (isinstance(a, PlaceholderNode) and isinstance(b, PlaceholderNode)) + or is_same_var_from_getattr(a, b) + ) + + +def is_trivial_body(block: Block) -> bool: + """Returns 'true' if the given body is "trivial" -- if it contains just a "pass", + "..." (ellipsis), or "raise NotImplementedError()". A trivial body may also + start with a statement containing just a string (e.g. a docstring). + + Note: Functions that raise other kinds of exceptions do not count as + "trivial". We use this function to help us determine when it's ok to + relax certain checks on body, but functions that raise arbitrary exceptions + are more likely to do non-trivial work. For example: + + def halt(self, reason: str = ...) -> NoReturn: + raise MyCustomError("Fatal error: " + reason, self.line, self.context) + + A function that raises just NotImplementedError is much less likely to be + this complex. + + Note: If you update this, you may also need to update + mypy.fastparse.is_possible_trivial_body! + """ + body = block.body + if not body: + # Functions have empty bodies only if the body is stripped or the function is + # generated or deserialized. In these cases the body is unknown. + return False + + # Skip a docstring + if isinstance(body[0], ExpressionStmt) and isinstance(body[0].expr, StrExpr): + body = block.body[1:] + + if len(body) == 0: + # There's only a docstring (or no body at all). + return True + elif len(body) > 1: + return False + + stmt = body[0] + + if isinstance(stmt, RaiseStmt): + expr = stmt.expr + if expr is None: + return False + if isinstance(expr, CallExpr): + expr = expr.callee + + return isinstance(expr, NameExpr) and expr.fullname == "builtins.NotImplementedError" + + return isinstance(stmt, PassStmt) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ) + + +def is_init_only(node: Var) -> bool: + return ( + isinstance(type := get_proper_type(node.type), Instance) + and type.type.fullname == "dataclasses.InitVar" + ) + + +def erase_func_annotations(func: FuncDef) -> None: + func.type_args = None + for arg in func.arguments: + arg.type_annotation = None + arg.variable.type = None + func.type = None + func.unanalyzed_type = None + + +def dotted_identifier_leftmost(s: str) -> str | None: + """The leftmost component of s, if s is a dotted identifier, else None. + + A dotted identifier is two or more identifiers joined by ".", such as + "builtins.tuple" or "typing.Mapping". A bare identifier is not a dotted + identifier: callers are expected to handle that case separately. + + Returns the leftmost component (which is never empty) so that callers + need not split s a second time to obtain it. + """ + # NOTE: Scanning with find() rather than s.split(".") avoids allocating a + # list, since only the leftmost component is ever needed. + dot = s.find(".") + if dot == -1: + return None + leftmost = s[:dot] + if not leftmost.isidentifier(): + return None + start = dot + 1 + while True: + dot = s.find(".", start) + if dot == -1: + if not s[start:].isidentifier(): + return None + return leftmost + if not s[start:dot].isidentifier(): + return None + start = dot + 1 + + +def has_nontype_char(s: str) -> bool: + """Whether s contains a character that cannot appear in a type expression. + + Callers must exclude strings that may contain a quoted Literal[...] + component first, since quoted content can hold arbitrary characters. + """ + # Look for _NONTYPE_CHARS + # NOTE: Iterating over s first (instead of _NONTYPE_CHARS) is NOT faster. + # NOTE: set.isdisjoint() is faster in interpreted-mypy (-22% runtime) + # but slower in compiled-mypy (+16% runtime) + for ch in _NONTYPE_CHARS: + if ch in s: + return True + + # Look for "-". + # It is valid only as a unary minus introducing a Literal[...] element, + # which is to say only where the preceding non-space character is a "[" + # (as in Literal[-1]) or a "," (as in Literal[-1, -2]). + # A "-" anywhere else means s cannot be a type. + i = s.find("-") + while i != -1: + # Look for a preceding "[" or "," (skipping whitespace) + j = i - 1 + while j >= 0 and s[j] == " ": + j -= 1 + if j < 0 or (s[j] != "[" and s[j] != ","): + return True + + # Continue to the next "-" + i = s.find("-", i + 1) + + return False diff --git a/mypy/typeanal.py b/mypy/typeanal.py new file mode 100644 index 000000000000..6717f4f18c90 --- /dev/null +++ b/mypy/typeanal.py @@ -0,0 +1,2961 @@ +"""Semantic analysis of types""" + +from __future__ import annotations + +import itertools +from collections.abc import Callable, Iterable, Iterator, Sequence +from contextlib import contextmanager +from typing import Final, Protocol, TypeVar + +from mypy import errorcodes as codes, message_registry, nodes +from mypy.errorcodes import ErrorCode +from mypy.errors import ErrorInfo +from mypy.expandtype import expand_type +from mypy.message_registry import ( + INVALID_PARAM_SPEC_LOCATION, + INVALID_PARAM_SPEC_LOCATION_NOTE, + TYPEDDICT_OVERRIDE_MERGE, +) +from mypy.messages import ( + MessageBuilder, + format_type, + format_type_bare, + quote_type_string, + wrong_type_arg_count, +) +from mypy.nodes import ( + ARG_NAMED, + ARG_NAMED_OPT, + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + MISSING_FALLBACK, + SYMBOL_FUNCBASE_TYPES, + ArgKind, + Context, + Decorator, + ImportFrom, + MypyFile, + ParamSpecExpr, + PlaceholderNode, + SymbolTableNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarLikeExpr, + TypeVarTupleExpr, + Var, + check_arg_kinds, + check_param_names, +) +from mypy.options import INLINE_TYPEDDICT, Options +from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface +from mypy.semanal_shared import ( + SemanticAnalyzerCoreInterface, + SemanticAnalyzerInterface, + paramspec_args, + paramspec_kwargs, +) +from mypy.state import state +from mypy.tvar_scope import TypeVarLikeScope +from mypy.types import ( + ANNOTATED_TYPE_NAMES, + ANY_STRATEGY, + CONCATENATE_TYPE_NAMES, + FINAL_TYPE_NAMES, + LITERAL_TYPE_NAMES, + MYPYC_NATIVE_INT_NAMES, + NEVER_NAMES, + TUPLE_NAMES, + TYPE_ALIAS_NAMES, + TYPE_NAMES, + UNPACK_TYPE_NAMES, + AnyType, + BoolTypeQuery, + CallableArgument, + CallableType, + CollectAliasesVisitor, + DeletedType, + EllipsisType, + ErasedType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecFlavor, + ParamSpecType, + PartialType, + PlaceholderType, + ProperType, + RawExpressionType, + ReadOnlyType, + RequiredType, + SyntheticTypeVisitor, + TrivialSyntheticTypeTranslator, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeList, + TypeOfAny, + TypeQuery, + TypeType, + TypeVarId, + TypeVarLikeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + callable_with_ellipsis, + find_unpack_in_list, + flatten_nested_tuples, + get_proper_type, + has_type_vars, +) +from mypy.types_utils import get_bad_type_type_item +from mypy.typevars import fill_typevars + +T = TypeVar("T") + +type_constructors: Final = { + "typing.Callable", + "typing.Optional", + "typing.Tuple", + "typing.Type", + "typing.Union", + *LITERAL_TYPE_NAMES, + *ANNOTATED_TYPE_NAMES, +} + +ARG_KINDS_BY_CONSTRUCTOR: Final = { + "mypy_extensions.Arg": ARG_POS, + "mypy_extensions.DefaultArg": ARG_OPT, + "mypy_extensions.NamedArg": ARG_NAMED, + "mypy_extensions.DefaultNamedArg": ARG_NAMED_OPT, + "mypy_extensions.VarArg": ARG_STAR, + "mypy_extensions.KwArg": ARG_STAR2, +} + +SELF_TYPE_NAMES: Final = {"typing.Self", "typing_extensions.Self"} + + +def analyze_type_alias( + type: Type, + api: SemanticAnalyzerCoreInterface, + tvar_scope: TypeVarLikeScope, + plugin: Plugin, + options: Options, + cur_mod_node: MypyFile, + is_typeshed_stub: bool, + allow_placeholder: bool = False, + in_dynamic_func: bool = False, + global_scope: bool = True, + allowed_alias_tvars: list[TypeVarLikeType] | None = None, + erase_tvar_defs: list[TypeVarType] | None = None, + alias_type_params_names: list[str] | None = None, + python_3_12_type_alias: bool = False, +) -> tuple[Type, set[str]]: + """Analyze r.h.s. of a (potential) type alias definition. + + If `node` is valid as a type alias rvalue, return the resulting type and a set of + full names of type aliases it depends on (directly or indirectly). + 'node' must have been semantically analyzed. + """ + analyzer = TypeAnalyser( + api, + tvar_scope, + plugin, + options, + cur_mod_node, + is_typeshed_stub, + defining_alias=True, + allow_placeholder=allow_placeholder, + prohibit_self_type="type alias target", + allowed_alias_tvars=allowed_alias_tvars, + erase_tvar_defs=erase_tvar_defs, + alias_type_params_names=alias_type_params_names, + python_3_12_type_alias=python_3_12_type_alias, + ) + analyzer.in_dynamic_func = in_dynamic_func + analyzer.global_scope = global_scope + res = analyzer.anal_type(type, nested=False) + return res, analyzer.aliases_used + + +class TypeAnalyser(SyntheticTypeVisitor[Type], TypeAnalyzerPluginInterface): + """Semantic analyzer for types. + + Converts unbound types into bound types. This is a no-op for already + bound types. + + If an incomplete reference is encountered, this does a defer. The + caller never needs to defer. + """ + + # Is this called from an untyped function definition? + in_dynamic_func: bool = False + # Is this called from global scope? + global_scope: bool = True + + def __init__( + self, + api: SemanticAnalyzerCoreInterface, + tvar_scope: TypeVarLikeScope, + plugin: Plugin, + options: Options, + cur_mod_node: MypyFile, + is_typeshed_stub: bool, + *, + defining_alias: bool = False, + python_3_12_type_alias: bool = False, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + allow_placeholder: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_final: bool = True, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + report_invalid_types: bool = True, + prohibit_self_type: str | None = None, + prohibit_special_class_field_types: str | None = None, + allowed_alias_tvars: list[TypeVarLikeType] | None = None, + erase_tvar_defs: list[TypeVarType] | None = None, + allow_type_any: bool = False, + alias_type_params_names: list[str] | None = None, + analyzing_tvar_def: bool = False, + ) -> None: + self.api = api + self.fail_func = api.fail + self.note_func = api.note + self.tvar_scope = tvar_scope + # Are we analysing a type alias definition rvalue? + self.defining_alias = defining_alias + self.python_3_12_type_alias = python_3_12_type_alias + self.allow_tuple_literal = allow_tuple_literal + # Positive if we are analyzing arguments of another (outer) type + self.nesting_level = 0 + # Should we accept unbound type variables? This is currently used for class bases, + # and alias right hand sides (before they are analyzed as type aliases). + self.allow_unbound_tvars = allow_unbound_tvars + if allowed_alias_tvars is None: + allowed_alias_tvars = [] + self.allowed_alias_tvars = allowed_alias_tvars + # Should we erase some type variables? This can be used to mass-erase type + # variables that were found to be invalid at the class/alias definition. + if erase_tvar_defs is None: + erase_tvar_defs = [] + self.erase_tvar_defs = erase_tvar_defs + self.alias_type_params_names = alias_type_params_names + # If false, record incomplete ref if we generate PlaceholderType. + self.allow_placeholder = allow_placeholder + # Are we in a context where Required[] is allowed? + self.allow_typed_dict_special_forms = allow_typed_dict_special_forms + # Set True when we analyze ClassVar else False + self.allow_final = allow_final + # Are we in a context where ParamSpec literals are allowed? + self.allow_param_spec_literals = allow_param_spec_literals + # Are we in context where literal "..." specifically is allowed? + self.allow_ellipsis = False + # Should we report an error whenever we encounter a RawExpressionType outside + # of a Literal context: e.g. whenever we encounter an invalid type? Normally, + # we want to report an error, but the caller may want to do more specialized + # error handling. + self.report_invalid_types = report_invalid_types + self.plugin = plugin + self.options = options + self.cur_mod_node = cur_mod_node + self.is_typeshed_stub = is_typeshed_stub + # Names of type aliases encountered while analysing a type will be collected here. + self.aliases_used: set[str] = set() + self.prohibit_self_type = prohibit_self_type + # Set when we analyze TypedDicts or NamedTuples, since they are special: + self.prohibit_special_class_field_types = prohibit_special_class_field_types + # Allow variables typed as Type[Any] and type (useful for base classes). + self.allow_type_any = allow_type_any + # Level of nesting at which a TypeVarTuple is allowed. Note we specify exact level + # to prohibit things like Unpack[list[Ts]], which are not supported. + self.allow_type_var_tuple = -1 + self.allow_unpack = allow_unpack + # Set when we are analyzing a default of a type variable. + self.analyzing_tvar_def = analyzing_tvar_def + + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + return self.api.lookup_qualified(name, ctx, suppress_errors) + + def lookup_fully_qualified(self, fullname: str) -> SymbolTableNode: + return self.api.lookup_fully_qualified(fullname) + + def visit_unbound_type(self, t: UnboundType, defining_literal: bool = False) -> Type: + typ = self.visit_unbound_type_nonoptional(t, defining_literal) + if t.optional: + # We don't need to worry about double-wrapping Optionals or + # wrapping Anys: Union simplification will take care of that. + return make_optional_type(typ) + return typ + + def not_declared_in_type_params(self, tvar_name: str) -> bool: + return ( + self.alias_type_params_names is not None + and tvar_name not in self.alias_type_params_names + ) + + def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) -> Type: + sym = self.lookup_qualified(t.name, t) + param_spec_name = None + if t.name.endswith((".args", ".kwargs")): + param_spec_name = t.name.rsplit(".", 1)[0] + maybe_param_spec = self.lookup_qualified(param_spec_name, t) + if maybe_param_spec and isinstance(maybe_param_spec.node, ParamSpecExpr): + sym = maybe_param_spec + else: + param_spec_name = None + + if sym is not None: + node = sym.node + if isinstance(node, PlaceholderNode): + if node.becomes_typeinfo: + # Reference to placeholder type. + if self.api.final_iteration: + self.cannot_resolve_type(t) + return AnyType(TypeOfAny.from_error) + elif self.allow_placeholder: + self.api.defer() + else: + self.api.record_incomplete_ref() + # Always allow ParamSpec for placeholders, if they are actually not valid, + # they will be reported later, after we resolve placeholders. + return PlaceholderType( + node.fullname, + self.anal_array( + t.args, + allow_param_spec=True, + allow_param_spec_literals=True, + allow_unpack=True, + ), + t.line, + ) + else: + if self.api.final_iteration: + self.cannot_resolve_type(t) + return AnyType(TypeOfAny.from_error) + else: + # Reference to an unknown placeholder node. + self.api.record_incomplete_ref() + return AnyType(TypeOfAny.special_form) + if node is None: + self.fail(f"Internal error (node is None, kind={sym.kind})", t) + return AnyType(TypeOfAny.special_form) + fullname = node.fullname + hook = self.plugin.get_type_analyze_hook(fullname) + if hook is not None: + return hook(AnalyzeTypeContext(t, t, self)) + tvar_def = self.tvar_scope.get_binding(sym) + if tvar_def is not None: + # We need to cover special-case explained in get_typevarlike_argument() here, + # since otherwise the deferral will not be triggered if the type variable is + # used in a different module. Using isinstance() should be safe for this purpose. + tvar_params = [tvar_def.upper_bound, tvar_def.default] + if isinstance(tvar_def, TypeVarType): + tvar_params += tvar_def.values + if any(isinstance(tp, PlaceholderType) for tp in tvar_params): + self.api.defer() + if isinstance(sym.node, ParamSpecExpr): + if tvar_def is None: + if self.allow_unbound_tvars: + return t + name = param_spec_name or t.name + if self.defining_alias and self.not_declared_in_type_params(t.name): + msg = f'ParamSpec "{name}" is not included in type_params' + else: + msg = f'ParamSpec "{name}" is unbound' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + assert isinstance(tvar_def, ParamSpecType) + if len(t.args) > 0: + self.fail( + f'ParamSpec "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) + if param_spec_name is not None and not self.allow_param_spec_literals: + self.fail( + "ParamSpec components are not allowed here", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + # Change the line number + return ParamSpecType( + tvar_def.name, + tvar_def.fullname, + tvar_def.id, + tvar_def.flavor, + tvar_def.upper_bound, + tvar_def.default, + line=t.line, + column=t.column, + ) + if ( + isinstance(sym.node, TypeVarExpr) + and self.defining_alias + and not defining_literal + and (tvar_def is None or tvar_def not in self.allowed_alias_tvars) + ): + if self.not_declared_in_type_params(t.name): + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{t.name}"' + ) + else: + msg = f'Type variable "{t.name}" is not included in type_params' + else: + msg = f'Can\'t use bound type variable "{t.name}" to define generic alias' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + if ( + isinstance(sym.node, TypeVarExpr) + and tvar_def is not None + and tvar_def in self.erase_tvar_defs + ): + # The caller should have already given a relevant error. + return AnyType(TypeOfAny.from_error) + if isinstance(sym.node, TypeVarExpr) and tvar_def is not None: + assert isinstance(tvar_def, TypeVarType) + if len(t.args) > 0: + self.fail( + f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) + # Change the line number + return tvar_def.copy_modified(line=t.line, column=t.column) + if isinstance(sym.node, TypeVarTupleExpr) and ( + tvar_def is not None + and self.defining_alias + and tvar_def not in self.allowed_alias_tvars + ): + if self.not_declared_in_type_params(t.name): + msg = f'Type variable "{t.name}" is not included in type_params' + else: + msg = f'Can\'t use bound type variable "{t.name}" to define generic alias' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + if isinstance(sym.node, TypeVarTupleExpr): + if tvar_def is None: + if self.allow_unbound_tvars: + return t + if self.defining_alias and self.not_declared_in_type_params(t.name): + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{t.name}"' + ) + else: + msg = f'TypeVarTuple "{t.name}" is not included in type_params' + else: + msg = f'TypeVarTuple "{t.name}" is unbound' + self.fail(msg, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + assert isinstance(tvar_def, TypeVarTupleType) + if self.allow_type_var_tuple != self.nesting_level: + self.fail( + f'TypeVarTuple "{t.name}" is only valid with an unpack', + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) > 0: + self.fail( + f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) + + # Change the line number + return TypeVarTupleType( + tvar_def.name, + tvar_def.fullname, + tvar_def.id, + tvar_def.upper_bound, + sym.node.tuple_fallback, + tvar_def.default, + line=t.line, + column=t.column, + ) + special = self.try_analyze_special_unbound_type(t, fullname) + if special is not None: + return special + if isinstance(node, TypeAlias): + self.aliases_used.add(fullname) + an_args = self.anal_array( + t.args, + allow_param_spec=True, + allow_param_spec_literals=node.has_param_spec_type, + allow_unpack=True, # Fixed length unpacks can be used for non-variadic aliases. + ) + if node.has_param_spec_type and len(node.alias_tvars) == 1: + an_args = self.pack_paramspec_args(an_args, t.empty_tuple_index) + + disallow_any = self.options.disallow_any_generics and not self.is_typeshed_stub + res, used_default = instantiate_type_alias( + node, + an_args, + self.fail, + self.note, + node.no_args, + t, + self.options, + unexpanded_type=t, + disallow_any=disallow_any, + empty_tuple_index=t.empty_tuple_index, + analyzing_tvar_def=self.analyzing_tvar_def, + ) + if self.analyzing_tvar_def and used_default and isinstance(res, TypeAliasType): + assert res.alias is not None + self.api.record_fixed_type(res.alias) + # The only case where instantiate_type_alias() can return an incorrect instance is + # when it is top-level instance, so no need to recurse. + if ( + isinstance(res, ProperType) + and isinstance(res, Instance) + and not (self.defining_alias and self.nesting_level == 0) + and not validate_instance(res, self.fail, t.empty_tuple_index) + ): + used_default = fix_instance( + res, + self.fail, + self.note, + disallow_any=disallow_any, + options=self.options, + use_generic_error=True, + unexpanded_type=t, + analyzing_tvar_def=self.analyzing_tvar_def, + ) + if self.analyzing_tvar_def and used_default: + self.api.record_fixed_type(res.type) + if node.eager: + res = get_proper_type(res) + return res + elif isinstance(node, TypeInfo): + return self.analyze_type_with_type_info(node, t.args, t, t.empty_tuple_index) + elif node.fullname in TYPE_ALIAS_NAMES: + return AnyType(TypeOfAny.special_form) + # Concatenate is an operator, no need for a proper type + elif node.fullname in CONCATENATE_TYPE_NAMES: + # We check the return type further up the stack for valid use locations + return self.apply_concatenate_operator(t) + else: + return self.analyze_unbound_type_without_type_info(t, sym, defining_literal) + else: # sym is None + return AnyType(TypeOfAny.special_form) + + def pack_paramspec_args(self, an_args: Sequence[Type], empty_tuple_index: bool) -> list[Type]: + # "Aesthetic" ParamSpec literals for single ParamSpec: C[int, str] -> C[[int, str]]. + # These do not support mypy_extensions VarArgs, etc. as they were already analyzed + # TODO: should these be re-analyzed to get rid of this inconsistency? + count = len(an_args) + if count == 0 and empty_tuple_index: + return [Parameters([], [], [])] + elif count == 0: + return [] + + if count == 1 and isinstance(get_proper_type(an_args[0]), AnyType): + # Single Any is interpreted as ..., rather that a single argument with Any type. + # I didn't find this in the PEP, but it sounds reasonable. + return list(an_args) + if any(isinstance(a, (Parameters, ParamSpecType)) for a in an_args): + if len(an_args) > 1: + first_wrong = next( + arg for arg in an_args if isinstance(arg, (Parameters, ParamSpecType)) + ) + self.fail( + "Nested parameter specifications are not allowed", + first_wrong, + code=codes.VALID_TYPE, + ) + return [AnyType(TypeOfAny.from_error)] + return list(an_args) + first = an_args[0] + return [ + Parameters( + an_args, [ARG_POS] * count, [None] * count, line=first.line, column=first.column + ) + ] + + def cannot_resolve_type(self, t: UnboundType) -> None: + # TODO: Move error message generation to messages.py. We'd first + # need access to MessageBuilder here. Also move the similar + # message generation logic in semanal.py. + self.api.fail(f'Cannot resolve name "{t.name}" (possible cyclic definition)', t) + if self.api.is_func_scope(): + self.note("Recursive types are not allowed at function scope", t) + + def apply_concatenate_operator(self, t: UnboundType) -> Type: + if len(t.args) == 0: + self.api.fail("Concatenate needs type arguments", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + + # Last argument has to be ParamSpec or Ellipsis. + ps = self.anal_type(t.args[-1], allow_param_spec=True, allow_ellipsis=True) + if not isinstance(ps, (ParamSpecType, Parameters)): + if isinstance(ps, UnboundType) and self.allow_unbound_tvars: + sym = self.lookup_qualified(ps.name, t) + if sym is not None and isinstance(sym.node, ParamSpecExpr): + return ps + self.api.fail( + "The last parameter to Concatenate needs to be a ParamSpec", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + elif isinstance(ps, ParamSpecType) and ps.prefix.arg_types: + self.api.fail("Nested Concatenates are invalid", t, code=codes.VALID_TYPE) + + args = self.anal_array(t.args[:-1]) + pre = ps.prefix if isinstance(ps, ParamSpecType) else ps + + # mypy can't infer this :( + names: list[str | None] = [None] * len(args) + + pre = Parameters( + args + pre.arg_types, + [ARG_POS] * len(args) + pre.arg_kinds, + names + pre.arg_names, + line=t.line, + column=t.column, + ) + return ps.copy_modified(prefix=pre) if isinstance(ps, ParamSpecType) else pre + + def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Type | None: + """Bind special type that is recognized through magic name such as 'typing.Any'. + + Return the bound type if successful, and return None if the type is a normal type. + """ + if fullname == "builtins.None": + return NoneType() + elif fullname == "typing.Any": + return AnyType(TypeOfAny.explicit, line=t.line, column=t.column) + elif fullname in FINAL_TYPE_NAMES: + if self.prohibit_special_class_field_types: + self.fail( + f"Final[...] can't be used inside a {self.prohibit_special_class_field_types}", + t, + code=codes.VALID_TYPE, + ) + else: + if not self.allow_final: + self.fail( + "Final can be only used as an outermost qualifier in a variable annotation", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + elif fullname in TUPLE_NAMES: + # Tuple is special because it is involved in builtin import cycle + # and may be not ready when used. + sym = self.api.lookup_fully_qualified_or_none("builtins.tuple") + if not sym or isinstance(sym.node, PlaceholderNode): + if self.api.is_incomplete_namespace("builtins"): + self.api.record_incomplete_ref() + else: + self.fail('Name "tuple" is not defined', t) + return AnyType(TypeOfAny.special_form) + if len(t.args) == 0 and not t.empty_tuple_index: + # Bare 'Tuple' is same as 'tuple' + any_type = self.get_omitted_any(t) + return self.named_type("builtins.tuple", [any_type], line=t.line, column=t.column) + if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): + # Tuple[T, ...] (uniform, variable-length tuple) + instance = self.named_type("builtins.tuple", [self.anal_type(t.args[0])]) + instance.line = t.line + return instance + return self.tuple_type( + self.anal_array(t.args, allow_unpack=True), line=t.line, column=t.column + ) + elif fullname == "typing.Union": + items = self.anal_array(t.args) + return UnionType.make_union(items, line=t.line, column=t.column) + elif fullname == "typing.Optional": + if len(t.args) != 1: + self.fail( + "Optional[...] must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + item = self.anal_type(t.args[0]) + return make_optional_type(item) + elif fullname == "typing.Callable": + return self.analyze_callable_type(t) + elif fullname in TYPE_NAMES: + if len(t.args) == 0: + if fullname == "typing.Type": + any_type = self.get_omitted_any(t) + return TypeType(any_type, line=t.line, column=t.column) + else: + # To prevent assignment of 'builtins.type' inferred as 'builtins.object' + # See https://github.com/python/mypy/issues/9476 for more information + return None + type_str = "Type[...]" if fullname == "typing.Type" else "type[...]" + if len(t.args) != 1: + self.fail( + f"{type_str} must have exactly one type argument", t, code=codes.VALID_TYPE + ) + item = self.anal_type(t.args[0]) + bad_item_name = get_bad_type_type_item(item) + if bad_item_name: + self.fail(f'{type_str} can\'t contain "{bad_item_name}"', t, code=codes.VALID_TYPE) + item = AnyType(TypeOfAny.from_error) + return TypeType.make_normalized(item, line=t.line, column=t.column) + elif fullname in ("typing_extensions.TypeForm", "typing.TypeForm"): + if len(t.args) == 0: + any_type = self.get_omitted_any(t) + return TypeType(any_type, line=t.line, column=t.column, is_type_form=True) + if len(t.args) != 1: + type_str = "TypeForm[...]" + self.fail( + type_str + " must have exactly one type argument", t, code=codes.VALID_TYPE + ) + item = self.anal_type(t.args[0]) + return TypeType.make_normalized(item, line=t.line, column=t.column, is_type_form=True) + elif fullname == "typing.ClassVar": + if self.nesting_level > 0: + self.fail( + "Invalid type: ClassVar nested inside other type", t, code=codes.VALID_TYPE + ) + if self.prohibit_special_class_field_types: + self.fail( + f"ClassVar[...] can't be used inside a {self.prohibit_special_class_field_types}", + t, + code=codes.VALID_TYPE, + ) + if self.defining_alias: + self.fail( + "ClassVar[...] can't be used inside a type alias", t, code=codes.VALID_TYPE + ) + if len(t.args) == 0: + return AnyType(TypeOfAny.from_omitted_generics, line=t.line, column=t.column) + if len(t.args) != 1: + self.fail( + "ClassVar[...] must have at most one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0], allow_final=self.options.python_version >= (3, 13)) + elif fullname in NEVER_NAMES: + return UninhabitedType() + elif fullname in LITERAL_TYPE_NAMES: + return self.analyze_literal_type(t) + elif fullname in ANNOTATED_TYPE_NAMES: + if len(t.args) < 2: + self.fail( + "Annotated[...] must have exactly one type argument" + " and at least one annotation", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + return self.anal_type( + t.args[0], allow_typed_dict_special_forms=self.allow_typed_dict_special_forms + ) + elif fullname in ("typing_extensions.Required", "typing.Required"): + if not self.allow_typed_dict_special_forms: + self.fail( + "Required[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + "Required[] must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return RequiredType( + self.anal_type(t.args[0], allow_typed_dict_special_forms=True), required=True + ) + elif fullname in ("typing_extensions.NotRequired", "typing.NotRequired"): + if not self.allow_typed_dict_special_forms: + self.fail( + "NotRequired[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + "NotRequired[] must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return RequiredType( + self.anal_type(t.args[0], allow_typed_dict_special_forms=True), required=False + ) + elif fullname in ("typing_extensions.ReadOnly", "typing.ReadOnly"): + if not self.allow_typed_dict_special_forms: + self.fail( + "ReadOnly[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + '"ReadOnly[]" must have exactly one type argument', t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return ReadOnlyType(self.anal_type(t.args[0], allow_typed_dict_special_forms=True)) + elif ( + self.anal_type_guard_arg(t, fullname) is not None + or self.anal_type_is_arg(t, fullname) is not None + ): + # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args) + return self.named_type("builtins.bool") + elif fullname in UNPACK_TYPE_NAMES: + if len(t.args) != 1: + self.fail("Unpack[...] requires exactly one type argument", t) + return AnyType(TypeOfAny.from_error) + if not self.allow_unpack: + self.fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + self.allow_type_var_tuple = self.nesting_level + 1 + result = UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) + self.allow_type_var_tuple = -1 + return result + elif fullname in SELF_TYPE_NAMES: + if t.args: + self.fail("Self type cannot have type arguments", t) + if self.prohibit_self_type is not None: + self.fail(f"Self type cannot be used in {self.prohibit_self_type}", t) + return AnyType(TypeOfAny.from_error) + if self.api.type is None: + self.fail("Self type is only allowed in annotations within class definition", t) + return AnyType(TypeOfAny.from_error) + if self.api.type.has_base("builtins.type"): + self.fail("Self type cannot be used in a metaclass", t) + if self.api.type.self_type is not None: + if self.api.type.is_final or self.api.type.is_enum and self.api.type.enum_members: + return fill_typevars(self.api.type) + return self.api.type.self_type.copy_modified(line=t.line, column=t.column) + # TODO: verify this is unreachable and replace with an assert? + self.fail("Unexpected Self type", t) + return AnyType(TypeOfAny.from_error) + return None + + def get_omitted_any(self, typ: Type, fullname: str | None = None) -> AnyType: + disallow_any = not self.is_typeshed_stub and self.options.disallow_any_generics + return get_omitted_any(disallow_any, self.fail, self.note, typ, self.options, fullname) + + def check_and_warn_deprecated(self, info: TypeInfo, ctx: Context) -> None: + """Similar logic to `TypeChecker.check_deprecated` and `TypeChecker.warn_deprecated.""" + + if ( + (deprecated := info.deprecated) + and not self.is_typeshed_stub + and not (self.api.type and (self.api.type.fullname == info.fullname)) + and not any( + info.fullname == p or info.fullname.startswith(f"{p}.") + for p in self.options.deprecated_calls_exclude + ) + ): + for imp in self.cur_mod_node.imports: + if isinstance(imp, ImportFrom) and any(info.name == n[0] for n in imp.names): + break + else: + warn = self.note if self.options.report_deprecated_as_note else self.fail + warn(deprecated, ctx, code=codes.DEPRECATED) + + def analyze_type_with_type_info( + self, info: TypeInfo, args: Sequence[Type], ctx: Context, empty_tuple_index: bool + ) -> Type: + """Bind unbound type when were able to find target TypeInfo. + + This handles simple cases like 'int', 'modname.UserClass[str]', etc. + """ + + self.check_and_warn_deprecated(info, ctx) + + if len(args) > 0 and info.fullname == "builtins.tuple": + fallback = Instance(info, [AnyType(TypeOfAny.special_form)], ctx.line) + return TupleType(self.anal_array(args, allow_unpack=True), fallback, ctx.line) + + # Analyze arguments and (usually) construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + # + # We allow ParamSpec literals based on a heuristic: it will be + # checked later anyways but the error message may be worse. + instance = Instance( + info, + self.anal_array( + args, + allow_param_spec=True, + allow_param_spec_literals=info.has_param_spec_type, + allow_unpack=True, # Fixed length tuples can be used for non-variadic types. + ), + ctx.line, + ctx.column, + ) + instance.end_line = ctx.end_line + instance.end_column = ctx.end_column + if len(info.type_vars) == 1 and info.has_param_spec_type: + instance.args = tuple(self.pack_paramspec_args(instance.args, empty_tuple_index)) + + if info.fullname == "librt.vecs.vec" and not check_vec_type_args( + instance.args, ctx, self.api + ): + return AnyType(TypeOfAny.from_error) + + # Check type argument count. + old_args = instance.args + instance.args = tuple(flatten_nested_tuples(instance.args)) + if old_args and not instance.args: + empty_tuple_index = True + if not (self.defining_alias and self.nesting_level == 0) and not validate_instance( + instance, self.fail, empty_tuple_index + ): + used_default = fix_instance( + instance, + self.fail, + self.note, + disallow_any=self.options.disallow_any_generics and not self.is_typeshed_stub, + options=self.options, + analyzing_tvar_def=self.analyzing_tvar_def, + ) + if self.analyzing_tvar_def and used_default: + self.api.record_fixed_type(info) + + tup = info.tuple_type + if tup is not None: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if info.special_alias: + res, used_default = instantiate_type_alias( + info.special_alias, + self.anal_array( + args, + allow_unpack=True, + allow_param_spec=True, + allow_param_spec_literals=True, + ), + self.fail, + self.note, + False, + ctx, + self.options, + use_standard_error=True, + empty_tuple_index=empty_tuple_index, + analyzing_tvar_def=self.analyzing_tvar_def, + ) + if self.analyzing_tvar_def and used_default: + # For convenience, we make default depend on the original TypeInfo, + # *not* on the special alias. + self.api.record_fixed_type(info) + return res + return tup.copy_modified( + items=self.anal_array(tup.items, allow_unpack=True), fallback=instance + ) + td = info.typeddict_type + if td is not None: + # The class has a TypedDict[...] base class so it will be + # represented as a typeddict type. + if info.special_alias: + res, used_default = instantiate_type_alias( + info.special_alias, + self.anal_array( + args, + allow_unpack=True, + allow_param_spec=True, + allow_param_spec_literals=True, + ), + self.fail, + self.note, + False, + ctx, + self.options, + use_standard_error=True, + analyzing_tvar_def=self.analyzing_tvar_def, + ) + if self.analyzing_tvar_def and used_default: + # For convenience, we make default depend on the original TypeInfo, + # *not* on the special alias. + self.api.record_fixed_type(info) + return res + # Create a named TypedDictType + return td.copy_modified( + item_types=self.anal_array(list(td.items.values())), fallback=instance + ) + + if info.fullname == "types.NoneType": + self.fail( + "NoneType should not be used as a type, please use None instead", + ctx, + code=codes.NONETYPE_TYPE, + ) + return NoneType(ctx.line, ctx.column) + + return instance + + def analyze_unbound_type_without_type_info( + self, t: UnboundType, sym: SymbolTableNode, defining_literal: bool + ) -> Type: + """Figure out what an unbound type that doesn't refer to a TypeInfo node means. + + This is something unusual. We try our best to find out what it is. + """ + name = sym.fullname + if name is None: + assert sym.node is not None + name = sym.node.name + # Option 1: + # Something with an Any type -- make it an alias for Any in a type + # context. This is slightly problematic as it allows using the type 'Any' + # as a base class -- however, this will fail soon at runtime so the problem + # is pretty minor. + if isinstance(sym.node, Var): + typ = get_proper_type(sym.node.type) + if isinstance(typ, AnyType): + return AnyType( + TypeOfAny.from_unimported_type, missing_import_name=typ.missing_import_name + ) + elif self.allow_type_any: + if isinstance(typ, Instance) and typ.type.fullname == "builtins.type": + return AnyType(TypeOfAny.special_form) + if isinstance(typ, TypeType) and isinstance(typ.item, AnyType): + return AnyType(TypeOfAny.from_another_any, source_any=typ.item) + # Option 2: + # Unbound type variable. Currently these may be still valid, + # for example when defining a generic type alias. + unbound_tvar = ( + isinstance(sym.node, (TypeVarExpr, TypeVarTupleExpr)) + and self.tvar_scope.get_binding(sym) is None + ) + if self.allow_unbound_tvars and unbound_tvar: + return t + + # Option 3: + # Enum value. Note: we only want to return a LiteralType when + # we're using this enum value specifically within context of + # a "Literal[...]" type. So, if `defining_literal` is not set, + # we bail out early with an error. + # + # If, in the distant future, we decide to permit things like + # `def foo(x: Color.RED) -> None: ...`, we can remove that + # check entirely. + if ( + isinstance(sym.node, Var) + and sym.node.info + and sym.node.info.is_enum + and sym.node.name in sym.node.info.enum_members + ): + value = sym.node.name + base_enum_short_name = sym.node.info.name + if not defining_literal: + msg = message_registry.INVALID_TYPE_RAW_ENUM_VALUE.format( + base_enum_short_name, value + ) + self.fail(msg.value, t, code=msg.code) + return AnyType(TypeOfAny.from_error) + return LiteralType( + value=value, + fallback=Instance(sym.node.info, [], line=t.line, column=t.column), + line=t.line, + column=t.column, + ) + + if isinstance(sym.node, Var) and sym.node.is_sentinel: + typ = get_proper_type(sym.node.type) + assert isinstance(typ, LiteralType) + return LiteralType( + value=typ.value, fallback=typ.fallback, line=t.line, column=t.column + ) + + # None of the above options worked. We parse the args (if there are any) + # to make sure there are no remaining semanal-only types, then give up. + t = t.copy_modified(args=self.anal_array(t.args)) + # TODO: Move this message building logic to messages.py. + notes: list[str] = [] + error_code = codes.VALID_TYPE + if isinstance(sym.node, Var): + notes.append( + "See https://mypy.readthedocs.io/en/" + "stable/common_issues.html#variables-vs-type-aliases" + ) + message = 'Variable "{}" is not valid as a type' + elif isinstance(sym.node, (SYMBOL_FUNCBASE_TYPES, Decorator)): + message = 'Function "{}" is not valid as a type' + if name == "builtins.any": + notes.append('Perhaps you meant "typing.Any" instead of "any"?') + elif name == "builtins.callable": + notes.append('Perhaps you meant "typing.Callable" instead of "callable"?') + else: + notes.append('Perhaps you need "Callable[...]" or a callback protocol?') + elif isinstance(sym.node, MypyFile): + message = 'Module "{}" is not valid as a type' + notes.append("Perhaps you meant to use a protocol matching the module structure?") + elif unbound_tvar: + assert isinstance(sym.node, TypeVarLikeExpr) + if sym.node.is_new_style: + # PEP 695 type parameters are never considered unbound -- they are undefined + # in contexts where they aren't valid, such as in argument default values. + message = 'Name "{}" is not defined' + name = name.split(".")[-1] + error_code = codes.NAME_DEFINED + else: + message = 'Type variable "{}" is unbound' + short = name.split(".")[-1] + notes.append( + f'(Hint: Use "Generic[{short}]" or "Protocol[{short}]" base class' + f' to bind "{short}" inside a class)' + ) + notes.append( + f'(Hint: Use "{short}" in function signature ' + f'to bind "{short}" inside a function)' + ) + else: + message = 'Cannot interpret reference "{}" as a type' + if not defining_literal: + # Literal check already gives a custom error. Avoid duplicating errors. + self.fail(message.format(name), t, code=error_code) + for note in notes: + self.note(note, t, code=error_code) + + # TODO: Would it be better to always return Any instead of UnboundType + # in case of an error? On one hand, UnboundType has a name so error messages + # are more detailed, on the other hand, some of them may be bogus, + # see https://github.com/python/mypy/issues/4987. + return t + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_none_type(self, t: NoneType) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + # This type should exist only temporarily during type inference + assert False, "Internal error: Unexpected erased type" + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + # Parameters literal (Z[[int, str, Whatever]]) + if self.allow_param_spec_literals: + params = self.analyze_callable_args(t) + if params: + ts, kinds, names = params + # bind these types + return Parameters(self.anal_array(ts), kinds, names, line=t.line, column=t.column) + else: + return AnyType(TypeOfAny.from_error) + else: + self.fail( + 'Bracketed expression "[...]" is not valid as a type', t, code=codes.VALID_TYPE + ) + if len(t.items) == 1: + self.note('Did you mean "List[...]"?', t) + return AnyType(TypeOfAny.from_error) + + def visit_callable_argument(self, t: CallableArgument) -> Type: + self.fail("Invalid type", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + + def visit_instance(self, t: Instance) -> Type: + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # TODO: should we do something here? + return t + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_param_spec(self, t: ParamSpecType) -> Type: + return t + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + return t + + def visit_unpack_type(self, t: UnpackType) -> Type: + if not self.allow_unpack: + self.fail(message_registry.INVALID_UNPACK_POSITION, t.type, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + self.allow_type_var_tuple = self.nesting_level + 1 + result = UnpackType(self.anal_type(t.type), from_star_syntax=t.from_star_syntax) + self.allow_type_var_tuple = -1 + return result + + def visit_parameters(self, t: Parameters) -> Type: + raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars") + + def visit_callable_type( + self, t: CallableType, nested: bool = True, namespace: str = "" + ) -> Type: + # Every Callable can bind its own type variables, if they're not in the outer scope + # TODO: attach namespace for nested free type variables (these appear in return type only). + with self.tvar_scope_frame(namespace=namespace): + unpacked_kwargs = t.unpack_kwargs + if self.defining_alias: + variables = t.variables + else: + variables, _ = self.bind_function_type_variables(t, t) + type_guard = self.anal_type_guard(t.ret_type) if t.type_guard is None else t.type_guard + type_is = self.anal_type_is(t.ret_type) if t.type_is is None else t.type_is + + arg_kinds = t.arg_kinds + arg_types = [] + param_spec_with_args = param_spec_with_kwargs = None + param_spec_invalid = False + for kind, ut in zip(arg_kinds, t.arg_types): + if kind == ARG_STAR: + param_spec_with_args, at = self.anal_star_arg_type(ut, kind, nested=nested) + elif kind == ARG_STAR2: + param_spec_with_kwargs, at = self.anal_star_arg_type(ut, kind, nested=nested) + else: + if param_spec_with_args: + param_spec_invalid = True + self.fail( + "Arguments not allowed after ParamSpec.args", t, code=codes.VALID_TYPE + ) + at = self.anal_type(ut, nested=nested, allow_unpack=False) + if isinstance(at, Parameters): + # A Parameters here comes from a misplaced Concatenate ending in + # an explicit ellipsis, e.g. Callable[[Concatenate[int, ...]], None]. + self.fail( + "Concatenate is only valid as the first argument to Callable", + ut, + code=codes.VALID_TYPE, + ) + at = AnyType(TypeOfAny.from_error) + arg_types.append(at) + + if nested and arg_types: + # If we've got a Callable[[Unpack[SomeTypedDict]], None], make sure + # Unpack is interpreted as `**` and not as `*`. + last = arg_types[-1] + if isinstance(last, UnpackType): + # TODO: it would be better to avoid this get_proper_type() call. + p_at = get_proper_type(last.type) + if isinstance(p_at, TypedDictType) and not last.from_star_syntax: + # Automatically detect Unpack[Foo] in Callable as backwards + # compatible syntax for **Foo, if Foo is a TypedDict. + arg_kinds[-1] = ARG_STAR2 + arg_types[-1] = p_at + unpacked_kwargs = True + arg_types = self.check_unpacks_in_list(arg_types) + + if not param_spec_invalid and param_spec_with_args != param_spec_with_kwargs: + # If already invalid, do not report more errors - definition has + # to be fixed anyway + name = param_spec_with_args or param_spec_with_kwargs + self.fail( + f'ParamSpec must have "*args" typed as "{name}.args" and "**kwargs" typed as "{name}.kwargs"', + t, + code=codes.VALID_TYPE, + ) + param_spec_invalid = True + + if param_spec_invalid: + if ARG_STAR in arg_kinds: + arg_types[arg_kinds.index(ARG_STAR)] = AnyType(TypeOfAny.from_error) + if ARG_STAR2 in arg_kinds: + arg_types[arg_kinds.index(ARG_STAR2)] = AnyType(TypeOfAny.from_error) + + # If there were multiple (invalid) unpacks, the arg types list will become shorter, + # we need to trim the kinds/names as well to avoid crashes. + arg_kinds = t.arg_kinds[: len(arg_types)] + arg_names = t.arg_names[: len(arg_types)] + + ret = t.copy_modified( + arg_types=arg_types, + arg_kinds=arg_kinds, + arg_names=arg_names, + ret_type=self.anal_type(t.ret_type, nested=nested), + # If the fallback isn't filled in yet, + # its type will be the falsey FakeInfo + fallback=(t.fallback if t.fallback.type else self.named_type("builtins.function")), + variables=self.anal_var_defs(variables), + type_guard=type_guard, + type_is=type_is, + unpack_kwargs=unpacked_kwargs, + ) + return ret + + def anal_type_guard(self, t: Type) -> Type | None: + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym is not None and sym.node is not None: + return self.anal_type_guard_arg(t, sym.node.fullname) + # TODO: What if it's an Instance? Then use t.type.fullname? + return None + + def anal_type_guard_arg(self, t: UnboundType, fullname: str) -> Type | None: + if fullname in ("typing_extensions.TypeGuard", "typing.TypeGuard"): + if len(t.args) != 1: + self.fail( + "TypeGuard must have exactly one type argument", t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0]) + return None + + def anal_type_is(self, t: Type) -> Type | None: + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym is not None and sym.node is not None: + return self.anal_type_is_arg(t, sym.node.fullname) + # TODO: What if it's an Instance? Then use t.type.fullname? + return None + + def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None: + if fullname in ("typing_extensions.TypeIs", "typing.TypeIs"): + if len(t.args) != 1: + self.fail("TypeIs must have exactly one type argument", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0]) + return None + + def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> tuple[str | None, Type]: + """Analyze signature argument type for *args and **kwargs argument.""" + if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args: + components = t.name.split(".") + tvar_name = ".".join(components[:-1]) + sym = self.lookup_qualified(tvar_name, t) + if sym is not None and isinstance(sym.node, ParamSpecExpr): + tvar_def = self.tvar_scope.get_binding(sym) + if isinstance(tvar_def, ParamSpecType): + if kind == ARG_STAR: + make_paramspec = paramspec_args + if components[-1] != "args": + self.fail( + f'Use "{tvar_name}.args" for variadic "*" parameter', + t, + code=codes.VALID_TYPE, + ) + elif kind == ARG_STAR2: + make_paramspec = paramspec_kwargs + if components[-1] != "kwargs": + self.fail( + f'Use "{tvar_name}.kwargs" for variadic "**" parameter', + t, + code=codes.VALID_TYPE, + ) + else: + assert False, kind + return tvar_name, make_paramspec( + tvar_def.name, + tvar_def.fullname, + tvar_def.id, + named_type_func=self.named_type, + line=t.line, + column=t.column, + ) + return None, self.anal_type(t, nested=nested, allow_unpack=True) + + def visit_overloaded(self, t: Overloaded) -> Type: + # Overloaded types are manually constructed in semanal.py by analyzing the + # AST and combining together the Callable types this visitor converts. + # + # So if we're ever asked to reanalyze an Overloaded type, we know it's + # fine to just return it as-is. + return t + + def visit_tuple_type(self, t: TupleType) -> Type: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if t.implicit and not self.allow_tuple_literal: + self.fail("Syntax error in type annotation", t, code=codes.SYNTAX) + if len(t.items) == 0: + self.note( + "Suggestion: Use Tuple[()] instead of () for an empty tuple, or " + "None for a function without a return value", + t, + code=codes.SYNTAX, + ) + elif len(t.items) == 1: + self.note("Suggestion: Is there a spurious trailing comma?", t, code=codes.SYNTAX) + else: + self.note( + "Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn)", + t, + code=codes.SYNTAX, + ) + return AnyType(TypeOfAny.from_error) + + any_type = AnyType(TypeOfAny.special_form) + # If the fallback isn't filled in yet, its type will be the falsey FakeInfo + fallback = ( + t.partial_fallback + if t.partial_fallback.type + else self.named_type("builtins.tuple", [any_type]) + ) + return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line) + + def visit_typeddict_type(self, t: TypedDictType) -> Type: + req_keys = set() + readonly_keys = set() + items = {} + for item_name, item_type in t.items.items(): + # TODO: rework + analyzed = self.anal_type(item_type, allow_typed_dict_special_forms=True) + if isinstance(analyzed, RequiredType): + if analyzed.required: + req_keys.add(item_name) + analyzed = analyzed.item + else: + # Keys are required by default. + req_keys.add(item_name) + if isinstance(analyzed, ReadOnlyType): + readonly_keys.add(item_name) + analyzed = analyzed.item + items[item_name] = analyzed + if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict + if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature: + self.fail( + "Inline TypedDict is experimental," + " must be enabled with --enable-incomplete-feature=InlineTypedDict", + t, + ) + is_closed = False + required_keys = req_keys + fallback = self.named_type("typing._TypedDict") + for typ in t.extra_items_from: + analyzed = self.analyze_type(typ) + p_analyzed = get_proper_type(analyzed) + if not isinstance(p_analyzed, TypedDictType): + if not isinstance(p_analyzed, (AnyType, PlaceholderType)): + self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE) + continue + for sub_item_name, sub_item_type in p_analyzed.items.items(): + if sub_item_name in items: + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t) + continue + items[sub_item_name] = sub_item_type + if sub_item_name in p_analyzed.required_keys: + req_keys.add(sub_item_name) + if sub_item_name in p_analyzed.readonly_keys: + readonly_keys.add(sub_item_name) + else: + readonly_keys = t.readonly_keys + required_keys = t.required_keys + fallback = t.fallback + is_closed = t.is_closed + return TypedDictType( + items, required_keys, readonly_keys, fallback, t.line, t.column, is_closed=is_closed + ) + + def visit_raw_expression_type(self, t: RawExpressionType) -> Type: + # We should never see a bare Literal. We synthesize these raw literals + # in the earlier stages of semantic analysis, but those + # "fake literals" should always be wrapped in an UnboundType + # corresponding to 'Literal'. + # + # Note: if at some point in the distant future, we decide to + # make signatures like "foo(x: 20) -> None" legal, we can change + # this method so it generates and returns an actual LiteralType + # instead. + + if self.report_invalid_types: + if t.base_type_name in ("builtins.int", "builtins.bool"): + # The only time it makes sense to use an int or bool is inside of + # a literal type. + msg = f"Invalid type: try using Literal[{repr(t.literal_value)}] instead?" + elif t.base_type_name in ("builtins.float", "builtins.complex"): + # We special-case warnings for floats and complex numbers. + msg = f"Invalid type: {t.simple_name()} literals cannot be used as a type" + else: + # And in all other cases, we default to a generic error message. + # Note: the reason why we use a generic error message for strings + # but not ints or bools is because whenever we see an out-of-place + # string, it's unclear if the user meant to construct a literal type + # or just misspelled a regular type. So we avoid guessing. + msg = "Invalid type comment or annotation" + + self.fail(msg, t, code=codes.VALID_TYPE) + if t.note is not None: + self.note(t.note, t, code=codes.VALID_TYPE) + + return AnyType(TypeOfAny.from_error, line=t.line, column=t.column) + + def visit_literal_type(self, t: LiteralType) -> Type: + return t + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.anal_array(t.items), t.line, uses_pep604_syntax=t.uses_pep604_syntax) + + def visit_partial_type(self, t: PartialType) -> Type: + assert False, "Internal error: Unexpected partial type" + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + if self.allow_ellipsis or self.allow_param_spec_literals: + any_type = AnyType(TypeOfAny.explicit) + return Parameters( + [any_type, any_type], [ARG_STAR, ARG_STAR2], [None, None], is_ellipsis_args=True + ) + else: + self.fail('Unexpected "..."', t) + return AnyType(TypeOfAny.from_error) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType.make_normalized( + self.anal_type(t.item), line=t.line, is_type_form=t.is_type_form + ) + + def visit_placeholder_type(self, t: PlaceholderType) -> Type: + n = ( + None + # No dot in fullname indicates we are at function scope, and recursive + # types are not supported there anyway, so we just give up. + if not t.fullname or "." not in t.fullname + else self.api.lookup_fully_qualified(t.fullname) + ) + if not n or isinstance(n.node, PlaceholderNode): + self.api.defer() # Still incomplete + return t + else: + # TODO: Handle non-TypeInfo + assert isinstance(n.node, TypeInfo) + return self.analyze_type_with_type_info(n.node, t.args, t, False) + + def analyze_callable_args_for_paramspec( + self, callable_args: Type, ret_type: Type, fallback: Instance + ) -> CallableType | None: + """Construct a 'Callable[P, RET]', where P is ParamSpec, return None if we cannot.""" + if not isinstance(callable_args, UnboundType): + return None + sym = self.lookup_qualified(callable_args.name, callable_args) + if sym is None: + return None + tvar_def = self.tvar_scope.get_binding(sym) + if not isinstance(tvar_def, ParamSpecType): + if ( + tvar_def is None + and self.allow_unbound_tvars + and isinstance(sym.node, ParamSpecExpr) + ): + # We are analyzing this type in runtime context (e.g. as type application). + # If it is not valid as a type in this position an error will be given later. + return callable_with_ellipsis( + AnyType(TypeOfAny.explicit), ret_type=ret_type, fallback=fallback + ) + return None + elif ( + self.defining_alias + and self.not_declared_in_type_params(tvar_def.name) + and tvar_def not in self.allowed_alias_tvars + ): + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{tvar_def.name}"' + ) + else: + msg = f'ParamSpec "{tvar_def.name}" is not included in type_params' + self.fail(msg, callable_args, code=codes.VALID_TYPE) + return callable_with_ellipsis( + AnyType(TypeOfAny.special_form), ret_type=ret_type, fallback=fallback + ) + + return CallableType( + [ + paramspec_args( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + paramspec_kwargs( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + ], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + ) + + def analyze_callable_args_for_concatenate( + self, callable_args: Type, ret_type: Type, fallback: Instance + ) -> CallableType | AnyType | None: + """Construct a 'Callable[C, RET]', where C is Concatenate[..., P], returning None if we + cannot. + """ + if not isinstance(callable_args, UnboundType): + return None + sym = self.lookup_qualified(callable_args.name, callable_args) + if sym is None: + return None + if sym.node is None: + return None + if sym.node.fullname not in CONCATENATE_TYPE_NAMES: + return None + + tvar_def = self.anal_type(callable_args, allow_param_spec=True) + if not isinstance(tvar_def, (ParamSpecType, Parameters)): + if self.allow_unbound_tvars and isinstance(tvar_def, UnboundType): + sym = self.lookup_qualified(tvar_def.name, callable_args) + if sym is not None and isinstance(sym.node, ParamSpecExpr): + # We are analyzing this type in runtime context (e.g. as type application). + # If it is not valid as a type in this position an error will be given later. + return callable_with_ellipsis( + AnyType(TypeOfAny.explicit), ret_type=ret_type, fallback=fallback + ) + # Error was already given, so prevent further errors. + return AnyType(TypeOfAny.from_error) + if isinstance(tvar_def, Parameters): + # This comes from Concatenate[int, ...] + return CallableType( + arg_types=tvar_def.arg_types, + arg_names=tvar_def.arg_names, + arg_kinds=tvar_def.arg_kinds, + ret_type=ret_type, + fallback=fallback, + from_concatenate=True, + ) + + # ick, CallableType should take ParamSpecType + prefix = tvar_def.prefix + # we don't set the prefix here as generic arguments will get updated at some point + # in the future. CallableType.param_spec() accounts for this. + return CallableType( + [ + *prefix.arg_types, + paramspec_args( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + paramspec_kwargs( + tvar_def.name, tvar_def.fullname, tvar_def.id, named_type_func=self.named_type + ), + ], + [*prefix.arg_kinds, nodes.ARG_STAR, nodes.ARG_STAR2], + [*prefix.arg_names, None, None], + ret_type=ret_type, + fallback=fallback, + from_concatenate=True, + ) + + def analyze_callable_type(self, t: UnboundType) -> Type: + fallback = self.named_type("builtins.function") + if len(t.args) == 0: + # Callable (bare). Treat as Callable[..., Any]. + any_type = self.get_omitted_any(t) + ret = callable_with_ellipsis(any_type, any_type, fallback) + elif len(t.args) == 2: + callable_args = t.args[0] + ret_type = t.args[1] + if isinstance(callable_args, TypeList): + # Callable[[ARG, ...], RET] (ordinary callable type) + analyzed_args = self.analyze_callable_args(callable_args) + if analyzed_args is None: + return AnyType(TypeOfAny.from_error) + args, kinds, names = analyzed_args + ret = CallableType(args, kinds, names, ret_type=ret_type, fallback=fallback) + elif isinstance(callable_args, EllipsisType): + # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) + ret = callable_with_ellipsis( + AnyType(TypeOfAny.explicit), ret_type=ret_type, fallback=fallback + ) + else: + # Callable[P, RET] (where P is ParamSpec) + with self.tvar_scope_frame(namespace=""): + # Temporarily bind ParamSpecs to allow code like this: + # my_fun: Callable[Q, Foo[Q]] + # We usually do this later in visit_callable_type(), but the analysis + # below happens at very early stage. + variables = [] + for name, tvar_expr in self.find_type_var_likes(callable_args): + variables.append( + self.tvar_scope.bind_new(name, tvar_expr, self.fail_func, t) + ) + maybe_ret = self.analyze_callable_args_for_paramspec( + callable_args, ret_type, fallback + ) or self.analyze_callable_args_for_concatenate( + callable_args, ret_type, fallback + ) + if isinstance(maybe_ret, CallableType): + maybe_ret = maybe_ret.copy_modified(variables=variables) + if maybe_ret is None: + # Callable[?, RET] (where ? is something invalid) + self.fail( + "The first argument to Callable must be a " + 'list of types, parameter specification, or "..."', + t, + code=codes.VALID_TYPE, + ) + self.note( + "See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas", + t, + ) + return AnyType(TypeOfAny.from_error) + elif isinstance(maybe_ret, AnyType): + return maybe_ret + ret = maybe_ret + else: + if self.options.disallow_any_generics: + self.fail('Please use "Callable[[], ]"', t) + else: + self.fail('Please use "Callable[[], ]" or "Callable"', t) + return AnyType(TypeOfAny.from_error) + assert isinstance(ret, CallableType) + return ret.accept(self) + + def refers_to_full_names(self, arg: UnboundType, names: Sequence[str]) -> bool: + sym = self.lookup_qualified(arg.name, arg) + if sym is not None: + if sym.fullname in names: + return True + return False + + def analyze_callable_args( + self, arglist: TypeList + ) -> tuple[list[Type], list[ArgKind], list[str | None]] | None: + args: list[Type] = [] + kinds: list[ArgKind] = [] + names: list[str | None] = [] + seen_unpack = False + unpack_types: list[Type] = [] + invalid_unpacks: list[Type] = [] + second_unpack_last = False + for i, arg in enumerate(arglist.items): + if isinstance(arg, CallableArgument): + args.append(arg.typ) + names.append(arg.name) + if arg.constructor is None: + return None + found = self.lookup_qualified(arg.constructor, arg) + if found is None: + # Looking it up already put an error message in + return None + elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: + self.fail(f'Invalid argument constructor "{found.fullname}"', arg) + return None + else: + assert found.fullname is not None + kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] + kinds.append(kind) + if arg.name is not None and kind.is_star(): + self.fail(f"{arg.constructor} arguments should not have names", arg) + return None + elif ( + isinstance(arg, UnboundType) + and self.refers_to_full_names(arg, UNPACK_TYPE_NAMES) + or isinstance(arg, UnpackType) + ): + if seen_unpack: + # Multiple unpacks, preserve them, so we can give an error later. + if i == len(arglist.items) - 1 and not invalid_unpacks: + # Special case: if there are just two unpacks, and the second one appears + # as last type argument, it can be still valid, if the second unpacked type + # is a TypedDict. This should be checked by the caller. + second_unpack_last = True + invalid_unpacks.append(arg) + continue + seen_unpack = True + unpack_types.append(arg) + else: + if seen_unpack: + unpack_types.append(arg) + else: + args.append(arg) + kinds.append(ARG_POS) + names.append(None) + if seen_unpack: + if len(unpack_types) == 1: + args.append(unpack_types[0]) + else: + first = unpack_types[0] + if isinstance(first, UnpackType): + # UnpackType doesn't have its own line/column numbers, + # so use the unpacked type for error messages. + first = first.type + args.append( + UnpackType(self.tuple_type(unpack_types, line=first.line, column=first.column)) + ) + kinds.append(ARG_STAR) + names.append(None) + for arg in invalid_unpacks: + args.append(arg) + kinds.append(ARG_STAR2 if second_unpack_last else ARG_STAR) + names.append(None) + # Note that arglist below is only used for error context. + check_param_names(names, [arglist] * len(args), self.fail, "Callable") + check_arg_kinds(kinds, [arglist] * len(args), self.fail) + return args, kinds, names + + def analyze_literal_type(self, t: UnboundType) -> Type: + if len(t.args) == 0: + self.fail("Literal[...] must have at least one parameter", t, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + + output: list[Type] = [] + for i, arg in enumerate(t.args): + analyzed_types = self.analyze_literal_param(i + 1, arg, t) + if analyzed_types is None: + return AnyType(TypeOfAny.from_error) + else: + output.extend(analyzed_types) + return UnionType.make_union(output, line=t.line) + + def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] | None: + # This UnboundType was originally defined as a string. + if ( + isinstance(arg, ProperType) + and isinstance(arg, (UnboundType, UnionType)) + and arg.original_str_expr is not None + ): + assert arg.original_str_fallback is not None + return [ + LiteralType( + value=arg.original_str_expr, + fallback=self.named_type(arg.original_str_fallback), + line=arg.line, + column=arg.column, + ) + ] + + # If arg is an UnboundType that was *not* originally defined as + # a string, try expanding it in case it's a type alias or something. + if isinstance(arg, UnboundType): + self.nesting_level += 1 + try: + arg = self.visit_unbound_type(arg, defining_literal=True) + finally: + self.nesting_level -= 1 + + # Literal[...] cannot contain Any. Give up and add an error message + # (if we haven't already). + arg = get_proper_type(arg) + if isinstance(arg, AnyType): + # Note: We can encounter Literals containing 'Any' under three circumstances: + # + # 1. If the user attempts use an explicit Any as a parameter + # 2. If the user is trying to use an enum value imported from a module with + # no type hints, giving it an implicit type of 'Any' + # 3. If there's some other underlying problem with the parameter. + # + # We report an error in only the first two cases. In the third case, we assume + # some other region of the code has already reported a more relevant error. + # + # TODO: Once we start adding support for enums, make sure we report a custom + # error for case 2 as well. + if arg.type_of_any not in (TypeOfAny.from_error, TypeOfAny.special_form): + self.fail( + f'Parameter {idx} of Literal[...] cannot be of type "Any"', + ctx, + code=codes.VALID_TYPE, + ) + return None + elif isinstance(arg, RawExpressionType): + # A raw literal. Convert it directly into a literal if we can. + if arg.literal_value is None: + name = arg.simple_name() + if name in ("float", "complex"): + msg = f'Parameter {idx} of Literal[...] cannot be of type "{name}"' + else: + msg = "Invalid type: Literal[...] cannot contain arbitrary expressions" + self.fail(msg, ctx, code=codes.VALID_TYPE) + # Note: we deliberately ignore arg.note here: the extra info might normally be + # helpful, but it generally won't make sense in the context of a Literal[...]. + return None + + # Remap bytes and unicode into the appropriate type for the correct Python version + fallback = self.named_type(arg.base_type_name) + assert isinstance(fallback, Instance) + return [LiteralType(arg.literal_value, fallback, line=arg.line, column=arg.column)] + elif isinstance(arg, (NoneType, LiteralType)): + # Types that we can just add directly to the literal/potential union of literals. + return [arg] + elif isinstance(arg, Instance) and arg.last_known_value is not None: + # Types generated from declarations like "var: Final = 4". + return [arg.last_known_value] + elif isinstance(arg, UnionType): + out = [] + for union_arg in arg.items: + union_result = self.analyze_literal_param(idx, union_arg, ctx) + if union_result is None: + return None + out.extend(union_result) + return out + else: + self.fail(f"Parameter {idx} of Literal[...] is invalid", ctx, code=codes.VALID_TYPE) + return None + + def analyze_type(self, typ: Type) -> Type: + return typ.accept(self) + + def fail(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + self.fail_func(msg, ctx, code=code) + + def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + self.note_func(msg, ctx, code=code) + + @contextmanager + def tvar_scope_frame(self, namespace: str) -> Iterator[None]: + old_scope = self.tvar_scope + self.tvar_scope = self.tvar_scope.method_frame(namespace) + yield + self.tvar_scope = old_scope + + def find_type_var_likes(self, t: Type) -> TypeVarLikeList: + visitor = FindTypeVarVisitor(self.api, self.tvar_scope) + t.accept(visitor) + return visitor.type_var_likes + + def infer_type_variables( + self, type: CallableType + ) -> tuple[list[tuple[str, TypeVarLikeExpr]], bool]: + """Infer type variables from a callable. + + Return tuple with these items: + - list of unique type variables referred to in a callable + - whether there is a reference to the Self type + """ + visitor = FindTypeVarVisitor(self.api, self.tvar_scope) + for arg in type.arg_types: + arg.accept(visitor) + + # When finding type variables in the return type of a function, don't + # look inside Callable types. Type variables only appearing in + # functions in the return type belong to those functions, not the + # function we're currently analyzing. + visitor.include_callables = False + type.ret_type.accept(visitor) + + return visitor.type_var_likes, visitor.has_self_type + + def bind_function_type_variables( + self, fun_type: CallableType, defn: Context + ) -> tuple[tuple[TypeVarLikeType, ...], bool]: + """Find the type variables of the function type and bind them in our tvar_scope""" + has_self_type = False + if fun_type.variables: + defs = [] + for var in fun_type.variables: + if self.api.type and self.api.type.self_type and var == self.api.type.self_type: + has_self_type = True + continue + var_node = self.lookup_qualified(var.name, defn) + assert var_node, "Binding for function type variable not found within function" + var_expr = var_node.node + assert isinstance(var_expr, TypeVarLikeExpr) + binding = self.tvar_scope.bind_new(var.name, var_expr, self.fail_func, fun_type) + defs.append(binding) + return tuple(defs), has_self_type + typevars, has_self_type = self.infer_type_variables(fun_type) + # Do not define a new type variable if already defined in scope. + typevars = [ + (name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn) + ] + defs = [] + for name, tvar in typevars: + if not self.tvar_scope.allow_binding(tvar.fullname): + err_msg = message_registry.TYPE_VAR_REDECLARED_IN_NESTED_CLASS.format(name) + self.fail(err_msg.value, defn, code=err_msg.code) + binding = self.tvar_scope.bind_new(name, tvar, self.fail_func, fun_type) + defs.append(binding) + + return tuple(defs), has_self_type + + def is_defined_type_var(self, tvar: str, context: Context) -> bool: + tvar_node = self.lookup_qualified(tvar, context) + if not tvar_node: + return False + return self.tvar_scope.get_binding(tvar_node) is not None + + def anal_array( + self, + a: Iterable[Type], + nested: bool = True, + *, + allow_param_spec: bool = False, + allow_param_spec_literals: bool = False, + allow_unpack: bool = False, + ) -> list[Type]: + old_allow_param_spec_literals = self.allow_param_spec_literals + self.allow_param_spec_literals = allow_param_spec_literals + res: list[Type] = [] + for t in a: + res.append( + self.anal_type( + t, nested, allow_param_spec=allow_param_spec, allow_unpack=allow_unpack + ) + ) + self.allow_param_spec_literals = old_allow_param_spec_literals + return self.check_unpacks_in_list(res) + + def anal_type( + self, + t: Type, + nested: bool = True, + *, + allow_param_spec: bool = False, + allow_unpack: bool = False, + allow_ellipsis: bool = False, + allow_typed_dict_special_forms: bool = False, + allow_final: bool = False, + ) -> Type: + if nested: + self.nesting_level += 1 + old_allow_typed_dict_special_forms = self.allow_typed_dict_special_forms + self.allow_typed_dict_special_forms = allow_typed_dict_special_forms + self.allow_final = allow_final + old_allow_ellipsis = self.allow_ellipsis + self.allow_ellipsis = allow_ellipsis + old_allow_unpack = self.allow_unpack + self.allow_unpack = allow_unpack + try: + analyzed = t.accept(self) + finally: + if nested: + self.nesting_level -= 1 + self.allow_typed_dict_special_forms = old_allow_typed_dict_special_forms + self.allow_ellipsis = old_allow_ellipsis + self.allow_unpack = old_allow_unpack + if ( + not allow_param_spec + and isinstance(analyzed, ParamSpecType) + and analyzed.flavor == ParamSpecFlavor.BARE + ): + if analyzed.prefix.arg_types: + self.fail("Invalid location for Concatenate", t, code=codes.VALID_TYPE) + self.note("You can use Concatenate as the first argument to Callable", t) + analyzed = AnyType(TypeOfAny.from_error) + else: + self.fail( + INVALID_PARAM_SPEC_LOCATION.format(format_type(analyzed, self.options)), + t, + code=codes.VALID_TYPE, + ) + self.note( + INVALID_PARAM_SPEC_LOCATION_NOTE.format(analyzed.name), + t, + code=codes.VALID_TYPE, + ) + analyzed = AnyType(TypeOfAny.from_error) + return analyzed + + def anal_var_def(self, var_def: TypeVarLikeType) -> TypeVarLikeType: + if isinstance(var_def, TypeVarType): + return TypeVarType( + name=var_def.name, + fullname=var_def.fullname, + id=var_def.id, + values=self.anal_array(var_def.values), + upper_bound=var_def.upper_bound.accept(self), + default=var_def.default.accept(self), + variance=var_def.variance, + line=var_def.line, + column=var_def.column, + ) + else: + return var_def + + def anal_var_defs(self, var_defs: Sequence[TypeVarLikeType]) -> list[TypeVarLikeType]: + return [self.anal_var_def(vd) for vd in var_defs] + + def named_type( + self, fullname: str, args: list[Type] | None = None, line: int = -1, column: int = -1 + ) -> Instance: + node = self.lookup_fully_qualified(fullname) + assert isinstance(node.node, TypeInfo) + any_type = AnyType(TypeOfAny.special_form) + if args is not None: + args = self.check_unpacks_in_list(args) + return Instance( + node.node, args or [any_type] * len(node.node.defn.type_vars), line=line, column=column + ) + + def check_unpacks_in_list(self, items: list[Type]) -> list[Type]: + new_items: list[Type] = [] + num_unpacks = 0 + final_unpack = None + for item in items: + # TODO: handle forward references here, they appear as Unpack[Any]. + if isinstance(item, UnpackType) and not isinstance( + get_proper_type(item.type), TupleType + ): + if not num_unpacks: + new_items.append(item) + num_unpacks += 1 + final_unpack = item + else: + new_items.append(item) + + if num_unpacks > 1: + assert final_unpack is not None + self.fail("More than one variadic Unpack in a type is not allowed", final_unpack.type) + return new_items + + def tuple_type(self, items: list[Type], line: int, column: int) -> TupleType: + any_type = AnyType(TypeOfAny.special_form) + return TupleType( + items, fallback=self.named_type("builtins.tuple", [any_type]), line=line, column=column + ) + + +TypeVarLikeList = list[tuple[str, TypeVarLikeExpr]] + + +class MsgCallback(Protocol): + def __call__( + self, __msg: str, __ctx: Context, *, code: ErrorCode | None = None + ) -> ErrorInfo | None: ... + + +def get_omitted_any( + disallow_any: bool, + fail: MsgCallback, + note: MsgCallback, + orig_type: Type, + options: Options, + fullname: str | None = None, + unexpanded_type: Type | None = None, + used_default: bool = False, +) -> AnyType: + if disallow_any: + typ = unexpanded_type or orig_type + type_str = typ.name if isinstance(typ, UnboundType) else format_type_bare(typ, options) + + fail( + message_registry.BARE_GENERIC.format(quote_type_string(type_str)), + typ, + code=codes.TYPE_ARG, + ) + if used_default: + note(message_registry.NO_CYCLIC_DEFAULT, typ, code=codes.TYPE_ARG) + + any_type = AnyType(TypeOfAny.from_error, line=typ.line, column=typ.column) + else: + any_type = AnyType( + TypeOfAny.from_omitted_generics, line=orig_type.line, column=orig_type.column + ) + return any_type + + +def fix_type_var_tuple_argument(t: Instance) -> None: + if t.type.has_type_var_tuple_type: + args = list(t.args) + assert t.type.type_var_tuple_prefix is not None + tvt = t.type.defn.type_vars[t.type.type_var_tuple_prefix] + assert isinstance(tvt, TypeVarTupleType) + args[t.type.type_var_tuple_prefix] = UnpackType( + Instance(tvt.tuple_fallback.type, [args[t.type.type_var_tuple_prefix]]) + ) + t.args = tuple(args) + + +def fix_instance( + t: Instance, + fail: MsgCallback, + note: MsgCallback, + disallow_any: bool, + options: Options, + use_generic_error: bool = False, + unexpanded_type: Type | None = None, + analyzing_tvar_def: bool = False, +) -> bool: + """Fix a malformed instance by replacing all type arguments with TypeVar default or Any. + + Also emit a suitable error if this is not due to implicit Any's. + """ + used_default = False + arg_count = len(t.args) + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in t.type.defn.type_vars + ) + max_tv_count = len(t.type.type_vars) + if arg_count < min_tv_count or arg_count > max_tv_count: + # Don't use existing args if arg_count doesn't match + if arg_count > max_tv_count: + # Already wrong arg count error, don't emit missing type parameters error as well. + disallow_any = False + t.args = () + + args: list[Type] = list(t.args) + any_type: AnyType | None = None + env: dict[TypeVarId, Type] = {} + tvt_no_default = False + + for tv, arg in itertools.zip_longest(t.type.defn.type_vars, t.args, fillvalue=None): + if tv is None: + continue + if arg is None: + use_any = False + if tv.has_default(): + arg = tv.default + if analyzing_tvar_def: + # Record the use of default only when analyzing another default. + used_default = True + if is_typevar_default_recursive(tv.fullname, t.type): + # If this results in infinite recursion, use Any instead. + use_any = True + else: + use_any = True + if use_any: + if any_type is None: + fullname = None if use_generic_error else t.type.fullname + any_type = get_omitted_any( + disallow_any, + fail, + note, + t, + options, + fullname, + unexpanded_type, + used_default, + ) + arg = any_type + else: + assert arg is not None + if use_any and isinstance(tv, TypeVarTupleType): + tvt_no_default = True + # Default such as *tuple[int, str] should be unpacked into individual items. + if isinstance(arg, UnpackType) and isinstance( + unpack := get_proper_type(arg.type), TupleType + ): + unpacked = unpack.items + else: + unpacked = [arg] + for arg in unpacked: + with state.strict_optional_set(True): + # Gradually expand defaults, as they may depend on previous variables. + if tv.has_default(): + arg = expand_type(arg, env) + env[tv.id] = arg + args.append(arg) + else: + env[tv.id] = arg + t.args = tuple(args) + if tvt_no_default: + fix_type_var_tuple_argument(t) + return used_default + + +def instantiate_type_alias( + node: TypeAlias, + args: list[Type], + fail: MsgCallback, + note: MsgCallback, + no_args: bool, + ctx: Context, + options: Options, + *, + unexpanded_type: Type | None = None, + disallow_any: bool = False, + use_standard_error: bool = False, + empty_tuple_index: bool = False, + analyzing_tvar_def: bool = False, +) -> tuple[Type, bool]: + """Create an instance of a (generic) type alias from alias node and type arguments. + + We are following the rules outlined in TypeAlias docstring. + Here: + node: type alias node (definition) + args: type arguments (types to be substituted in place of type variables + when expanding the alias) + fail: error reporter callback + no_args: whether original definition used a bare generic `A = List` + ctx: context where expansion happens + unexpanded_type, disallow_any, use_standard_error: used to customize error messages + """ + # Type aliases are special, since they can be expanded during semantic analysis, + # so we need to normalize them as soon as possible. + # TODO: can this cause an infinite recursion? + old_args = args + args = flatten_nested_tuples(args) + if old_args and not args: + empty_tuple_index = True + if any(unknown_unpack(a) for a in args): + # This type is not ready to be validated, because of unknown total count. + # Note that we keep the kind of Any for consistency. + return set_any_tvars(node, [], ctx.line, ctx.column, options, special_form=True) + + if ( + no_args + and isinstance(node.target, ProperType) + and isinstance(node.target, Instance) + and node.target.type.fullname == "builtins.tuple" + and len(args) + ): + no_args = False + + max_tv_count = len(node.alias_tvars) + act_len = len(args) + if ( + max_tv_count > 0 + and act_len == 0 + and not (empty_tuple_index and node.tvar_tuple_index is not None) + ): + # Interpret bare Alias same as normal generic, i.e., Alias[Any, Any, ...] + return set_any_tvars( + node, + args, + ctx.line, + ctx.column, + options, + disallow_any=disallow_any, + fail=fail, + note=note, + unexpanded_type=unexpanded_type, + analyzing_tvar_def=analyzing_tvar_def, + ) + if max_tv_count == 0 and act_len == 0: + if no_args: + assert isinstance(node.target, Instance) # type: ignore[misc] + # Note: this is the only case where we use an eager expansion. See more info about + # no_args aliases like L = List in the docstring for TypeAlias class. + return Instance(node.target.type, [], line=ctx.line, column=ctx.column), False + return TypeAliasType(node, [], line=ctx.line, column=ctx.column), False + if ( + max_tv_count == 0 + and act_len > 0 + and isinstance(node.target, Instance) # type: ignore[misc] + and no_args + ): + tp = Instance(node.target.type, args) + tp.line = ctx.line + tp.column = ctx.column + tp.end_line = ctx.end_line + tp.end_column = ctx.end_column + return tp, False + if node.tvar_tuple_index is None: + if any(isinstance(a, UnpackType) for a in args): + # A variadic unpack in fixed size alias (fixed unpacks must be flattened by the caller) + fail(message_registry.INVALID_UNPACK_POSITION, ctx, code=codes.VALID_TYPE) + return set_any_tvars(node, [], ctx.line, ctx.column, options, from_error=True) + min_tv_count = sum(not tv.has_default() for tv in node.alias_tvars) + fill_typevars = act_len != max_tv_count + correct = min_tv_count <= act_len <= max_tv_count + else: + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in node.alias_tvars + ) + correct = act_len >= min_tv_count + for a in args: + if isinstance(a, UnpackType): + unpacked = get_proper_type(a.type) + if isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple": + # Variadic tuple is always correct. + correct = True + fill_typevars = not correct + if fill_typevars: + if not correct: + if use_standard_error: + # This is used if type alias is an internal representation of another type, + # for example a generic TypedDict or NamedTuple. + msg = wrong_type_arg_count(max_tv_count, max_tv_count, str(act_len), node.name) + else: + if node.tvar_tuple_index is not None: + msg = ( + "Bad number of arguments for type alias," + f" expected at least {min_tv_count}, given {act_len}" + ) + elif min_tv_count != max_tv_count: + msg = ( + "Bad number of arguments for type alias," + f" expected between {min_tv_count} and {max_tv_count}, given {act_len}" + ) + else: + msg = ( + "Bad number of arguments for type alias," + f" expected {min_tv_count}, given {act_len}" + ) + fail(msg, ctx, code=codes.TYPE_ARG) + args = [] + return set_any_tvars( + node, + args, + ctx.line, + ctx.column, + options, + disallow_any=disallow_any, + fail=fail, + note=note, + from_error=not correct, + analyzing_tvar_def=analyzing_tvar_def, + ) + elif node.tvar_tuple_index is not None: + # We also need to check if we are not performing a type variable tuple split. + unpack = find_unpack_in_list(args) + if unpack is not None: + unpack_arg = args[unpack] + assert isinstance(unpack_arg, UnpackType) + if isinstance(unpack_arg.type, TypeVarTupleType): + exp_prefix = node.tvar_tuple_index + act_prefix = unpack + exp_suffix = len(node.alias_tvars) - node.tvar_tuple_index - 1 + act_suffix = len(args) - unpack - 1 + if act_prefix < exp_prefix or act_suffix < exp_suffix: + fail("TypeVarTuple cannot be split", ctx, code=codes.TYPE_ARG) + return set_any_tvars(node, [], ctx.line, ctx.column, options, from_error=True) + # TODO: we need to check args validity w.r.t alias.alias_tvars. + # Otherwise invalid instantiations will be allowed in runtime context. + # Note: in type context, these will be still caught by semanal_typeargs. + typ = TypeAliasType(node, args, ctx.line, ctx.column) + assert typ.alias is not None + # HACK: Implement FlexibleAlias[T, typ] by expanding it to typ here. + if ( + isinstance(typ.alias.target, Instance) # type: ignore[misc] + and typ.alias.target.type.fullname == "mypy_extensions.FlexibleAlias" + ): + exp = get_proper_type(typ) + assert isinstance(exp, Instance) + return exp.args[-1], False + return typ, False + + +def set_any_tvars( + node: TypeAlias, + args: list[Type], + newline: int, + newcolumn: int, + options: Options, + *, + from_error: bool = False, + disallow_any: bool = False, + special_form: bool = False, + fail: MsgCallback | None = None, + note: MsgCallback | None = None, + unexpanded_type: Type | None = None, + analyzing_tvar_def: bool = False, +) -> tuple[TypeAliasType, bool]: + used_default = False + if from_error or disallow_any: + type_of_any = TypeOfAny.from_error + elif special_form: + type_of_any = TypeOfAny.special_form + else: + type_of_any = TypeOfAny.from_omitted_generics + any_type = AnyType(type_of_any, line=newline, column=newcolumn) + + env: dict[TypeVarId, Type] = {} + used_any_type = False + for tv, arg in itertools.zip_longest(node.alias_tvars, args, fillvalue=None): + if tv is None: + continue + if arg is None: + if tv.has_default(): + arg = tv.default + # Same as for instances, record and avoid infinite recursion. + if analyzing_tvar_def: + used_default = True + if is_typevar_default_recursive(tv.fullname, node): + arg = any_type + used_any_type = True + else: + arg = any_type + used_any_type = True + if used_any_type and isinstance(tv, TypeVarTupleType): + arg = UnpackType(Instance(tv.tuple_fallback.type, [any_type])) + # Default such as *tuple[int, str] should be unpacked into individual items. + if isinstance(arg, UnpackType) and isinstance( + unpack := get_proper_type(arg.type), TupleType + ): + unpacked = unpack.items + else: + unpacked = [arg] + for arg in unpacked: + with state.strict_optional_set(options.strict_optional): + # Gradually expand defaults, as they may depend on previous variables. + if tv.has_default(): + arg = expand_type(arg, env) + env[tv.id] = arg + args.append(arg) + else: + env[tv.id] = arg + t = TypeAliasType(node, args, newline, newcolumn) + + if used_any_type and disallow_any and node.alias_tvars and not from_error: + assert fail is not None + if unexpanded_type: + type_str = ( + unexpanded_type.name + if isinstance(unexpanded_type, UnboundType) + else format_type_bare(unexpanded_type, options) + ) + else: + type_str = node.name + + fail( + message_registry.BARE_GENERIC.format(quote_type_string(type_str)), + Context(newline, newcolumn), + code=codes.TYPE_ARG, + ) + if used_default: + assert note is not None + note( + message_registry.NO_CYCLIC_DEFAULT, + Context(newline, newcolumn), + code=codes.TYPE_ARG, + ) + return t, used_default + + +def is_typevar_default_recursive(tv_fname: str, start: TypeInfo | TypeAlias) -> bool: + """Check if the type variable can lead to infinite recursion via defaults.""" + if tv_fname not in start.default_depends: + return False + todo = start.default_depends[tv_fname].copy() + seen: set[TypeAlias | TypeInfo] = set() + while todo: + node = todo.pop() + if node is start: + return True + if node in seen: + # We don't return True here, since we are interested only in + # recursion via the original type variable. + continue + seen.add(node) + for dep_nodes in node.default_depends.values(): + todo |= dep_nodes + return False + + +class DivergingAliasDetector(TrivialSyntheticTypeTranslator): + """See docstring of detect_diverging_alias() for details.""" + + # TODO: this doesn't really need to be a translator, but we don't have a trivial visitor. + def __init__(self, seen_nodes: set[TypeAlias]) -> None: + super().__init__() + self.seen_nodes = seen_nodes + self.diverging = False + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + assert t.alias is not None, f"Unfixed type alias {t.type_ref}" + if t.alias in self.seen_nodes: + for arg in t.args: + if not ( + isinstance(arg, TypeVarLikeType) + or isinstance(arg, UnpackType) + and isinstance(arg.type, TypeVarLikeType) + ) and has_type_vars(arg): + self.diverging = True + return t + # All clear for this expansion chain. + return t + new_nodes = self.seen_nodes | {t.alias} + visitor = DivergingAliasDetector(new_nodes) + _ = get_proper_type(t).accept(visitor) + if visitor.diverging: + self.diverging = True + return t + + +def detect_diverging_alias(node: TypeAlias, target: Type) -> bool: + """This detects type aliases that will diverge during type checking. + + For example F = Something[..., F[List[T]]]. At each expansion step this will produce + *new* type aliases: e.g. F[List[int]], F[List[List[int]]], etc. So we can't detect + recursion. It is a known problem in the literature, recursive aliases and generic types + don't always go well together. It looks like there is no known systematic solution yet. + + # TODO: should we handle such aliases using type_recursion counter and some large limit? + They may be handy in rare cases, e.g. to express a union of non-mixed nested lists: + Nested = Union[T, Nested[List[T]]] ~> Union[T, List[T], List[List[T]], ...] + """ + is_recursive = node._is_recursive + if is_recursive is None: + is_recursive = node in node.target.accept(CollectAliasesVisitor()) + if not is_recursive: + # Fast path: this is not a recursive alias at all. + return False + # Note we only cache positive case, caching negative case is risky, as this type alias + # (or more importantly any other alias it uses) may be not ready yet. + node._is_recursive = True + visitor = DivergingAliasDetector({node}) + _ = target.accept(visitor) + return visitor.diverging + + +def check_for_explicit_any( + typ: Type | None, + options: Options, + is_typeshed_stub: bool, + msg: MessageBuilder, + context: Context, +) -> None: + if options.disallow_any_explicit and not is_typeshed_stub and typ and has_explicit_any(typ): + msg.explicit_any(context) + + +def has_explicit_any(t: Type) -> bool: + """ + Whether this type is or type it contains is an Any coming from explicit type annotation + """ + return t.accept(HasExplicitAny()) + + +class HasExplicitAny(BoolTypeQuery): + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_any(self, t: AnyType) -> bool: + return t.type_of_any == TypeOfAny.explicit + + def visit_typeddict_type(self, t: TypedDictType) -> bool: + # typeddict is checked during TypedDict declaration, so don't typecheck it here. + return False + + +def has_any_from_unimported_type(t: Type) -> bool: + """Return true if this type is Any because an import was not followed. + + If type t is such Any type or has type arguments that contain such Any type + this function will return true. + """ + return t.accept(HasAnyFromUnimportedType()) + + +class HasAnyFromUnimportedType(BoolTypeQuery): + def __init__(self) -> None: + super().__init__(ANY_STRATEGY) + + def visit_any(self, t: AnyType) -> bool: + return t.type_of_any == TypeOfAny.from_unimported_type + + def visit_typeddict_type(self, t: TypedDictType) -> bool: + # typeddict is checked during TypedDict declaration, so don't typecheck it here + return False + + +def collect_all_inner_types(t: Type) -> list[Type]: + """ + Return all types that `t` contains + """ + return t.accept(CollectAllInnerTypesQuery()) + + +class CollectAllInnerTypesQuery(TypeQuery[list[Type]]): + def query_types(self, types: Iterable[Type]) -> list[Type]: + return self.strategy([t.accept(self) for t in types]) + list(types) + + def strategy(self, items: Iterable[list[Type]]) -> list[Type]: + return list(itertools.chain.from_iterable(items)) + + +def make_optional_type(t: Type) -> Type: + """Return the type corresponding to Optional[t]. + + Note that we can't use normal union simplification, since this function + is called during semantic analysis and simplification only works during + type checking. + """ + if isinstance(t, ProperType) and isinstance(t, NoneType): + return t + elif isinstance(t, ProperType) and isinstance(t, UnionType): + # Eagerly expanding aliases is not safe during semantic analysis. + items = [item for item in t.items if not isinstance(get_proper_type(item), NoneType)] + return UnionType(items + [NoneType()], t.line, t.column) + else: + return UnionType([t, NoneType()], t.line, t.column) + + +def validate_instance(t: Instance, fail: MsgCallback, indexed: bool) -> bool: + """Check if this is a well-formed instance with respect to argument count/positions.""" + # TODO: combine logic with instantiate_type_alias(). + if any(unknown_unpack(a) for a in t.args): + # This type is not ready to be validated, because of unknown total count. + # TODO: is it OK to fill with TypeOfAny.from_error instead of special form? + return False + empty_tuple_index = indexed and not t.args + if t.type.has_type_var_tuple_type: + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in t.type.defn.type_vars + ) + correct = len(t.args) >= min_tv_count + if any( + isinstance(a, UnpackType) and isinstance(get_proper_type(a.type), Instance) + for a in t.args + ): + correct = True + if not t.args: + if not (empty_tuple_index and len(t.type.type_vars) == 1): + # The Any arguments should be set by the caller. + if empty_tuple_index and min_tv_count: + fail( + f"At least {min_tv_count} type argument(s) expected, none given", + t, + code=codes.TYPE_ARG, + ) + return False + elif not correct: + fail( + f"Bad number of arguments, expected: at least {min_tv_count}, given: {len(t.args)}", + t, + code=codes.TYPE_ARG, + ) + return False + else: + # We also need to check if we are not performing a type variable tuple split. + unpack = find_unpack_in_list(t.args) + if unpack is not None: + unpack_arg = t.args[unpack] + assert isinstance(unpack_arg, UnpackType) + if isinstance(unpack_arg.type, TypeVarTupleType): + assert t.type.type_var_tuple_prefix is not None + assert t.type.type_var_tuple_suffix is not None + exp_prefix = t.type.type_var_tuple_prefix + act_prefix = unpack + exp_suffix = t.type.type_var_tuple_suffix + act_suffix = len(t.args) - unpack - 1 + if act_prefix < exp_prefix or act_suffix < exp_suffix: + fail("TypeVarTuple cannot be split", t, code=codes.TYPE_ARG) + return False + elif any(isinstance(a, UnpackType) for a in t.args): + # A variadic unpack in fixed size instance (fixed unpacks must be flattened by the caller) + fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) + t.args = () + return False + elif len(t.args) != len(t.type.type_vars): + # Invalid number of type parameters. + arg_count = len(t.args) + min_tv_count = sum(not tv.has_default() for tv in t.type.defn.type_vars) + max_tv_count = len(t.type.type_vars) + if (arg_count or empty_tuple_index) and ( + arg_count < min_tv_count or arg_count > max_tv_count + ): + fail( + wrong_type_arg_count(min_tv_count, max_tv_count, str(arg_count), t.type.name), + t, + code=codes.TYPE_ARG, + ) + return False + return True + + +def find_self_type(typ: Type, lookup: Callable[[str], SymbolTableNode | None]) -> bool: + return typ.accept(HasSelfType(lookup)) + + +class HasSelfType(BoolTypeQuery): + def __init__(self, lookup: Callable[[str], SymbolTableNode | None]) -> None: + self.lookup = lookup + super().__init__(ANY_STRATEGY) + + def visit_unbound_type(self, t: UnboundType) -> bool: + sym = self.lookup(t.name) + if sym and sym.fullname in SELF_TYPE_NAMES: + return True + return super().visit_unbound_type(t) + + +def unknown_unpack(t: Type) -> bool: + """Check if a given type is an unpack of an unknown type. + + Unfortunately, there is no robust way to distinguish forward references from + genuine undefined names here. But this worked well so far, although it looks + quite fragile. + """ + if isinstance(t, UnpackType): + unpacked = get_proper_type(t.type) + if isinstance(unpacked, AnyType) and unpacked.type_of_any == TypeOfAny.special_form: + return True + return False + + +class FindTypeVarVisitor(SyntheticTypeVisitor[None]): + """Type visitor that looks for type variable types and self types.""" + + def __init__(self, api: SemanticAnalyzerCoreInterface, scope: TypeVarLikeScope) -> None: + self.api = api + self.scope = scope + self.type_var_likes: list[tuple[str, TypeVarLikeExpr]] = [] + self.has_self_type = False + self.include_callables = True + + def _seems_like_callable(self, type: UnboundType) -> bool: + if not type.args: + return False + return isinstance(type.args[0], (EllipsisType, TypeList, ParamSpecType)) + + def visit_unbound_type(self, t: UnboundType) -> None: + name = t.name + node = self.api.lookup_qualified(name, t) + if node and node.fullname in SELF_TYPE_NAMES: + self.has_self_type = True + if ( + node + and isinstance(node.node, TypeVarLikeExpr) + and self.scope.get_binding(node) is None + ): + if (name, node.node) not in self.type_var_likes: + self.type_var_likes.append((name, node.node)) + elif not self.include_callables and self._seems_like_callable(t): + if find_self_type( + t, lambda name: self.api.lookup_qualified(name, t, suppress_errors=True) + ): + self.has_self_type = True + return + elif node and node.fullname in LITERAL_TYPE_NAMES: + return + elif node and node.fullname in ANNOTATED_TYPE_NAMES and t.args: + # Don't query the second argument to Annotated for TypeVars + self.process_types([t.args[0]]) + elif t.args: + self.process_types(t.args) + + def visit_type_list(self, t: TypeList) -> None: + self.process_types(t.items) + + def visit_callable_argument(self, t: CallableArgument) -> None: + t.typ.accept(self) + + def visit_any(self, t: AnyType) -> None: + pass + + def visit_uninhabited_type(self, t: UninhabitedType) -> None: + pass + + def visit_none_type(self, t: NoneType) -> None: + pass + + def visit_erased_type(self, t: ErasedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_var(self, t: TypeVarType) -> None: + self.process_types([t.upper_bound, t.default] + t.values) + + def visit_param_spec(self, t: ParamSpecType) -> None: + self.process_types([t.upper_bound, t.default, t.prefix]) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + self.process_types([t.upper_bound, t.default]) + + def visit_unpack_type(self, t: UnpackType) -> None: + self.process_types([t.type]) + + def visit_parameters(self, t: Parameters) -> None: + self.process_types(t.arg_types) + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_instance(self, t: Instance) -> None: + self.process_types(t.args) + + def visit_callable_type(self, t: CallableType) -> None: + # FIX generics + self.process_types(t.arg_types) + t.ret_type.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + self.process_types(t.items) + + def visit_typeddict_type(self, t: TypedDictType) -> None: + self.process_types(list(t.items.values())) + + def visit_raw_expression_type(self, t: RawExpressionType) -> None: + pass + + def visit_literal_type(self, t: LiteralType) -> None: + pass + + def visit_union_type(self, t: UnionType) -> None: + self.process_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> None: + for it in t.items: + it.accept(self) + + def visit_type_type(self, t: TypeType) -> None: + t.item.accept(self) + + def visit_ellipsis_type(self, t: EllipsisType) -> None: + pass + + def visit_placeholder_type(self, t: PlaceholderType) -> None: + return self.process_types(t.args) + + def visit_type_alias_type(self, t: TypeAliasType) -> None: + self.process_types(t.args) + + def process_types(self, types: list[Type] | tuple[Type, ...]) -> None: + # Redundant type check helps mypyc. + if isinstance(types, list): + for t in types: + t.accept(self) + else: + for t in types: + t.accept(self) + + +class TypeVarDefaultTranslator(TrivialSyntheticTypeTranslator): + """Type translate visitor that replaces UnboundTypes with in-scope TypeVars.""" + + def __init__( + self, api: SemanticAnalyzerInterface, tvar_expr_name: str, context: Context + ) -> None: + super().__init__() + self.api = api + self.tvar_expr_name = tvar_expr_name + self.context = context + + def visit_unbound_type(self, t: UnboundType) -> Type: + sym = self.api.lookup_qualified(t.name, t, suppress_errors=True) + if sym is not None: + if type_var := self.api.tvar_scope.get_binding(sym): + return type_var + if isinstance(sym.node, TypeVarLikeExpr): + self.api.fail( + f'Type parameter "{self.tvar_expr_name}" has a default type ' + "that refers to one or more type variables that are out of scope", + self.context, + ) + return AnyType(TypeOfAny.from_error) + return super().visit_unbound_type(t) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # TypeAliasTypes are analyzed separately already, just return it + return t + + +def check_vec_type_args( + args: tuple[Type, ...] | list[Type], ctx: Context, api: SemanticAnalyzerCoreInterface +) -> bool: + """Report an error if type args for 'vec' are invalid. + + Return False on error. + """ + ok = True + if len(args) != 1: + ok = False + else: + arg = get_proper_type(args[0]) + if isinstance(arg, Instance): + if arg.type.fullname == "builtins.int": + # A fixed-width integer such as 'i64' must be used instead of plain 'int' + ok = False + elif isinstance(arg, UnionType): + non_optional = None + items = [get_proper_type(item) for item in arg.items] + if len(items) != 2: + ok = False + elif isinstance(items[0], NoneType): + if not check_vec_type_args([items[1]], ctx, api): + # Error has already been reported so it's fine to return + return False + non_optional = items[1] + elif isinstance(items[1], NoneType): + if not check_vec_type_args([items[0]], ctx, api): + # Error has already been reported so it's fine to return + return False + non_optional = items[0] + else: + ok = False + if isinstance(non_optional, Instance) and ( + non_optional.type.fullname in MYPYC_NATIVE_INT_NAMES + or non_optional.type.fullname + in ("builtins.int", "builtins.float", "builtins.bool", "librt.vecs.vec") + ): + ok = False + elif isinstance(arg, TypeVarType): + # Generic vec types aren't supported in type checked Python code, but + # they can be provided in libraries implemented in C (e.g. append). + if not api.is_stub_file: + ok = False + else: + ok = False + if not ok: + api.fail('Invalid item type for "vec"', ctx) + return ok From 08ce848a1652aff420d362a353e9dca850c5502c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:03:17 +0000 Subject: [PATCH 04/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/annotate.py | 3 +-- mypyc/codegen/emitclass.py | 1 - mypyc/codegen/emitmodule.py | 3 +-- mypyc/codegen/emitwrapper.py | 1 - mypyc/ir/func_ir.py | 1 - mypyc/irbuild/ast_helpers.py | 1 - mypyc/irbuild/builder.py | 5 ++--- mypyc/irbuild/classdef.py | 1 - mypyc/irbuild/constant_fold.py | 3 +-- mypyc/irbuild/context.py | 1 - mypyc/irbuild/env_class.py | 1 - mypyc/irbuild/expression.py | 1 - mypyc/irbuild/for_helpers.py | 1 - mypyc/irbuild/format_str_tokenizer.py | 3 +-- mypyc/irbuild/function.py | 1 - mypyc/irbuild/generator.py | 1 - mypyc/irbuild/ll_builder.py | 3 +-- mypyc/irbuild/main.py | 3 +-- mypyc/irbuild/mapper.py | 1 - mypyc/irbuild/match.py | 1 - mypyc/irbuild/missingtypevisitor.py | 1 - mypyc/irbuild/prebuildvisitor.py | 1 - mypyc/irbuild/prepare.py | 3 +-- mypyc/irbuild/specialize.py | 1 - mypyc/irbuild/statement.py | 1 - mypyc/irbuild/util.py | 1 - mypyc/irbuild/visitor.py | 1 - mypyc/test/test_statement.py | 3 +-- mypyc/test/testutil.py | 3 +-- 29 files changed, 11 insertions(+), 40 deletions(-) diff --git a/mypyc/annotate.py b/mypyc/annotate.py index cba886f1acaa..bc282fc3ea6c 100644 --- a/mypyc/annotate.py +++ b/mypyc/annotate.py @@ -11,6 +11,7 @@ from html import escape from typing import Final +from mypy.build import BuildResult from mypy.nodes import ( AssignmentStmt, CallExpr, @@ -38,8 +39,6 @@ Var, WithStmt, ) - -from mypy.build import BuildResult from mypy.traverser import TraverserVisitor from mypy.types import AnyType, Instance, ProperType, Type, TypeOfAny, get_proper_type from mypy.util import FancyFormatter diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 51970849f82b..953fd8a9e435 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -5,7 +5,6 @@ from collections.abc import Callable, Mapping from mypy.nodes import ARG_STAR, ARG_STAR2 - from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import ( Emitter, diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index f2895c164c4f..1b21b51709db 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -11,8 +11,6 @@ from collections.abc import Iterable from typing import TypeVar -from mypy.nodes import MypyFile, TypeInfo - from mypy.build import ( BuildResult, BuildSource, @@ -25,6 +23,7 @@ ) from mypy.errors import CompileError from mypy.fscache import FileSystemCache +from mypy.nodes import MypyFile, TypeInfo from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 9918e8c5969a..5213de19011f 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -15,7 +15,6 @@ from collections.abc import Sequence from mypy.nodes import ARG_NAMED, ARG_NAMED_OPT, ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, ArgKind - from mypy.operators import op_methods_to_symbols, reverse_op_method_names, reverse_op_methods from mypyc.codegen.emit import AssignHandler, Emitter, ErrorHandler, GotoHandler, ReturnHandler from mypyc.common import ( diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py index ca72e393a5c7..d11fef42feb5 100644 --- a/mypyc/ir/func_ir.py +++ b/mypyc/ir/func_ir.py @@ -7,7 +7,6 @@ from typing import Final from mypy.nodes import ARG_POS, ArgKind, Block, FuncDef - from mypyc.common import BITMAP_BITS, JsonDict, bitmap_name, get_id_from_name, short_id_from_name from mypyc.ir.ops import ( Assign, diff --git a/mypyc/irbuild/ast_helpers.py b/mypyc/irbuild/ast_helpers.py index 74b516152145..8fff4b19b33d 100644 --- a/mypyc/irbuild/ast_helpers.py +++ b/mypyc/irbuild/ast_helpers.py @@ -20,7 +20,6 @@ UnaryExpr, Var, ) - from mypyc.ir.ops import BasicBlock from mypyc.ir.rtypes import is_fixed_width_rtype, is_tagged from mypyc.irbuild.builder import IRBuilder diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 6413373fdbc2..957cc2704441 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -9,6 +9,8 @@ from contextlib import contextmanager from typing import Any, Final, overload +from mypy.build import Graph +from mypy.maptype import map_instance_to_supertype from mypy.nodes import ( ARG_NAMED, ARG_POS, @@ -48,9 +50,6 @@ YieldExpr, YieldFromExpr, ) - -from mypy.build import Graph -from mypy.maptype import map_instance_to_supertype from mypy.traverser import TraverserVisitor from mypy.types import ( AnyType, diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 8490c872963d..3eb617cf8655 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -29,7 +29,6 @@ Var, is_class_var, ) - from mypy.types import Instance, UnboundType, get_proper_type from mypyc.common import GENERATOR_HELPER_NAME, MYPYC_DEFAULTS_SETUP, PROPSET_PREFIX from mypyc.ir.class_ir import ClassIR, NonExtClassInfo diff --git a/mypyc/irbuild/constant_fold.py b/mypyc/irbuild/constant_fold.py index 96b2d441533e..53274dd3f971 100644 --- a/mypyc/irbuild/constant_fold.py +++ b/mypyc/irbuild/constant_fold.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Final +from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op from mypy.nodes import ( BytesExpr, ComplexExpr, @@ -25,8 +26,6 @@ UnaryExpr, Var, ) - -from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op from mypyc.irbuild.util import bytes_from_str if TYPE_CHECKING: diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index dbc4a3664048..c1997a921333 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -3,7 +3,6 @@ from __future__ import annotations from mypy.nodes import FuncItem - from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import INVALID_FUNC_DEF from mypyc.ir.ops import BasicBlock, Value diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 92927b8fcedb..4e5c58158cae 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -18,7 +18,6 @@ def g() -> int: from __future__ import annotations from mypy.nodes import Argument, FuncDef, SymbolNode, Var - from mypyc.common import ( BITMAP_BITS, ENV_ATTR_NAME, diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 6092225cde49..8bc4568e0d47 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -49,7 +49,6 @@ UnaryExpr, Var, ) - from mypy.types import ( AnyType, Instance, diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 9d08fd19c369..f1536fb3111a 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -27,7 +27,6 @@ TypeAlias, Var, ) - from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types from mypyc.common import GENERATOR_HELPER_NAME, IS_FREE_THREADED from mypyc.ir.ops import ( diff --git a/mypyc/irbuild/format_str_tokenizer.py b/mypyc/irbuild/format_str_tokenizer.py index e2716c6cfed6..f4c6167023bf 100644 --- a/mypyc/irbuild/format_str_tokenizer.py +++ b/mypyc/irbuild/format_str_tokenizer.py @@ -5,8 +5,6 @@ from enum import Enum, unique from typing import Final -from mypy.nodes import Context, Expression - from mypy.checkstrformat import ( ConversionSpecifier, parse_conversion_specifiers, @@ -14,6 +12,7 @@ ) from mypy.errors import Errors from mypy.messages import MessageBuilder +from mypy.nodes import Context, Expression from mypy.options import Options from mypyc.ir.ops import Integer, Value from mypyc.ir.rtypes import ( diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 0f7c457f0895..588e1cd51dbb 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -28,7 +28,6 @@ TypeInfo, Var, ) - from mypy.types import CallableType, Type, UnboundType, get_proper_type from mypyc.common import FAST_PREFIX, LAMBDA_NAME, PROPSET_PREFIX, SELF_NAME from mypyc.ir.class_ir import ClassIR, NonExtClassInfo diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 31e9f0449a22..7cd4f7de2846 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -13,7 +13,6 @@ from collections.abc import Callable from mypy.nodes import ARG_OPT, FuncDef, Var - from mypyc.common import ( ENV_ATTR_NAME, GENERATOR_ATTRIBUTE_PREFIX, diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index da0d863252e1..aafdb73fed84 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -11,9 +11,8 @@ from contextlib import contextmanager from typing import Final, TypeGuard, cast -from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind - from mypy.argmap import map_actuals_to_formals +from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind from mypy.operators import op_methods, unary_op_methods from mypy.types import AnyType, TypeOfAny from mypyc.common import ( diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index 4c7d5632ce0e..bc5354c34203 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -25,9 +25,8 @@ def f(x: int) -> int: from collections.abc import Callable from typing import Any, TypeVar, cast -from mypy.nodes import ClassDef, Expression, FuncDef, MypyFile - from mypy.build import Graph +from mypy.nodes import ClassDef, Expression, FuncDef, MypyFile from mypy.state import state from mypy.types import Type from mypyc.analysis.attrdefined import analyze_always_defined_attrs diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 62a96020e67c..523c718c9b86 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -3,7 +3,6 @@ from __future__ import annotations from mypy.nodes import ARG_STAR, ARG_STAR2, GDEF, ArgKind, FuncDef, RefExpr, SymbolNode, TypeInfo - from mypy.types import ( AnyType, CallableType, diff --git a/mypyc/irbuild/match.py b/mypyc/irbuild/match.py index fcd2c6046c89..7b08cf1437ca 100644 --- a/mypyc/irbuild/match.py +++ b/mypyc/irbuild/match.py @@ -4,7 +4,6 @@ from contextlib import contextmanager from mypy.nodes import MatchStmt, NameExpr, TypeInfo - from mypy.patterns import ( AsPattern, ClassPattern, diff --git a/mypyc/irbuild/missingtypevisitor.py b/mypyc/irbuild/missingtypevisitor.py index c251f38d1899..e655d270a4a4 100644 --- a/mypyc/irbuild/missingtypevisitor.py +++ b/mypyc/irbuild/missingtypevisitor.py @@ -1,7 +1,6 @@ from __future__ import annotations from mypy.nodes import Expression, Node - from mypy.traverser import ExtendedTraverserVisitor from mypy.types import AnyType, Type, TypeOfAny diff --git a/mypyc/irbuild/prebuildvisitor.py b/mypyc/irbuild/prebuildvisitor.py index 39f822dce5e1..2d9535f06a52 100644 --- a/mypyc/irbuild/prebuildvisitor.py +++ b/mypyc/irbuild/prebuildvisitor.py @@ -22,7 +22,6 @@ TupleExpr, Var, ) - from mypy.traverser import ExtendedTraverserVisitor, TraverserVisitor from mypy.types import Type from mypyc.errors import Errors diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 312237c7d4fc..99ce7de8df1f 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -17,6 +17,7 @@ from collections.abc import Iterable from typing import NamedTuple +from mypy.build import Graph from mypy.nodes import ( ARG_STAR, ARG_STAR2, @@ -37,8 +38,6 @@ Var, ) from mypy.semanal import refers_to_fullname - -from mypy.build import Graph from mypy.traverser import TraverserVisitor from mypy.types import Instance, Type, get_proper_type from mypyc.common import ( diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 8dd220787fa3..b2ff57847d6a 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -36,7 +36,6 @@ TupleExpr, Var, ) - from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( BasicBlock, diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index cbb0c0333a5c..a797b8f5ab34 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,6 @@ YieldExpr, YieldFromExpr, ) - from mypyc.common import ( GENERATOR_HELPER_NAME, KEEP_ALIVE_SHORT_LIVED, diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index c28b5f373660..a2378fa55bcc 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -32,7 +32,6 @@ is_class_var, ) from mypy.semanal import refers_to_fullname - from mypy.types import FINAL_DECORATOR_NAMES from mypyc.errors import Errors from mypyc.ir.class_ir import ClassIR diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index a5caac41be3b..bb8aa4932e39 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -84,7 +84,6 @@ YieldExpr, YieldFromExpr, ) - from mypyc.ir.ops import Value from mypyc.irbuild.builder import IRBuilder, IRVisitor, UnsupportedException from mypyc.irbuild.classdef import transform_class_def diff --git a/mypyc/test/test_statement.py b/mypyc/test/test_statement.py index a571ddbd6039..35773f934122 100644 --- a/mypyc/test/test_statement.py +++ b/mypyc/test/test_statement.py @@ -3,9 +3,8 @@ import unittest from typing import cast -from mypy.nodes import Import, MypyFile - from mypy.build import Graph +from mypy.nodes import Import, MypyFile from mypy.options import Options from mypyc.errors import Errors from mypyc.irbuild.builder import IRBuilder diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 6f51e54fbe5a..2d139ace7c8e 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -9,11 +9,10 @@ import shutil from collections.abc import Callable, Iterator -from mypy.nodes import Expression, MypyFile - from mypy import build from mypy.errors import CompileError from mypy.main import process_options +from mypy.nodes import Expression, MypyFile from mypy.options import Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite From 4e5a0449a58e01fcfdd202f30e45fcc1d5dcecde Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Thu, 24 Sep 2026 19:30:38 +0200 Subject: [PATCH 05/24] Restore #21998 implementation lost in bad merge The merge in 55d210e5 deleted mypy/checker.py, mypy/nodes.py, mypy/semanal.py and mypy/typeanal.py; the follow-up restore brought back plain upstream copies, dropping the #21998 implementation (DeferredBaseClassVar deferral in semanal/typeanal, deferred_base_classes on TypeInfo, and TypeChecker.check_deferred_base_classes / is_explicit_any). Re-apply the implementation from c56f1038 on top of the current upstream files. Verified locally (Hatch VM): testSubclassInferredAnyVariable and testSubclassInferredNonAnyVariable pass, and the full check-classes.test suite is green (612 passed, 1 xfailed). --- mypy/checker.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ mypy/nodes.py | 7 +++++ mypy/semanal.py | 38 ++++++++++++++++++++------ mypy/typeanal.py | 25 +++++++++++++++++ 4 files changed, 132 insertions(+), 8 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f03d06d965c3..6a40b3ee6124 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -207,6 +207,7 @@ def __init__(self) -> None: WithStmt, YieldExpr, get_func_def, + get_member_expr_fullname, is_final_node, ) from mypy.operators import flip_ops, int_op_to_method, neg_ops @@ -2807,9 +2808,78 @@ def check__exit__return_type(self, defn: FuncItem) -> None: ): self.msg.incorrect__exit__return(defn) + def is_explicit_any(self, typ: ProperType) -> bool: + """Is this an Any type that comes from an explicit annotation? + + Unlike ``Any`` that leaks in from unannotated (dynamically typed) code, + an explicitly written ``Any`` means the user opted into dynamic typing, + so it is treated like an explicitly ``Any``-typed variable. + """ + while ( + isinstance(typ, AnyType) + and typ.type_of_any == TypeOfAny.from_another_any + and typ.source_any is not None + ): + typ = get_proper_type(typ.source_any) + return isinstance(typ, AnyType) and typ.type_of_any != TypeOfAny.unannotated + + def check_deferred_base_classes(self, defn: ClassDef) -> None: + """Validate base classes deferred from semantic analysis. + + A base class that is a variable with an inferred (not declared) type -- + e.g. ``x = args[1]`` where ``args[1]`` is ``Any`` -- could not be + validated during semantic analysis, since the type is only inferred by + the type checker. Such bases were provisionally treated as ``Any``; + now that the type is known, accept it if it is ``Any`` (like an + explicitly ``Any``-typed variable) and otherwise report the errors that + semantic analysis deferred. + """ + for var, base_expr in defn.info.deferred_base_classes: + typ = get_proper_type(var.type) + # Mirror the cases semantic analysis accepts for an explicitly + # typed variable (see TypeAnalyser.analyze_unbound_type_without_type_info). + # An inferred plain ``Any`` is only accepted if it comes from an + # explicit annotation; ``Any`` leaking in from unannotated code is + # still rejected, as before. + is_any = ( + self.is_explicit_any(typ) + or (isinstance(typ, Instance) and typ.type.fullname == "builtins.type") + or (isinstance(typ, TypeType) and self.is_explicit_any(get_proper_type(typ.item))) + ) + if is_any: + if self.options.disallow_subclassing_any: + if isinstance(base_expr, (NameExpr, MemberExpr)): + msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' + else: + msg = 'Class cannot subclass value of type "Any"' + self.fail(msg, base_expr) + continue + if isinstance(base_expr, NameExpr): + name_repr: str | None = base_expr.name + elif isinstance(base_expr, MemberExpr): + name_repr = get_member_expr_fullname(base_expr) + else: + name_repr = None + self.fail( + f'Variable "{var.fullname}" is not valid as a type', + base_expr, + code=codes.VALID_TYPE, + ) + self.note( + "See https://mypy.readthedocs.io/en/stable/common_issues.html" + "#variables-vs-type-aliases", + base_expr, + code=codes.VALID_TYPE, + ) + msg = "Invalid base class" + if name_repr: + msg += f' "{name_repr}"' + self.fail(msg, base_expr) + def visit_class_def(self, defn: ClassDef) -> None: """Type check a class definition.""" typ = defn.info + self.check_deferred_base_classes(defn) for base in typ.mro[1:]: if base.is_final: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) diff --git a/mypy/nodes.py b/mypy/nodes.py index dc86550cebe4..57e1ea18772b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3686,6 +3686,7 @@ class is generic then it will be a type constructor of higher kind. "type_object_type", "default_depends", "typeddict_data", + "deferred_base_classes", ) _fullname: str # Fully qualified name @@ -3920,6 +3921,12 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.dataclass_transform_spec = None self.is_type_check_only = False self.deprecated = None + # Base classes that are variables whose types had not been inferred yet + # during semantic analysis. Each entry is a (variable, expression) tuple. + # The type checker validates them once the types are known (see + # TypeChecker.check_deferred_base_classes). This is not serialized; it + # is only meaningful within a single build. + self.deferred_base_classes: list[tuple[Var, Expression]] = [] self.type_object_type = None self.default_depends = {} self.typeddict_data = None diff --git a/mypy/semanal.py b/mypy/semanal.py index 5ef5223c2346..d0d76aa6be21 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -243,6 +243,7 @@ from mypy.tvar_scope import TypeVarLikeScope from mypy.typeanal import ( SELF_TYPE_NAMES, + DeferredBaseClassVar, FindTypeVarVisitor, TypeAnalyser, TypeVarDefaultTranslator, @@ -2034,7 +2035,7 @@ def analyze_class(self, defn: ClassDef) -> None: self.mark_incomplete(defn.name, defn) return - base_types, base_error = bases_result + base_types, base_error, deferred_bases = bases_result if any(isinstance(base, PlaceholderType) for base, _ in base_types): # We need to know the TypeInfo of each base to construct the MRO. Placeholder types # are okay in nested positions, since they can't affect the MRO. @@ -2071,6 +2072,7 @@ def analyze_class(self, defn: ClassDef) -> None: defn.info.default_depends = default_depends if base_error: defn.info.fallback_to_any = True + defn.info.deferred_base_classes = deferred_bases if any_meta: defn.info.meta_fallback_to_any = True @@ -2622,7 +2624,7 @@ def get_name_repr_of_expr(self, expr: Expression) -> str | None: def analyze_base_classes( self, cls_name: str, base_type_exprs: list[Expression] - ) -> tuple[list[tuple[ProperType, Expression]], bool] | None: + ) -> tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression]]] | None: """Analyze base class types. Return None if some definition was incomplete. Otherwise, return a tuple @@ -2630,9 +2632,15 @@ def analyze_base_classes( * List of (analyzed type, original expression) tuples * Boolean indicating whether one of the bases had a semantic analysis error + * List of (variable, original expression) tuples for bases whose validity + could not be checked yet, because the variable's type has not been + inferred (inference happens in the type checker). These are + provisionally treated as having an Any base; the type checker + validates them once the type is known. """ is_error = False bases = [] + deferred: list[tuple[Var, Expression]] = [] for i, base_expr in enumerate(base_type_exprs): if ( isinstance(base_expr, RefExpr) @@ -2656,6 +2664,13 @@ def analyze_base_classes( allow_type_any=True, unique_name=inline_base(cls_name, i), ) + except DeferredBaseClassVar as e: + # The base is a variable whose type will only be known after + # type checking (e.g. `x = args[1]` where `args[1]` is `Any`). + # Provisionally treat it as Any; the type checker verifies the + # inferred type (see TypeChecker.check_deferred_base_classes). + deferred.append((e.var, base_expr)) + base = AnyType(TypeOfAny.special_form) except TypeTranslationError: name = self.get_name_repr_of_expr(base_expr) if isinstance(base_expr, CallExpr): @@ -2671,7 +2686,7 @@ def analyze_base_classes( return None base = get_proper_type(base) bases.append((base, base_expr)) - return bases, is_error + return bases, is_error, deferred def configure_base_classes( self, defn: ClassDef, bases: list[tuple[ProperType, Expression]] @@ -2695,11 +2710,18 @@ def configure_base_classes( base_types.append(base) elif isinstance(base, AnyType): if self.options.disallow_subclassing_any: - if isinstance(base_expr, (NameExpr, MemberExpr)): - msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' - else: - msg = 'Class cannot subclass value of type "Any"' - self.fail(msg, base_expr) + # A deferred base class is provisionally treated as Any; + # the type checker reports this error once the actual + # inferred type is known, if it is Any. + is_deferred = any( + expr is base_expr for _, expr in defn.info.deferred_base_classes + ) + if not is_deferred: + if isinstance(base_expr, (NameExpr, MemberExpr)): + msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' + else: + msg = 'Class cannot subclass value of type "Any"' + self.fail(msg, base_expr) info.fallback_to_any = True elif isinstance(base, TypedDictType): base_types.append(base.fallback) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6717f4f18c90..9e638f1c30bb 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -186,6 +186,25 @@ def analyze_type_alias( return res, analyzer.aliases_used +class DeferredBaseClassVar(Exception): + """A base class is a variable whose type hasn't been inferred yet. + + Semantic analysis runs before type checking, so when a class uses a variable + with an inferred (not declared) type as a base class -- e.g. ``x = args[1]`` + where ``args[1]`` is ``Any`` -- the variable's type is not yet known. The + class is provisionally treated as having an ``Any`` base; the type checker + validates the inferred type once it is known (see + ``TypeChecker.check_deferred_base_classes``). + + The variable is carried on the exception so the checker can inspect its + inferred type without re-resolving the base class expression. + """ + + def __init__(self, var: nodes.Var) -> None: + super().__init__() + self.var = var + + class TypeAnalyser(SyntheticTypeVisitor[Type], TypeAnalyzerPluginInterface): """Semantic analyzer for types. @@ -1018,6 +1037,12 @@ def analyze_unbound_type_without_type_info( return AnyType(TypeOfAny.special_form) if isinstance(typ, TypeType) and isinstance(typ.item, AnyType): return AnyType(TypeOfAny.from_another_any, source_any=typ.item) + if typ is None: + # The variable's type has not been inferred yet (inference + # happens in the type checker, after semantic analysis). + # Defer the validity check; the type checker will verify + # the inferred type. Provisionally treat the base as Any. + raise DeferredBaseClassVar(sym.node) # Option 2: # Unbound type variable. Currently these may be still valid, # for example when defining a generic type alias. From e7381cac67b9d99dd2c91aa2b0cf7e8873875f8b Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Thu, 24 Sep 2026 20:59:03 +0200 Subject: [PATCH 06/24] Fix self-check errors in check_deferred_base_classes The mypyc-compiled CI jobs build mypy with its own type checker, which flagged three errors in the new code: redundant get_proper_type() on AnyType.source_any and TypeType.item (both already ProperType), and is_explicit_any() called with ProperType | None from var.type. Drop the redundant calls and narrow out None before the is_any check. No behavior change: get_proper_type is identity on proper types, and None still falls through to the deferred-error branch. Verified: mypy self-check on checker.py is clean; testSubclassInferredAnyVariable and testSubclassInferredNonAnyVariable pass. --- mypy/checker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 6a40b3ee6124..f3e6b1ff70e8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2820,7 +2820,7 @@ def is_explicit_any(self, typ: ProperType) -> bool: and typ.type_of_any == TypeOfAny.from_another_any and typ.source_any is not None ): - typ = get_proper_type(typ.source_any) + typ = typ.source_any return isinstance(typ, AnyType) and typ.type_of_any != TypeOfAny.unannotated def check_deferred_base_classes(self, defn: ClassDef) -> None: @@ -2835,16 +2835,17 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: semantic analysis deferred. """ for var, base_expr in defn.info.deferred_base_classes: - typ = get_proper_type(var.type) + var_type = var.type + typ = get_proper_type(var_type) if var_type is not None else None # Mirror the cases semantic analysis accepts for an explicitly # typed variable (see TypeAnalyser.analyze_unbound_type_without_type_info). # An inferred plain ``Any`` is only accepted if it comes from an # explicit annotation; ``Any`` leaking in from unannotated code is # still rejected, as before. - is_any = ( + is_any = typ is not None and ( self.is_explicit_any(typ) or (isinstance(typ, Instance) and typ.type.fullname == "builtins.type") - or (isinstance(typ, TypeType) and self.is_explicit_any(get_proper_type(typ.item))) + or (isinstance(typ, TypeType) and self.is_explicit_any(typ.item)) ) if is_any: if self.options.disallow_subclassing_any: From 5b2acbc119ee97d897274cd1621e4d6a91148deb Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 02:28:28 +0200 Subject: [PATCH 07/24] test: add tuple-index Any base class cases from #22002 --- test-data/unit/check-classes.test | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 16141bc132f6..8fcdc04f7a51 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9760,3 +9760,209 @@ def f(x: Any) -> None: class Sub2(z): pass # E: Variable "z" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ # E: Invalid base class "z" + +[case testSubclassAnyFromIndexedTuple] +# https://github.com/python/mypy/issues/21998 +from typing import Any, Tuple + +def from_tuple(t: Tuple[int, Any]) -> None: + class Sub(t[1]): # E: Variable "t" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "t" \ + # E: Invalid type: try using Literal[1] instead? + pass + +def from_star_any(*args: Any) -> None: + class Sub(args[0]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[0] instead? + pass + +def from_any_param(base: Any) -> None: + class Sub(base): + pass + +def from_annotated_index(*args: Any) -> None: + x: Any = args[0] + class Sub(x): + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassAnyFromUnpackedStarArgs] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def from_unpacked(*args: *tuple[int, Any]) -> None: + class Sub(args[1]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[1] instead? + pass + +def from_unpacked_type_annotation(*args: *tuple[int, Any]) -> None: + x: type = args[1] + class Sub(x): + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassAnyFromNestedUnpackedStarArgs] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def from_unpacked(*args: *tuple[int, *tuple[Any]]) -> None: + reveal_type(args) # N: Revealed type is "tuple[builtins.int, Any]" + class Sub(args[1]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[1] instead? + pass + +def from_unpacked_type_annotation(*args: *tuple[int, *tuple[Any]]) -> None: + reveal_type(args) # N: Revealed type is "tuple[builtins.int, Any]" + x: type = args[1] + class Sub(x): + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassIndexedNonAnyStillInvalid] +# https://github.com/python/mypy/issues/21998 +from typing import Any, Tuple + +def from_str_tuple(t: Tuple[int, str]) -> None: + class Sub(t[1]): pass # E: Variable "t" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "t" \ + # E: Invalid type: try using Literal[1] instead? + +def from_int_index(t: Tuple[int, Any]) -> None: + class Sub(t[0]): pass # E: Variable "t" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "t" \ + # E: Invalid type: try using Literal[0] instead? + +def from_int_star(*args: int) -> None: + class Sub(args[0]): pass # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[0] instead? + +def unannotated_local(base: Any) -> None: + # An unannotated local initialized from an Any value is inferred as + # explicit Any, so it is accepted as a base class. + x = base + class Sub(x): pass +[builtins fixtures/tuple.pyi] + +[case testSubclassAnyFromNestedUnpackedUnboundedStarArgs] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def from_unpacked(*args: *tuple[int, *tuple[int, ...], Any]) -> None: + reveal_type(args[-1]) # N: Revealed type is "Any" + class Sub(args[-1]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[-1] instead? + pass + +def from_unpacked_type_annotation(*args: *tuple[int, *tuple[int, ...], Any]) -> None: + reveal_type(args[-1]) # N: Revealed type is "Any" + x : type = args[-1] + class Sub(x): + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassAnyFromUnpackedStarArgsAmbiguousIndex] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def before_the_unpack(*args: *tuple[Any, *tuple[int, ...], int]) -> None: + class Sub(args[0]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[0] instead? + pass + +def inside_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: + class Sub(args[1]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[1] instead? + pass + +def across_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: + class Sub(args[-2]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[-2] instead? + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassAnyFromNestedUnpackedUnboundedStarArgsIndexInsideUnambiguous] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def from_unpacked(*args: *tuple[Any, ...]) -> None: + reveal_type(args[0]) # N: Revealed type is "Any" + reveal_type(args[1]) # N: Revealed type is "Any" + class Sub(args[0]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[0] instead? + pass + class Sub2(args[1]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[1] instead? + pass + +def from_unpacked2(*args: *tuple[*tuple[Any, ...]]) -> None: + reveal_type(args[0]) # N: Revealed type is "Any" + reveal_type(args[1]) # N: Revealed type is "Any" + class Sub(args[0]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[0] instead? + pass + class Sub2(args[1]): # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[1] instead? + pass + +def from_unpacked_type_annotation(*args: *tuple[Any, ...]) -> None: + reveal_type(args[0]) # N: Revealed type is "Any" + reveal_type(args[1]) # N: Revealed type is "Any" + x: type = args[0] + class Sub(x): + pass + x1: type = args[1] + class Sub2(x1): + pass + +def from_unpacked2_type_annotation(*args: *tuple[*tuple[Any, ...]]) -> None: + reveal_type(args[0]) # N: Revealed type is "Any" + reveal_type(args[1]) # N: Revealed type is "Any" + x: type = args[0] + class Sub(x): + pass + x1: type = args[1] + class Sub2(x1): + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassFromSequenceIndexed] +from typing import Any, Sequence + +def f(y: Sequence[Any]) -> None: + class A(y[0]): pass # E: Variable "y" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "y" \ + # E: Invalid type: try using Literal[0] instead? +[builtins fixtures/tuple.pyi] From a5d92f38a9c97b3f932695c2729ca35c086674fa Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 02:31:11 +0200 Subject: [PATCH 08/24] test: pin rejection of Any leaking from unannotated code Covers the remaining negative direction of the deferred base-class check: an inferred Any that does not come from an explicit annotation is still rejected as a base class. --- test-data/unit/check-classes.test | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 8fcdc04f7a51..4d048770392d 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9761,6 +9761,15 @@ def f(x: Any) -> None: # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ # E: Invalid base class "z" +[case testSubclassInferredLeakedAnyStillInvalid] +from typing import Any +def g(a) -> None: + # Any leaking in from unannotated code is still rejected. + x = a + class Sub(x): pass # E: Variable "x" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "x" + [case testSubclassAnyFromIndexedTuple] # https://github.com/python/mypy/issues/21998 from typing import Any, Tuple From febb05a1a7dc00d368e20ca948059e72312b170a Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 12:24:42 +0200 Subject: [PATCH 09/24] Do not report deferred base class errors where semanal would not (1/5: mypy/checker.py) --- mypy/checker.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index f3e6b1ff70e8..03a7379f348a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2834,7 +2834,15 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: explicitly ``Any``-typed variable) and otherwise report the errors that semantic analysis deferred. """ - for var, base_expr in defn.info.deferred_base_classes: + if not self.in_checked_function(): + # Semantic analysis silently drops these errors in unchecked + # functions; do the same here to avoid new reports. + return + for var, base_expr, defining_literal in defn.info.deferred_base_classes: + if defining_literal: + # Semantic analysis does not report "not valid as a type" for + # variables inside Literal[...]; do the same here. + continue var_type = var.type typ = get_proper_type(var_type) if var_type is not None else None # Mirror the cases semantic analysis accepts for an explicitly From 28683a3a1f711460cdb69411bba39d5618c0e84c Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 12:24:44 +0200 Subject: [PATCH 10/24] Do not report deferred base class errors where semanal would not (2/5: mypy/nodes.py) --- mypy/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 57e1ea18772b..b683ec6db2b0 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3926,7 +3926,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None # The type checker validates them once the types are known (see # TypeChecker.check_deferred_base_classes). This is not serialized; it # is only meaningful within a single build. - self.deferred_base_classes: list[tuple[Var, Expression]] = [] + self.deferred_base_classes: list[tuple[Var, Expression, bool]] = [] self.type_object_type = None self.default_depends = {} self.typeddict_data = None From aafedde2028d814a821384151cbad927b2e39b74 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 12:24:45 +0200 Subject: [PATCH 11/24] Do not report deferred base class errors where semanal would not (3/5: mypy/semanal.py) --- mypy/semanal.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d0d76aa6be21..f7695265216c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2624,7 +2624,7 @@ def get_name_repr_of_expr(self, expr: Expression) -> str | None: def analyze_base_classes( self, cls_name: str, base_type_exprs: list[Expression] - ) -> tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression]]] | None: + ) -> tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression, bool]]] | None: """Analyze base class types. Return None if some definition was incomplete. Otherwise, return a tuple @@ -2632,15 +2632,15 @@ def analyze_base_classes( * List of (analyzed type, original expression) tuples * Boolean indicating whether one of the bases had a semantic analysis error - * List of (variable, original expression) tuples for bases whose validity - could not be checked yet, because the variable's type has not been - inferred (inference happens in the type checker). These are - provisionally treated as having an Any base; the type checker - validates them once the type is known. + * List of (variable, original expression, defining_literal) tuples for + bases whose validity could not be checked yet, because the + variable's type has not been inferred (inference happens in the + type checker). These are provisionally treated as having an Any + base; the type checker validates them once the type is known. """ is_error = False bases = [] - deferred: list[tuple[Var, Expression]] = [] + deferred: list[tuple[Var, Expression, bool]] = [] for i, base_expr in enumerate(base_type_exprs): if ( isinstance(base_expr, RefExpr) @@ -2669,7 +2669,7 @@ def analyze_base_classes( # type checking (e.g. `x = args[1]` where `args[1]` is `Any`). # Provisionally treat it as Any; the type checker verifies the # inferred type (see TypeChecker.check_deferred_base_classes). - deferred.append((e.var, base_expr)) + deferred.append((e.var, base_expr, e.defining_literal)) base = AnyType(TypeOfAny.special_form) except TypeTranslationError: name = self.get_name_repr_of_expr(base_expr) @@ -2714,7 +2714,7 @@ def configure_base_classes( # the type checker reports this error once the actual # inferred type is known, if it is Any. is_deferred = any( - expr is base_expr for _, expr in defn.info.deferred_base_classes + expr is base_expr for _, expr, _ in defn.info.deferred_base_classes ) if not is_deferred: if isinstance(base_expr, (NameExpr, MemberExpr)): From 7337b492cff425e8c0b7f4872bc62654d0feb2b0 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 12:24:47 +0200 Subject: [PATCH 12/24] Do not report deferred base class errors where semanal would not (4/5: mypy/typeanal.py) --- mypy/typeanal.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 9e638f1c30bb..3980fd453fe2 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -200,9 +200,14 @@ class is provisionally treated as having an ``Any`` base; the type checker inferred type without re-resolving the base class expression. """ - def __init__(self, var: nodes.Var) -> None: + def __init__(self, var: nodes.Var, defining_literal: bool) -> None: super().__init__() self.var = var + # Whether the base class expression was being analyzed as part of a + # Literal[...] type. Semantic analysis does not report "not valid as a + # type" for variables in that position, so the type checker must not + # either. + self.defining_literal = defining_literal class TypeAnalyser(SyntheticTypeVisitor[Type], TypeAnalyzerPluginInterface): @@ -1042,7 +1047,7 @@ def analyze_unbound_type_without_type_info( # happens in the type checker, after semantic analysis). # Defer the validity check; the type checker will verify # the inferred type. Provisionally treat the base as Any. - raise DeferredBaseClassVar(sym.node) + raise DeferredBaseClassVar(sym.node, defining_literal) # Option 2: # Unbound type variable. Currently these may be still valid, # for example when defining a generic type alias. From 6e20719608b7f75f174899b7cc8506aaea38c578 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 28 Sep 2026 12:24:48 +0200 Subject: [PATCH 13/24] Do not report deferred base class errors where semanal would not (5/5: test-data/unit/check-classes.test) --- test-data/unit/check-classes.test | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 4d048770392d..5c7320d4e451 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9975,3 +9975,26 @@ def f(y: Sequence[Any]) -> None: # E: Invalid base class "y" \ # E: Invalid type: try using Literal[0] instead? [builtins fixtures/tuple.pyi] + +[case testDeferredBaseClassUncheckedFunction] +# Dynamic base classes in unchecked functions were silently accepted before; +# the deferred checker must not report new errors there. +def f(base): + class A(base): + pass + return A +def g(x): + y = x + class B(y): + pass +[builtins fixtures/tuple.pyi] + +[case testDeferredBaseClassInLiteral] +# A variable inside Literal[...] in a base class: semanal does not report +# "not valid as a type" there, so the deferred checker must not either. +from typing import Literal, Generic, TypeVar +T = TypeVar("T") +class B(Generic[T]): pass +x = 1 +class C(B["Literal[x]"]): pass # E: Parameter 1 of Literal[...] is invalid +[builtins fixtures/tuple.pyi] From 1a41018d8c6bdd701675f12a3a4e492c78bbd725 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:26:26 +0000 Subject: [PATCH 14/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/semanal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f7695265216c..2b4279b48ea6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2624,7 +2624,9 @@ def get_name_repr_of_expr(self, expr: Expression) -> str | None: def analyze_base_classes( self, cls_name: str, base_type_exprs: list[Expression] - ) -> tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression, bool]]] | None: + ) -> ( + tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression, bool]]] | None + ): """Analyze base class types. Return None if some definition was incomplete. Otherwise, return a tuple From 17fab6cbc7b7302a225667f890044970a651b220 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Thu, 8 Oct 2026 00:57:11 +0200 Subject: [PATCH 15/24] Address review nits: use dataclass for deferred bases, simplify is_explicit_any --- mypy/checker.py | 20 ++++++++++++-------- mypy/nodes.py | 22 +++++++++++++++++----- mypy/semanal.py | 19 ++++++++++--------- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 03a7379f348a..3bf016a8c36c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -216,6 +216,7 @@ def __init__(self) -> None: from mypy.plugin import Plugin from mypy.plugins import dataclasses as dataclasses_plugin from mypy.scope import Scope +from mypy.stats import get_original_any from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS from mypy.semanal_shared import SemanticAnalyzerCoreInterface @@ -2815,13 +2816,13 @@ def is_explicit_any(self, typ: ProperType) -> bool: an explicitly written ``Any`` means the user opted into dynamic typing, so it is treated like an explicitly ``Any``-typed variable. """ - while ( - isinstance(typ, AnyType) - and typ.type_of_any == TypeOfAny.from_another_any - and typ.source_any is not None - ): - typ = typ.source_any - return isinstance(typ, AnyType) and typ.type_of_any != TypeOfAny.unannotated + if not isinstance(typ, AnyType): + return False + # Unwrap Any types that come from another Any to get the original. + typ = get_original_any(typ) + # Only accept explicitly written Any, not Any from unannotated code + # or other sources. + return typ.type_of_any == TypeOfAny.explicit def check_deferred_base_classes(self, defn: ClassDef) -> None: """Validate base classes deferred from semantic analysis. @@ -2838,7 +2839,10 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: # Semantic analysis silently drops these errors in unchecked # functions; do the same here to avoid new reports. return - for var, base_expr, defining_literal in defn.info.deferred_base_classes: + for deferred in defn.info.deferred_base_classes: + var = deferred.var + base_expr = deferred.expr + defining_literal = deferred.defining_literal if defining_literal: # Semantic analysis does not report "not valid as a type" for # variables inside Literal[...]; do the same here. diff --git a/mypy/nodes.py b/mypy/nodes.py index b683ec6db2b0..6dcea6df232f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -7,6 +7,7 @@ from collections import defaultdict from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager +from dataclasses import dataclass from enum import Enum, unique from typing import ( TYPE_CHECKING, @@ -3627,6 +3628,18 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: EXCLUDED_ENUM_ATTRIBUTES: Final = frozenset({"_ignore_", "_order_", "__order__"}) +@dataclass +class DeferredBaseClass: + """A base class that is a variable whose type was not yet inferred. + + The type checker validates these once the variable's type is known. + """ + + var: Var + expr: Expression + defining_literal: bool + + class TypeInfo(SymbolNode): """The type structure of a single class. @@ -3922,11 +3935,10 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.is_type_check_only = False self.deprecated = None # Base classes that are variables whose types had not been inferred yet - # during semantic analysis. Each entry is a (variable, expression) tuple. - # The type checker validates them once the types are known (see - # TypeChecker.check_deferred_base_classes). This is not serialized; it - # is only meaningful within a single build. - self.deferred_base_classes: list[tuple[Var, Expression, bool]] = [] + # during semantic analysis. The type checker validates them once the + # types are known (see TypeChecker.check_deferred_base_classes). + # This is not serialized; it is only meaningful within a single build. + self.deferred_base_classes: list[DeferredBaseClass] = [] self.type_object_type = None self.default_depends = {} self.typeddict_data = None diff --git a/mypy/semanal.py b/mypy/semanal.py index 2b4279b48ea6..ee84acc401ec 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -110,6 +110,7 @@ ContinueStmt, DataclassTransformSpec, Decorator, + DeferredBaseClass, DelStmt, DictExpr, DictionaryComprehension, @@ -2625,7 +2626,7 @@ def get_name_repr_of_expr(self, expr: Expression) -> str | None: def analyze_base_classes( self, cls_name: str, base_type_exprs: list[Expression] ) -> ( - tuple[list[tuple[ProperType, Expression]], bool, list[tuple[Var, Expression, bool]]] | None + tuple[list[tuple[ProperType, Expression]], bool, list[DeferredBaseClass]] | None ): """Analyze base class types. @@ -2634,15 +2635,15 @@ def analyze_base_classes( * List of (analyzed type, original expression) tuples * Boolean indicating whether one of the bases had a semantic analysis error - * List of (variable, original expression, defining_literal) tuples for - bases whose validity could not be checked yet, because the - variable's type has not been inferred (inference happens in the - type checker). These are provisionally treated as having an Any - base; the type checker validates them once the type is known. + * List of DeferredBaseClass for bases whose validity could not be + checked yet, because the variable's type has not been inferred + (inference happens in the type checker). These are provisionally + treated as having an Any base; the type checker validates them once + the type is known. """ is_error = False bases = [] - deferred: list[tuple[Var, Expression, bool]] = [] + deferred: list[DeferredBaseClass] = [] for i, base_expr in enumerate(base_type_exprs): if ( isinstance(base_expr, RefExpr) @@ -2671,7 +2672,7 @@ def analyze_base_classes( # type checking (e.g. `x = args[1]` where `args[1]` is `Any`). # Provisionally treat it as Any; the type checker verifies the # inferred type (see TypeChecker.check_deferred_base_classes). - deferred.append((e.var, base_expr, e.defining_literal)) + deferred.append(DeferredBaseClass(e.var, base_expr, e.defining_literal)) base = AnyType(TypeOfAny.special_form) except TypeTranslationError: name = self.get_name_repr_of_expr(base_expr) @@ -2716,7 +2717,7 @@ def configure_base_classes( # the type checker reports this error once the actual # inferred type is known, if it is Any. is_deferred = any( - expr is base_expr for _, expr, _ in defn.info.deferred_base_classes + d.expr is base_expr for d in defn.info.deferred_base_classes ) if not is_deferred: if isinstance(base_expr, (NameExpr, MemberExpr)): From 4eb00047b2d79993ccef5b20d3fb4818859adba8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:58:46 +0000 Subject: [PATCH 16/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/checker.py | 2 +- mypy/semanal.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3bf016a8c36c..7afc564572d0 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -216,12 +216,12 @@ def __init__(self) -> None: from mypy.plugin import Plugin from mypy.plugins import dataclasses as dataclasses_plugin from mypy.scope import Scope -from mypy.stats import get_original_any from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS from mypy.semanal_shared import SemanticAnalyzerCoreInterface from mypy.sharedparse import BINARY_MAGIC_METHODS from mypy.state import state +from mypy.stats import get_original_any from mypy.subtypes import ( find_member, infer_class_variances, diff --git a/mypy/semanal.py b/mypy/semanal.py index ee84acc401ec..ef72546619f2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2625,9 +2625,7 @@ def get_name_repr_of_expr(self, expr: Expression) -> str | None: def analyze_base_classes( self, cls_name: str, base_type_exprs: list[Expression] - ) -> ( - tuple[list[tuple[ProperType, Expression]], bool, list[DeferredBaseClass]] | None - ): + ) -> tuple[list[tuple[ProperType, Expression]], bool, list[DeferredBaseClass]] | None: """Analyze base class types. Return None if some definition was incomplete. Otherwise, return a tuple @@ -2716,9 +2714,7 @@ def configure_base_classes( # A deferred base class is provisionally treated as Any; # the type checker reports this error once the actual # inferred type is known, if it is Any. - is_deferred = any( - d.expr is base_expr for d in defn.info.deferred_base_classes - ) + is_deferred = any(d.expr is base_expr for d in defn.info.deferred_base_classes) if not is_deferred: if isinstance(base_expr, (NameExpr, MemberExpr)): msg = f'Class cannot subclass "{base_expr.name}" (has type "Any")' From 902472610ee4b4122df790783555bb6b614a1cfb Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Fri, 23 Oct 2026 22:29:16 +0200 Subject: [PATCH 17/24] Support subscripted variable bases with Any item type --- mypy/checker.py | 118 +++++++++++++++++++++++++++++- mypy/typeanal.py | 7 ++ test-data/unit/check-classes.test | 66 +++++------------ 3 files changed, 142 insertions(+), 49 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 77e5b6fd09b9..540d63fb63c1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -268,6 +268,7 @@ def __init__(self) -> None: MYPYC_NATIVE_INT_NAMES, NOT_IMPLEMENTED_TYPE_NAMES, OVERLOAD_NAMES, + TUPLE_NAMES, AnyType, BoolTypeQuery, CallableType, @@ -298,6 +299,7 @@ def __init__(self) -> None: UnpackType, extend_args_for_prefix_and_suffix, find_unpack_in_list, + flatten_nested_tuples, flatten_nested_unions, get_proper_type, get_proper_types, @@ -2866,6 +2868,10 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: now that the type is known, accept it if it is ``Any`` (like an explicitly ``Any``-typed variable) and otherwise report the errors that semantic analysis deferred. + + A base class may also subscript a variable directly, e.g. + ``class Sub(args[1])``. Then the item type at the subscript is what + gets validated, in the same way. """ if not self.in_checked_function(): # Semantic analysis silently drops these errors in unchecked @@ -2879,8 +2885,11 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: # Semantic analysis does not report "not valid as a type" for # variables inside Literal[...]; do the same here. continue - var_type = var.type - typ = get_proper_type(var_type) if var_type is not None else None + if isinstance(base_expr, IndexExpr): + typ = self.deferred_index_base_type(var, base_expr) + else: + var_type = var.type + typ = get_proper_type(var_type) if var_type is not None else None # Mirror the cases semantic analysis accepts for an explicitly # typed variable (see TypeAnalyser.analyze_unbound_type_without_type_info). # An inferred plain ``Any`` is only accepted if it comes from an @@ -2903,6 +2912,8 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: name_repr: str | None = base_expr.name elif isinstance(base_expr, MemberExpr): name_repr = get_member_expr_fullname(base_expr) + elif isinstance(base_expr, IndexExpr): + name_repr = var.name else: name_repr = None self.fail( @@ -2920,6 +2931,109 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: if name_repr: msg += f' "{name_repr}"' self.fail(msg, base_expr) + if isinstance(base_expr, IndexExpr): + index_value = self.index_expr_int_value(base_expr.index) + if index_value is not None: + # Mirror the hint semantic analysis reports for an + # integer used as a type argument. + self.fail( + f"Invalid type: try using Literal[{index_value}] instead?", + base_expr, + code=codes.VALID_TYPE, + ) + + @staticmethod + def index_expr_int_value(index: Expression) -> int | None: + """Value of a constant integer index expression, or None.""" + if isinstance(index, IntExpr): + return index.value + if isinstance(index, UnaryExpr) and index.op == "-" and isinstance(index.expr, IntExpr): + return -index.expr.value + return None + + def deferred_index_base_type(self, var: Var, base_expr: IndexExpr) -> ProperType | None: + """Type of a deferred subscripted base class, e.g. ``args[1]``. + + ``var`` is the subscripted variable. Return the type of the item at + the subscript, or None if it cannot be determined statically; the + base class is then rejected, as it was before deferred validation. + Only a constant integer index is supported, and only into a tuple + or sequence type, mirroring what can be known about the item type. + """ + if not isinstance(base_expr.base, NameExpr): + return None + index_value = self.index_expr_int_value(base_expr.index) + if index_value is None: + return None + var_type = get_proper_type(var.type) if var.type is not None else None + if isinstance(var_type, TupleType): + return self.tuple_item_at_index(var_type, index_value) + if isinstance(var_type, Instance): + if var_type.type.fullname in TUPLE_NAMES and var_type.args: + # Homogeneous tuple[T, ...]: every item has type T. + return get_proper_type(var_type.args[0]) + sequence_info = var_type.type.get_base("typing.Sequence") + if sequence_info is not None: + mapped = map_instance_to_supertype(var_type, sequence_info) + if mapped.args: + return get_proper_type(mapped.args[0]) + return None + + def tuple_item_at_index(self, tuple_type: TupleType, index: int) -> ProperType | None: + """Type of a tuple item at a constant index, or None if ambiguous. + + If the tuple type has a variadic (unpacked) item, an index may land + on more than one item. The item type is only determined if the index + lands on a single fixed item, or if every item it may land on is + ``Any``; otherwise the index is ambiguous and None is returned. + """ + items = [get_proper_type(item) for item in flatten_nested_tuples(tuple_type.items)] + unpack_index = find_unpack_in_list(items) + if unpack_index is None: + pos = index if index >= 0 else index + len(items) + if 0 <= pos < len(items): + return items[pos] + return None + unpack_item = items[unpack_index] + assert isinstance(unpack_item, UnpackType) + unpacked = get_proper_type(unpack_item.type) + if not ( + isinstance(unpacked, Instance) + and unpacked.type.fullname in TUPLE_NAMES + and unpacked.args + ): + return None + variadic_item = get_proper_type(unpacked.args[0]) + prefix = items[:unpack_index] + suffix = items[unpack_index + 1 :] + possible: list[ProperType] + if index >= 0: + if index < len(prefix): + possible = [prefix[index]] + else: + possible = [variadic_item] + # A suffix item sits at position len(prefix) + + k, and the variadic length is unknown, so it may + # also land on this index. + for k, item in enumerate(suffix): + if len(prefix) + k <= index: + possible.append(item) + else: + from_end = -index + if from_end <= len(suffix): + possible = [suffix[len(suffix) - from_end]] + else: + possible = [variadic_item] + # A prefix item sits at distance len(suffix) + + (len(prefix) - k) from the end. + for k, item in enumerate(prefix): + if len(suffix) + (len(prefix) - k) <= from_end: + possible.append(item) + if all(isinstance(item, AnyType) for item in possible): + return possible[0] + if len(possible) == 1: + return possible[0] + return None def visit_class_def(self, defn: ClassDef) -> None: """Type check a class definition.""" diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 8e89b8475aeb..c1821cb7774f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1048,6 +1048,13 @@ def analyze_unbound_type_without_type_info( # Defer the validity check; the type checker will verify # the inferred type. Provisionally treat the base as Any. raise DeferredBaseClassVar(sym.node, defining_literal) + if t.args: + # The base class subscripts the variable, e.g. `args[1]` + # where `args` is a *args tuple. Whether the base is valid + # depends on the type of the subscripted item, which the + # type checker computes (the variable's type may itself be + # inferred). Defer the validity check, as above. + raise DeferredBaseClassVar(sym.node, defining_literal) # Option 2: # Unbound type variable. Currently these may be still valid, # for example when defining a generic type alias. diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index c1127e3275d6..7f3d9f672331 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9827,17 +9827,7 @@ def g(a) -> None: from typing import Any, Tuple def from_tuple(t: Tuple[int, Any]) -> None: - class Sub(t[1]): # E: Variable "t" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "t" \ - # E: Invalid type: try using Literal[1] instead? - pass - -def from_star_any(*args: Any) -> None: - class Sub(args[0]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[0] instead? + class Sub(t[1]): pass def from_any_param(base: Any) -> None: @@ -9850,16 +9840,22 @@ def from_annotated_index(*args: Any) -> None: pass [builtins fixtures/tuple.pyi] +[case testSubclassAnyFromStarAny] +# https://github.com/python/mypy/issues/21998 +from typing import Any + +def from_star_any(*args: Any) -> None: + class Sub(args[0]): + pass +[builtins fixtures/tuple.pyi] + [case testSubclassAnyFromUnpackedStarArgs] # https://github.com/python/mypy/issues/21998 # flags: --python-version 3.12 from typing import Any def from_unpacked(*args: *tuple[int, Any]) -> None: - class Sub(args[1]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[1] instead? + class Sub(args[1]): pass def from_unpacked_type_annotation(*args: *tuple[int, Any]) -> None: @@ -9875,10 +9871,7 @@ from typing import Any def from_unpacked(*args: *tuple[int, *tuple[Any]]) -> None: reveal_type(args) # N: Revealed type is "tuple[builtins.int, Any]" - class Sub(args[1]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[1] instead? + class Sub(args[1]): pass def from_unpacked_type_annotation(*args: *tuple[int, *tuple[Any]]) -> None: @@ -9924,10 +9917,7 @@ from typing import Any def from_unpacked(*args: *tuple[int, *tuple[int, ...], Any]) -> None: reveal_type(args[-1]) # N: Revealed type is "Any" - class Sub(args[-1]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[-1] instead? + class Sub(args[-1]): pass def from_unpacked_type_annotation(*args: *tuple[int, *tuple[int, ...], Any]) -> None: @@ -9943,10 +9933,7 @@ def from_unpacked_type_annotation(*args: *tuple[int, *tuple[int, ...], Any]) -> from typing import Any def before_the_unpack(*args: *tuple[Any, *tuple[int, ...], int]) -> None: - class Sub(args[0]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[0] instead? + class Sub(args[0]): pass def inside_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: @@ -9972,29 +9959,17 @@ from typing import Any def from_unpacked(*args: *tuple[Any, ...]) -> None: reveal_type(args[0]) # N: Revealed type is "Any" reveal_type(args[1]) # N: Revealed type is "Any" - class Sub(args[0]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[0] instead? + class Sub(args[0]): pass - class Sub2(args[1]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[1] instead? + class Sub2(args[1]): pass def from_unpacked2(*args: *tuple[*tuple[Any, ...]]) -> None: reveal_type(args[0]) # N: Revealed type is "Any" reveal_type(args[1]) # N: Revealed type is "Any" - class Sub(args[0]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[0] instead? + class Sub(args[0]): pass - class Sub2(args[1]): # E: Variable "args" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "args" \ - # E: Invalid type: try using Literal[1] instead? + class Sub2(args[1]): pass def from_unpacked_type_annotation(*args: *tuple[Any, ...]) -> None: @@ -10022,10 +9997,7 @@ def from_unpacked2_type_annotation(*args: *tuple[*tuple[Any, ...]]) -> None: from typing import Any, Sequence def f(y: Sequence[Any]) -> None: - class A(y[0]): pass # E: Variable "y" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ - # E: Invalid base class "y" \ - # E: Invalid type: try using Literal[0] instead? + class A(y[0]): pass [builtins fixtures/tuple.pyi] [case testDeferredBaseClassUncheckedFunction] From 4e6dc32b0fe4cafd1454c51d3c4eb275730a7e56 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Sun, 25 Oct 2026 20:09:25 +0100 Subject: [PATCH 18/24] Support type | Any deferred base classes, fold index expressions generally tuple_item_at_index now returns the union of candidate items for an ambiguous index instead of None, and deferred base validation accepts a union whose items are all acceptable (explicit Any or a type object), so type | Any bases pass while Any | int still errors. index_expr_int_value now uses constant_fold_expr, so indexes like -(-1) work instead of only literal ints and single negations. Tests: Sub3 with args[-(-1)], reveal_type assertions on the ambiguous error cases, before/after_the_unpack moved to an unambiguous case, one-or-more Any cases, and the two type | Any cases. --- mypy/checker.py | 44 +++++++++++++++++++--------- test-data/unit/check-classes.test | 48 ++++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 540d63fb63c1..ddfbe3eb85a9 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -89,6 +89,7 @@ def __init__(self) -> None: is_instance_var, ) from mypy.checkpattern import PatternChecker +from mypy.constant_fold import constant_fold_expr from mypy.constraints import SUPERTYPE_OF from mypy.erasetype import ( erase_type, @@ -2895,11 +2896,7 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: # An inferred plain ``Any`` is only accepted if it comes from an # explicit annotation; ``Any`` leaking in from unannotated code is # still rejected, as before. - is_any = typ is not None and ( - self.is_explicit_any(typ) - or (isinstance(typ, Instance) and typ.type.fullname == "builtins.type") - or (isinstance(typ, TypeType) and self.is_explicit_any(typ.item)) - ) + is_any = typ is not None and self.is_valid_deferred_base_type(typ) if is_any: if self.options.disallow_subclassing_any: if isinstance(base_expr, (NameExpr, MemberExpr)): @@ -2942,13 +2939,31 @@ def check_deferred_base_classes(self, defn: ClassDef) -> None: code=codes.VALID_TYPE, ) + def is_valid_deferred_base_type(self, typ: ProperType) -> bool: + """Can a deferred base class have this inferred type? + + Mirror what semantic analysis accepts for an explicitly typed + variable: ``Any`` from an explicit annotation, a type object, or a + union of those (e.g. ``type | Any``, when an ambiguous tuple index + may land on either). Anything else is rejected. + """ + if isinstance(typ, UnionType): + return all( + self.is_valid_deferred_base_type(get_proper_type(item)) + for item in typ.items + ) + return ( + self.is_explicit_any(typ) + or (isinstance(typ, Instance) and typ.type.fullname == "builtins.type") + or (isinstance(typ, TypeType) and self.is_explicit_any(typ.item)) + ) + @staticmethod def index_expr_int_value(index: Expression) -> int | None: """Value of a constant integer index expression, or None.""" - if isinstance(index, IntExpr): - return index.value - if isinstance(index, UnaryExpr) and index.op == "-" and isinstance(index.expr, IntExpr): - return -index.expr.value + value = constant_fold_expr(index, "") + if isinstance(value, int) and not isinstance(value, bool): + return value return None def deferred_index_base_type(self, var: Var, base_expr: IndexExpr) -> ProperType | None: @@ -2983,9 +2998,10 @@ def tuple_item_at_index(self, tuple_type: TupleType, index: int) -> ProperType | """Type of a tuple item at a constant index, or None if ambiguous. If the tuple type has a variadic (unpacked) item, an index may land - on more than one item. The item type is only determined if the index - lands on a single fixed item, or if every item it may land on is - ``Any``; otherwise the index is ambiguous and None is returned. + on more than one item. Then the union of the items it may land on + is returned, and the caller decides whether that union is an + acceptable base class type. A single fixed item (or ``Any``, if + every candidate is ``Any``) is returned directly. """ items = [get_proper_type(item) for item in flatten_nested_tuples(tuple_type.items)] unpack_index = find_unpack_in_list(items) @@ -3033,7 +3049,9 @@ def tuple_item_at_index(self, tuple_type: TupleType, index: int) -> ProperType | return possible[0] if len(possible) == 1: return possible[0] - return None + # The index may land on several items; return their union and let + # the caller decide whether it is an acceptable base class type. + return make_simplified_union(possible) def visit_class_def(self, defn: ClassDef) -> None: """Type check a class definition.""" diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 7f3d9f672331..71baaf83c538 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9932,11 +9932,8 @@ def from_unpacked_type_annotation(*args: *tuple[int, *tuple[int, ...], Any]) -> # flags: --python-version 3.12 from typing import Any -def before_the_unpack(*args: *tuple[Any, *tuple[int, ...], int]) -> None: - class Sub(args[0]): - pass - def inside_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: + reveal_type(args[1]) # N: Revealed type is "Any | builtins.int" class Sub(args[1]): # E: Variable "args" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ # E: Invalid base class "args" \ @@ -9944,6 +9941,7 @@ def inside_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: pass def across_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: + reveal_type(args[-2]) # N: Revealed type is "builtins.int | Any" class Sub(args[-2]): # E: Variable "args" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ # E: Invalid base class "args" \ @@ -9951,6 +9949,46 @@ def across_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: pass [builtins fixtures/tuple.pyi] +[case testSubclassAnyFromUnpackedStarArgsUnambiguousIndex] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def before_the_unpack(*args: *tuple[Any, *tuple[int, ...], int]) -> None: + class Sub(args[0]): + pass + +def after_the_unpack(*args: *tuple[int, *tuple[int, ...], Any]) -> None: + class Sub(args[-1]): + pass + +def across_or_inside_the_unpack_one_or_more_any(*args: *tuple[int, *tuple[Any, ...], Any, int]) -> None: + reveal_type(args[1]) # N: Revealed type is "Any" + class Sub(args[1]): + pass + +def across_or_inside_the_unpack_one_or_more_any2(*args: *tuple[int, Any, *tuple[Any, ...], int]) -> None: + reveal_type(args[-2]) # N: Revealed type is "Any" + class Sub(args[-2]): + pass +[builtins fixtures/tuple.pyi] + +[case testSubclassTypeOrAnyFromUnpackedStarArgsAmbiguousIndex] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def across_the_unpack_ambiguous_with_type_should_be_fine(*args: *tuple[type, *tuple[Any, ...], int]) -> None: + reveal_type(args[-2]) # N: Revealed type is "builtins.type | Any" + class Sub(args[-2]): + pass + +def across_the_unpack_ambiguous_with_type_should_be_fine2(*args: *tuple[int, *tuple[Any, ...], type]) -> None: + reveal_type(args[1]) # N: Revealed type is "Any | builtins.type" + class Sub(args[1]): + pass +[builtins fixtures/tuple.pyi] + [case testSubclassAnyFromNestedUnpackedUnboundedStarArgsIndexInsideUnambiguous] # https://github.com/python/mypy/issues/21998 # flags: --python-version 3.12 @@ -9971,6 +10009,8 @@ def from_unpacked2(*args: *tuple[*tuple[Any, ...]]) -> None: pass class Sub2(args[1]): pass + class Sub3(args[-(-1)]): + pass def from_unpacked_type_annotation(*args: *tuple[Any, ...]) -> None: reveal_type(args[0]) # N: Revealed type is "Any" From 4b224fd635ef448c118d4bb5089068c4cf0b01c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:11:01 +0000 Subject: [PATCH 19/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/checker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ddfbe3eb85a9..8ea20f76295e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2949,8 +2949,7 @@ def is_valid_deferred_base_type(self, typ: ProperType) -> bool: """ if isinstance(typ, UnionType): return all( - self.is_valid_deferred_base_type(get_proper_type(item)) - for item in typ.items + self.is_valid_deferred_base_type(get_proper_type(item)) for item in typ.items ) return ( self.is_explicit_any(typ) From 60a8ea58d6e88eb36247457ecf4980947ffeefae Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Sun, 25 Oct 2026 22:25:48 +0100 Subject: [PATCH 20/24] add reveal_type to before/after unpack cases --- test-data/unit/check-classes.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 71baaf83c538..fdabf25fe20b 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9955,10 +9955,12 @@ def across_the_unpack(*args: *tuple[int, *tuple[Any, ...], int]) -> None: from typing import Any def before_the_unpack(*args: *tuple[Any, *tuple[int, ...], int]) -> None: + reveal_type(args[0]) # N: Revealed type is "Any" class Sub(args[0]): pass def after_the_unpack(*args: *tuple[int, *tuple[int, ...], Any]) -> None: + reveal_type(args[-1]) # N: Revealed type is "Any" class Sub(args[-1]): pass From 4297367bc30fb152419a1b0052b43fe8bfc7e7b1 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Mon, 26 Oct 2026 00:31:36 +0100 Subject: [PATCH 21/24] Do not defer variables nested in base class type arguments A variable appearing as a type argument of a base class (e.g. the T in tuple[T]) was deferred as if it were the base class itself, and the checker then reported Invalid base class for it. Only defer when the variable is the base class expression itself (nesting level 0). --- mypy/typeanal.py | 8 ++++++-- test-data/unit/check-classes.test | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index c1821cb7774f..59bf3ddcabcb 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1042,13 +1042,17 @@ def analyze_unbound_type_without_type_info( return AnyType(TypeOfAny.special_form) if isinstance(typ, TypeType) and isinstance(typ.item, AnyType): return AnyType(TypeOfAny.from_another_any, source_any=typ.item) - if typ is None: + if typ is None and self.nesting_level == 0: # The variable's type has not been inferred yet (inference # happens in the type checker, after semantic analysis). # Defer the validity check; the type checker will verify # the inferred type. Provisionally treat the base as Any. + # Only the base class itself may be deferred: a variable + # nested inside another base type (e.g. a type argument, + # as in `tuple[T]`) is not a base class, so it keeps the + # normal "not valid as a type" error below. raise DeferredBaseClassVar(sym.node, defining_literal) - if t.args: + if t.args and self.nesting_level == 0: # The base class subscripts the variable, e.g. `args[1]` # where `args` is a *args tuple. Whether the base is valid # depends on the type of the subscripted item, which the diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index fdabf25fe20b..f7d4cf61ad9f 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -10065,6 +10065,19 @@ x = 1 class C(B["Literal[x]"]): pass # E: Parameter 1 of Literal[...] is invalid [builtins fixtures/tuple.pyi] +[case testSubclassVariableNestedInBaseClassArgs] +# A variable nested inside a base class's type arguments is not itself a +# base class, so it must not be deferred: it gets the normal "not valid as +# a type" error. Regression test for the mypy_primer diff on prefect, +# where `class BaseAnnotation(tuple[T])` gained an "Invalid base class" +# error for the type argument. +from typing import Any +def f(v: Any) -> None: + x = v + class Sub(tuple[x]): pass # E: Variable "x" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/tuple.pyi] + [case testTypeVarDefaultErrorLocation] from typing import Generic From 829cb4d81045cc5b030e3214540391784a761e73 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Tue, 27 Oct 2026 01:20:51 +0100 Subject: [PATCH 22/24] Support a second subscript on a deferred base class class Sub(args[1][0]) was rejected even when the inner item is list[Any]: converting the base to a type raised TypeTranslationError for a subscript whose base already has arguments, so the validity check was never deferred. Flatten an index chain rooted at a plain variable into one UnboundType in exprtotype (subscripting an actual type, e.g. list[int][0], still errors), and resolve the item type inside out in the checker's deferred index handling. One more level, e.g. args[1][0][0] into list[list[Any]], works the same way. Test cases from the review: subclassing args[1][0] from *tuple[int, list[Any]] and a[0] from list[Any] pass, and the list[int] version is rejected with the usual errors. --- mypy/checker.py | 11 ++++++--- mypy/exprtotype.py | 36 +++++++++++++++++++++++++++- test-data/unit/check-classes.test | 39 +++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 8ea20f76295e..3b5a6fb1a263 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2973,13 +2973,18 @@ def deferred_index_base_type(self, var: Var, base_expr: IndexExpr) -> ProperType base class is then rejected, as it was before deferred validation. Only a constant integer index is supported, and only into a tuple or sequence type, mirroring what can be known about the item type. + A subscript may itself be subscripted, e.g. ``args[1][0]``; then the + item type is computed inside out, one index at a time. """ - if not isinstance(base_expr.base, NameExpr): - return None index_value = self.index_expr_int_value(base_expr.index) if index_value is None: return None - var_type = get_proper_type(var.type) if var.type is not None else None + if isinstance(base_expr.base, NameExpr): + var_type = get_proper_type(var.type) if var.type is not None else None + elif isinstance(base_expr.base, IndexExpr): + var_type = self.deferred_index_base_type(var, base_expr.base) + else: + return None if isinstance(var_type, TupleType): return self.tuple_item_at_index(var_type, index_value) if isinstance(var_type, Instance): diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 8c42d205d568..d8920479dc87 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -26,6 +26,7 @@ SymbolTableNode, TupleExpr, UnaryExpr, + Var, get_member_expr_fullname, ) from mypy.options import Options @@ -105,7 +106,40 @@ def expr_to_unanalyzed_type( ) if isinstance(base, UnboundType): if base.args: - raise TypeTranslationError() + # A variable subscripted more than once, e.g. `args[1][0]` + # used as a base class. Flatten the index chain into a + # single UnboundType, so the type analyser can defer it + # like a single subscript. Subscripting an actual type, + # e.g. `list[int][0]`, is not a valid type and keeps the + # error: only a plain variable may root the chain. + root: Expression = expr.base + while isinstance(root, IndexExpr): + root = root.base + if not (isinstance(root, NameExpr) and isinstance(root.node, Var)): + raise TypeTranslationError() + if isinstance(expr.index, TupleExpr): + outer_args: list[Expression] = list(expr.index.items) + else: + outer_args = [expr.index] + return UnboundType( + base.name, + [ + *base.args, + *( + expr_to_unanalyzed_type( + arg, + options, + allow_new_syntax, + expr, + allow_unpack=True, + lookup_qualified=lookup_qualified, + ) + for arg in outer_args + ), + ], + line=base.line, + column=base.column, + ) if isinstance(expr.index, TupleExpr): args = expr.index.items else: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index f7d4cf61ad9f..dd48097b3e4f 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -10042,6 +10042,45 @@ def f(y: Sequence[Any]) -> None: class A(y[0]): pass [builtins fixtures/tuple.pyi] +[case testSubclassFromUnpackedStarArgsAnyTypedList] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def fun_varargs_list_wrapped_element_to_type_used_for_inherit( + *args: *tuple[int, list[Any]], +) -> Any: + class Sub(args[1][0]): pass + +[builtins fixtures/tuple.pyi] + +[case testSubclassFromAnyTypedList] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def fun_list_wrapped_element_to_type_used_for_inherit( + a: list[Any], +) -> Any: + class Sub(a[0]): pass + +[builtins fixtures/tuple.pyi] + +[case testSubclassFromUnpackedStarArgsNonAnyTypedList] +# https://github.com/python/mypy/issues/21998 +# flags: --python-version 3.12 +from typing import Any + +def fun_varargs_int_list_element_used_for_inherit( + *args: *tuple[int, list[int]], +) -> Any: + class Sub(args[1][0]): pass # E: Variable "args" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "args" \ + # E: Invalid type: try using Literal[0] instead? + +[builtins fixtures/tuple.pyi] + [case testDeferredBaseClassUncheckedFunction] # Dynamic base classes in unchecked functions were silently accepted before; # the deferred checker must not report new errors there. From e0576e0ef05333c9c65f834b7056cfa677e4c896 Mon Sep 17 00:00:00 2001 From: Dextheking1 Date: Thu, 29 Oct 2026 04:15:40 +0100 Subject: [PATCH 23/24] Support string subscripts on deferred base classes Covers the **kwargs case from #22309: `class Sub(kwargs['some_any_field'])` where kwargs is an Unpack[TypedDict] with an Any field. - checker: deferred_index_base_type now also resolves a constant string index, into a TypedDict (item type by key) or a mapping (value type). - semanal/typeanal: the type variable scan over base classes no longer looks up subscript arguments as type names; a string key was parsed as a type string and reported as an undefined name. - tests: testSubclassAnyFromKwargsTypedDictEntry, testSubclassAnyFromKwargsDict, testSubclassNonAnyFromKwargsTypedDictEntry. --- mypy/checker.py | 63 +++++++++++++++++++++++-------- mypy/semanal.py | 7 +++- mypy/typeanal.py | 9 ++++- test-data/unit/check-classes.test | 40 ++++++++++++++++++++ 4 files changed, 100 insertions(+), 19 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3b5a6fb1a263..4bab0f909041 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2965,37 +2965,68 @@ def index_expr_int_value(index: Expression) -> int | None: return value return None + @staticmethod + def index_expr_str_value(index: Expression) -> str | None: + """Value of a constant string index expression, or None.""" + value = constant_fold_expr(index, "") + if isinstance(value, str): + return value + return None + def deferred_index_base_type(self, var: Var, base_expr: IndexExpr) -> ProperType | None: """Type of a deferred subscripted base class, e.g. ``args[1]``. ``var`` is the subscripted variable. Return the type of the item at the subscript, or None if it cannot be determined statically; the base class is then rejected, as it was before deferred validation. - Only a constant integer index is supported, and only into a tuple - or sequence type, mirroring what can be known about the item type. + Only a constant index is supported: an integer index into a tuple + or sequence type, or a string index into a TypedDict or mapping + type (e.g. ``kwargs['some_any_field']`` for ``**kwargs``), mirroring + what can be known about the item type. A subscript may itself be subscripted, e.g. ``args[1][0]``; then the item type is computed inside out, one index at a time. """ - index_value = self.index_expr_int_value(base_expr.index) - if index_value is None: - return None if isinstance(base_expr.base, NameExpr): var_type = get_proper_type(var.type) if var.type is not None else None elif isinstance(base_expr.base, IndexExpr): var_type = self.deferred_index_base_type(var, base_expr.base) else: return None - if isinstance(var_type, TupleType): - return self.tuple_item_at_index(var_type, index_value) - if isinstance(var_type, Instance): - if var_type.type.fullname in TUPLE_NAMES and var_type.args: - # Homogeneous tuple[T, ...]: every item has type T. - return get_proper_type(var_type.args[0]) - sequence_info = var_type.type.get_base("typing.Sequence") - if sequence_info is not None: - mapped = map_instance_to_supertype(var_type, sequence_info) - if mapped.args: - return get_proper_type(mapped.args[0]) + if var_type is None: + return None + index_value = self.index_expr_int_value(base_expr.index) + if index_value is not None: + if isinstance(var_type, TupleType): + return self.tuple_item_at_index(var_type, index_value) + if isinstance(var_type, Instance): + if var_type.type.fullname in TUPLE_NAMES and var_type.args: + # Homogeneous tuple[T, ...]: every item has type T. + return get_proper_type(var_type.args[0]) + sequence_info = var_type.type.get_base("typing.Sequence") + if sequence_info is not None: + mapped = map_instance_to_supertype(var_type, sequence_info) + if mapped.args: + return get_proper_type(mapped.args[0]) + return None + key = self.index_expr_str_value(base_expr.index) + if key is not None: + if isinstance(var_type, TypedDictType): + item_type = var_type.items.get(key) + if item_type is not None: + return get_proper_type(item_type) + return None + if isinstance(var_type, Instance): + # Look the base up in the MRO directly: get_base() asserts + # if it is missing, which can happen with partial stubs. + mapping_info = next( + (base for base in var_type.type.mro if base.fullname == "typing.Mapping"), + None, + ) + if mapping_info is not None: + mapped = map_instance_to_supertype(var_type, mapping_info) + if len(mapped.args) >= 2: + return get_proper_type(mapped.args[1]) + return None return None def tuple_item_at_index(self, tuple_type: TupleType, index: int) -> ProperType | None: diff --git a/mypy/semanal.py b/mypy/semanal.py index ebcd1f6f5c0b..777718eb76ec 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2489,8 +2489,11 @@ def analyze_unbound_tvar_impl( assert isinstance(sym.node, TypeVarExpr) return t.name, sym.node - def find_type_var_likes(self, t: Type) -> TypeVarLikeList: + def find_type_var_likes( + self, t: Type, *, suppress_name_errors: bool = False + ) -> TypeVarLikeList: visitor = FindTypeVarVisitor(self, self.tvar_scope) + visitor.suppress_name_errors = suppress_name_errors t.accept(visitor) return visitor.type_var_likes @@ -2506,7 +2509,7 @@ def get_all_bases_tvars( except TypeTranslationError: # This error will be caught later. continue - base_tvars = self.find_type_var_likes(base) + base_tvars = self.find_type_var_likes(base, suppress_name_errors=True) tvars.extend(base_tvars) return remove_dups(tvars) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 59bf3ddcabcb..d15207da7d7e 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2806,6 +2806,13 @@ def __init__(self, api: SemanticAnalyzerCoreInterface, scope: TypeVarLikeScope) self.type_var_likes: list[tuple[str, TypeVarLikeExpr]] = [] self.has_self_type = False self.include_callables = True + # Only set when scanning base classes for type variables: an argument + # of a subscripted base may be a value, not a type name (e.g. the + # string key in `kwargs['some_any_field']`, converted to an + # UnboundType by parse_type_string), so an unknown name there is not + # an error. Undefined type names in bases are reported when the + # bases are analyzed. + self.suppress_name_errors = False def _seems_like_callable(self, type: UnboundType) -> bool: if not type.args: @@ -2814,7 +2821,7 @@ def _seems_like_callable(self, type: UnboundType) -> bool: def visit_unbound_type(self, t: UnboundType) -> None: name = t.name - node = self.api.lookup_qualified(name, t) + node = self.api.lookup_qualified(name, t, suppress_errors=self.suppress_name_errors) if node and node.fullname in SELF_TYPE_NAMES: self.has_self_type = True if ( diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index dd48097b3e4f..81ca3e5f02f6 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -10117,6 +10117,46 @@ def f(v: Any) -> None: # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases [builtins fixtures/tuple.pyi] +[case testSubclassAnyFromKwargsTypedDictEntry] +# https://github.com/python/mypy/issues/22309 +from typing import Any, TypedDict +from typing_extensions import Unpack + +class Person(TypedDict): + name: str + age: int + some_any_field: Any + +def foo(arg: bool, **kwargs: Unpack[Person]) -> None: + class Sub(kwargs["some_any_field"]): pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testSubclassAnyFromKwargsDict] +# https://github.com/python/mypy/issues/22309 +from typing import Any + +def foo(**kwargs: Any) -> None: + class Sub(kwargs["some_any_field"]): pass +[builtins fixtures/dict.pyi] + +[case testSubclassNonAnyFromKwargsTypedDictEntry] +# https://github.com/python/mypy/issues/22309 +from typing import Any, TypedDict +from typing_extensions import Unpack + +class Person(TypedDict): + name: str + age: int + some_any_field: Any + +def foo(arg: bool, **kwargs: Unpack[Person]) -> None: + class Sub(kwargs["name"]): pass # E: Variable "kwargs" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases \ + # E: Invalid base class "kwargs" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + [case testTypeVarDefaultErrorLocation] from typing import Generic From bb3171d34237bb9a5fd8ac38c88a31efde74437d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:17:18 +0000 Subject: [PATCH 24/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/checker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4bab0f909041..14d0a69793dc 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3019,8 +3019,7 @@ def deferred_index_base_type(self, var: Var, base_expr: IndexExpr) -> ProperType # Look the base up in the MRO directly: get_base() asserts # if it is missing, which can happen with partial stubs. mapping_info = next( - (base for base in var_type.type.mro if base.fullname == "typing.Mapping"), - None, + (base for base in var_type.type.mro if base.fullname == "typing.Mapping"), None ) if mapping_info is not None: mapped = map_instance_to_supertype(var_type, mapping_info)