Lenient handling of *tuple[Any, ...] (part 1)

mypy/subtypes.py

@@ -67,6 +67,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    extend_args_for_prefix_and_suffix,
     find_unpack_in_list,
     flatten_nested_unions,
     get_proper_type,
@@ -509,18 +510,17 @@ def visit_instance(self, left: Instance) -> bool:
                     if isinstance(unpacked, Instance):
                         return self._is_subtype(left, unpacked)
             if left.type.has_base(right.partial_fallback.type.fullname):
+                mapped = map_instance_to_supertype(left, right.partial_fallback.type)
+                # Special cases to consider:
+                #   * tuple[Any, ...] instance is a (non-proper) subtype of all tuple types.
+                #   * Foo[*tuple[X, ...]] (normalized) instance is a subtype of all
+                #     tuples with appropriate fallback (e.g. for variadic NamedTuples).
                 if not self.proper_subtype:
-                    # Special cases to consider:
-                    #   * Plain tuple[Any, ...] instance is a subtype of all tuple types.
-                    #   * Foo[*tuple[Any, ...]] (normalized) instance is a subtype of all
-                    #     tuples with fallback to Foo (e.g. for variadic NamedTuples).
-                    mapped = map_instance_to_supertype(left, right.partial_fallback.type)
-                    if is_erased_instance(mapped):
-                        if (
-                            mapped.type.fullname == "builtins.tuple"
-                            or mapped.type.has_type_var_tuple_type
-                        ):
-                            return True
+                    if is_erased_instance(mapped) and mapped.type.fullname == "builtins.tuple":
+                        return True
+                if is_normalized_instance(mapped):
+                    if self._is_subtype(mapped, right.partial_fallback):
+                        return True
             return False
         if isinstance(right, TypeVarTupleType):
             # tuple[Any, ...] is like Any in the world of tuples (see special case above).
@@ -819,9 +819,12 @@ def visit_tuple_type(self, left: TupleType) -> bool:
         elif isinstance(right, TupleType):
             # If right has a variadic unpack this needs special handling. If there is a TypeVarTuple
             # unpack, item count must coincide. If the left has variadic unpack but right
-            # doesn't have one, we will fall through to False down the line.
+            # doesn't have one, we will fall through.
             if self.variadic_tuple_subtype(left, right):
                 return True
+            # The only case where variadic can be subtype of fixed is when left variadic item is Any.
+            # Otherwise, the original left will be returned, causing fall through to False.
+            left = self.adjust_left_if_possible(left, right)
             if len(left.items) != len(right.items):
                 return False
             if any(not self._is_subtype(l, r) for l, r in zip(left.items, right.items)):
@@ -841,6 +844,57 @@ def visit_tuple_type(self, left: TupleType) -> bool:
         else:
             return False
 
+    def adjust_left_if_possible(self, left: TupleType, right: TupleType) -> TupleType:
+        """Adjust shape of left containing *tuple[Any, ...] to match right.
+
+        Note: this only works if right is fixed size (including *Ts), the variadic
+        right are handled by the caller, currently with variadic_tuple_subtype().
+        """
+        left_variadic = self.get_variadic_item(left)
+        if left_variadic is None:
+            return left
+        left_unpack_index, left_item = left_variadic
+        if not isinstance(get_proper_type(left_item), AnyType):
+            return left
+        right_unpack_index = find_unpack_in_list(right.items)
+        if right_unpack_index is None:
+            if len(left.items) > len(right.items) + 1:
+                return left
+            num_anys = len(right.items) - len(left.items) + 1
+            return left.copy_modified(
+                items=left.items[:left_unpack_index]
+                + [left_item] * num_anys
+                + left.items[left_unpack_index + 1 :]
+            )
+        right_unpack = right.items[right_unpack_index]
+        assert isinstance(right_unpack, UnpackType)
+        right_unpacked = get_proper_type(right_unpack.type)
+        if isinstance(right_unpacked, Instance):
+            return left
+        right_prefix = right_unpack_index
+        right_suffix = len(right.items) - right_prefix - 1
+        left_prefix = left_unpack_index
+        left_suffix = len(left.items) - left_prefix - 1
+        if left_prefix > right_prefix or left_suffix > right_suffix:
+            return left
+        new_items = extend_args_for_prefix_and_suffix(
+            tuple(left.items), right_prefix, right_suffix
+        )
+        return left.copy_modified(items=list(new_items))
+
+    def get_variadic_item(self, tup: TupleType) -> tuple[int, Type] | None:
+        """If this is tuple[X, *tuple[Y, ...], Z], return Y, otherwise None."""
+        unpack_index = find_unpack_in_list(tup.items)
+        if unpack_index is None:
+            return None
+        unpack = tup.items[unpack_index]
+        assert isinstance(unpack, UnpackType)
+        unpacked = get_proper_type(unpack.type)
+        if not isinstance(unpacked, Instance):
+            return None
+        assert unpacked.type.fullname == "builtins.tuple"
+        return unpack_index, unpacked.args[0]
+
     def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool:
         """Check subtyping between two potentially variadic tuples.
 
@@ -850,18 +904,11 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool:
         Note: the cases where right is fixed or has *Ts unpack should be handled
         by the caller.
         """
-        right_unpack_index = find_unpack_in_list(right.items)
-        if right_unpack_index is None:
+        right_variadic = self.get_variadic_item(right)
+        if right_variadic is None:
             # This case should be handled by the caller.
             return False
-        right_unpack = right.items[right_unpack_index]
-        assert isinstance(right_unpack, UnpackType)
-        right_unpacked = get_proper_type(right_unpack.type)
-        if not isinstance(right_unpacked, Instance):
-            # This case should be handled by the caller.
-            return False
-        assert right_unpacked.type.fullname == "builtins.tuple"
-        right_item = right_unpacked.args[0]
+        right_unpack_index, right_item = right_variadic
         right_prefix = right_unpack_index
         right_suffix = len(right.items) - right_prefix - 1
         left_unpack_index = find_unpack_in_list(left.items)
@@ -883,16 +930,16 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool:
                 return False
             return all(self._is_subtype(li, right_item) for li in middle)
         else:
-            if len(left.items) < len(right.items):
-                # There are some items on the left that will never have a matching length
-                # on the right.
-                return False
             left_prefix = left_unpack_index
             left_suffix = len(left.items) - left_prefix - 1
             left_unpack = left.items[left_unpack_index]
             assert isinstance(left_unpack, UnpackType)
             left_unpacked = get_proper_type(left_unpack.type)
             if not isinstance(left_unpacked, Instance):
+                if len(left.items) < len(right.items):
+                    # There are some items on the left that will never have a matching length
+                    # on the right.
+                    return False
                 # *Ts unpack can't be split, except if it is all mapped to Anys or objects.
                 if self.is_top_type(right_item):
                     right_prefix_types, middle, right_suffix_types = split_with_prefix_and_suffix(
@@ -914,6 +961,12 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool:
             # subtyping: *each* item on the left, must be a subtype of *some* item on the right.
             # For this we first check the "asymptotic case", i.e. that both unpacks a subtypes,
             # and then check subtyping for all finite overlaps.
+            # Note: if the left item is Any we use any() semantics instead of all().
+            use_any = isinstance(get_proper_type(left_item), AnyType)
+            if not use_any and len(left.items) < len(right.items):
+                # There are some items on the left that will never have a matching length
+                # on the right.
+                return False
             if not self._is_subtype(left_item, right_item):
                 return False
             max_overlap = max(0, right_prefix - left_prefix, right_suffix - left_suffix)
@@ -923,8 +976,11 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool:
                     repr_items += left.items[-left_suffix:]
                 left_repr = left.copy_modified(items=repr_items)
                 if not self._is_subtype(left_repr, right):
-                    return False
-            return True
+                    if not use_any:
+                        return False
+                elif use_any:
+                    return True
+            return not use_any
 
     def is_top_type(self, typ: Type) -> bool:
         if not self.proper_subtype and isinstance(get_proper_type(typ), AnyType):
@@ -2384,3 +2440,21 @@ def is_erased_instance(t: Instance) -> bool:
         elif not isinstance(get_proper_type(arg), AnyType):
             return False
     return True
+
+
+def is_normalized_instance(t: Instance) -> bool:
+    """Is this instance type a normalized representation of a tuple type?
+
+    Type like class C[T, *Ts](tuple[T, *Ts]) are internally represented as
+    tuple types, so that e.g. C[int, str] is tuple[int, str, fallback=C[int, str]].
+    However, in case of a variadic argument they are normalized, similar to how
+    tuple[*tuple[int, ...]] (TupleType) is normalized to tuple[int, ...] (Instance).
+    For example, C[int, *tuple[str, ...]] is represented as an instance. This
+    function detects such instances, when they need special-casing.
+    """
+    if not t.args:
+        return False
+    if not t.type.tuple_type:
+        return False
+    expanded = expand_type_by_instance(t.type.tuple_type, t)
+    return isinstance(expanded, Instance)

test-data/unit/check-typevar-tuple.test

@@ -2917,3 +2917,139 @@ def g(x: K) -> None:
     assert d is not None
     reveal_type(d)  # N: Revealed type is "tuple[K | list[int], ...] | tuple[*tuple[int | list[int], ...], int]"
 [builtins fixtures/tuple.pyi]
+
+[case testVariadicTupleSubtypingWithAnyBasic]
+from typing import Any, Unpack
+
+Ge1 = tuple[int, Unpack[tuple[Any, ...]]]
+
+def ge1_to_0(shape: Ge1) -> tuple[()]:
+    return shape  # E: Incompatible return value type (got "tuple[int, Unpack[tuple[Any, ...]]]", expected "tuple[()]")
+def ge1_to_1(shape: Ge1) -> tuple[int]:
+    return shape
+def ge1_to_2(shape: Ge1) -> tuple[int, int]:
+    return shape
+[builtins fixtures/tuple.pyi]
+
+[case testVariadicTupleSubtypingWithAnyExtra]
+from typing import Any, Unpack
+
+def foo(x: tuple[int, str]) -> None: ...
+def bar(x: tuple[int, int]) -> None: ...
+
+t1: tuple[int, Unpack[tuple[Any, ...]], str]
+t2: tuple[Unpack[tuple[Any, ...]], int, str]
+t3: tuple[int, str, Unpack[tuple[Any, ...]]]
+t4: tuple[int, Unpack[tuple[Any, ...]], int]
+foo(t1)
+foo(t2)
+foo(t3)
+foo(t4)  # E: Argument 1 to "foo" has incompatible type "tuple[int, Unpack[tuple[Any, ...]], int]"; expected "tuple[int, str]"
+
+tt1: tuple[int, Unpack[tuple[Any, ...]], int, int]
+tt2: tuple[int, int, Unpack[tuple[Any, ...]], int]
+tt3: tuple[Unpack[tuple[Any, ...]], int]
+tt4: tuple[Unpack[tuple[Any, ...]], str]
+bar(tt1)  # E: Argument 1 to "bar" has incompatible type "tuple[int, Unpack[tuple[Any, ...]], int, int]"; expected "tuple[int, int]"
+bar(tt2)  # E: Argument 1 to "bar" has incompatible type "tuple[int, int, Unpack[tuple[Any, ...]], int]"; expected "tuple[int, int]"
+bar(tt3)
+bar(tt4)  # E: Argument 1 to "bar" has incompatible type "tuple[Unpack[tuple[Any, ...]], str]"; expected "tuple[int, int]"
+[builtins fixtures/tuple.pyi]
+
+[case testVariadicTupleSubtypingWithAnyExtraTypeVar]
+from typing import Any, Unpack, TypeVarTuple
+
+t1: tuple[int, Unpack[tuple[Any, ...]], int]
+t2: tuple[int, int, Unpack[tuple[Any, ...]], int, int]
+t3: tuple[int, int, int, Unpack[tuple[Any, ...]], int, int, int]
+
+Ts = TypeVarTuple("Ts")
+def test(x: tuple[int, int, Unpack[Ts], int, int]) -> None:
+    x = t1
+    x = t2
+    x = t3  # E: Incompatible types in assignment (expression has type "tuple[int, int, int, Unpack[tuple[Any, ...]], int, int, int]", variable has type "tuple[int, int, Unpack[Ts], int, int]")
+[builtins fixtures/tuple.pyi]
+
+[case testVariadicTupleSubtypingWithAnyExtraVariadic]
+from typing import Any, Unpack
+
+def foo(x: tuple[int, Unpack[tuple[int, ...]], int]) -> None: ...
+
+t1: tuple[int, Unpack[tuple[Any, ...]]]
+t2: tuple[Unpack[tuple[Any, ...]], int]
+t3: tuple[Unpack[tuple[Any, ...]], str]
+t4: tuple[Unpack[tuple[Any, ...]], int, int, int]
+
+foo(t1)
+foo(t2)
+foo(t3)  # E: Argument 1 to "foo" has incompatible type "tuple[Unpack[tuple[Any, ...]], str]"; expected "tuple[int, Unpack[tuple[int, ...]], int]"
+foo(t4)
+[builtins fixtures/tuple.pyi]
+
+[case testVariadicTupleSubclassSubtypingWithAny]
+from typing import Any, Generic, Unpack, TypeVar, TypeVarTuple
+
+T = TypeVar("T")
+Ts = TypeVarTuple("Ts")
+class Array(tuple[Unpack[Ts]], Generic[T, Unpack[Ts]]): ...
+
+Float2D = Array[float, int, int]
+FloatND = Array[float, Unpack[tuple[Any, ...]]]
+
+def from_2d(a: Float2D) -> FloatND:
+    return a
+
+def into_2d(a: FloatND) -> Float2D:
+    return a
+[builtins fixtures/tuple.pyi]
+
+[case testVariadicInstanceSubtypingWithAny]
+from typing import Any, Generic, Unpack, TypeVarTuple
+
+Ts = TypeVarTuple("Ts")
+class C(Generic[Unpack[Ts]]): ...
+
+def foo(x: C[int, str]) -> None: ...
+def bar(x: C[int, int]) -> None: ...
+def baz(x: C[int, int, int]) -> None: ...
+
+t1: C[int, Unpack[tuple[Any, ...]], str]
+t2: C[Unpack[tuple[Any, ...]], int, str]
+t3: C[int, str, Unpack[tuple[Any, ...]]]
+t4: C[int, Unpack[tuple[Any, ...]], int]
+foo(t1)
+foo(t2)
+foo(t3)
+foo(t4)  # E: Argument 1 to "foo" has incompatible type "C[int, Unpack[tuple[Any, ...]], int]"; expected "C[int, str]"
+
+tt1: C[int, Unpack[tuple[Any, ...]], int, int]
+tt2: C[int, int, Unpack[tuple[Any, ...]], int]
+tt3: C[Unpack[tuple[Any, ...]], int]
+tt4: C[Unpack[tuple[Any, ...]], str]
+bar(tt1)  # E: Argument 1 to "bar" has incompatible type "C[int, Unpack[tuple[Any, ...]], int, int]"; expected "C[int, int]"
+bar(tt2)  # E: Argument 1 to "bar" has incompatible type "C[int, int, Unpack[tuple[Any, ...]], int]"; expected "C[int, int]"
+bar(tt3)
+bar(tt4)  # E: Argument 1 to "bar" has incompatible type "C[Unpack[tuple[Any, ...]], str]"; expected "C[int, int]"
+
+baz(tt3)
+baz(t4)
+[builtins fixtures/tuple.pyi]
+
+[case testVariadicInstanceSubtypingWithAnyVariadic]
+from typing import Any, Generic, Unpack, TypeVarTuple
+
+Ts = TypeVarTuple("Ts")
+class C(Generic[Unpack[Ts]]): ...
+
+def foo(x: C[int, Unpack[tuple[int, ...]], int]) -> None: ...
+
+t1: C[int, Unpack[tuple[Any, ...]]]
+t2: C[Unpack[tuple[Any, ...]], int]
+t3: C[Unpack[tuple[Any, ...]], str]
+t4: C[Unpack[tuple[Any, ...]], int, int, int]
+
+foo(t1)
+foo(t2)
+foo(t3)  # E: Argument 1 to "foo" has incompatible type "C[Unpack[tuple[Any, ...]], str]"; expected "C[int, Unpack[tuple[int, ...]], int]"
+foo(t4)
+[builtins fixtures/tuple.pyi]