Enforces having unique test ids in a single namespace (#11687)

I found lots of duplicate test names. It might hide important errors, be hard to reproduce 
or rerun.

Refs #11662

Co-authored-by: Shantanu <[email protected]>

Author: sobolevn

mypy/test/data.py

@@ -578,8 +578,13 @@ def split_test_cases(parent: 'DataFileCollector', suite: 'DataSuite',
                      data,
                      flags=re.DOTALL | re.MULTILINE)
     line_no = cases[0].count('\n') + 1
+    test_names = set()
     for i in range(1, len(cases), NUM_GROUPS):
         name, writescache, only_when, platform_flag, skip, xfail, data = cases[i:i + NUM_GROUPS]
+        if name in test_names:
+            raise RuntimeError('Found a duplicate test name "{}" in {} on line {}'.format(
+                name, parent.name, line_no,
+            ))
         platform = platform_flag[1:] if platform_flag else None
         yield DataDrivenTestCase.from_parent(
             parent=parent,
@@ -596,6 +601,9 @@ def split_test_cases(parent: 'DataFileCollector', suite: 'DataSuite',
         )
         line_no += data.count('\n') + 1
 
+        # Record existing tests to prevent duplicates:
+        test_names.update({name})
+
 
 class DataSuiteCollector(pytest.Class):
     def collect(self) -> Iterator['DataFileCollector']:

test-data/unit/check-abstract.test

@@ -600,7 +600,7 @@ a.f(B()) # E: No overload variant of "f" of "A" matches argument type "B" \
          # N:     def f(self, x: int) -> int \
          # N:     def f(self, x: str) -> str
 
-[case testOverloadedAbstractMethodVariantMissingDecorator1]
+[case testOverloadedAbstractMethodVariantMissingDecorator0]
 from foo import *
 [file foo.pyi]
 from abc import abstractmethod, ABCMeta

test-data/unit/check-attr.test

@@ -502,7 +502,7 @@ reveal_type(sub_str.attr)  # N: Revealed type is "builtins.str*"
 [builtins fixtures/bool.pyi]
 
 
-[case testAttrsGenericInheritance]
+[case testAttrsGenericInheritance2]
 from typing import Generic, TypeVar
 import attr
 

test-data/unit/check-class-namedtuple.test

@@ -270,7 +270,7 @@ a = A(B())
 a = A(1)  # E: Argument 1 to "A" has incompatible type "int"; expected "B"
 [builtins fixtures/tuple.pyi]
 
-[case testNewNamedTupleProperty]
+[case testNewNamedTupleProperty36]
 # flags: --python-version 3.6
 from typing import NamedTuple
 

test-data/unit/check-classes.test

@@ -6463,7 +6463,7 @@ class C(B):
 [out]
 main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int")
 
-[case testIgnorePrivateMethodsTypeCheck]
+[case testIgnorePrivateMethodsTypeCheck2]
 class A:
     def __foo_(self) -> int: ...
 class B:

test-data/unit/check-functions.test

@@ -1485,7 +1485,7 @@ x = None # type: Any
 if x:
     def f(x: int) -> None: pass # E: All conditional function variants must have identical signatures
 
-[case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition1]
+[case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition2]
 from typing import Any
 def f(x: int) -> None: pass # N: "f" defined here
 x = None # type: Any
@@ -2212,7 +2212,7 @@ from typing import Callable
 class A:
     def f(self) -> None:
         # In particular, test that the error message contains "g" of "A".
-        self.g() # E: Too few arguments for "g" of "A"   
+        self.g() # E: Too few arguments for "g" of "A"
         self.g(1)
     @dec
     def g(self, x: str) -> None: pass

test-data/unit/check-generics.test

@@ -52,7 +52,7 @@ class C: pass
 [out]
 main:8: error: Incompatible types in assignment (expression has type "C", variable has type "B")
 
-[case testGenericMemberVariable]
+[case testGenericMemberVariable2]
 from typing import TypeVar, Generic
 T = TypeVar('T')
 a, b, c = None, None, None # type: (A[B], B, C)
@@ -2118,7 +2118,7 @@ class B(A[T], Generic[T, S]):
 reveal_type(B.foo)  # N: Revealed type is "def [T, S] () -> Tuple[T`1, __main__.B[T`1, S`2]]"
 [builtins fixtures/classmethod.pyi]
 
-[case testGenericClassAlternativeConstructorPrecise]
+[case testGenericClassAlternativeConstructorPrecise2]
 from typing import Generic, TypeVar, Type, Tuple, Any
 
 T = TypeVar('T')

test-data/unit/check-inference.test

@@ -2614,7 +2614,7 @@ class C(B):
 reveal_type(B.x)  # N: Revealed type is "None"
 reveal_type(C.x)  # N: Revealed type is "None"
 
-[case testLocalPartialTypesWithInheritance2]
+[case testLocalPartialTypesWithInheritance3]
 # flags: --local-partial-types
 from typing import Optional
 
@@ -2652,7 +2652,7 @@ class C:
     def f(self, x) -> None:
         C.a.y  # E: Item "None" of "Optional[Any]" has no attribute "y"
 
-[case testLocalPartialTypesAccessPartialNoneAttribute]
+[case testLocalPartialTypesAccessPartialNoneAttribute2]
 # flags: --local-partial-types
 class C:
     a = None  # E: Need type annotation for "a"

test-data/unit/check-modules.test

@@ -1,7 +1,7 @@
 -- Type checker test cases dealing with modules and imports.
 -- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages).
 
-[case testAccessImportedDefinitions]
+[case testAccessImportedDefinitions0]
 import m
 import typing
 m.f()           # E: Missing positional argument "a" in call to "f"
@@ -14,7 +14,7 @@ class A: pass
 def f(a: A) -> None: pass
 x = A()
 
-[case testAccessImportedDefinitions]
+[case testAccessImportedDefinitions1]
 import m
 import typing
 m.f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A"

test-data/unit/check-multiple-inheritance.test

@@ -502,7 +502,7 @@ class A(Base1, Base2):
 [out]
 main:10: error: Incompatible types in assignment (expression has type "GenericBase[Base2]", base class "Base1" defined the type as "GenericBase[Base1]")
 
-[case testMultipleInheritance_NestedVariableOverriddenWithCompatibleType]
+[case testMultipleInheritance_NestedVariableOverriddenWithCompatibleType2]
 from typing import TypeVar, Generic
 T = TypeVar('T', covariant=True)
 class GenericBase(Generic[T]):

test-data/unit/check-newsemanal.test

@@ -2314,7 +2314,7 @@ reveal_type(x)  # N: Revealed type is "builtins.int"
 
 C = int
 
-[case testNewAnalyzerCastForward2]
+[case testNewAnalyzerCastForward3]
 from typing import cast, NamedTuple
 
 x = cast('C', None)

test-data/unit/check-overloading.test

@@ -4314,7 +4314,7 @@ class Wrapper3:
     def foo(x: Union[int, str]): pass  # E: Self argument missing for a non-static method (or an invalid type for self)
 [builtins fixtures/staticmethod.pyi]
 
-[case testOverloadWithSwappedDecorators]
+[case testOverloadWithSwappedDecorators2]
 from typing import overload
 
 class Wrapper1:

test-data/unit/check-typeddict.test

@@ -1264,7 +1264,7 @@ from mypy_extensions import TypedDict
 Point = TypedDict('Point', {'x': 1, 'y': 1})  # E: Invalid type: try using Literal[1] instead?
 [builtins fixtures/dict.pyi]
 
-[case testCannotCreateTypedDictTypeWithInvalidName]
+[case testCannotCreateTypedDictTypeWithInvalidName2]
 from mypy_extensions import TypedDict
 X = TypedDict('Y', {'x': int})  # E: First argument "Y" to TypedDict() does not match variable name "X"
 [builtins fixtures/dict.pyi]

test-data/unit/check-unions.test

@@ -241,7 +241,7 @@ def f(x: M[C]) -> None:
     y = x.get(None)
     reveal_type(y) # N: Revealed type is "__main__.C"
 
-[case testUnionSimplificationSpecialCases]
+[case testUnionSimplificationSpecialCases2]
 from typing import Any, TypeVar, Union
 
 class C(Any): pass

test-data/unit/check-varargs.test

@@ -735,7 +735,7 @@ f(a) # E: Argument 1 to "f" has incompatible type "List[int]"; expected "Listene
 g(b) # E: Argument 1 to "g" has incompatible type "Dict[str, int]"; expected "DictReader"
 [builtins fixtures/dict.pyi]
 
-[case testInvariantTypeConfusingNames]
+[case testInvariantTypeConfusingNames2]
 from typing import Iterable, Generic, TypeVar, List
 
 T = TypeVar('T')

test-data/unit/deps-expressions.test

@@ -191,7 +191,7 @@ def g(a: A) -> int:
 <m.f1> -> m.g
 <m.f2> -> m.g
 
-[case testIndexExpr]
+[case testIndexExpr2]
 class A:
     def __getitem__(self, x: int) -> int: pass
 

test-data/unit/fine-grained-blockers.test

@@ -338,7 +338,7 @@ main:1: error: Cannot find implementation or library stub for module named "a"
 main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 b.py:1: error: Cannot find implementation or library stub for module named "a"
 
-[case testDeleteFileWithBlockingError-only_when_cache]
+[case testDeleteFileWithBlockingError2-only_when_cache]
 -- Different cache/no-cache tests because:
 -- Error message ordering differs
 import a
@@ -521,7 +521,7 @@ a.py:1: error: invalid syntax. Perhaps you forgot a comma?
 b.py:2: error: Incompatible return value type (got "str", expected "int")
 ==
 
-[case testDecodeErrorBlocker-posix]
+[case testDecodeErrorBlocker1-posix]
 import a
 a.f(1)
 [file a.py]
@@ -537,7 +537,7 @@ mypy: can't decode file 'tmp/a.py': 'ascii' codec can't decode byte 0xc3 in posi
 ==
 main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
 
-[case testDecodeErrorBlocker-windows]
+[case testDecodeErrorBlocker2-windows]
 import a
 a.f(1)
 [file a.py]

test-data/unit/fine-grained-follow-imports.test

@@ -428,7 +428,7 @@ main.py:3: error: Missing positional argument "x" in call to "f"
 main.py:3: error: Missing positional argument "x" in call to "f"
 ==
 
-[case testFollowImportsNormalPackage-only_when_cache]
+[case testFollowImportsNormalPackage2-only_when_cache]
 # flags: --follow-imports=normal
 # cmd: mypy main.py
 
@@ -681,7 +681,7 @@ import bar
 src/bar.py:1: error: "int" not callable
 src/foo.py:2: error: "str" not callable
 
-[case testFollowImportsNormalSearchPathUpdate-only_when_cache]
+[case testFollowImportsNormalSearchPathUpdate2-only_when_cache]
 # flags: --follow-imports=normal
 # cmd: mypy main.py
 # cmd2: mypy main.py src/foo.py

test-data/unit/fine-grained-modules.test

@@ -410,7 +410,7 @@ main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missin
 
 -- TODO: Fix this bug. It is a real bug that was been papered over
 -- by the test harness.
-[case testDeletionOfSubmoduleTriggersImportFrom1-only_when_cache-skip]
+[case testDeletionOfSubmoduleTriggersImportFrom1_2-only_when_cache-skip]
 -- Different cache/no-cache tests because:
 -- missing module error message mismatch
 from p import q
@@ -1752,7 +1752,7 @@ class Foo:
 ==
 a.py:3: error: Argument 1 to "foo" of "Foo" has incompatible type "int"; expected "str"
 
-[case testAddAndUseClass4]
+[case testAddAndUseClass4_2]
 [file a.py]
 [file a.py.2]
 from p.b import *

test-data/unit/fine-grained.test

@@ -361,7 +361,7 @@ n.py:2: error: "A" has no attribute "g"
 ==
 n.py:2: error: "A" has no attribute "g"
 
-[case testContinueToReportErrorAtTopLevel-only_when_cache]
+[case testContinueToReportErrorAtTopLevel2-only_when_cache]
 -- Different cache/no-cache tests because:
 -- Error message ordering differs
 import n
@@ -7171,7 +7171,7 @@ main:4: note:          def f(self, arg: int) -> int
 main:4: note:          @overload
 main:4: note:          def f(self, arg: str) -> str
 
-[case testOverloadedMethodSupertype-only_when_nocache]
+[case testOverloadedMethodSupertype2-only_when_nocache]
 -- Different cache/no-cache tests because
 -- CallableType.def_extras.first_arg differs ("self"/None)
 from typing import overload, Any
@@ -8411,7 +8411,7 @@ a.py:3: note:          def meth(self, x: str) -> str
 a.py:3: note:      Subclass:
 a.py:3: note:          def meth(self) -> None
 
-[case testFinalBodyReprocessedAndStillFinalOverloaded-only_when_nocache]
+[case testFinalBodyReprocessedAndStillFinalOverloaded2-only_when_nocache]
 -- Different cache/no-cache tests because
 -- CallableType.def_extras.first_arg differs ("self"/None)
 import a

test-data/unit/parse-errors.test

@@ -295,7 +295,7 @@ file:1: error: invalid syntax
 [out]
 file:1: error: invalid syntax
 
-[case testInvalidConditionInConditionalExpression2]
+[case testInvalidConditionInConditionalExpression3]
 1 if x else for y in z
 [out]
 file:1: error: invalid syntax

test-data/unit/parse.test

@@ -3123,7 +3123,7 @@ MypyFile:1(
         NameExpr(x)
         NameExpr(y)))))
 
-[case testConditionalExpressionInListComprehension]
+[case testConditionalExpressionInListComprehension2]
 a = [ 1 if x else 2 for x in y ]
 [out]
 MypyFile:1(
@@ -3329,7 +3329,7 @@ MypyFile:1()
 [out]
 MypyFile:1()
 
-[case testLatinUnixEncoding]
+[case testLatinUnixEncoding2]
 # coding: iso-latin-1
 [out]
 MypyFile:1()

test-data/unit/semanal-basic.test

@@ -286,7 +286,7 @@ MypyFile:1(
         NameExpr(x* [l])
         NameExpr(None [builtins.None])))))
 
-[case testGlobalDeclScope]
+[case testGlobalDeclScope2]
 x = None
 def f():
     global x

test-data/unit/semanal-modules.test

@@ -206,7 +206,7 @@ MypyFile:1(
       NameExpr(_n)
       x [_n.x])))
 
-[case testAccessingImportedModule]
+[case testAccessingImportedModule2]
 import _m
 _m._n.x
 [file _m.py]
@@ -385,7 +385,7 @@ MypyFile:1(
     NameExpr(None [builtins.None])
     m._n.c))
 
-[case testSubmodulesAndTypes]
+[case testSubmodulesAndTypes2]
 from m._n import c
 x = None # type: c
 [file m/__init__.py]

test-data/unit/semanal-namedtuple.test

@@ -207,7 +207,7 @@ N = NamedTuple(name='N', fields=[]) # E: Unexpected arguments to "NamedTuple()"
 
 -- NOTE: The following code works at runtime but is not yet supported by mypy.
 --       Keyword arguments may potentially be supported in the future.
-[case testNamedTupleWithNonpositionalArgs]
+[case testNamedTupleWithNonpositionalArgs2]
 from collections import namedtuple
 N = namedtuple(typename='N', field_names=['x']) # E: Unexpected arguments to "namedtuple()"
 [builtins fixtures/tuple.pyi]

test-data/unit/stubgen.test

@@ -483,7 +483,7 @@ import re as re
 
 x: int
 
-[case testExportModule_import]
+[case testExportModule2_import]
 import re
 __all__ = ['re', 'x']
 x = 1
@@ -701,7 +701,7 @@ import x
 
 class D(x.C): ...
 
-[case testArbitraryBaseClass]
+[case testArbitraryBaseClass2]
 import x.y
 class D(x.y.C): ...
 [out]
@@ -837,7 +837,7 @@ import collections
 x: collections.defaultdict
 
 
-[case testAnnotationImports]
+[case testAnnotationImports2]
 from typing import List
 import collections
 x: List[collections.defaultdict]

test-data/unit/typexport-basic.test

@@ -727,7 +727,7 @@ f = lambda: [1]
 LambdaExpr(3) : def () -> builtins.list[builtins.int]
 NameExpr(3) : def () -> builtins.list[builtins.int]
 
-[case testLambdaWithInferredType2]
+[case testLambdaWithInferredType3]
 from typing import List, Callable
 f = lambda x: [] # type: Callable[[B], List[A]]
 class A: pass