diff --git a/test_cases/README.md b/test_cases/README.md index d343f9ac0..a27e9f65c 100644 --- a/test_cases/README.md +++ b/test_cases/README.md @@ -1,9 +1,9 @@ ## Regression tests for typeshed -This directory contains regression tests for the stubs found elsewhere in the -typeshed repo. +This directory contains code samples that act as a regression test for the +standard library stubs found elsewhere in the typeshed repo. -**This directory should *only* contain tests for functions and classes which +**This directory should *only* contain test cases for functions and classes which are known to have caused problems in the past, where the stubs are difficult to get right.** 100% test coverage for typeshed is neither necessary nor desirable, as it would lead to code duplication. Moreover, typeshed has @@ -11,21 +11,25 @@ multiple other mechanisms for spotting errors in the stubs. ### The purpose of these tests -Different tests in this directory serve different purposes. For some stubs in +Different test cases in this directory serve different purposes. For some stubs in typeshed, the type annotations are complex enough that it's useful to have basic sanity checks that test whether a type checker understands the intent of the annotations correctly. Examples of tests like these are -`stdlib/builtins/test_pow.py` and `stdlib/asyncio/test_gather.py`. +`stdlib/builtins/check_pow.py` and `stdlib/asyncio/check_gather.py`. -Other tests, such as the tests for `ExitStack` in `stdlib/test_contextlib.py` -and the tests for `LogRecord` in `stdlib/test_logging.py`, do not relate to +Other test cases, such as the samples for `ExitStack` in `stdlib/check_contextlib.py` +and the samples for `LogRecord` in `stdlib/check_logging.py`, do not relate to stubs where the annotations are particularly complex, but they *do* relate to stubs where decisions have been taken that might be slightly unusual. These -tests serve a different purpose: to check that type checkers do not emit +test cases serve a different purpose: to check that type checkers do not emit false-positive errors for idiomatic usage of these classes. ### How the tests work +The code in this directory is not intended to be directly executed. Instead, +type checkers are run on the code, in order to check whether typing errors are +emitted at the places where we want them to be. + Some files in this directory simply contain samples of idiomatic Python, which should not (if the stubs are correct) cause a type checker to emit any errors. @@ -52,19 +56,20 @@ provides a useful guide. ### Naming convention Use the same top-level name for the module / package you would like to test. -Use `test_${thing}.py` naming pattern for individual test files. +Use the `check_${thing}.py` naming pattern for individual test files. -By default, tests go into a test file with the same name as the stub file, prefixed with `test_`. -For example: `stdlib/test_contextlib.py`. +By default, test cases go into a file with the same name as the stub file, prefixed with `check_`. +For example: `stdlib/check_contextlib.py`. If that file becomes too big, we instead create a directory with files named after individual objects being tested. -For example: `stdlib/builtins/test_dict.py`. +For example: `stdlib/builtins/check_dict.py`. ### Differences to the rest of typeshed Unlike the rest of typeshed, this directory largely contains `.py` files. This is because the purpose of this folder is to test the implications of typeshed -changes for end users. +changes for end users, who will mainly be using `.py` files rather than `.pyi` +files. Another difference to the rest of typeshed is that the test cases in this directory cannot always use modern syntax for type hints. diff --git a/test_cases/stdlib/asyncio/test_coroutines.py b/test_cases/stdlib/asyncio/check_coroutines.py similarity index 100% rename from test_cases/stdlib/asyncio/test_coroutines.py rename to test_cases/stdlib/asyncio/check_coroutines.py diff --git a/test_cases/stdlib/asyncio/test_gather.py b/test_cases/stdlib/asyncio/check_gather.py similarity index 100% rename from test_cases/stdlib/asyncio/test_gather.py rename to test_cases/stdlib/asyncio/check_gather.py diff --git a/test_cases/stdlib/builtins/test_dict.py b/test_cases/stdlib/builtins/check_dict.py similarity index 100% rename from test_cases/stdlib/builtins/test_dict.py rename to test_cases/stdlib/builtins/check_dict.py diff --git a/test_cases/stdlib/builtins/test_iteration.py b/test_cases/stdlib/builtins/check_iteration.py similarity index 100% rename from test_cases/stdlib/builtins/test_iteration.py rename to test_cases/stdlib/builtins/check_iteration.py diff --git a/test_cases/stdlib/builtins/test_list.py b/test_cases/stdlib/builtins/check_list.py similarity index 100% rename from test_cases/stdlib/builtins/test_list.py rename to test_cases/stdlib/builtins/check_list.py diff --git a/test_cases/stdlib/builtins/test_object.py b/test_cases/stdlib/builtins/check_object.py similarity index 100% rename from test_cases/stdlib/builtins/test_object.py rename to test_cases/stdlib/builtins/check_object.py diff --git a/test_cases/stdlib/builtins/test_pow.py b/test_cases/stdlib/builtins/check_pow.py similarity index 100% rename from test_cases/stdlib/builtins/test_pow.py rename to test_cases/stdlib/builtins/check_pow.py diff --git a/test_cases/stdlib/builtins/test_sum.py b/test_cases/stdlib/builtins/check_sum.py similarity index 100% rename from test_cases/stdlib/builtins/test_sum.py rename to test_cases/stdlib/builtins/check_sum.py diff --git a/test_cases/stdlib/builtins/test_tuple.py b/test_cases/stdlib/builtins/check_tuple.py similarity index 100% rename from test_cases/stdlib/builtins/test_tuple.py rename to test_cases/stdlib/builtins/check_tuple.py diff --git a/test_cases/stdlib/test_codecs.py b/test_cases/stdlib/check_codecs.py similarity index 100% rename from test_cases/stdlib/test_codecs.py rename to test_cases/stdlib/check_codecs.py diff --git a/test_cases/stdlib/test_contextlib.py b/test_cases/stdlib/check_contextlib.py similarity index 100% rename from test_cases/stdlib/test_contextlib.py rename to test_cases/stdlib/check_contextlib.py diff --git a/test_cases/stdlib/test_logging.py b/test_cases/stdlib/check_logging.py similarity index 100% rename from test_cases/stdlib/test_logging.py rename to test_cases/stdlib/check_logging.py diff --git a/test_cases/stdlib/test_threading.py b/test_cases/stdlib/check_threading.py similarity index 100% rename from test_cases/stdlib/test_threading.py rename to test_cases/stdlib/check_threading.py diff --git a/test_cases/stdlib/test_unittest.py b/test_cases/stdlib/check_unittest.py similarity index 100% rename from test_cases/stdlib/test_unittest.py rename to test_cases/stdlib/check_unittest.py diff --git a/test_cases/stdlib/typing/test_all.py b/test_cases/stdlib/typing/check_all.py similarity index 100% rename from test_cases/stdlib/typing/test_all.py rename to test_cases/stdlib/typing/check_all.py diff --git a/test_cases/stdlib/typing/test_pattern.py b/test_cases/stdlib/typing/check_pattern.py similarity index 100% rename from test_cases/stdlib/typing/test_pattern.py rename to test_cases/stdlib/typing/check_pattern.py diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 9fc27b8d1..a4e9fdcb6 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -59,9 +59,9 @@ def check_stubs() -> None: def check_test_cases() -> None: assert_consistent_filetypes(Path("test_cases"), kind=".py", allowed={"README.md"}) - bad_test_case_filename = 'Files in the `test_cases` directory must have names starting with "test_"; got "{}"' + bad_test_case_filename = 'Files in the `test_cases` directory must have names starting with "check_"; got "{}"' for file in Path("test_cases").rglob("*.py"): - assert file.stem.startswith("test_"), bad_test_case_filename.format(file) + assert file.stem.startswith("check_"), bad_test_case_filename.format(file) def check_no_symlinks() -> None: