VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2025-12-25 13:21:11 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add pyright testcases / regression tests to the runtests script (#10002)

Browse Source 
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
This commit is contained in:
Avasam
2023-04-14 06:11:21 -04:00
committed by GitHub
parent f66769a6f5
commit 08e6e4ced9
1 changed files with 55 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
scripts/runtests.py
View File

@@ -19,6 +19,10 @@ except ImportError:
_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
_TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"
_TESTCASES = "test_cases"
_NPX_ERROR_PATTERN = r"error (runn|find)ing npx"
_NPX_ERROR_MESSAGE = colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow")
_SUCCESS = colored("Success", "green")
_SKIPPED = colored("Skipped", "yellow")
_FAILED = colored("Failed", "red")
@@ -89,14 +93,15 @@ def main() -> None:
print("\nRunning check_new_syntax.py...")
check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
print(f"\nRunning Pyright on Python {_PYTHON_VERSION}...")
strict_params = _get_strict_params(path)
print(f"\nRunning Pyright ({'stricter' if strict_params else 'base' } configs) for Python {_PYTHON_VERSION}...")
pyright_result = subprocess.run(
[sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + _get_strict_params(path),
stderr=subprocess.PIPE,
[sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + strict_params,
capture_output=True,
text=True,
)
if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
print(colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow"))
if re.match(_NPX_ERROR_PATTERN, pyright_result.stderr):
print(_NPX_ERROR_MESSAGE)
pyright_returncode = 0
pyright_skipped = True
else:
@@ -133,19 +138,47 @@ def main() -> None:
print("\nRunning pytype...")
pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
print(f"\nRunning regression tests for Python {_PYTHON_VERSION}...")
regr_test_result = subprocess.run(
[sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
stderr=subprocess.PIPE,
text=True,
)
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
if "No test cases found" in regr_test_result.stderr:
test_cases_path = Path(path) / "@tests" / _TESTCASES if folder == "stubs" else Path(_TESTCASES)
if not test_cases_path.exists():
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
print(colored(f"\nRegression tests: No {_TESTCASES} folder for {stub!r}!", "green"))
pyright_testcases_returncode = 0
pyright_testcases_skipped = False
regr_test_returncode = 0
print(colored(f"\nNo test cases found for {stub!r}!", "green"))
else:
regr_test_returncode = regr_test_result.returncode
print(regr_test_result.stderr)
print(f"\nRunning Pyright regression tests for Python {_PYTHON_VERSION}...")
command = [
sys.executable,
"tests/pyright_test.py",
str(test_cases_path),
"--pythonversion",
_PYTHON_VERSION,
"-p",
_TESTCASES_CONFIG_FILE,
]
pyright_testcases_result = subprocess.run(command, capture_output=True, text=True)
if re.match(_NPX_ERROR_PATTERN, pyright_testcases_result.stderr):
print(_NPX_ERROR_MESSAGE)
pyright_testcases_returncode = 0
pyright_testcases_skipped = True
else:
print(pyright_result.stderr)
pyright_testcases_returncode = pyright_testcases_result.returncode
pyright_testcases_skipped = False
print(f"\nRunning mypy regression tests for Python {_PYTHON_VERSION}...")
regr_test_result = subprocess.run(
[sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
capture_output=True,
text=True,
)
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
if "No test cases found" in regr_test_result.stderr:
regr_test_returncode = 0
print(colored(f"\nNo test cases found for {stub!r}!", "green"))
else:
regr_test_returncode = regr_test_result.returncode
print(regr_test_result.stderr)
any_failure = any(
[
@@ -156,6 +189,7 @@ def main() -> None:
mypy_result.returncode,
getattr(stubtest_result, "returncode", 0),
getattr(pytype_result, "returncode", 0),
pyright_testcases_returncode,
regr_test_returncode,
]
)
@@ -180,7 +214,11 @@ def main() -> None:
print("pytype:", _SKIPPED)
else:
print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
if pyright_testcases_skipped:
print("Pyright regression tests:", _SKIPPED)
else:
print("Pyright regression tests:", _SUCCESS if pyright_testcases_returncode == 0 else _FAILED)
print("mypy regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
sys.exit(int(any_failure))