diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f56ace97f..9b5783e01 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -105,6 +105,13 @@ jobs:
           python-platform: ${{ matrix.python-platform }}
           python-version: ${{ matrix.python-version }}
           no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy.
+          project: ./pyrightconfig.testcases.json
+      - uses: jakebailey/pyright-action@v1
+        with:
+          version: ${{ env.PYRIGHT_VERSION }}
+          python-platform: ${{ matrix.python-platform }}
+          python-version: ${{ matrix.python-version }}
+          no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }}  # Having each job create the same comment is too noisy.
 
   stubtest-third-party:
     name: Check third party stubs with stubtest
diff --git a/pyrightconfig.json b/pyrightconfig.json
index 923f815eb..0851d5654 100644
--- a/pyrightconfig.json
+++ b/pyrightconfig.json
@@ -2,8 +2,7 @@
     "typeshedPath": ".",
     "include": [
         "stdlib",
-        "stubs",
-        "test_cases"
+        "stubs"
     ],
     "typeCheckingMode": "basic",
     "strictListInference": true,
diff --git a/pyrightconfig.stricter.json b/pyrightconfig.stricter.json
index f257208ec..c8ad0c8bb 100644
--- a/pyrightconfig.stricter.json
+++ b/pyrightconfig.stricter.json
@@ -3,8 +3,7 @@
     "typeshedPath": ".",
     "include": [
         "stdlib",
-        "stubs",
-        "test_cases"
+        "stubs"
     ],
     "exclude": [
         "stdlib/distutils/command",
diff --git a/pyrightconfig.testcases.json b/pyrightconfig.testcases.json
new file mode 100644
index 000000000..0e0ec73f0
--- /dev/null
+++ b/pyrightconfig.testcases.json
@@ -0,0 +1,51 @@
+{
+    "$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
+    "typeshedPath": ".",
+    "include": [
+        "test_cases"
+    ],
+    "typeCheckingMode": "basic",
+    "strictListInference": true,
+    "strictDictionaryInference": true,
+    "strictParameterNoneValue": true,
+    "reportFunctionMemberAccess": "error",
+    "reportMissingModuleSource": "none",
+    "reportMissingTypeStubs": "error",
+    "reportUnusedImport": "error",
+    "reportUnusedClass": "error",
+    "reportUnusedFunction": "error",
+    "reportUnusedVariable": "error",
+    "reportDuplicateImport": "error",
+    "reportOptionalSubscript": "error",
+    "reportOptionalMemberAccess": "error",
+    "reportOptionalCall": "error",
+    "reportOptionalIterable": "error",
+    "reportOptionalContextManager": "error",
+    "reportOptionalOperand": "error",
+    "reportUntypedFunctionDecorator": "error",
+    "reportUntypedClassDecorator": "error",
+    "reportUntypedBaseClass": "error",
+    "reportUntypedNamedTuple": "error",
+    "reportPrivateUsage": "none",
+    "reportConstantRedefinition": "error",
+    "reportInvalidStringEscapeSequence": "error",
+    "reportUnknownParameterType": "error",
+    "reportUnknownArgumentType": "error",
+    "reportUnknownLambdaType": "error",
+    "reportUnknownVariableType": "error",
+    "reportUnknownMemberType": "error",
+    "reportMissingTypeArgument": "error",
+    "reportUndefinedVariable": "error",
+    "reportUnboundVariable": "error",
+    "reportInvalidStubStatement": "error",
+    "reportInvalidTypeVarUse": "error",
+    "reportPropertyTypeMismatch": "error",
+    "reportSelfClsParameterName": "error",
+    "reportUnsupportedDunderAll": "error",
+    "reportIncompatibleMethodOverride": "error",
+    "reportIncompatibleVariableOverride": "error",
+    "reportOverlappingOverload": "error",
+    "reportUnnecessaryTypeIgnoreComment": "error",
+    "reportMissingTypeStubs": "error",
+    "reportMatchNotExhaustive": "error"
+}
diff --git a/test_cases/README.md b/test_cases/README.md
index d15382e15..d343f9ac0 100644
--- a/test_cases/README.md
+++ b/test_cases/README.md
@@ -40,11 +40,8 @@ mypy's
 setting and pyright's
 [`reportUnnecessaryTypeIgnoreComment`](https://github.com/microsoft/pyright/blob/main/docs/configuration.md#type-check-diagnostics-settings)
 setting) to test instances where a type checker *should* emit some kind of
-error, if the stubs are correct. Note that mypy's `--warn-unused-ignores`
-setting is enabled for the entire subdirectory; however, the pyright setting
-must be enabled on a per-file basis with
-`# pyright: reportUnnecessaryTypeIgnoreComment=true` at the the top of the
-file.
+error, if the stubs are correct. Both settings are enabled by default for the entire
+subdirectory.
 
 For more information on using `assert_type` and
 `--warn-unused-ignores`/`reportUnnecessaryTypeIgnoreComment` to test type
diff --git a/test_cases/stdlib/builtins/test_dict.py b/test_cases/stdlib/builtins/test_dict.py
index 08f763333..b2def8b9a 100644
--- a/test_cases/stdlib/builtins/test_dict.py
+++ b/test_cases/stdlib/builtins/test_dict.py
@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 from typing import Dict, Generic, Iterable, Tuple, TypeVar
 from typing_extensions import assert_type
 
diff --git a/test_cases/stdlib/builtins/test_pow.py b/test_cases/stdlib/builtins/test_pow.py
index 732dfca58..c1ff7497e 100644
--- a/test_cases/stdlib/builtins/test_pow.py
+++ b/test_cases/stdlib/builtins/test_pow.py
@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 from decimal import Decimal
 from fractions import Fraction
 from typing import Any
diff --git a/test_cases/stdlib/builtins/test_sum.py b/test_cases/stdlib/builtins/test_sum.py
index 421116767..0f256fd2f 100644
--- a/test_cases/stdlib/builtins/test_sum.py
+++ b/test_cases/stdlib/builtins/test_sum.py
@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 from typing import Any, List, Union
 from typing_extensions import Literal, assert_type
 
diff --git a/test_cases/stdlib/test_codecs.py b/test_cases/stdlib/test_codecs.py
index dfbc46db7..0657416bc 100644
--- a/test_cases/stdlib/test_codecs.py
+++ b/test_cases/stdlib/test_codecs.py
@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 import codecs
 from typing_extensions import assert_type
 
diff --git a/test_cases/stdlib/test_unittest.py b/test_cases/stdlib/test_unittest.py
index 5d1efc1b0..b8a16e7ef 100644
--- a/test_cases/stdlib/test_unittest.py
+++ b/test_cases/stdlib/test_unittest.py
@@ -1,5 +1,3 @@
-# pyright: reportUnnecessaryTypeIgnoreComment=true
-
 import unittest
 from datetime import datetime, timedelta
 from decimal import Decimal