Improve pre-commit config (#9563)

- Add a few more hooks. These are all very fast, and I've found them useful in other projects:
  - Autofixes:
    - `trailing-whitespace`: fixes trailing whitespace
    - `requirements-txt-fixer`: alphabetises items in `requirements.txt` files
    - `end-of-file-fixer`: makes sure every file ends with a single newline character
    - `mixed-line-ending`: Makes sure Windows users don't accidentally introduce CRLF line endings into a file that uses LF line endings
  - None-autofixes:
    - `check-yaml`: loads YAML files to validate syntax
    - `check-toml`: loads TOML files to validate syntax
    - `check-merge-conflict`: detects merge-conflict strings in files and blocks them from accidentally being committed
    - `check-case-conflict`: checks for files with names that would conflict on a case-insensitive filesystem like MacOS HFS+ or Windows FAT; blocks them from being committed.
  - Change the bot schedule to quarterly, to reduce noisy PRs
  - Change the `black` language target-version to Python 3.10, synching the setting here with the changes that were made to our `pyproject.toml` file in #7538

.pre-commit-config.yaml

@@ -1,4 +1,15 @@
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0 # must match requirements-tests.txt
+    hooks:
+      - id: trailing-whitespace
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: mixed-line-ending
+      - id: check-case-conflict
   - repo: https://github.com/hadialqattan/pycln
     rev: v2.1.3 # must match requirements-tests.txt
     hooks:
@@ -8,7 +19,7 @@ repos:
     rev: 22.12.0 # must match requirements-tests.txt
     hooks:
       - id: black
-        language_version: python3.9
+        language_version: python3.10
   - repo: https://github.com/pycqa/isort
     rev: 5.11.4 # must match requirements-tests.txt
     hooks:
@@ -33,6 +44,6 @@ ci:
   autofix_commit_msg: "[pre-commit.ci] auto fixes from pre-commit.com hooks"
   autofix_prs: true
   autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate"
-  autoupdate_schedule: weekly
+  autoupdate_schedule: quarterly
   skip: [flake8]
   submodules: false

LICENSE

@@ -235,4 +235,3 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.
 
 = = = = =
-

requirements-tests.txt

@@ -8,9 +8,10 @@ isort==5.11.4                                     # must match .pre-commit-confi
 mypy==0.991
 packaging==23.0
 pathspec>=0.10.3
+pre-commit-hooks==4.4.0                           # must match .pre-commit-config.yaml
 pycln==2.1.3                                      # must match .pre-commit-config.yaml
-pyyaml==6.0
 pytype==2023.1.17; platform_system != "Windows" and python_version < "3.11"
+pyyaml==6.0
 termcolor>=2
 tomli==2.0.1
 tomlkit==0.11.6

stubs/aiofiles/@tests/stubtest_allowlist.txt

@@ -63,4 +63,3 @@ aiofiles.tempfile.AiofilesContextManagerTempDir
 
 # Helper decorator, too complex to type
 aiofiles.os.wrap
-

stubs/croniter/@tests/stubtest_allowlist.txt

@@ -2,4 +2,4 @@
 croniter.croniter.EXPANDERS
 croniter.croniter.HashExpander
 croniter.croniter.VALID_LEN_EXPRESSION
-croniter.croniter.timedelta_to_seconds
+croniter.croniter.timedelta_to_seconds

stubs/editdistance/METADATA.toml

@@ -2,4 +2,3 @@ version = "0.6.*"
 
 [tool.stubtest]
 ignore_missing_stub = false
-

stubs/humanfriendly/@tests/requirements-stubtest.txt

@@ -1,2 +1,2 @@
 docutils
-mock
+mock

stubs/oauthlib/@tests/stubtest_allowlist.txt

@@ -1 +1 @@
-oauthlib.oauth1.rfc5849.parameters.prepare_headers
+oauthlib.oauth1.rfc5849.parameters.prepare_headers

stubs/openpyxl/@tests/stubtest_allowlist.txt

@@ -1,2 +1,2 @@
 # "cls" argument has wrong name in implementation.
-openpyxl.descriptors.slots.AutoSlotProperties.__new__ 
+openpyxl.descriptors.slots.AutoSlotProperties.__new__

stubs/psycopg2/METADATA.toml

@@ -1 +1 @@
-version = "2.9.*"
+version = "2.9.*"

stubs/tensorflow/@tests/stubtest_allowlist.txt

@@ -1,4 +1,4 @@
-# Some methods are dynamically patched onto to instances as they 
+# Some methods are dynamically patched onto to instances as they
 # may depend on whether code is executed in graph/eager/v1/v2/etc.
 # Tensorflow supports multiple modes of execution which changes some
 # of the attributes/methods/even class hierachies.
@@ -17,6 +17,6 @@ tensorflow.Operation.__getattr__
 tensorflow.Variable.__getattr__
 # Internal undocumented API
 tensorflow.RaggedTensor.__init__
-# Has an undocumented extra argument that tf.Variable which acts like subclass 
+# Has an undocumented extra argument that tf.Variable which acts like subclass
 # (by dynamically patching tf.Tensor methods) does not preserve.
-tensorflow.Tensor.__getitem__
+tensorflow.Tensor.__getitem__

stubs/tqdm/@tests/requirements-stubtest.txt

@@ -1,4 +1,4 @@
-tensorflow
-pandas
 dask
+pandas
 rich
+tensorflow

tests/check_consistent.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 import os
 import re
 import sys
+import urllib.parse
 from pathlib import Path
 
 import tomli
@@ -220,8 +221,9 @@ def get_precommit_requirements() -> dict[str, SpecifierSet]:
         if not repo.get("python_requirement", True):
             continue
         hook = repo["hooks"][0]
-        package_name, package_rev = hook["id"], repo["rev"]
-        package_specifier = SpecifierSet(f"=={package_rev.removeprefix('v')}")
+        package_name = Path(urllib.parse.urlparse(repo["repo"]).path).name
+        package_rev = repo["rev"].removeprefix("v")
+        package_specifier = SpecifierSet(f"=={package_rev}")
         precommit_requirements[package_name] = package_specifier
         for additional_req in hook.get("additional_dependencies", []):
             req = Requirement(additional_req)

tests/stubtest_allowlists/darwin-py311.txt

@@ -10,4 +10,4 @@ socket.TCP_CONNECTION_INFO
 (locale.dcgettext)
 (locale.dgettext)?
 (locale.gettext)?
-(locale.textdomain)?
+(locale.textdomain)?