diff --git a/.github/workflows/stubsabot.yml b/.github/workflows/stubsabot.yml new file mode 100644 index 000000000..70089e0dd --- /dev/null +++ b/.github/workflows/stubsabot.yml @@ -0,0 +1,48 @@ +name: Run stubsabot weekly # TODO: change to daily + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 5" # TODO: change to daily + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + stubsabot: + name: Upgrade stubs with stubsabot + if: github.repository == 'python/typeshed' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: git config + run: | + git config --global user.name stubsabot + git config --global user.email '<>' + - name: Install dependencies + run: pip install aiohttp packaging termcolor tomli tomlkit + - name: Run stubsabot + run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python scripts/stubsabot.py --action-level everything + + # https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2 + create-issue-on-failure: + name: Create an issue if stubsabot failed + runs-on: ubuntu-latest + needs: [stubsabot] + if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubsabot.result == 'failure') }} + steps: + - uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.issues.create({ + owner: "python", + repo: "typeshed", + title: `Stubsabot failed on ${new Date().toDateString()}`, + body: "Stubsabot runs are listed here: https://github.com/python/typeshed/actions/workflows/stubsabot.yml", + } diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 7efc069c2..99228867b 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -16,7 +16,7 @@ import urllib.parse import zipfile from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar import aiohttp import packaging.specifiers @@ -24,6 +24,15 @@ import packaging.version import tomli import tomlkit +if TYPE_CHECKING: + + def colored(__str: str, __style: str) -> str: + ... + +else: + from termcolor import colored + + ActionLevelSelf = TypeVar("ActionLevelSelf", bound="ActionLevel") @@ -36,7 +45,8 @@ class ActionLevel(enum.IntEnum): nothing = 0, "make no changes" local = 1, "make changes that affect local repo" - everything = 2, "do everything, e.g. open PRs" + fork = 2, "make changes that affect remote repo, but won't open PRs against upstream" + everything = 3, "do everything, e.g. open PRs" @dataclass @@ -208,10 +218,10 @@ TYPESHED_OWNER = "python" @functools.lru_cache() def get_origin_owner() -> str: - output = subprocess.check_output(["git", "remote", "get-url", "origin"], text=True) - match = re.search(r"(git@github.com:|https://github.com/)(?P[^/]+)/(?P[^/]+).git", output) - assert match is not None - assert match.group("repo") == "typeshed" + output = subprocess.check_output(["git", "remote", "get-url", "origin"], text=True).strip() + match = re.match(r"(git@github.com:|https://github.com/)(?P[^/]+)/(?P[^/\s]+)", output) + assert match is not None, f"Couldn't identify origin's owner: {output!r}" + assert match.group("repo").removesuffix(".git") == "typeshed", f'Unexpected repo: {match.group("repo")!r}' return match.group("owner") @@ -254,6 +264,31 @@ async def create_or_update_pull_request(*, title: str, body: str, branch_name: s response.raise_for_status() +def origin_branch_has_changes(branch: str) -> bool: + assert not branch.startswith("origin/") + try: + # number of commits on origin/branch that are not on branch or are + # patch equivalent to a commit on branch + output = subprocess.check_output( + ["git", "rev-list", "--right-only", "--cherry-pick", "--count", f"{branch}...origin/{branch}"], + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + # origin/branch does not exist + return False + return int(output) > 0 + + +class RemoteConflict(Exception): + pass + + +def somewhat_safe_force_push(branch: str) -> None: + if origin_branch_has_changes(branch): + raise RemoteConflict(f"origin/{branch} has changes not on {branch}!") + subprocess.check_call(["git", "push", "origin", branch, "--force"]) + + def normalize(name: str) -> str: # PEP 503 normalization return re.sub(r"[-_.]+", "-", name).lower() @@ -280,7 +315,9 @@ async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession subprocess.check_call(["git", "commit", "--all", "-m", title]) if action_level <= ActionLevel.local: return - subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) + somewhat_safe_force_push(branch_name) + if action_level <= ActionLevel.fork: + return body = "\n".join(f"{k}: {v}" for k, v in update.links.items()) body += """ @@ -309,7 +346,9 @@ async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientS subprocess.check_call(["git", "commit", "--all", "-m", title]) if action_level <= ActionLevel.local: return - subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) + somewhat_safe_force_push(branch_name) + if action_level <= ActionLevel.fork: + return body = "\n".join(f"{k}: {v}" for k, v in obsolete.links.items()) await create_or_update_pull_request(title=title, body=body, branch_name=branch_name, session=session) @@ -333,12 +372,15 @@ async def main() -> None: ) args = parser.parse_args() - if args.action_level > ActionLevel.local: + if args.action_level > ActionLevel.fork: if os.environ.get("GITHUB_TOKEN") is None: raise ValueError("GITHUB_TOKEN environment variable must be set") denylist = {"gdb"} # gdb is not a pypi distribution + if args.action_level >= ActionLevel.fork: + subprocess.check_call(["git", "fetch", "--prune", "--all"]) + try: conn = aiohttp.TCPConnector(limit_per_host=10) async with aiohttp.ClientSession(connector=conn) as session: @@ -357,15 +399,19 @@ async def main() -> None: continue if args.action_count_limit is not None and action_count >= args.action_count_limit: - print("... but we've reached action count limit") + print(colored("... but we've reached action count limit", "red")) continue action_count += 1 - if isinstance(update, Update): - await suggest_typeshed_update(update, session, action_level=args.action_level) - continue - if isinstance(update, Obsolete): - await suggest_typeshed_obsolete(update, session, action_level=args.action_level) + try: + if isinstance(update, Update): + await suggest_typeshed_update(update, session, action_level=args.action_level) + continue + if isinstance(update, Obsolete): + await suggest_typeshed_obsolete(update, session, action_level=args.action_level) + continue + except RemoteConflict as e: + print(colored(f"... but ran into {type(e).__qualname__}: {e}", "red")) continue raise AssertionError finally: