VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2026-09-30 15:12:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: VimPlug/typeshed:notimplemented-not-any
Branches Tags
VimPlug/typeshed:jedi VimPlug/typeshed:jedi020 VimPlug/typeshed:notimplemented-not-any VimPlug/typeshed:old_jedi VimPlug/typeshed:sqlite3_cursor VimPlug/typeshed:master
VimPlug/typeshed:jedi_v0.16.0 VimPlug/typeshed:jedi_v0.15.0
..
pull from: VimPlug/typeshed:sqlite3_cursor
Branches Tags
VimPlug/typeshed:jedi020 VimPlug/typeshed:notimplemented-not-any VimPlug/typeshed:jedi VimPlug/typeshed:old_jedi VimPlug/typeshed:sqlite3_cursor VimPlug/typeshed:master
VimPlug/typeshed:jedi_v0.16.0 VimPlug/typeshed:jedi_v0.15.0
1 Commits
notimplemented-not-any .. sqlite3_cursor
Author SHA1 Message Date
Dave Halter 74b5289e33 Make sure that the context manager for sqlite3.Connection works 
This was originally reported in https://github.com/davidhalter/jedi/issues/1084.
 2019-12-14 02:00:09 +01:00
6434 changed files with 69402 additions and 324784 deletions
Expand all files Collapse all files
.editorconfig
-16
View File
@@ -1,16 +0,0 @@
root = true
[*]
trim_trailing_whitespace = true
insert_final_newline = true
indent_style = space
end_of_line = lf
indent_size = 2
[*.md]
max_line_length = 79
trim_trailing_whitespace = false
[*.{py,pyi,toml,json}]
max_line_length = 130
indent_size = 4
.flake8
+22 -18
View File
@@ -1,19 +1,23 @@
[flake8]
# NQA: Ruff won't warn about redundant `# noqa: Y`
# Y: Flake8 is only used to run flake8-pyi, everything else is in Ruff
# F821: Typeshed is a testing ground for flake8-pyi, which monkeypatches F821
select = NQA, Y, F821
# Ignore rules normally excluded by default
extend-ignore = Y090
per-file-ignores =
# We should only need to noqa Y and F821 codes in .pyi files
*.py: NQA
# Generated protobuf files:
# Y021: Include docstrings
# Y023: Alias typing as typing_extensions
# Y026: Have implicit type aliases
# Y053: have literals >50 characters long
stubs/*_pb2.pyi: Y021, Y023, Y026, Y053
# Some PEP8 deviations are considered irrelevant to stub files:
# (error counts as of 2017-05-22)
# 17952 E704 multiple statements on one line (def)
# 12197 E301 expected 1 blank line
# 7155 E302 expected 2 blank lines
# 1463 F401 imported but unused
# 967 E701 multiple statements on one line (colon)
# 457 F811 redefinition (should be fixed in pyflakes 2.1.2)
# 390 E305 expected 2 blank lines
# 4 E741 ambiguous variable name
exclude = .venv*,.git
noqa_require_code = true
# Nice-to-haves ignored for now
# 2307 E501 line too long
# Other ignored warnings
# W504 line break after binary operator
[flake8]
ignore = F401, F403, F405, F811, E301, E302, E305, E501, E701, E704, E741, B303, W504
# We are checking with Python 3 but many of the stubs are Python 2 stubs.
builtins = StandardError,apply,basestring,buffer,cmp,coerce,execfile,file,intern,long,raw_input,reduce,reload,unichr,unicode,xrange
exclude = .venv*,@*,.git
max-line-length = 130
.gitattributes
-9
View File
@@ -1,9 +0,0 @@
# Normalize EOF
* autocrlf=false
* eol=lf
# Set linguist-language to support comments syntax highlight
**/stubtest_allowlist*.txt linguist-language=ini
**/stubtest_allowlists/*.txt linguist-language=ini
tests/pytype_exclude_list.txt linguist-language=ini
pyrightconfig*.json linguist-language=jsonc
.vscode/*.json linguist-language=jsonc
.github/renovate.json
-35
View File
@@ -1,35 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"dependencyDashboard": true,
"suppressNotifications": ["prEditedNotification"],
"extends": ["config:recommended"],
"labels": ["bot: dependencies"],
"semanticCommits": "disabled",
"separateMajorMinor": false,
"prHourlyLimit": 10,
"pre-commit": {
"enabled": true
},
"packageRules": [
{
"groupName": "GitHub Actions",
"matchManagers": ["github-actions"],
"description": "Quarterly update of GitHub Action dependencies",
"schedule": ["every 3 months on the first day of the month"]
},
{
"groupName": "most test/lint dependencies",
"matchManagers": ["pip_requirements", "pre-commit"],
"excludePackageNames": ["pytype", "pyright"],
"description": "Quarterly update of most test dependencies",
"schedule": ["every 3 months on the first day of the month"]
},
{
"groupName": "pytype and pyright",
"matchManagers": ["pip_requirements"],
"matchPackageNames": ["pytype", "pyright"],
"description": "Daily update of pyright and pytype",
"schedule": ["before 4am"]
}
]
}
.github/workflows/daily.yml
-150
View File
@@ -1,150 +0,0 @@
name: Daily test
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
pull_request:
paths:
- "requirements-tests.txt"
- ".github/workflows/daily.yml"
# Please keep the permissions minimal, as stubtest runs arbitrary code from pypi.
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
# A few env vars to speedup brew install
HOMEBREW_NO_ANALYTICS: 1
HOMEBREW_NO_AUTOUPDATE: 1
HOMEBREW_NO_INSTALL_CLEANUP: 1 # Environments are isolated, no need to cleanup old versions
NONINTERACTIVE: 1 # Required for brew install on CI
PIP_DISABLE_PIP_VERSION_CHECK: 1
FORCE_COLOR: 1
TERM: xterm-256color # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
jobs:
stubtest-stdlib:
name: Check stdlib with stubtest
if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
# As of 2024-10-18, ubuntu-latest can refer to different Ubuntu versions,
# which can can cause problems with os module constants.
os: ["ubuntu-24.04", "windows-latest", "macos-latest"]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: requirements-tests.txt
allow-prereleases: true
check-latest: true
- name: Install dependencies
run: pip install -r requirements-tests.txt
- name: Run stubtest
run: python tests/stubtest_stdlib.py
stubtest-third-party:
name: Check third party stubs with stubtest
if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
# As of 2024-10-18, ubuntu-latest can refer to different Ubuntu versions,
# which causes problems when testing gdb.
os: ["ubuntu-24.04", "windows-latest", "macos-latest"]
shard-index: [0, 1, 2, 3]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
# TODO: Use Python 3.12. As of 2024-03-08, several third-party
# packages fail to install with Python 3.12.
python-version: "3.11"
cache: pip
cache-dependency-path: |
requirements-tests.txt
stubs/**/METADATA.toml
- name: Install dependencies
run: pip install -r requirements-tests.txt
- name: Run stubtest
shell: bash
run: |
PACKAGES=$(python tests/get_stubtest_system_requirements.py)
if [ "${{ runner.os }}" = "Linux" ]; then
if [ -n "$PACKAGES" ]; then
sudo apt-get update && sudo apt-get install -y $PACKAGES
fi
PYTHON_EXECUTABLE="xvfb-run python"
else
if [ "${{ runner.os }}" = "macOS" ] && [ -n "$PACKAGES" ]; then
brew install -q $PACKAGES
fi
if [ "${{ runner.os }}" = "Windows" ] && [ -n "$PACKAGES" ]; then
choco install -y $PACKAGES
fi
PYTHON_EXECUTABLE="python"
fi
$PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only --num-shards 4 --shard-index ${{ matrix.shard-index }}
stub-uploader:
name: Run the stub_uploader tests
if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
steps:
- name: Checkout typeshed
uses: actions/checkout@v4
with:
path: typeshed
- name: Checkout stub_uploader
uses: actions/checkout@v4
with:
repository: typeshed-internal/stub_uploader
path: stub_uploader
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: astral-sh/setup-uv@v5
- name: Run tests
run: |
cd stub_uploader
uv pip install -r requirements.txt --system
python -m pytest tests
# https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2
create-issue-on-failure:
name: Create an issue if daily tests failed
runs-on: ubuntu-latest
needs: [stubtest-stdlib, stubtest-third-party, stub-uploader]
if: ${{ github.repository == 'python/typeshed' && always() && github.event_name == 'schedule' && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure' || needs.stub-uploader.result == 'failure') }}
permissions:
issues: write
steps:
- uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.create({
owner: "python",
repo: "typeshed",
title: `Daily tests failed on ${new Date().toDateString()}`,
body: "Run listed here: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
labels: ["help wanted"],
})
.github/workflows/meta_tests.yml
-80
View File
@@ -1,80 +0,0 @@
# This workflow is for testing typeshed's scripts and tests themselves
name: Meta-tests
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
paths:
- "scripts/**"
- "tests/**"
- "lib/**"
- ".github/workflows/meta_tests.yml"
- "requirements-tests.txt"
- "pyproject.toml"
permissions:
contents: read
env:
PIP_DISABLE_PIP_VERSION_CHECK: 1
FORCE_COLOR: 1
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
mypy:
name: Run mypy against the scripts and tests directories
runs-on: ubuntu-latest
strategy:
matrix:
platform: ["linux", "win32"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- run: uv pip install -r requirements-tests.txt --system
- run: python ./tests/typecheck_typeshed.py --platform=${{ matrix.platform }}
pyright:
name: Run pyright against the scripts and tests directories
runs-on: ubuntu-latest
strategy:
matrix:
python-platform: ["Linux", "Windows"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- run: uv pip install -r requirements-tests.txt --system
- name: Run pyright on typeshed
uses: jakebailey/pyright-action@v2
with:
version: PATH
python-platform: ${{ matrix.python-platform }}
python-version: "3.9" # Oldest version supported for running scripts and tests
project: ./pyrightconfig.scripts_and_tests.json
stubsabot-dry-run:
name: Stubsabot dry run
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: Git config
run: |
git config --global user.name stubsabot
git config --global user.email '<>'
- run: uv pip install -r requirements-tests.txt --system
- run: python scripts/stubsabot.py --action-level local
.github/workflows/mypy_primer.yml
-91
View File
@@ -1,91 +0,0 @@
name: Run mypy_primer
on:
# Only run on PR, since we diff against main
pull_request:
paths:
- "stdlib/**"
- "stubs/**/*.pyi"
- ".github/workflows/mypy_primer.yml"
- ".github/workflows/mypy_primer_comment.yml"
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
jobs:
mypy_primer:
name: Run
runs-on: ubuntu-latest
strategy:
matrix:
shard-index: [0, 1, 2, 3]
fail-fast: false
steps:
- uses: actions/checkout@v4
with:
path: typeshed_to_test
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Install dependencies
run: pip install git+https://github.com/hauntsaninja/mypy_primer.git
- name: Run mypy_primer
shell: bash
run: |
cd typeshed_to_test
MYPY_VERSION=$(grep mypy== requirements-tests.txt | cut -d = -f 3)
echo "new commit"
git rev-list --format=%s --max-count=1 $GITHUB_SHA
git checkout -b upstream_main origin/main
echo "base commit"
git rev-list --format=%s --max-count=1 upstream_main
echo ''
cd ..
# fail action if exit code isn't zero or one
(
mypy_primer \
--new v${MYPY_VERSION} --old v${MYPY_VERSION} \
--custom-typeshed-repo typeshed_to_test \
--new-typeshed $GITHUB_SHA --old-typeshed upstream_main \
--num-shards 4 --shard-index ${{ matrix.shard-index }} \
--debug \
--output concise \
| tee diff_${{ matrix.shard-index }}.txt
) || [ $? -eq 1 ]
- if: ${{ matrix.shard-index == 0 }}
name: Save PR number
run: |
echo ${{ github.event.pull_request.number }} | tee pr_number.txt
- name: Upload mypy_primer diff + PR number
uses: actions/upload-artifact@v4
if: ${{ matrix.shard-index == 0 }}
with:
name: mypy_primer_diffs-${{ matrix.shard-index }}
path: |
diff_${{ matrix.shard-index }}.txt
pr_number.txt
- name: Upload mypy_primer diff
uses: actions/upload-artifact@v4
if: ${{ matrix.shard-index != 0 }}
with:
name: mypy_primer_diffs-${{ matrix.shard-index }}
path: diff_${{ matrix.shard-index }}.txt
join_artifacts:
name: Join artifacts
runs-on: ubuntu-latest
needs: [mypy_primer]
permissions:
contents: read
steps:
- name: Merge artifacts
uses: actions/upload-artifact/merge@v4
with:
name: mypy_primer_diffs
pattern: mypy_primer_diffs-*
delete-merged: true
.github/workflows/mypy_primer_comment.yml
-87
View File
@@ -1,87 +0,0 @@
name: Comment with mypy_primer diff
on:
workflow_run:
workflows:
- Run mypy_primer
types:
- completed
permissions:
contents: read
pull-requests: write
jobs:
comment:
name: Comment PR from mypy_primer
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'success' }}
steps:
- name: Download diffs
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{ github.event.workflow_run.id }},
});
const [matchArtifact] = artifacts.data.artifacts.filter((artifact) =>
artifact.name == "mypy_primer_diffs");
const download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: "zip",
});
fs.writeFileSync("diff.zip", Buffer.from(download.data));
- run: unzip diff.zip
- run: |
cat diff_*.txt | tee fulldiff.txt
- name: Post comment
id: post-comment
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs')
let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' })
// Maximum comment length is 65536 characters. We need much less than 236 for extra text.
const MAX_LENGTH = 65300
if (data.length > MAX_LENGTH) {
let truncated_data = data.substring(0, MAX_LENGTH)
let lines_truncated = data.split('\n').length - truncated_data.split('\n').length
data = truncated_data + `\n\n... (truncated ${lines_truncated} lines) ...\n`
}
console.log("Diff from mypy_primer:")
console.log(data)
let body
if (data.trim()) {
body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```'
} else {
body = 'According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change has no effect on the checked open source code. 🤖🎉'
}
const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" }))
await github.rest.issues.createComment({
issue_number: prNumber,
owner: context.repo.owner,
repo: context.repo.repo,
body
})
return prNumber
- name: Hide old comments
# v0.4.0
uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
leave_visible: 1
issue_number: ${{ steps.post-comment.outputs.result }}
.github/workflows/stubsabot.yml
-56
View File
@@ -1,56 +0,0 @@
name: Run stubsabot daily
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
permissions:
contents: write
issues: write
pull-requests: write
env:
FORCE_COLOR: 1
jobs:
stubsabot:
name: Upgrade stubs with stubsabot
if: github.repository == 'python/typeshed'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
# use an ssh key so that checks automatically run on stubsabot PRs
ssh-key: ${{ secrets.STUBSABOT_SSH_PRIVATE_KEY }}
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: git config
run: |
git config --global user.name stubsabot
git config --global user.email '<>'
- name: Install dependencies
run: uv pip install -r requirements-tests.txt --system
- name: Run stubsabot
run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python scripts/stubsabot.py --action-level everything
# https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2
create-issue-on-failure:
name: Create an issue if stubsabot failed
runs-on: ubuntu-latest
needs: [stubsabot]
if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubsabot.result == 'failure') }}
steps:
- uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.create({
owner: "python",
repo: "typeshed",
title: `Stubsabot failed on ${new Date().toDateString()}`,
body: "Stubsabot run is listed here: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
})
.github/workflows/stubtest_stdlib.yml
-52
View File
@@ -1,52 +0,0 @@
name: Stdlib stubtest
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
paths:
- "stdlib/**"
- ".github/workflows/stubtest_stdlib.yml"
- "tests/**"
# When requirements.txt changes, we run `daily.yml`, which includes stdlib stubtest
permissions:
contents: read
env:
PIP_DISABLE_PIP_VERSION_CHECK: 1
FORCE_COLOR: 1
TERM: xterm-256color # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
stubtest-stdlib:
name: Check stdlib with stubtest
runs-on: ${{ matrix.os }}
strategy:
matrix:
# As of 2024-10-18, ubuntu-latest can refer to different Ubuntu versions,
# which can can cause problems with os module constants.
os: ["ubuntu-24.04", "windows-latest", "macos-latest"]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: requirements-tests.txt
allow-prereleases: true
check-latest: true
- name: Install dependencies
run: pip install -r requirements-tests.txt
- name: Run stubtest
run: python tests/stubtest_stdlib.py
.github/workflows/stubtest_third_party.yml
-95
View File
@@ -1,95 +0,0 @@
name: Third-party stubtest
on:
pull_request:
paths:
- "stubs/**"
- ".github/workflows/stubtest_third_party.yml"
- "tests/**"
# When requirements.txt changes, we run `daily.yml`, which includes third-party stubtest
permissions:
contents: read
env:
# A few env vars to speedup brew install
HOMEBREW_NO_ANALYTICS: 1
HOMEBREW_NO_AUTOUPDATE: 1
HOMEBREW_NO_INSTALL_CLEANUP: 1 # Environments are isolated, no need to cleanup old versions
NONINTERACTIVE: 1 # Required for brew install on CI
PIP_DISABLE_PIP_VERSION_CHECK: 1
FORCE_COLOR: 1
TERM: xterm-256color # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
stubtest-third-party:
name: Check third party stubs with stubtest
runs-on: ${{ matrix.os }}
strategy:
matrix:
# As of 2024-10-18, ubuntu-latest can refer to different Ubuntu versions,
# which causes problems when testing gdb.
os: ["ubuntu-24.04", "windows-latest", "macos-latest"]
fail-fast: false
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
# TODO: Use Python 3.12. As of 2024-03-08, several third-party
# packages fail to install with Python 3.12.
python-version: "3.11"
cache: pip
cache-dependency-path: |
requirements-tests.txt
stubs/**/METADATA.toml
- name: Install dependencies
run: pip install -r requirements-tests.txt
- name: Run stubtest
shell: bash
run: |
# This only runs stubtest on changed stubs, because it is much faster.
# Use the daily.yml workflow to run stubtest on all third party stubs.
function find_stubs {
git diff --name-only origin/${{ github.base_ref }} HEAD | \
egrep ^stubs/ | cut -d "/" -f 2 | sort -u | \
(while read stub; do [ -d "stubs/$stub" ] && echo "$stub" || true; done)
}
STUBS=$(find_stubs || echo '')
if [ -n "$STUBS" ]; then
echo "Testing $STUBS..."
PACKAGES=$(python tests/get_stubtest_system_requirements.py $STUBS)
if [ "${{ runner.os }}" = "Linux" ]; then
if [ -n "$PACKAGES" ]; then
echo "Installing apt packages: $PACKAGES"
sudo apt-get update && sudo apt-get install -y $PACKAGES
fi
PYTHON_EXECUTABLE="xvfb-run python"
else
if [ "${{ runner.os }}" = "macOS" ] && [ -n "$PACKAGES" ]; then
echo "Installing Homebrew packages: $PACKAGES"
brew install -q $PACKAGES
fi
if [ "${{ runner.os }}" = "Windows" ] && [ -n "$PACKAGES" ]; then
echo "Installing Chocolatey packages: $PACKAGES"
choco install -y $PACKAGES
fi
PYTHON_EXECUTABLE="python"
fi
$PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only $STUBS
else
echo "Nothing to test"
fi
.github/workflows/tests.yml
-167
View File
@@ -1,167 +0,0 @@
name: Test
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
paths-ignore:
- "**/*.md"
- "scripts/**"
permissions:
contents: read
env:
PIP_DISABLE_PIP_VERSION_CHECK: 1
FORCE_COLOR: 1
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
typeshed-structure:
name: Check typeshed structure
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- run: uv pip install -r requirements-tests.txt --system
- run: python ./tests/check_typeshed_structure.py
pytype:
name: Run pytype against the stubs
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
# Max supported Python version as of pytype 2024.9.13
python-version: "3.12"
- uses: astral-sh/setup-uv@v5
- run: uv pip install -r requirements-tests.txt --system
- name: Install external dependencies for 3rd-party stubs
run: |
DEPENDENCIES=$( python tests/get_external_stub_requirements.py )
if [ -n "$DEPENDENCIES" ]; then
printf "Installing packages:\n $(echo $DEPENDENCIES | sed 's/ /\n /g')\n"
uv pip install --system $DEPENDENCIES
fi
- run: uv pip freeze
- run: ./tests/pytype_test.py --print-stderr
mypy:
name: Run mypy against the stubs
runs-on: ubuntu-latest
strategy:
matrix:
platform: ["linux", "win32", "darwin"]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- run: uv pip install -r requirements-tests.txt --system
- run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }}
regression-tests:
name: Run mypy on the test cases
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
# Use py311 for now, as py312 seems to be around 30s slower in CI
# TODO: figure out why that is (#11590)
python-version: "3.11"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- run: uv pip install -r requirements-tests.txt --system
- run: python ./tests/regr_test.py --all --verbosity QUIET
pyright:
name: Test typeshed with pyright
runs-on: ubuntu-latest
strategy:
matrix:
python-platform: ["Linux", "Windows", "Darwin"]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: astral-sh/setup-uv@v5
- name: Install typeshed test-suite requirements
# Install these so we can run `get_external_stub_requirements.py`
run: uv pip install -r requirements-tests.txt --system
- name: Create an isolated venv for testing
run: uv venv .venv
- name: Install 3rd-party stub dependencies
run: |
DEPENDENCIES=$( python tests/get_external_stub_requirements.py )
if [ -n "$DEPENDENCIES" ]; then
printf "Installing packages:\n $(echo $DEPENDENCIES | sed 's/ /\n /g')\n"
# TODO: We need to specify the platform here, but the platforms
# strings supported by uv are different from the ones supported by
# pyright.
uv pip install --python-version ${{ matrix.python-version }} $DEPENDENCIES
fi
- name: Activate the isolated venv for the rest of the job
run: echo "$PWD/.venv/bin" >> $GITHUB_PATH
- name: List 3rd-party stub dependencies installed
run: uv pip freeze
- name: Run pyright with basic settings on all the stubs
uses: jakebailey/pyright-action@v2
with:
version: PATH
python-platform: ${{ matrix.python-platform }}
python-version: ${{ matrix.python-version }}
annotate: ${{ matrix.python-version == '3.12' && matrix.python-platform == 'Linux' }} # Having each job create the same comment is too noisy.
- name: Run pyright with stricter settings on some of the stubs
uses: jakebailey/pyright-action@v2
with:
version: PATH
python-platform: ${{ matrix.python-platform }}
python-version: ${{ matrix.python-version }}
annotate: ${{ matrix.python-version == '3.12' && matrix.python-platform == 'Linux' }} # Having each job create the same comment is too noisy.
project: ./pyrightconfig.stricter.json
- name: Run pyright on the test cases
uses: jakebailey/pyright-action@v2
with:
version: PATH
python-platform: ${{ matrix.python-platform }}
python-version: ${{ matrix.python-version }}
annotate: ${{ matrix.python-version == '3.12' && matrix.python-platform == 'Linux' }} # Having each job create the same comment is too noisy.
project: ./pyrightconfig.testcases.json
stub-uploader:
name: Run the stub_uploader tests
runs-on: ubuntu-latest
steps:
- name: Checkout typeshed
uses: actions/checkout@v4
with:
path: typeshed
- name: Checkout stub_uploader
uses: actions/checkout@v4
with:
repository: typeshed-internal/stub_uploader
path: stub_uploader
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: astral-sh/setup-uv@v5
- name: Run tests
run: |
cd stub_uploader
uv pip install -r requirements.txt --system
python -m pytest tests
.gitignore
+12 -20
View File
@@ -7,16 +7,17 @@ __pycache__/
# Distribution / packaging
.Python
/env/
/lib/build/
/develop-eggs/
/dist/
/downloads/
/eggs/
/lib64/
/parts/
/sdist/
/var/
env/
build/
develop-eggs/
dist/
downloads/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
@@ -38,7 +39,6 @@ htmlcov/
.cache
nosetests.xml
coverage.xml
stubtest-output*
# Translations
*.mo
@@ -59,17 +59,9 @@ analyze.py
# Editor backup files
*~
.*.sw?
.vscode/*
!.vscode/settings.default.json
!.vscode/extensions.json
.vscode/
.idea/
.venv*/
# Mypy cache
.mypy_cache/
# pyenv local python version
.python-version
# deliberately local test configuration files
stdlib/@tests/stubtest_allowlists/*.local
.pre-commit-config.yaml
-52
View File
@@ -1,52 +0,0 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-toml
- id: check-merge-conflict
- id: mixed-line-ending
args: [--fix=lf]
- id: check-case-conflict
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.9.3 # must match requirements-tests.txt
hooks:
- id: ruff
name: Run ruff on stubs, tests and scripts
args: ["--exit-non-zero-on-fix"]
- id: ruff
# Very few rules are useful to run on our test cases;
# we explicitly enumerate them here:
name: Run ruff on the test cases
args:
- "--exit-non-zero-on-fix"
- "--select=FA,I,RUF100"
- "--no-force-exclude"
- "--unsafe-fixes"
files: '.*test_cases/.+\.py$'
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 25.1.0
hooks:
- id: black
- repo: https://github.com/pycqa/flake8
rev: 7.1.1
hooks:
- id: flake8
language: python
additional_dependencies:
- "flake8-noqa==1.4.0"
- "flake8-pyi==24.9.0"
types: [file]
types_or: [python, pyi]
- repo: meta
hooks:
- id: check-hooks-apply
ci:
autofix_commit_msg: "[pre-commit.ci] auto fixes from pre-commit.com hooks"
autofix_prs: true
autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate"
autoupdate_schedule: quarterly
submodules: false
.travis.yml
+31
View File
@@ -0,0 +1,31 @@
dist: bionic
language: python
python: 3.8
jobs:
include:
- name: "pytype"
python: 3.6
install: pip install -r requirements-tests-py3.txt
script: ./tests/pytype_test.py
- name: "mypy (typed-ast)"
python: 3.7
install: pip install -U git+git://github.com/python/mypy git+git://github.com/python/typed_ast
script: ./tests/mypy_test.py --platform=linux
- name: "mypy (ast)"
python: 3.8
install: pip install -U git+git://github.com/python/mypy
script: ./tests/mypy_test.py --platform=linux
- name: "mypy (Windows)"
install: pip install -U git+git://github.com/python/mypy
script: ./tests/mypy_test.py --platform=win32
- name: "mypy (Darwin)"
install: pip install -U git+git://github.com/python/mypy
script: ./tests/mypy_test.py --platform=darwin
- name: "mypy self test"
script: ./tests/mypy_selftest.py
- name: "check file consistency"
script: ./tests/check_consistent.py
- name: "flake8"
install: pip install -r requirements-tests-py3.txt
script: flake8
.vscode/extensions.json
Vendored
-41
View File
@@ -1,41 +0,0 @@
// ⚠ Disclaimer: The typeshed team doesn't commit to maintaining this file. It exists purely for your ease of use.
// Keep in alphabetical order
{
"recommendations": [
"bierner.github-markdown-preview",
"charliermarsh.ruff",
"editorconfig.editorconfig",
"ms-python.black-formatter",
"ms-python.flake8",
"ms-python.mypy-type-checker",
"ms-python.python",
"ms-python.vscode-pylance",
"tamasfe.even-better-toml",
"redhat.vscode-yaml",
],
"unwantedRecommendations": [
/*
* Don't recommend by default for this workspace
*/
"christian-kohler.npm-intellisense",
/*
* Must disable in this workspace
* https://github.com/microsoft/vscode/issues/40239
*/
// even-better-toml has format on save
"bungcip.better-toml",
// Don't use two mypy extensions simultaneously
"matangover.mypy",
// Use Ruff instead
"ms-python.isort",
// We use Black
"ms-python.autopep8",
// Not using pylint
"ms-python.pylint",
// VSCode has implemented an optimized version
"coenraads.bracket-pair-colorizer",
"coenraads.bracket-pair-colorizer-2",
// Obsoleted by Pylance
"ms-pyright.pyright",
],
}
.vscode/settings.default.json
Vendored
-125
View File
@@ -1,125 +0,0 @@
/*
* Copy this file as `.vscode/settings.json` to configure VSCode for this workspace.
* Unfortunately, VSCode doesn't (yet) offer any way to have "workspace defaults" or "user-worspace settings",
* so offering defaults to copy is the best we can do at the moment.
*
* ⚠ Disclaimer: The typeshed team doesn't commit to maintaining this file. It exists purely for your ease of use.
*/
{
// Don't format on save for formatters we don't explicitely control
"editor.formatOnSave": false,
"editor.codeActionsOnSave": {
"source.fixAll": "never"
},
// Set file associations to support comments syntax highlight
"files.associations": {
"settings.default.json": "jsonc",
"pyrightconfig*.json": "jsonc",
".flake8": "properties",
"stubtest_allowlist*.txt": "properties",
"**/stubtest_allowlists/*.txt": "properties",
"pytype_exclude_list.txt": "properties"
},
"files.exclude": {
"**/.*_cache": true, // mypy and Ruff cache
"**/__pycache__": true
},
"files.eol": "\n",
"files.insertFinalNewline": true,
"files.trimFinalNewlines": true,
"files.trimTrailingWhitespace": true,
"editor.comments.insertSpace": true,
"editor.insertSpaces": true,
"editor.detectIndentation": false,
"editor.tabSize": 2,
"[json][jsonc][python][toml]": {
"editor.tabSize": 4
},
"editor.rulers": [
90,
130
],
"[git-commit]": {
"editor.rulers": [
72
]
},
// Format on save for formatters we explicitely control
"[json][jsonc][yaml][python][toml]": {
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.fixAll": "explicit"
}
},
"[json][jsonc]": {
"editor.defaultFormatter": "vscode.json-language-features"
},
"[yaml]": {
"editor.defaultFormatter": "redhat.vscode-yaml"
},
"[toml]": {
"editor.rulers": [
90
],
"editor.defaultFormatter": "tamasfe.even-better-toml"
},
"[python]": {
"editor.rulers": [
130
],
"editor.defaultFormatter": "ms-python.black-formatter",
"editor.codeActionsOnSave": {
// Let Ruff lint fixes handle imports
"source.organizeImports": "never",
"source.unusedImports": "never"
}
},
// python.analysis is Pylance (pyright) configurations
"python.analysis.fixAll": [
// Explicitly omiting "source.convertImportFormat", some stubs use relative imports
// Explicitly omiting "source.unusedImports", Let Ruff lint fixes handle imports
],
"python.analysis.typeshedPaths": [
"${workspaceFolder}"
],
"python.analysis.extraPaths": [
"tests"
],
"mypy-type-checker.importStrategy": "fromEnvironment",
"mypy-type-checker.args": [
"--custom-typeshed-dir=${workspaceFolder}",
// We only guarantee all of our tests can be run if you're on Python 3.9 or higher
"--python-version=3.9",
"--strict",
// Needed because a library stubbed in typeshed won't necessarily be installed in the dev's environment
"--ignore-missing-imports"
],
// Ensure typeshed's configs are used, and not user's VSCode settings
"flake8.args": [
"--config=.flake8"
],
"flake8.importStrategy": "fromEnvironment",
"black-formatter.importStrategy": "fromEnvironment",
// Using Ruff instead of isort
"isort.check": false,
"ruff.importStrategy": "fromEnvironment",
"ruff.fixAll": true,
"ruff.organizeImports": true,
"evenBetterToml.formatter.alignComments": false,
"evenBetterToml.formatter.alignEntries": false,
"evenBetterToml.formatter.allowedBlankLines": 1,
"evenBetterToml.formatter.arrayAutoCollapse": true,
"evenBetterToml.formatter.arrayAutoExpand": true,
"evenBetterToml.formatter.arrayTrailingComma": true,
"evenBetterToml.formatter.columnWidth": 90,
"evenBetterToml.formatter.compactArrays": true,
"evenBetterToml.formatter.compactEntries": false,
"evenBetterToml.formatter.compactInlineTables": false,
"evenBetterToml.formatter.indentEntries": false,
"evenBetterToml.formatter.indentTables": false,
"evenBetterToml.formatter.inlineTableExpand": false,
"evenBetterToml.formatter.reorderArrays": true,
"evenBetterToml.formatter.trailingNewline": true,
// We like keeping TOML keys in a certain non-alphabetical order that feels more natural
"evenBetterToml.formatter.reorderKeys": false
}
CONTRIBUTING.md
+298 -418
View File
@@ -1,234 +1,95 @@
# Contributing to typeshed
Welcome! typeshed is a community project that aims to work for a wide
range of Python users and Python codebases. If you're trying a type
Welcome! typeshed is a community project that aims to work for a wide
range of Python users and Python codebases. If you're trying a type
checker on your Python code, your experience and what you can contribute
are important to the project's success.
## The contribution process at a glance
1. [Prepare your environment](#preparing-the-environment).
2. Find out [where to make your changes](#where-to-make-changes).
1. Read the [README.md file](README.md).
2. Set up your environment to be able to [run all tests](README.md#running-the-tests). They should pass.
3. [Prepare your changes](#preparing-changes):
* Small fixes and additions can be submitted directly as pull requests,
but [contact us](README.md#discussion) before starting significant work.
* Create your stubs, considering [what to include](#what-to-include) and
conforming to the [coding style](#stub-file-coding-style).
4. Optionally [format and check your stubs](#code-formatting).
5. Optionally [run the tests](tests/README.md).
6. [Submit your changes](#submitting-changes) by opening a pull request.
7. Make sure that all tests in CI are passing.
but [contact us](#discussion) before starting significant work.
* Create your stubs [conforming to the coding style](#stub-file-coding-style).
* Make sure your tests pass cleanly on `mypy`, `pytype`, and `flake8`.
4. [Submit your changes](#submitting-changes) by opening a pull request.
5. You can expect a reply within a few days:
* Diffs are merged when considered ready by the core team.
* Feel free to ping the core team if your pull request goes without
a reply for more than a few days.
You can expect a reply within a few days, but please be patient when
it takes a bit longer. For more details, read below.
For more details, read below.
## Preparing the environment
### Code away!
## Discussion
Typeshed runs continuous integration (CI) on all pull requests. This means that
if you file a pull request (PR), our full test suite
-- including our linter, [Flake8](https://github.com/PyCQA/flake8) --
is run on your PR. It also means that bots will automatically apply
changes to your PR (using [Black](https://github.com/psf/black) and
[Ruff](https://github.com/astral-sh/ruff)) to fix any formatting issues.
This frees you up to ignore all local setup on your side, focus on the
code and rely on the CI to fix everything, or point you to the places that
need fixing.
If you've run into behavior in the type checker that suggests the type
stubs for a given library are incorrect or incomplete,
we want to hear from you!
### ... Or create a local development environment
Our main forum for discussion is the project's [GitHub issue
tracker](https://github.com/python/typeshed/issues). This is the right
place to start a discussion of any of the above or most any other
topic concerning the project.
If you prefer to run the tests and formatting locally, it's
possible too. Follow platform-specific instructions below.
For more information about our available tests, see
[tests/README.md](tests/README.md).
For less formal discussion, try the typing chat room on
[gitter.im](https://gitter.im/python/typing). Some Mypy core developers
are almost always present; feel free to find us there and we're happy
to chat. Substantive technical discussion will be directed to the
issue tracker.
Whichever platform you're using, you will need a
virtual environment. If you're not familiar with what it is and how it works,
please refer to this
[documentation](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/).
### Code of Conduct
Note that some tests require extra setup steps to install the required dependencies.
Everyone participating in the typeshed community, and in particular in
our issue tracker, pull requests, and IRC channel, is expected to treat
other people with respect and more generally to follow the guidelines
articulated in the [Python Community Code of
Conduct](https://www.python.org/psf/codeofconduct/).
### Linux/Mac OS/WSL
On Linux and Mac OS, you will be able to run the full test suite on Python
3.9-3.12.
To install the necessary requirements, run the following commands from a
terminal window:
## Submitting Changes
```bash
$ python3 -m venv .venv
$ source .venv/bin/activate
(.venv)$ pip install -U pip
(.venv)$ pip install -r requirements-tests.txt
```
Even more excellent than a good bug report is a fix for a bug, or the
implementation of a much-needed stub. We'd love to have
your contributions.
### Windows
We use the usual GitHub pull-request flow, which may be familiar to
you if you've contributed to other projects on GitHub. For the
mechanics, see [Mypy's git and GitHub workflow help page](https://github.com/python/mypy/wiki/Using-Git-And-GitHub),
or [GitHub's own documentation](https://help.github.com/articles/using-pull-requests/).
Run the following commands from a Windows terminal to install all requirements:
Anyone interested in type stubs may review your code. One of the core
developers will merge your pull request when they think it's ready.
For every pull request, we aim to promptly either merge it or say why
it's not yet ready; if you go a few days without a reply, please feel
free to ping the thread by adding a new comment.
```powershell
> python -m venv .venv
> .venv\Scripts\activate
(.venv) > pip install -U pip
(.venv) > pip install -r "requirements-tests.txt"
```
To get your pull request merged sooner, you should explain why you are
making the change. For example, you can point to a code sample that is
processed incorrectly by a type checker. It is also helpful to add
links to online documentation or to the implementation of the code
you are changing.
To be able to run pytype tests, you'll also need to install it manually
as it's currently excluded from the requirements file:
Also, do not squash your commits after you have submitted a pull request, as this
erases context during review. We will squash commits when the pull request is merged.
```powershell
(.venv) > pip install -U pytype
```
At present the core developers are (alphabetically):
* David Fisher (@ddfisher)
* Łukasz Langa (@ambv)
* Jukka Lehtosalo (@JukkaL)
* Ivan Levkivskyi (@ilevkivskyi)
* Matthias Kramm (@matthiaskramm)
* Greg Price (@gnprice)
* Sebastian Rittau (@srittau)
* Guido van Rossum (@gvanrossum)
* Jelle Zijlstra (@JelleZijlstra)
## Code formatting
The code is formatted using [`Black`](https://github.com/psf/black).
Various other autofixes and lint rules are
also performed by [`Ruff`](https://github.com/astral-sh/ruff) and
[`Flake8`](https://github.com/pycqa/flake8),
with plugins [`flake8-pyi`](https://github.com/pycqa/flake8-pyi),
and [`flake8-noqa`](https://github.com/plinss/flake8-noqa).
The repository is equipped with a [pre-commit.ci](https://pre-commit.ci/)
configuration file. This means that you don't *need* to do anything yourself to
run the code formatters or linters. When you push a commit, a bot will run
those for you right away and add any autofixes to your PR. Anything
that can't be autofixed will show up as a CI failure, hopefully with an error
message that will make it clear what's gone wrong.
That being said, if you *want* to run the formatters and linters locally
when you commit, you're free to do so. To use the same configuration as we use
in CI, we recommend doing this via pre-commit:
```bash
(.venv)$ pre-commit run --all-files
```
## Where to make changes
### Standard library stubs
The `stdlib` directory contains stubs for modules in the
Python standard library — which
includes pure Python modules, dynamically loaded extension modules,
hard-linked extension modules, and the builtins. The `VERSIONS` file lists
the versions of Python where the module is available.
We accept changes for future versions of Python after the first beta for that
version was released. We drop support for a Python version three months
after it reaches [end-of-life](https://devguide.python.org/versions/). This
means that we will no longer actively test the stubs against that version.
After six months, we will remove the stubs for that version and start
to use syntax and typing features not supported by that version.
### Third-party library stubs
We accept stubs for third-party packages into typeshed as long as:
* the package is publicly available on the [Python Package Index](https://pypi.org/);
* the package supports any Python version supported by typeshed; and
* the package does not ship with its own stubs or type annotations.
The fastest way to generate new stubs is to use `scripts/create_baseline_stubs.py` (see below).
Stubs for third-party packages go into the `stubs` directory. Each subdirectory
there represents a PyPI distribution, and contains the following:
* `METADATA.toml`, describing the package. See below for details.
* Stubs (i.e. `*.pyi` files) for packages and modules that are shipped in the
source distribution.
* (Rarely) some docs specific to a given type stub package in `README` file.
When a third party stub is added or
modified, an updated version of the corresponding distribution will be
automatically uploaded to PyPI within a few hours.
Each time this happens the least significant
version level is incremented. For example, if `stubs/foo/METADATA.toml` has
`version = "x.y"` the package on PyPI will be updated from `types-foo-x.y.n`
to `types-foo-x.y.n+1`.
*Note:* In its current implementation, typeshed cannot contain stubs for
multiple versions of the same third-party library. Prefer to generate
stubs for the latest version released on PyPI at the time of your
stubbing.
#### The `METADATA.toml` file
The metadata file describes the stubs package using the
[TOML file format](https://toml.io/en/). Currently, the following keys are
supported:
* `version`: The versions of the library that the stubs support. Two
formats are supported:
- A concrete version. This is especially suited for libraries that
use [Calendar Versioning](https://calver.org/).
- A version range ending in `.*`. This is suited for libraries that
reflect API changes in the version number only, where the API-independent
part is represented by the asterisk. In the case
of [Semantic Versioning](https://semver.org/), this version could look
like this: `2.7.*`.
When the stubs are updated to a newer version
of the library, the version of the stub should be bumped (note that
previous versions are still available on PyPI).
* `requires` (optional): A list of other stub packages or packages with type
information that are imported by the stubs in this package. Only packages
generated by typeshed or required by the upstream package are allowed to
be listed here, for security reasons. See
[this issue](https://github.com/typeshed-internal/stub_uploader/issues/90)
for more information about what external dependencies are allowed.
* `extra_description` (optional): Can be used to add a custom description to
the package's long description. It should be a multi-line string in
Markdown format.
* `stub_distribution` (optional): Distribution name to be uploaded to PyPI.
This defaults to `types-<distribution>` and should only be set in special
cases.
* `upstream_repository` (recommended): The URL of the upstream repository.
* `obsolete_since` (optional): This field is part of our process for
[removing obsolete third-party libraries](#third-party-library-removal-policy).
It contains the first version of the corresponding library that ships
its own `py.typed` file.
* `no_longer_updated` (optional): This field is set to `true` before removing
stubs for other reasons than the upstream library shipping with type
information.
* `upload` (optional): This field is set to `false` to prevent automatic
uploads to PyPI. This should only used in special cases, e.g. when the stubs
break the upload.
* `partial_stub` (optional): This field marks the type stub package as
[partial](https://peps.python.org/pep-0561/#partial-stub-packages). This is for
3rd-party stubs that don't cover the entirety of the package's public API.
* `requires_python` (optional): The minimum version of Python required to install
the type stub package. It must be in the form `>=3.*`. If omitted, the oldest
Python version supported by typeshed is used.
In addition, we specify configuration for stubtest in the `tool.stubtest` table.
This has the following keys:
* `skip` (default: `false`): Whether stubtest should be run against this
package. Please avoid setting this to `true`, and add a comment if you have
to.
* `ignore_missing_stub`: When set to `true`, this will add the
`--ignore_missing_stub` option to the stubtest call. See
[tests/README.md](./tests/README.md) for more information. In most cases,
this field should be identical to `partial_stub`.
* `stubtest_requirements` (default: `[]`): A list of Python packages that need
to be installed for stubtest to run successfully. These packages are installed
in addition to the requirements in the `requires` field.
* `apt_dependencies` (default: `[]`): A list of Ubuntu APT packages
that need to be installed for stubtest to run successfully.
* `brew_dependencies` (default: `[]`): A list of MacOS Homebrew packages
that need to be installed for stubtest to run successfully
* `choco_dependencies` (default: `[]`): A list of Windows Chocolatey packages
that need to be installed for stubtest to run successfully
* `platforms` (default: `["linux"]`): A list of OSes on which to run stubtest.
Can contain `win32`, `linux`, and `darwin` values.
If not specified, stubtest is run only on `linux`.
Only add extra OSes to the test
if there are platform-specific branches in a stubs package.
`*_dependencies` are usually packages needed to `pip install` the implementation
distribution.
The format of all `METADATA.toml` files can be checked by running
`python3 ./tests/check_typeshed_structure.py`.
NOTE: the process for preparing and submitting changes also applies to
core developers. This ensures high quality contributions and keeps
everybody on the same page. Avoid direct pushes to the repository.
## Preparing Changes
@@ -240,12 +101,6 @@ recommend starting by opening an issue laying out what you want to do.
That lets a conversation happen early in case other contributors disagree
with what you'd like to do or have ideas that will help you do it.
### Stub Content and Style
Each Python module is represented by a .pyi "stub file". This is a syntactically valid Python file, where all methods are empty and [type annotations](https://typing.readthedocs.io/en/latest/spec/annotations.html) are used to describe function signatures and variable types.
Typeshed follows the standard type system guidelines for [stub content](https://typing.readthedocs.io/en/latest/guides/writing_stubs.html#stub-content) and [coding style](https://typing.readthedocs.io/en/latest/guides/writing_stubs.html#style-guide).
### What to include
Stubs should include the complete interface (classes, functions,
@@ -261,27 +116,229 @@ or if they are not prefixed with an underscore. This means
that typeshed will generally accept contributions that add missing
objects, even if they are undocumented. Undocumented objects should
be marked with a comment of the form ``# undocumented``.
### Incomplete Annotations
When submitting new stubs, it is not necessary to annotate all arguments,
return types, and fields. Such items should either be left unannotated or
use `_typeshed.Incomplete` if this is not possible:
Example:
```python
from _typeshed import Incomplete
field: Incomplete # unannotated
def foo(x): ... # unannotated argument and return type
def list2cmdline(seq: Sequence[str]) -> str: ... # undocumented
```
`Incomplete` can also be used for partially known types:
We accept such undocumented objects because omitting objects can confuse
users. Users who see an error like "module X has no attribute Y" will
not know whether the error appeared because their code had a bug or
because the stub is wrong. Although it may also be helpful for a type
checker to point out usage of private objects, we usually prefer false
negatives (no errors for wrong code) over false positives (type errors
for correct code). In addition, even for private objects a type checker
can be helpful in pointing out that an incorrect type was used.
### Incomplete stubs
We accept partial stubs, especially for larger packages. These need to
follow the following guidelines:
* Included functions and methods must list all arguments, but the arguments
can be left unannotated. Do not use `Any` to mark unannotated arguments
or return values.
* Partial classes must include a `__getattr__()` method marked with an
`# incomplete` comment (see example below).
* Partial modules (i.e. modules that are missing some or all classes,
functions, or attributes) must include a top-level `__getattr__()`
function marked with an `# incomplete` comment (see example below).
* Partial packages (i.e. packages that are missing one or more sub-modules)
must have a `__init__.pyi` stub that is marked as incomplete (see above).
A better alternative is to create empty stubs for all sub-modules and
mark them as incomplete individually.
Example of a partial module with a partial class `Foo` and a partially
annotated function `bar()`:
```python
def foo(x: Incomplete | None = None) -> list[Incomplete]: ...
def __getattr__(name: str) -> Any: ... # incomplete
class Foo:
def __getattr__(self, name: str) -> Any: # incomplete
x: int
y: str
def bar(x: str, y, *, z=...): ...
```
### Using stubgen
Mypy includes a tool called [stubgen](https://mypy.readthedocs.io/en/latest/stubgen.html)
that auto-generates stubs for Python and C modules using static analysis,
Sphinx docs, and runtime introspection. It can be used to get a starting
point for your stubs. Note that this generator is currently unable to
determine most argument and return types and omits them or uses ``Any`` in
their place. Fill out manually the types that you know.
### Stub file coding style
#### Syntax example
The below is an excerpt from the types for the `datetime` module.
```python
MAXYEAR: int
MINYEAR: int
class date:
def __init__(self, year: int, month: int, day: int) -> None: ...
@classmethod
def fromtimestamp(cls, timestamp: float) -> date: ...
@classmethod
def today(cls) -> date: ...
@classmethod
def fromordinal(cls, ordinal: int) -> date: ...
@property
def year(self) -> int: ...
def replace(self, year: int = ..., month: int = ..., day: int = ...) -> date: ...
def ctime(self) -> str: ...
def weekday(self) -> int: ...
```
#### Conventions
Stub files are *like* Python files and you should generally expect them
to look the same. Your tools should be able to successfully treat them
as regular Python files. However, there are a few important differences
you should know about.
Style conventions for stub files are different from PEP 8. The general
rule is that they should be as concise as possible. Specifically:
* lines can be up to 130 characters long;
* functions and methods that don't fit in one line should be split up
with one argument per line;
* all function bodies should be empty;
* prefer ``...`` over ``pass``;
* prefer ``...`` on the same line as the class/function signature;
* avoid vertical whitespace between consecutive module-level functions,
names, or methods and fields within a single class;
* use a single blank line between top-level class definitions, or none
if the classes are very small;
* do not use docstrings;
* use variable annotations instead of type comments, even for stubs
that target older versions of Python;
* for arguments with a type and a default, use spaces around the `=`.
The code formatter [black](https://github.com/psf/black) will format
stubs according to this standard.
Stub files should only contain information necessary for the type
checker, and leave out unnecessary detail:
* for arguments with a default, use `...` instead of the actual
default;
* for arguments that default to `None`, use `Optional[]` explicitly
(see below for details);
* use `float` instead of `Union[int, float]`.
Some further tips for good type hints:
* avoid invariant collection types (`List`, `Dict`) in argument
positions, in favor of covariant types like `Mapping` or `Sequence`;
* avoid Union return types: https://github.com/python/mypy/issues/1693;
* in Python 2, whenever possible, use `unicode` if that's the only
possible type, and `Text` if it can be either `unicode` or `bytes`;
* use platform checks like `if sys.platform == 'win32'` to denote
platform-dependent APIs.
Imports in stubs are considered private (not part of the exported API)
unless:
* they use the form ``from library import name as name`` (sic, using
explicit ``as`` even if the name stays the same); or
* they use the form ``from library import *`` which means all names
from that library are exported.
When adding type hints, avoid using the `Any` type when possible. Reserve
the use of `Any` for when:
* the correct type cannot be expressed in the current type system; and
* to avoid Union returns (see above).
Note that `Any` is not the correct type to use if you want to indicate
that some function can accept literally anything: in those cases use
`object` instead.
For arguments with type and a default value of `None`, PEP 484
prescribes that the type automatically becomes `Optional`. However we
prefer explicit over implicit in this case, and require the explicit
`Optional[]` around the type. The mypy tests enforce this (through
the use of --no-implicit-optional) and the error looks like
`Incompatible types in assignment (expression has type None, variable
has type "Blah") `.
Stub files support forward references natively. In other words, the
order of class declarations and type aliases does not matter in
a stub file. You can also use the name of the class within its own
body. Focus on making your stubs clear to the reader. Avoid using
string literals in type annotations.
Type variables and aliases you introduce purely for legibility reasons
should be prefixed with an underscore to make it obvious to the reader
they are not part of the stubbed API.
When adding type annotations for context manager classes, annotate
the return type of `__exit__` as bool only if the context manager
sometimes suppresses annotations -- if it sometimes returns `True`
at runtime. If the context manager never suppresses exceptions,
have the return type be either `None` or `Optional[bool]`. If you
are not sure whether exceptions are suppressed or not or if the
context manager is meant to be subclassed, pick `Optional[bool]`.
See https://github.com/python/mypy/issues/7214 for more details.
NOTE: there are stubs in this repository that don't conform to the
style described above. Fixing them is a great starting point for new
contributors.
### Stub versioning
There are separate directories for `stdlib` and `third_party` stubs.
Within those, there are separate directories for different versions of
Python the stubs target.
The directory name indicates the major version of Python that a stub targets
and optionally the lowest minor version, with the exception of the `2and3`
directory which applies to both Python 2 and 3.
For example, stubs in the `3` directory will be applied to all versions of
Python 3, though stubs in the `3.7` directory will only be applied to versions
3.7 and above. However, stubs in the `2` directory will not be applied to
Python 3.
It is preferred to use a single stub in the more generic directory that
conditionally targets specific versions when needed, as opposed
to maintaining multiple stub files within more specific directories. Similarly,
if the given library works on both Python 2 and Python 3, prefer to put your
stubs in the `2and3` directory, unless the types are so different that the stubs
become unreadable that way.
You can use checks like `if sys.version_info >= (3, 8):` to denote new
functionality introduced in a given Python version or solve type
differences. When doing so, only use one-tuples or two-tuples. This is
because:
* mypy doesn't support more fine-grained version checks; and more
importantly
* the micro versions of a Python release will change over time in your
checking environment and the checker should return consistent results
regardless of the micro version used.
Because of this, if a given functionality was introduced in, say, Python
3.7.4, your check:
* should be expressed as `if sys.version_info >= (3, 7):`
* should NOT be expressed as `if sys.version_info >= (3, 7, 4):`
* should NOT be expressed as `if sys.version_info >= (3, 8):`
This makes the type checker assume the functionality was also available
in 3.7.0 - 3.7.3, which while *technically* incorrect is relatively
harmless. This is a strictly better compromise than using the latter
two forms, which would generate false positive errors for correct use
under Python 3.7.4.
Note: in its current implementation, typeshed cannot contain stubs for
multiple versions of the same third-party library. Prefer to generate
stubs for the latest version released on PyPI at the time of your
stubbing.
### What to do when a project's documentation and implementation disagree
Type stubs are meant to be external type annotations for a given
@@ -291,205 +348,28 @@ documentation. Whenever you find them disagreeing, model the type
information after the actual implementation and file an issue on the
project's tracker to fix their documentation.
### Docstrings
## Issue-tracker conventions
Typeshed stubs should not include duplicated docstrings from the source code.
We aim to reply to all new issues promptly. We'll assign one or more
labels to indicate we've triaged an issue, but most typeshed issues
are relatively simple (stubs for a given module or package are
missing, incomplete or incorrect) and we won't add noise to the
tracker by labeling all of them. Please see the
[list of all labels](https://github.com/python/typeshed/issues/labels)
for a detailed description of the labels we use.
### Auto-generating stub files
Sometimes a PR can't make progress until some external issue is
addressed. We indicate this by editing the subject to add a ``[WIP]``
prefix. (This should be removed before committing the issue once
unblocked!)
Typeshed includes `scripts/create_baseline_stubs.py`.
It generates stubs automatically using a tool called
[stubgen](https://mypy.readthedocs.io/en/latest/stubgen.html) that comes with mypy.
### Core developer guidelines
To get started, fork typeshed, clone your fork, and then
[create a virtualenv](#-or-create-a-local-development-environment).
You can then install the library with `pip` into the virtualenv and run the script below,
replacing `$INSERT_LIBRARY_NAME_HERE` with the name of the library:
```bash
(.venv)$ pip install $INSERT_LIBRARY_NAME_HERE
(.venv)$ python3 scripts/create_baseline_stubs.py $INSERT_LIBRARY_NAME_HERE
```
When the script has finished running, it will print instructions telling you what to do next.
If it has been a while since you set up the virtualenv, make sure you have
the latest mypy (`pip install -r requirements-tests.txt`) before running the script.
### Supported type system features
Since PEP 484 was accepted, there have been many other PEPs that added
new features to the Python type system. In general, new features can
be used in typeshed as soon as the PEP has been accepted and implemented
and most type checkers support the new feature.
Supported features include:
- [PEP 544](https://peps.python.org/pep-0544/) (Protocol)
- [PEP 585](https://peps.python.org/pep-0585/) (builtin generics)
- [PEP 586](https://peps.python.org/pep-0586/) (Literal)
- [PEP 591](https://peps.python.org/pep-0591/) (Final/@final)
- [PEP 589](https://peps.python.org/pep-0589/) (TypedDict)
- [PEP 604](https://peps.python.org/pep-0604/) (`Foo | Bar` union syntax)
- [PEP 612](https://peps.python.org/pep-0612/) (ParamSpec)
- [PEP 647](https://peps.python.org/pep-0647/) (TypeGuard):
see [#5406](https://github.com/python/typeshed/issues/5406)
- [PEP 655](https://peps.python.org/pep-0655/) (`Required` and `NotRequired`)
- [PEP 673](https://peps.python.org/pep-0673/) (`Self`)
- [PEP 675](https://peps.python.org/pep-0675/) (`LiteralString`)
- [PEP 702](https://peps.python.org/pep-0702/) (`@deprecated()`)
Features from the `typing` module that are not present in all
supported Python versions must be imported from `typing_extensions`
instead in typeshed stubs. This currently affects:
- `TypeAlias` (new in Python 3.10)
- `Concatenate` (new in Python 3.10)
- `ParamSpec` (new in Python 3.10)
- `TypeGuard` (new in Python 3.10)
- `Self` (new in Python 3.11)
- `Never` (new in Python 3.11)
- `LiteralString` (new in Python 3.11)
- `TypeVarTuple` and `Unpack` (new in Python 3.11)
- `Required` and `NotRequired` (new in Python 3.11)
- `Buffer` (new in Python 3.12; in the `collections.abc` module)
- `@deprecated` (new in Python 3.13; in the `warnings` module)
Some type checkers implicitly promote the `bytearray` and
`memoryview` types to `bytes`.
[PEP 688](https://www.python.org/dev/peps/pep-0688/) removes
this implicit promotion.
Typeshed stubs should be written assuming that these promotions
do not happen, so a parameter that accepts either `bytes` or
`bytearray` should be typed as `bytes | bytearray`.
Often one of the aliases from `_typeshed`, such as
`_typeshed.ReadableBuffer`, can be used instead.
## Submitting Changes
Even more excellent than a good bug report is a fix for a bug, or the
implementation of a much-needed stub. We'd love to have
your contributions.
We use the usual GitHub pull-request flow, which may be familiar to
you if you've contributed to other projects on GitHub. For the
mechanics, see [Mypy's git and GitHub workflow help page](https://github.com/python/mypy/wiki/Using-Git-And-GitHub),
or [GitHub's own documentation](https://help.github.com/articles/using-pull-requests/).
Anyone interested in type stubs may review your code. One of the
maintainers will merge your pull request when they think it's ready.
For every pull request, we aim to promptly either merge it or say why
it's not yet ready; if you go a few days without a reply, please feel
free to ping the thread by adding a new comment.
To get your pull request merged sooner, you should explain why you are
making the change. For example, you can point to a code sample that is
processed incorrectly by a type checker. It is also helpful to add
links to online documentation or to the implementation of the code
you are changing.
As the author of the pull request, it is your responsibility to make
sure all CI tests pass and that any feedback is addressed. The typeshed
maintainers will probably provide some help and may even push changes
to your PR to fix any minor issues, but this is not always possible.
If a PR lingers with unresolved problems for too long, we may close it
([see below](#closing-stale-prs)).
Also, do not squash your commits or use `git commit --amend` after you have submitted a pull request, as this
erases context during review. We will squash commits when the pull request is merged.
This way, your pull request will appear as a single commit in our git history, even
if it consisted of several smaller commits.
## Third-party library removal policy
Third-party stubs are generally removed from typeshed when one of the
following criteria is met:
* The upstream package ships a `py.typed` file for at least six months,
and the upstream type annotations are of a comparable standard to those in
typeshed, or
* the upstream package was declared or appears to be unmaintained, and
retaining the stubs causes maintenance issues in typeshed.
If a package ships its own `py.typed` file, please follow these steps:
1. Open an issue with the earliest month of removal in the subject.
2. A maintainer will add the
["stubs: removal" label](https://github.com/python/typeshed/labels/stubs%3A%20removal).
3. Open a PR that sets the `obsolete_since` field in the `METADATA.toml`
file to the first version of the package that shipped `py.typed`.
4. After at least six months, open a PR to remove the stubs.
If third-party stubs should be removed for other reasons, please follow these
steps:
1. Open an issue explaining why the stubs should be removed.
2. A maintainer will add the
["stubs: removal" label](https://github.com/python/typeshed/labels/stubs%3A%20removal).
3. Open a PR that sets the `no_longer_updated` field in the `METADATA.toml`
file to `true`.
4. When a new version of the package was automatically uploaded to PyPI
(which can take up to a day), open a PR to remove the stubs.
If feeling kindly, please update [mypy](https://github.com/python/mypy/blob/master/mypy/stubinfo.py)
for any stub obsoletions or removals.
## Maintainer guidelines
The process for preparing and submitting changes also applies to
maintainers. This ensures high quality contributions and keeps
everybody on the same page. Avoid direct pushes to the repository.
When reviewing pull requests, follow these guidelines:
* Typing is hard. Try to be helpful and explain issues with the PR,
especially to new contributors.
* When reviewing auto-generated stubs, just scan for red flags and obvious
errors. Leave possible manual improvements for separate PRs.
* When reviewing large, hand-crafted PRs, you only need to look for red flags
and general issues, and do a few spot checks.
* Review smaller, hand-crafted PRs thoroughly.
When merging pull requests, follow these guidelines:
Core developers should follow these rules when processing pull requests:
* Always wait for tests to pass before merging PRs.
* Use "[Squash and merge](https://github.com/blog/2141-squash-your-commits)" to merge PRs.
* Make sure the commit message is meaningful. For example, remove irrelevant
* Delete branches for merged PRs (by core devs pushing to the main repo).
* Make sure commit messages to master are meaningful. For example, remove irrelevant
intermediate commit messages.
* The commit message for third-party stubs is used to generate the changelog.
It should be valid Markdown, be comprehensive, read like a changelog entry,
and assume that the reader has no access to the diff.
* Delete branches for merged PRs (by maintainers pushing to the main repo).
### Marking PRs as "deferred"
We sometimes use the ["status: deferred" label](https://github.com/python/typeshed/labels/status%3A%20deferred)
to mark PRs and issues that we'd like to accept, but that are blocked by some
external factor. Blockers can include:
- An unambiguous bug in a type checker (i.e., a case where the
type checker is not implementing [the typing spec](https://typing.readthedocs.io/en/latest/spec/index.html)).
- A dependency on a typing PEP that is still under consideration.
- A pending change in a related project, such as stub-uploader.
PRs should only be marked as "deferred" if there is a clear path towards getting
the blocking issue resolved within a reasonable time frame. If a PR depends on
a more amorphous change, such as a type system change that has not yet reached
the PEP stage, it should instead be closed.
Maintainers who add the "deferred" label should state clearly what exactly the
blocker is, usually with a link to an open issue in another project.
### Closing stale PRs
To keep the number of open PRs manageable, we may close PRs when they have been
open for too long. Specifically, we close open PRs that either have failures in CI,
serious merge conflicts or unaddressed feedback, and that have not seen any
activity in three months.
We want to maintain a welcoming atmosphere for contributors, so use a friendly
message when closing the PR. Example message:
Thanks for contributing! I'm closing this PR for now, because it still
<fails some tests OR has unresolved review feedback OR has a merge conflict>
after three months of inactivity. If you are still interested, please feel free to open
a new PR (or ping us to reopen this one).
* If stubs for a new library are submitted, notify the library's maintainers.
LICENSE
+1
View File
@@ -235,3 +235,4 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
= = = = =
MAINTAINERS.md
-29
View File
@@ -1,29 +0,0 @@
At present the active maintainers are (alphabetically):
* Rebecca Chen (@rchen152)
* Jukka Lehtosalo (@JukkaL)
* Ivan Levkivskyi (@ilevkivskyi)
* Sebastian Rittau (@srittau)
* Guido van Rossum (@gvanrossum)
* Shantanu (@hauntsaninja)
* Nikita Sobolev (@sobolevn)
* Samuel Therrien (@Avasam)
* Aku Viljanen (@Akuli)
* Alex Waygood (@AlexWaygood)
* Jelle Zijlstra (@JelleZijlstra)
Former maintainers include:
* David Fisher (@ddfisher)
* Matthias Kramm (@matthiaskramm)
* Łukasz Langa (@ambv)
* Greg Price (@gnprice)
* Rune Tynan (@CraftSpider)
For security reasons, maintainers who haven't been active for twelve months
(no PR reviews or merges, no opened PRs, no significant participation in
issues or typing-related discussion) will have their access rights removed.
They will also be moved to the "former maintainers" section here.
Former maintainers who want their access rights restored should open
an issue or mail one of the active maintainers.
README.md
+128 -102
View File
@@ -1,8 +1,8 @@
# typeshed
[![Tests](https://github.com/python/typeshed/actions/workflows/tests.yml/badge.svg)](https://github.com/python/typeshed/actions/workflows/tests.yml)
[![Build Status](https://travis-ci.org/python/typeshed.svg?branch=master)](https://travis-ci.org/python/typeshed)
[![Chat at https://gitter.im/python/typing](https://badges.gitter.im/python/typing.svg)](https://gitter.im/python/typing?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Pull Requests Welcome](https://img.shields.io/badge/pull%20requests-welcome-brightgreen.svg)](https://github.com/python/typeshed/blob/main/CONTRIBUTING.md)
[![Pull Requests Welcome](https://img.shields.io/badge/pull%20requests-welcome-brightgreen.svg)](https://github.com/python/typeshed/blob/master/CONTRIBUTING.md)
## About
@@ -10,115 +10,141 @@ Typeshed contains external type annotations for the Python standard library
and Python builtins, as well as third party packages as contributed by
people external to those projects.
This data can e.g. be used for static analysis, type checking, type inference,
and autocompletion.
This data can e.g. be used for static analysis, type checking or type inference.
For information on how to use typeshed, read below. Information for
For information on how to use `typeshed`, read below. Information for
contributors can be found in [CONTRIBUTING.md](CONTRIBUTING.md). **Please read
it before submitting pull requests; do not report issues with annotations to
the project the stubs are for, but instead report them here to typeshed.**
Further documentation on stub files, typeshed, and Python's typing system in
general, can also be found at https://typing.readthedocs.io/en/latest/.
Typeshed supports Python versions 3.9 to 3.13.
Typeshed supports Python versions 2.7 and 3.5 and up.
## Using
If you're just using a type checker ([mypy](https://github.com/python/mypy/),
[pyright](https://github.com/microsoft/pyright),
[pytype](https://github.com/google/pytype/), PyCharm, ...), as opposed to
If you're just using mypy (or pytype or PyCharm), as opposed to
developing it, you don't need to interact with the typeshed repo at
all: a copy of standard library part of typeshed is bundled with type checkers.
And type stubs for third party packages and modules you are using can
be installed from PyPI. For example, if you are using `html5lib` and `requests`,
you can install the type stubs using
all: a copy of typeshed is bundled with mypy.
```bash
$ pip install types-html5lib types-requests
When you use a checked-out clone of the mypy repo, a copy of typeshed
should be included as a submodule, using
$ git clone --recurse-submodules https://github.com/python/mypy.git
or
$ git clone https://github.com/python/mypy.git
$ cd mypy
$ git submodule init
$ git submodule update
and occasionally you will have to repeat the final command (`git
submodule update`) to pull in changes made in the upstream typeshed
repo.
PyCharm and pytype similarly include a copy of typeshed. The one in
pytype can be updated in the same way if you are working with the
pytype repo.
## Format
Each Python module is represented by a `.pyi` "stub file". This is a
syntactically valid Python file, although it usually cannot be run by
Python 3 (since forward references don't require string quotes). All
the methods are empty.
Python function annotations ([PEP 3107](https://www.python.org/dev/peps/pep-3107/))
are used to describe the signature of each function or method.
See [PEP 484](http://www.python.org/dev/peps/pep-0484/) for the exact
syntax of the stub files and [CONTRIBUTING.md](CONTRIBUTING.md) for the
coding style used in typeshed.
## Directory structure
### stdlib
This contains stubs for modules the Python standard library -- which
includes pure Python modules, dynamically loaded extension modules,
hard-linked extension modules, and the builtins.
### third_party
Modules that are not shipped with Python but have a type description in Python
go into `third_party`. Since these modules can behave differently for different
versions of Python, `third_party` has version subdirectories, just like
`stdlib`.
NOTE: When you're contributing a new stub for a package that you did
not develop, please obtain consent of the package owner (this is
specified in [PEP
484](https://www.python.org/dev/peps/pep-0484/#the-typeshed-repo)).
The best way to obtain consent is to file an issue in the third-party
package's tracker and include the link to a positive response in your PR
for typeshed.
For more information on directory structure and stub versioning, see
[the relevant section of CONTRIBUTING.md](
https://github.com/python/typeshed/blob/master/CONTRIBUTING.md#stub-versioning).
## Contributing
Please read [CONTRIBUTING.md](CONTRIBUTING.md) before submitting pull
requests. If you have questions related to contributing, drop by the [typing Gitter](https://gitter.im/python/typing).
## Running the tests
The tests are automatically run by Travis CI on every PR and push to
the repo. There are several sets of tests: `tests/mypy_test.py`
runs tests against [mypy](https://github.com/python/mypy/), while
`tests/pytype_test.py` runs tests against
[pytype](https://github.com/google/pytype/).
Both sets of tests are shallow -- they verify that all stubs can be
imported but they don't check whether stubs match their implementation
(in the Python standard library or a third-party package). Also note
that each set of tests has a blacklist of modules that are not tested
at all. The blacklists also live in the tests directory.
In addition, you can run `tests/mypy_selftest.py` to run mypy's own
test suite using the typeshed code in your repo. This will sometimes
catch issues with incorrectly typed stubs, but is much slower than the
other tests.
To manually run the mypy tests, you need to have Python 3.5 or higher;
Python 3.6.1 or higher is recommended.
Run:
```
$ python3.6 -m venv .venv3
$ source .venv3/bin/activate
(.venv3)$ pip3 install -r requirements-tests-py3.txt
```
This will install mypy (you need the latest master branch from GitHub),
typed-ast, flake8, and pytype. You can then run mypy, flake8, and pytype tests
by invoking:
```
(.venv3)$ python3 tests/mypy_test.py
...
(.venv3)$ python3 tests/mypy_selftest.py
...
(.venv3)$ flake8
...
(.venv3)$ python3 tests/pytype_test.py
...
```
Note that flake8 only works with Python 3.6 or higher, and that to run the
pytype tests, you will need Python 2.7 and Python 3.6 interpreters. Pytype will
find these automatically if they're in `PATH`, but otherwise you must point to
them with the `--python27-exe` and `--python36-exe` arguments, respectively.
These PyPI packages follow [PEP 561](http://www.python.org/dev/peps/pep-0561/)
and are automatically released (up to once a day) by
[typeshed internal machinery](https://github.com/typeshed-internal/stub_uploader).
Type checkers should be able to use these stub packages when installed. For more
details, see the documentation for your type checker.
### Package versioning for third-party stubs
Version numbers of third-party stub packages consist of at least four parts.
All parts of the stub version, except for the last part, correspond to the
version of the runtime package being stubbed. For example, if the `types-foo`
package has version `1.2.0.20240309`, this guarantees that the `types-foo` package
contains stubs targeted against `foo==1.2.*` and tested against the latest
version of `foo` matching that specifier. In this example, the final element
of the version number (20240309) indicates that the stub package was pushed on
March 9, 2024.
At typeshed, we try to keep breaking changes to a minimum. However, due to the
nature of stubs, any version bump can introduce changes that might make your
code fail to type check.
There are several strategies available for specifying the version of a stubs
package you're using, each with its own tradeoffs:
1. Use the same bounds that you use for the package being stubbed. For example,
if you use `requests>=2.30.0,<2.32`, you can use
`types-requests>=2.30.0,<2.32`. This ensures that the stubs are compatible
with the package you are using, but it carries a small risk of breaking
type checking due to changes in the stubs.
Another risk of this strategy is that stubs often lag behind
the package being stubbed. You might want to force the package being stubbed
to a certain minimum version because it fixes a critical bug, but if
correspondingly updated stubs have not been released, your type
checking results may not be fully accurate.
2. Pin the stubs to a known good version and update the pin from time to time
(either manually, or using a tool such as dependabot or renovate).
For example, if you use `types-requests==2.31.0.1`, you can have confidence
that upgrading dependencies will not break type checking. However, you will
miss out on improvements in the stubs that could potentially improve type
checking until you update the pin. This strategy also has the risk that the
stubs you are using might become incompatible with the package being stubbed.
3. Don't pin the stubs. This is the option that demands the least work from
you when it comes to updating version pins, and has the advantage that you
will automatically benefit from improved stubs whenever a new version of the
stubs package is released. However, it carries the risk that the stubs
become incompatible with the package being stubbed.
For example, if a new major version of the package is released, there's a
chance the stubs might be updated to reflect the new version of the runtime
package before you update the package being stubbed.
You can also switch between the different strategies as needed. For example,
you could default to strategy (1), but fall back to strategy (2) when
a problem arises that can't easily be fixed.
### The `_typeshed` package
typeshed includes a package `_typeshed` as part of the standard library.
This package and its submodules contain utility types, but are not
available at runtime. For more information about how to use this package,
[see the `stdlib/_typeshed` directory](https://github.com/python/typeshed/tree/main/stdlib/_typeshed).
## Discussion
If you've run into behavior in the type checker that suggests the type
stubs for a given library are incorrect or incomplete,
we want to hear from you!
Our main forum for discussion is the project's [GitHub issue
tracker](https://github.com/python/typeshed/issues). This is the right
place to start a discussion of any of the above or most any other
topic concerning the project.
If you have general questions about typing with Python, or you need
a review of your type annotations or stubs outside of typeshed, head over to
[our discussion forum](https://github.com/python/typing/discussions).
For less formal discussion, try the typing chat room on
[gitter.im](https://gitter.im/python/typing). Some typeshed maintainers
are almost always present; feel free to find us there and we're happy
to chat. Substantive technical discussion will be directed to the
issue tracker.
For mypy, if you are in the typeshed repo that is submodule of the
mypy repo (so `..` refers to the mypy repo), there's a shortcut to run
the mypy tests that avoids installing mypy:
```bash
$ PYTHONPATH=../.. python3 tests/mypy_test.py
```
You can mypy tests to a single version by passing `-p2` or `-p3.5` e.g.
```bash
$ PYTHONPATH=../.. python3 tests/mypy_test.py -p3.5
running mypy --python-version 3.5 --strict-optional # with 342 files
```
lib/pyproject.toml
-1
View File
@@ -1 +0,0 @@
# Utilities for typeshed infrastructure scripts.
lib/ts_utils/__init__.py
-1
View File
@@ -1 +0,0 @@
"""Utilities for typeshed infrastructure scripts."""
lib/ts_utils/metadata.py
-382
View File
@@ -1,382 +0,0 @@
# This module is made specifically to abstract away those type errors
# pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false
"""Tools to help parse and validate information stored in METADATA.toml files."""
from __future__ import annotations
import re
import urllib.parse
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final, NamedTuple, final
from typing_extensions import Annotated, TypeGuard
import tomli
import tomlkit
from packaging.requirements import Requirement
from packaging.specifiers import Specifier
from .paths import PYPROJECT_PATH, STUBS_PATH, distribution_path
from .utils import cache
__all__ = [
"NoSuchStubError",
"PackageDependencies",
"StubMetadata",
"StubtestSettings",
"get_recursive_requirements",
"read_dependencies",
"read_metadata",
"read_stubtest_settings",
]
_STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
# Some older websites have a bad pattern of using query params for navigation.
_QUERY_URL_ALLOWLIST = {"sourceware.org"}
def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]:
return isinstance(obj, list) and all(isinstance(item, str) for item in obj)
@cache
def _get_oldest_supported_python() -> str:
with PYPROJECT_PATH.open("rb") as config:
val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"]
assert type(val) is str
return val
def metadata_path(distribution: str) -> Path:
"""Return the path to the METADATA.toml file of a third-party distribution."""
return distribution_path(distribution) / "METADATA.toml"
@final
@dataclass(frozen=True)
class StubtestSettings:
"""The stubtest settings for a single stubs distribution.
Don't construct instances directly; use the `read_stubtest_settings` function.
"""
skip: bool
apt_dependencies: list[str]
brew_dependencies: list[str]
choco_dependencies: list[str]
extras: list[str]
ignore_missing_stub: bool
platforms: list[str]
stubtest_requirements: list[str]
def system_requirements_for_platform(self, platform: str) -> list[str]:
assert platform in _STUBTEST_PLATFORM_MAPPING, f"Unrecognised platform {platform!r}"
ret = getattr(self, _STUBTEST_PLATFORM_MAPPING[platform])
assert _is_list_of_strings(ret)
return ret
@cache
def read_stubtest_settings(distribution: str) -> StubtestSettings:
"""Return an object describing the stubtest settings for a single stubs distribution."""
with metadata_path(distribution).open("rb") as f:
data: dict[str, object] = tomli.load(f).get("tool", {}).get("stubtest", {})
skip: object = data.get("skip", False)
apt_dependencies: object = data.get("apt_dependencies", [])
brew_dependencies: object = data.get("brew_dependencies", [])
choco_dependencies: object = data.get("choco_dependencies", [])
extras: object = data.get("extras", [])
ignore_missing_stub: object = data.get("ignore_missing_stub", False)
specified_platforms: object = data.get("platforms", ["linux"])
stubtest_requirements: object = data.get("stubtest_requirements", [])
assert type(skip) is bool
assert type(ignore_missing_stub) is bool
# It doesn't work for type-narrowing if we use a for loop here...
assert _is_list_of_strings(specified_platforms)
assert _is_list_of_strings(apt_dependencies)
assert _is_list_of_strings(brew_dependencies)
assert _is_list_of_strings(choco_dependencies)
assert _is_list_of_strings(extras)
assert _is_list_of_strings(stubtest_requirements)
unrecognised_platforms = set(specified_platforms) - _STUBTEST_PLATFORM_MAPPING.keys()
assert not unrecognised_platforms, f"Unrecognised platforms specified for {distribution!r}: {unrecognised_platforms}"
for platform, dep_key in _STUBTEST_PLATFORM_MAPPING.items():
if platform not in specified_platforms:
assert dep_key not in data, (
f"Stubtest is not run on {platform} in CI for {distribution!r}, "
f"but {dep_key!r} are specified in METADATA.toml"
)
return StubtestSettings(
skip=skip,
apt_dependencies=apt_dependencies,
brew_dependencies=brew_dependencies,
choco_dependencies=choco_dependencies,
extras=extras,
ignore_missing_stub=ignore_missing_stub,
platforms=specified_platforms,
stubtest_requirements=stubtest_requirements,
)
@final
@dataclass(frozen=True)
class StubMetadata:
"""The metadata for a single stubs distribution.
Don't construct instances directly; use the `read_metadata` function.
"""
distribution: Annotated[str, "The name of the distribution on PyPI"]
version_spec: Annotated[Specifier, "Upstream versions that the stubs are compatible with"]
requires: Annotated[list[Requirement], "The parsed requirements as listed in METADATA.toml"]
extra_description: str | None
stub_distribution: Annotated[str, "The name under which the distribution is uploaded to PyPI"]
upstream_repository: Annotated[str, "The URL of the upstream repository"] | None
obsolete_since: Annotated[str, "A string representing a specific version"] | None
no_longer_updated: bool
uploaded_to_pypi: Annotated[bool, "Whether or not a distribution is uploaded to PyPI"]
partial_stub: Annotated[bool, "Whether this is a partial type stub package as per PEP 561."]
stubtest_settings: StubtestSettings
requires_python: Annotated[Specifier, "Versions of Python supported by the stub package"]
@property
def is_obsolete(self) -> bool:
return self.obsolete_since is not None
_KNOWN_METADATA_FIELDS: Final = frozenset(
{
"version",
"requires",
"extra_description",
"stub_distribution",
"upstream_repository",
"obsolete_since",
"no_longer_updated",
"upload",
"tool",
"partial_stub",
"requires_python",
}
)
_KNOWN_METADATA_TOOL_FIELDS: Final = {
"stubtest": {
"skip",
"apt_dependencies",
"brew_dependencies",
"choco_dependencies",
"extras",
"ignore_missing_stub",
"platforms",
"stubtest_requirements",
}
}
_DIST_NAME_RE: Final = re.compile(r"^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$", re.IGNORECASE)
class NoSuchStubError(ValueError):
"""Raise NoSuchStubError to indicate that a stubs/{distribution} directory doesn't exist."""
@cache
def read_metadata(distribution: str) -> StubMetadata:
"""Return an object describing the metadata of a stub as given in the METADATA.toml file.
This function does some basic validation,
but does no parsing, transforming or normalization of the metadata.
Use `read_dependencies` if you need to parse the dependencies
given in the `requires` field, for example.
"""
try:
with metadata_path(distribution).open("rb") as f:
data: dict[str, object] = tomli.load(f)
except FileNotFoundError:
raise NoSuchStubError(f"Typeshed has no stubs for {distribution!r}!") from None
unknown_metadata_fields = data.keys() - _KNOWN_METADATA_FIELDS
assert not unknown_metadata_fields, f"Unexpected keys in METADATA.toml for {distribution!r}: {unknown_metadata_fields}"
assert "version" in data, f"Missing 'version' field in METADATA.toml for {distribution!r}"
version = data["version"]
assert isinstance(version, str) and len(version) > 0, f"Invalid 'version' field in METADATA.toml for {distribution!r}"
# Check that the version spec parses
if version[0].isdigit():
version = f"=={version}"
version_spec = Specifier(version)
assert version_spec.operator in {"==", "~="}, f"Invalid 'version' field in METADATA.toml for {distribution!r}"
requires_s: object = data.get("requires", [])
assert isinstance(requires_s, list)
requires = [parse_requires(distribution, req) for req in requires_s]
extra_description: object = data.get("extra_description")
assert isinstance(extra_description, (str, type(None)))
if "stub_distribution" in data:
stub_distribution = data["stub_distribution"]
assert isinstance(stub_distribution, str)
assert _DIST_NAME_RE.fullmatch(stub_distribution), f"Invalid 'stub_distribution' value for {distribution!r}"
else:
stub_distribution = f"types-{distribution}"
upstream_repository: object = data.get("upstream_repository")
assert isinstance(upstream_repository, (str, type(None)))
if isinstance(upstream_repository, str):
parsed_url = urllib.parse.urlsplit(upstream_repository)
assert parsed_url.scheme == "https", f"{distribution}: URLs in the upstream_repository field should use https"
no_www_please = (
f"{distribution}: `World Wide Web` subdomain (`www.`) should be removed from URLs in the upstream_repository field"
)
assert not parsed_url.netloc.startswith("www."), no_www_please
no_query_params_please = (
f"{distribution}: Query params (`?`) should be removed from URLs in the upstream_repository field"
)
assert parsed_url.hostname in _QUERY_URL_ALLOWLIST or (not parsed_url.query), no_query_params_please
no_fragments_please = f"{distribution}: Fragments (`#`) should be removed from URLs in the upstream_repository field"
assert not parsed_url.fragment, no_fragments_please
if parsed_url.netloc == "github.com":
cleaned_url_path = parsed_url.path.strip("/")
num_url_path_parts = len(Path(cleaned_url_path).parts)
bad_github_url_msg = (
f"Invalid upstream_repository for {distribution!r}: "
"URLs for GitHub repositories always have two parts in their paths"
)
assert num_url_path_parts == 2, bad_github_url_msg
obsolete_since: object = data.get("obsolete_since")
assert isinstance(obsolete_since, (str, type(None)))
no_longer_updated: object = data.get("no_longer_updated", False)
assert type(no_longer_updated) is bool
uploaded_to_pypi: object = data.get("upload", True)
assert type(uploaded_to_pypi) is bool
partial_stub: object = data.get("partial_stub", True)
assert type(partial_stub) is bool
requires_python_str: object = data.get("requires_python")
oldest_supported_python = _get_oldest_supported_python()
oldest_supported_python_specifier = Specifier(f">={oldest_supported_python}")
if requires_python_str is None:
requires_python = oldest_supported_python_specifier
else:
assert type(requires_python_str) is str
requires_python = Specifier(requires_python_str)
assert requires_python != oldest_supported_python_specifier, f'requires_python="{requires_python}" is redundant'
# Check minimum Python version is not less than the oldest version of Python supported by typeshed
assert oldest_supported_python_specifier.contains(
requires_python.version
), f"'requires_python' contains versions lower than typeshed's oldest supported Python ({oldest_supported_python})"
assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'"
empty_tools: dict[object, object] = {}
tools_settings: object = data.get("tool", empty_tools)
assert isinstance(tools_settings, dict)
assert tools_settings.keys() <= _KNOWN_METADATA_TOOL_FIELDS.keys(), f"Unrecognised tool for {distribution!r}"
for tool, tk in _KNOWN_METADATA_TOOL_FIELDS.items():
settings_for_tool: object = tools_settings.get(tool, {}) # pyright: ignore[reportUnknownMemberType]
assert isinstance(settings_for_tool, dict)
for key in settings_for_tool:
assert key in tk, f"Unrecognised {tool} key {key!r} for {distribution!r}"
return StubMetadata(
distribution=distribution,
version_spec=version_spec,
requires=requires,
extra_description=extra_description,
stub_distribution=stub_distribution,
upstream_repository=upstream_repository,
obsolete_since=obsolete_since,
no_longer_updated=no_longer_updated,
uploaded_to_pypi=uploaded_to_pypi,
partial_stub=partial_stub,
stubtest_settings=read_stubtest_settings(distribution),
requires_python=requires_python,
)
def update_metadata(distribution: str, **new_values: object) -> tomlkit.TOMLDocument:
"""Update a distribution's METADATA.toml.
Return the updated TOML dictionary for use without having to open the file separately.
"""
path = metadata_path(distribution)
try:
with path.open("rb") as file:
data = tomlkit.load(file)
except FileNotFoundError:
raise NoSuchStubError(f"Typeshed has no stubs for {distribution!r}!") from None
data.update(new_values) # pyright: ignore[reportUnknownMemberType] # tomlkit.TOMLDocument.update is partially typed
with path.open("w", encoding="UTF-8") as file:
tomlkit.dump(data, file) # pyright: ignore[reportUnknownMemberType] # tomlkit.dump has partially unknown Mapping type
return data
def parse_requires(distribution: str, req: object) -> Requirement:
assert isinstance(req, str), f"Invalid requirement {req!r} for {distribution!r}"
return Requirement(req)
class PackageDependencies(NamedTuple):
typeshed_pkgs: tuple[Requirement, ...]
external_pkgs: tuple[Requirement, ...]
@cache
def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]:
return {read_metadata(dir.name).stub_distribution: dir.name for dir in STUBS_PATH.iterdir()}
@cache
def read_dependencies(distribution: str) -> PackageDependencies:
"""Read the dependencies listed in a METADATA.toml file for a stubs package.
Once the dependencies have been read,
determine which dependencies are typeshed-internal dependencies,
and which dependencies are external (non-types) dependencies.
For typeshed dependencies, translate the "dependency name" into the "package name";
for external dependencies, leave them as they are in the METADATA.toml file.
Note that this function may consider things to be typeshed stubs
even if they haven't yet been uploaded to PyPI.
If a typeshed stub is removed, this function will consider it to be an external dependency.
"""
pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping()
typeshed: list[Requirement] = []
external: list[Requirement] = []
for dependency in read_metadata(distribution).requires:
if dependency.name in pypi_name_to_typeshed_name_mapping:
req = Requirement(str(dependency)) # copy the requirement
req.name = pypi_name_to_typeshed_name_mapping[dependency.name]
typeshed.append(req)
else:
external.append(dependency)
return PackageDependencies(tuple(typeshed), tuple(external))
@cache
def get_recursive_requirements(package_name: str) -> PackageDependencies:
"""Recursively gather dependencies for a single stubs package.
For example, if the stubs for `caldav`
declare a dependency on typeshed's stubs for `requests`,
and the stubs for requests declare a dependency on typeshed's stubs for `urllib3`,
`get_recursive_requirements("caldav")` will determine that the stubs for `caldav`
have both `requests` and `urllib3` as typeshed-internal dependencies.
"""
typeshed: set[Requirement] = set()
external: set[Requirement] = set()
non_recursive_requirements = read_dependencies(package_name)
typeshed.update(non_recursive_requirements.typeshed_pkgs)
external.update(non_recursive_requirements.external_pkgs)
for pkg in non_recursive_requirements.typeshed_pkgs:
reqs = get_recursive_requirements(pkg.name)
typeshed.update(reqs.typeshed_pkgs)
external.update(reqs.external_pkgs)
return PackageDependencies(tuple(typeshed), tuple(external))
lib/ts_utils/paths.py
-39
View File
@@ -1,39 +0,0 @@
from pathlib import Path
from typing import Final
# TODO: Use base path relative to this file. Currently, ts_utils gets
# installed into the user's virtual env, so we can't determine the path
# to typeshed. Installing ts_utils editable would solve that, see
# https://github.com/python/typeshed/pull/12806.
TS_BASE_PATH: Final = Path("")
STDLIB_PATH: Final = TS_BASE_PATH / "stdlib"
STUBS_PATH: Final = TS_BASE_PATH / "stubs"
PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml"
REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt"
TESTS_DIR: Final = "@tests"
TEST_CASES_DIR: Final = "test_cases"
def distribution_path(distribution_name: str) -> Path:
"""Return the path to the directory of a third-party distribution."""
return STUBS_PATH / distribution_name
def tests_path(distribution_name: str) -> Path:
if distribution_name == "stdlib":
return STDLIB_PATH / TESTS_DIR
else:
return STUBS_PATH / distribution_name / TESTS_DIR
def test_cases_path(distribution_name: str) -> Path:
return tests_path(distribution_name) / TEST_CASES_DIR
def allowlists_path(distribution_name: str) -> Path:
if distribution_name == "stdlib":
return tests_path("stdlib") / "stubtest_allowlists"
else:
return tests_path(distribution_name)
lib/ts_utils/requirements.py
-28
View File
@@ -1,28 +0,0 @@
from __future__ import annotations
import itertools
import os
import sys
from collections.abc import Iterable
from packaging.requirements import Requirement
from ts_utils.metadata import read_dependencies, read_stubtest_settings
from ts_utils.paths import STUBS_PATH
def get_external_stub_requirements(distributions: Iterable[str] = ()) -> set[Requirement]:
if not distributions:
distributions = os.listdir(STUBS_PATH)
return set(itertools.chain.from_iterable([read_dependencies(distribution).external_pkgs for distribution in distributions]))
def get_stubtest_system_requirements(distributions: Iterable[str] = (), platform: str = sys.platform) -> list[str]:
if not distributions:
distributions = os.listdir(STUBS_PATH)
requirements: list[str] = []
for distribution in distributions:
requirements.extend(read_stubtest_settings(distribution).system_requirements_for_platform(platform))
return requirements
lib/ts_utils/utils.py
-229
View File
@@ -1,229 +0,0 @@
"""Utilities that are imported by multiple scripts in the tests directory."""
from __future__ import annotations
import re
import sys
from collections.abc import Iterable, Mapping
from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, Final, NamedTuple, Tuple
from typing_extensions import TypeAlias
import pathspec
from packaging.requirements import Requirement
try:
from termcolor import colored as colored # pyright: ignore[reportAssignmentType]
except ImportError:
def colored(text: str, color: str | None = None, **kwargs: Any) -> str: # type: ignore[misc] # noqa: ARG001
return text
from .paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path
PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
# A backport of functools.cache for Python <3.9
# This module is imported by mypy_test.py, which needs to run on 3.8 in CI
cache = lru_cache(None)
def strip_comments(text: str) -> str:
return text.split("#")[0].strip()
# ====================================================================
# Printing utilities
# ====================================================================
def print_command(cmd: str | Iterable[str]) -> None:
if not isinstance(cmd, str):
cmd = " ".join(cmd)
print(colored(f"Running: {cmd}", "blue"))
def print_info(message: str) -> None:
print(colored(message, "blue"))
def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None:
error_split = error.split("\n")
old, new = fix_path
for line in error_split[:-1]:
print(colored(line.replace(old, new), "red"))
print(colored(error_split[-1], "red"), end=end)
def print_success_msg() -> None:
print(colored("success", "green"))
def print_divider() -> None:
"""Print a row of * symbols across the screen.
This can be useful to divide terminal output into separate sections.
"""
print()
print("*" * 70)
print()
# ====================================================================
# Dynamic venv creation
# ====================================================================
@cache
def venv_python(venv_dir: Path) -> Path:
if sys.platform == "win32":
return venv_dir / "Scripts" / "python.exe"
return venv_dir / "bin" / "python"
# ====================================================================
# Parsing the requirements file
# ====================================================================
@cache
def parse_requirements() -> Mapping[str, Requirement]:
"""Return a dictionary of requirements from the requirements file."""
with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file:
stripped_lines = map(strip_comments, requirements_file)
stripped_more = [li for li in stripped_lines if not li.startswith("-")]
requirements = map(Requirement, filter(None, stripped_more))
return {requirement.name: requirement for requirement in requirements}
def get_mypy_req() -> str:
return str(parse_requirements()["mypy"])
# ====================================================================
# Parsing the stdlib/VERSIONS file
# ====================================================================
VersionTuple: TypeAlias = Tuple[int, int]
SupportedVersionsDict: TypeAlias = Dict[str, Tuple[VersionTuple, VersionTuple]]
VERSIONS_PATH = STDLIB_PATH / "VERSIONS"
VERSION_LINE_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$")
VERSION_RE = re.compile(r"^([23])\.(\d+)$")
def parse_stdlib_versions_file() -> SupportedVersionsDict:
result: dict[str, tuple[VersionTuple, VersionTuple]] = {}
with VERSIONS_PATH.open(encoding="UTF-8") as f:
for line in f:
line = strip_comments(line)
if line == "":
continue
m = VERSION_LINE_RE.match(line)
assert m, f"invalid VERSIONS line: {line}"
mod: str = m.group(1)
assert mod not in result, f"Duplicate module {mod} in VERSIONS"
min_version = _parse_version(m.group(2))
max_version = _parse_version(m.group(3)) if m.group(3) else (99, 99)
result[mod] = min_version, max_version
return result
def supported_versions_for_module(module_versions: SupportedVersionsDict, module_name: str) -> tuple[VersionTuple, VersionTuple]:
while "." in module_name:
if module_name in module_versions:
return module_versions[module_name]
module_name = ".".join(module_name.split(".")[:-1])
return module_versions[module_name]
def _parse_version(v_str: str) -> tuple[int, int]:
m = VERSION_RE.match(v_str)
assert m, f"invalid version: {v_str}"
return int(m.group(1)), int(m.group(2))
# ====================================================================
# Test Directories
# ====================================================================
class DistributionTests(NamedTuple):
name: str
test_cases_path: Path
@property
def is_stdlib(self) -> bool:
return self.name == "stdlib"
def distribution_info(distribution_name: str) -> DistributionTests:
if distribution_name == "stdlib":
return DistributionTests("stdlib", test_cases_path("stdlib"))
test_path = test_cases_path(distribution_name)
if test_path.is_dir():
if not list(test_path.iterdir()):
raise RuntimeError(f"{distribution_name!r} has a '{TEST_CASES_DIR}' directory but it is empty!")
return DistributionTests(distribution_name, test_path)
raise RuntimeError(f"No test cases found for {distribution_name!r}!")
def get_all_testcase_directories() -> list[DistributionTests]:
testcase_directories: list[DistributionTests] = []
for distribution_path in STUBS_PATH.iterdir():
try:
pkg_info = distribution_info(distribution_path.name)
except RuntimeError:
continue
testcase_directories.append(pkg_info)
return [distribution_info("stdlib"), *sorted(testcase_directories)]
def allowlists(distribution_name: str) -> list[str]:
prefix = "" if distribution_name == "stdlib" else "stubtest_allowlist_"
version_id = f"py{sys.version_info.major}{sys.version_info.minor}"
platform_allowlist = f"{prefix}{sys.platform}.txt"
version_allowlist = f"{prefix}{version_id}.txt"
combined_allowlist = f"{prefix}{sys.platform}-{version_id}.txt"
local_version_allowlist = version_allowlist + ".local"
if distribution_name == "stdlib":
return ["common.txt", platform_allowlist, version_allowlist, combined_allowlist, local_version_allowlist]
else:
return ["stubtest_allowlist.txt", platform_allowlist]
# ====================================================================
# Parsing .gitignore
# ====================================================================
@cache
def get_gitignore_spec() -> pathspec.PathSpec:
with open(".gitignore", encoding="UTF-8") as f:
return pathspec.PathSpec.from_lines("gitwildmatch", f.readlines())
def spec_matches_path(spec: pathspec.PathSpec, path: Path) -> bool:
normalized_path = path.as_posix()
if path.is_dir():
normalized_path += "/"
return spec.match_file(normalized_path)
# ====================================================================
# mypy/stubtest call
# ====================================================================
def allowlist_stubtest_arguments(distribution_name: str) -> list[str]:
stubtest_arguments: list[str] = []
for allowlist in allowlists(distribution_name):
path = allowlists_path(distribution_name) / allowlist
if path.exists():
stubtest_arguments.extend(["--allowlist", str(path)])
return stubtest_arguments
pyproject.toml
+9 -172
View File
@@ -1,174 +1,11 @@
[tool.black]
line-length = 130
target-version = ["py310"]
skip-magic-trailing-comma = true
# Exclude protobuf files because they have long line lengths
# that can't be autofixed. Like docstrings and import aliases.
# Ideally, we could configure Black to allow longer line lengths
# for just these files, but doesn't seem possible yet.
force-exclude = ".*_pb2.pyi"
line_length = 130
target_version = ["py37"]
[tool.ruff]
line-length = 130
# Oldest supported Python version
target-version = "py38"
fix = true
exclude = [
# virtual environment
".env",
".venv",
"env",
# cache directories, etc.:
".git",
".mypy_cache",
".pytype",
]
[tool.ruff.lint]
# Disable all rules on test cases by default:
# test cases often deliberately contain code
# that might not be considered idiomatic or modern.
#
# Note: some rules that are specifically useful to the test cases
# are invoked via separate runs of ruff in pre-commit:
# see our .pre-commit-config.yaml file for details
exclude = ["**/test_cases/**/*.py"]
# We still use flake8-pyi and flake8-noqa to check these (see .flake8 config file);
# tell ruff not to flag these as e.g. "unused noqa comments"
external = ["F821", "NQA", "Y"]
select = [
"ARG", # flake8-unused-arguments
"B", # flake8-bugbear
"D", # pydocstyle
"EXE", # flake8-executable
"FA", # flake8-future-annotations
"I", # isort
"N", # pep8-naming
"PGH", # pygrep-hooks
"RUF", # Ruff-specific and unused-noqa
"TRY", # tryceratops
"UP", # pyupgrade
"YTT", # flake8-2020
# Flake8 base rules
"E", # pycodestyle Error
"F", # Pyflakes
"W", # pycodestyle Warning
# Only include flake8-annotations rules that are autofixable. Otherwise leave this to mypy+pyright
"ANN2",
# Don't include TC rules that create a TYPE_CHECKING block or stringifies annotations
"TC004", # Move import `{qualified_name}` out of type-checking block. Import is used for more than type hinting.
"TC005", # Found empty type-checking block
# "TC008", # TODO: Enable when out of preview
"TC010", # Invalid string member in `X | Y`-style union type
# Most refurb rules are in preview and can be opinionated,
# consider them individually as they come out of preview (last check: 0.8.4)
"FURB105", # Unnecessary empty string passed to `print`
"FURB129", # Instead of calling `readlines()`, iterate over file object directly
"FURB136", # Replace `if` expression with `{min_max}` call
"FURB167", # Use of regular expression alias `re.{}`
"FURB168", # Prefer `is` operator over `isinstance` to check if an object is `None`
"FURB169", # Compare the identities of `{object}` and None instead of their respective types
"FURB177", # Prefer `Path.cwd()` over `Path().resolve()` for current-directory lookups
"FURB187", # Use of assignment of `reversed` on list `{name}`
# PYI: only enable rules that have autofixes and that we always want to fix (even manually),
# avoids duplicate # noqa with flake8-pyi and flake8-noqa flagging `PYI` codes
# See https://github.com/plinss/flake8-noqa/issues/22
"PYI009", # Empty body should contain `...`, not pass
"PYI010", # Function body must contain only `...`
"PYI012", # Class bodies must not contain `pass`
"PYI013", # Non-empty class bodies must not contain `...`
"PYI014", # Only simple default values allowed for arguments
"PYI015", # Only simple default values allowed for assignments
"PYI016", # Duplicate union member `{}`
"PYI019", # Methods like `{method_name}` should return `Self` instead of a custom `TypeVar`
"PYI020", # Quoted annotations should not be included in stubs
"PYI025", # Use `from collections.abc import Set as AbstractSet` to avoid confusion with the `set` builtin
# "PYI026", Waiting for this mypy bug to be fixed: https://github.com/python/mypy/issues/16581
"PYI030", # Multiple literal members in a union. Use a single literal, e.g. `Literal[{}]`
"PYI032", # Prefer `object` to `Any` for the second parameter to `{method_name}`
"PYI034", # `__new__` methods usually return self at runtime
"PYI036", # Star-args in `{method_name}` should be annotated with `object`
"PYI044", # `from __future__ import annotations` has no effect in stub files, since type checkers automatically treat stubs as having those semantics
"PYI055", # Multiple `type[T]` usages in a union. Combine them into one, e.g., `type[{union_str}]`.
"PYI058", # Use `{return_type}` as the return value for simple `{method}` methods
# "PYI061", # TODO: Enable when out of preview
"PYI062", # Duplicate literal member `{}`
"PYI064", # `Final[Literal[{literal}]]` can be replaced with a bare Final
]
extend-safe-fixes = [
"UP036", # Remove unnecessary `sys.version_info` blocks
]
ignore = [
###
# Rules that can conflict with the formatter (Black)
# https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
###
"E111", # indentation-with-invalid-multiple
"E114", # indentation-with-invalid-multiple-comment
"E117", # over-indented
"W191", # tab-indentation
###
# Rules we don't want or don't agree with
###
# We're not a library, no need to document everything
"D1", # Missing docstring in ...
# Doesn't support split "summary line"
"D205", # 1 blank line required between summary line and description
# Used for direct, non-subclass type comparison, for example: `type(val) is str`
# see https://github.com/astral-sh/ruff/issues/6465
"E721", # Do not compare types, use `isinstance()`
# Mostly from scripts and tests, it's ok to have messages passed directly to exceptions
"TRY003", # Avoid specifying long messages outside the exception class
# Slower and more verbose https://github.com/astral-sh/ruff/issues/7871
"UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)`
###
# False-positives, but already checked by type-checkers
###
# Ruff doesn't support multi-file analysis yet: https://github.com/astral-sh/ruff/issues/5295
"RUF013", # PEP 484 prohibits implicit `Optional`
]
[tool.ruff.lint.per-file-ignores]
"*.pyi" = [
# A lot of stubs are incomplete on purpose, and that's configured through pyright
# Some ANN204 (special method) are autofixable in stubs, but not all.
"ANN2", # Missing return type annotation for ...
# Most pep8-naming rules don't apply for third-party stubs like typeshed.
# N811 to N814 could apply, but we often use them to disambiguate a name whilst making it look like a more common one
"N8",
# Rules that are out of the control of stub authors:
"F403", # `from . import *` used; unable to detect undefined names
# Stubs can sometimes re-export entire modules.
# Issues with using a star-imported name will be caught by type-checkers.
"F405", # may be undefined, or defined from star imports
# Ruff 0.8.0 added sorting of __all__ and __slots_.
# There is no consensus on whether we want to apply this to stubs, so keeping the status quo.
# See https://github.com/python/typeshed/pull/13108
"RUF022",
"RUF023",
]
"*_pb2.pyi" = [
# Leave the docstrings as-is, matching source
"D", # pydocstyle
# See comment on black's force-exclude config above
"E501", # Line too long
]
[tool.ruff.lint.pydocstyle]
convention = "pep257" # https://docs.astral.sh/ruff/settings/#lint_pydocstyle_convention
[tool.ruff.lint.isort]
split-on-trailing-comma = false
combine-as-imports = true
extra-standard-library = [
# Group these with stdlib
"_typeshed",
"typing_extensions",
# Extra modules not recognized by Ruff
# Added in Python 3.9
"zoneinfo",
]
known-first-party = ["_utils", "ts_utils"]
[tool.typeshed]
oldest_supported_python = "3.9"
[tool.isort]
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
combine_as_imports = true
line_length = 130
pyrightconfig.json
-50
View File
@@ -1,50 +0,0 @@
{
"$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
"typeshedPath": ".",
"include": [
"stdlib",
"stubs",
],
"exclude": [
// Stubs that don't work in all Python versions
"stubs/seaborn",
"stubs/shapely",
"stubs/geopandas",
// test cases use a custom config file
"**/@tests/test_cases",
],
"typeCheckingMode": "strict",
// Allowed in base settings for incomplete stubs, checked in stricter settings
"reportIncompleteStub": "none",
"reportMissingParameterType": "none",
"reportUnknownMemberType": "none",
"reportUnknownParameterType": "none",
"reportUnknownVariableType": "none",
// Extra strict settings
"reportCallInDefaultInitializer": "error",
"reportUnnecessaryTypeIgnoreComment": "error",
// Leave "type: ignore" comments to mypy
"enableTypeIgnoreComments": false,
// No effect in stubs
"reportMissingSuperCall": "none",
"reportUninitializedInstanceVariable": "none",
// stdlib stubs trigger reportShadowedImports
"reportShadowedImports": "none",
// Stubs are allowed to use private variables
"reportPrivateUsage": "none",
// Stubs don't need the actual modules to be installed
"reportMissingModuleSource": "none",
// Incompatible overrides and property type mismatches are out of typeshed's control
// as they are inherited from the implementation.
"reportIncompatibleMethodOverride": "none",
"reportIncompatibleVariableOverride": "none",
"reportPropertyTypeMismatch": "none",
// Overlapping overloads are often necessary in a stub, meaning pyright's check
// (which is stricter than mypy's; see mypy issue #10143 and #10157)
// would cause many false positives and catch few bugs.
"reportOverlappingOverload": "none",
// The name of the self/cls parameter is out of typeshed's control.
"reportSelfClsParameterName": "none",
// Not actionable in typeshed
"reportDeprecated": "none",
}
pyrightconfig.scripts_and_tests.json
-26
View File
@@ -1,26 +0,0 @@
{
"$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
"typeshedPath": ".",
"include": [
"lib",
"scripts",
"tests",
],
"extraPaths": [
"lib",
],
"typeCheckingMode": "strict",
// More of a lint. Unwanted for typeshed's own code.
"reportImplicitStringConcatenation": "none",
// Extra strict settings
"reportMissingModuleSource": "error",
"reportShadowedImports": "error",
"reportCallInDefaultInitializer": "error",
"reportPropertyTypeMismatch": "error",
"reportUninitializedInstanceVariable": "error",
"reportUnnecessaryTypeIgnoreComment": "error",
// Leave "type: ignore" comments to mypy
"enableTypeIgnoreComments": false,
// Too strict
"reportMissingSuperCall": "none",
}
pyrightconfig.stricter.json
-139
View File
@@ -1,139 +0,0 @@
{
"$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
"typeshedPath": ".",
"include": [
"stdlib",
"stubs",
],
"exclude": [
// test cases use a custom pyrightconfig file
"**/@tests/test_cases",
"stdlib/distutils/command",
"stdlib/distutils/dist.pyi",
"stdlib/importlib/readers.pyi",
"stdlib/lib2to3/fixes/*.pyi",
"stdlib/numbers.pyi",
"stdlib/optparse.pyi",
"stdlib/_tkinter.pyi",
"stdlib/tkinter/__init__.pyi",
"stdlib/tkinter/commondialog.pyi",
"stdlib/tkinter/filedialog.pyi",
"stdlib/tkinter/dialog.pyi",
"stdlib/tkinter/messagebox.pyi",
"stdlib/tkinter/scrolledtext.pyi",
"stdlib/tkinter/tix.pyi",
"stdlib/tkinter/ttk.pyi",
"stdlib/xml/dom/NodeFilter.pyi",
"stdlib/xml/dom/expatbuilder.pyi",
"stdlib/xml/dom/minidom.pyi",
"stdlib/xml/dom/pulldom.pyi",
"stdlib/xml/sax",
"stubs/aiofiles/aiofiles/tempfile/temptypes.pyi",
"stubs/antlr4-python3-runtime",
"stubs/Authlib",
"stubs/aws-xray-sdk",
"stubs/beautifulsoup4",
"stubs/bleach/bleach/sanitizer.pyi",
"stubs/boltons",
"stubs/braintree",
"stubs/caldav",
"stubs/cffi",
"stubs/click-default-group",
"stubs/click-web",
"stubs/commonmark",
"stubs/corus",
"stubs/dateparser",
"stubs/defusedxml",
"stubs/docker",
"stubs/docutils",
"stubs/Flask-SocketIO",
"stubs/fpdf2",
"stubs/gdb",
"stubs/geopandas",
"stubs/google-cloud-ndb",
"stubs/hdbcli/hdbcli/dbapi.pyi",
"stubs/html5lib",
"stubs/httplib2",
"stubs/humanfriendly",
"stubs/hvac",
"stubs/icalendar",
"stubs/influxdb-client",
"stubs/jmespath",
"stubs/jsonschema",
"stubs/jwcrypto",
"stubs/ldap3",
"stubs/m3u8",
"stubs/Markdown",
"stubs/mysqlclient",
"stubs/netaddr/netaddr/core.pyi",
"stubs/netaddr/netaddr/ip/__init__.pyi",
"stubs/netaddr/netaddr/ip/iana.pyi",
"stubs/networkx",
"stubs/oauthlib",
"stubs/openpyxl",
"stubs/parsimonious/parsimonious/nodes.pyi",
"stubs/passlib",
"stubs/peewee",
"stubs/pexpect",
"stubs/pika",
"stubs/protobuf",
"stubs/psutil",
"stubs/psycopg2",
"stubs/pyasn1",
"stubs/pycurl",
"stubs/pyflakes",
"stubs/Pygments",
"stubs/PyMySQL",
"stubs/python-crontab",
"stubs/python-dateutil",
"stubs/python-http-client",
"stubs/python-jose",
"stubs/pywin32",
"stubs/pyxdg",
"stubs/PyYAML",
"stubs/reportlab",
"stubs/requests",
"stubs/requests-oauthlib",
"stubs/seaborn",
"stubs/setuptools/setuptools",
"stubs/shapely",
"stubs/tensorflow",
"stubs/tqdm",
"stubs/ttkthemes",
"stubs/vobject",
"stubs/workalendar",
"stubs/wurlitzer",
],
"typeCheckingMode": "strict",
// TODO: Complete incomplete stubs
"reportIncompleteStub": "none",
// Extra strict settings
"reportCallInDefaultInitializer": "error",
// implicit string concatenation is useful for long deprecation messages
"reportImplicitStringConcatenation": "none",
"reportUnnecessaryTypeIgnoreComment": "error",
// Leave "type: ignore" comments to mypy
"enableTypeIgnoreComments": false,
// No effect in stubs
"reportMissingSuperCall": "none",
"reportUninitializedInstanceVariable": "none",
// stdlib stubs trigger reportShadowedImports
"reportShadowedImports": "none",
// Stubs are allowed to use private variables
"reportPrivateUsage": "none",
// Stubs don't need the actual modules to be installed
"reportMissingModuleSource": "none",
// Incompatible overrides and property type mismatches are out of typeshed's control
// as they are inherited from the implementation.
"reportIncompatibleMethodOverride": "none",
"reportIncompatibleVariableOverride": "none",
"reportPropertyTypeMismatch": "none",
// Overlapping overloads are often necessary in a stub, meaning pyright's check
// (which is stricter than mypy's; see mypy issue #10143 and #10157)
// would cause many false positives and catch few bugs.
"reportOverlappingOverload": "none",
// The name of the self/cls parameter is out of typeshed's control.
"reportSelfClsParameterName": "none",
// Not actionable in typeshed
"reportDeprecated": "none",
}
pyrightconfig.testcases.json
-32
View File
@@ -1,32 +0,0 @@
{
"$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
"typeshedPath": ".",
"include": [
"**/@tests/test_cases",
],
"typeCheckingMode": "strict",
// Extra strict settings
"reportShadowedImports": "error", // Don't accidentally name a file something that shadows stdlib
"reportImplicitStringConcatenation": "error",
"reportUninitializedInstanceVariable": "error",
"reportUnnecessaryTypeIgnoreComment": "error",
// Using unspecific `type: ignore` comments in test_cases.
// See https://github.com/python/typeshed/pull/8083
"enableTypeIgnoreComments": true,
// If a test case uses this anti-pattern, there's likely a reason and annoying to `type: ignore`.
// Let Ruff flag it (B006)
"reportCallInDefaultInitializer": "none",
// Too strict and not needed for type testing
"reportMissingSuperCall": "none",
// Stubs are allowed to use private variables. We may want to test those.
"reportPrivateUsage": "none",
// Stubs don't need the actual modules to be installed
"reportMissingModuleSource": "none",
// Incompatible property type mismatches may be out of typeshed's control
// when they are inherited from the implementation.
"reportPropertyTypeMismatch": "none",
// isinstance checks are still needed when validating inputs outside of typeshed's control
"reportUnnecessaryIsInstance": "none",
// The name of the self/cls parameter is out of typeshed's control.
"reportSelfClsParameterName": "none",
}
requirements-tests-py3.txt
+8
View File
@@ -0,0 +1,8 @@
git+https://github.com/python/mypy.git@master
typed-ast>=1.0.4
black==19.3b0
flake8==3.7.8
flake8-bugbear==19.8.0
flake8-pyi==19.3.0
isort==4.3.21
pytype>=2019.10.17
requirements-tests.txt
-25
View File
@@ -1,25 +0,0 @@
# Type checkers that we test our stubs against. These should always
# be pinned to a specific version to make failure reproducible.
mypy==1.15.0
pyright==1.1.389
# pytype can be installed on Windows, but requires building wheels, let's not do that on the CI
pytype==2024.10.11; platform_system != "Windows" and python_version >= "3.10" and python_version < "3.13"
# Libraries used by our various scripts.
aiohttp==3.10.11
grpcio-tools>=1.66.2 # For grpc_tools.protoc
mypy-protobuf==3.6.0
packaging==24.2
pathspec>=0.11.1
pre-commit
# Required by create_baseline_stubs.py. Must match .pre-commit-config.yaml.
ruff==0.9.3
stubdefaulter==0.1.0
termcolor>=2.3
tomli==2.2.1
tomlkit==0.13.2
typing_extensions>=4.12.0rc1
uv==0.5.14
# Utilities for typeshed infrastructure scripts.
ts_utils @ file:lib
scripts/create_baseline_stubs.py
-250
View File
@@ -1,250 +0,0 @@
#!/usr/bin/env python3
"""Script to generate unannotated baseline stubs using stubgen.
Basic usage:
$ python3 scripts/create_baseline_stubs.py <project on PyPI>
Run with -h for more help.
"""
from __future__ import annotations
import argparse
import asyncio
import glob
import os.path
import re
import subprocess
import sys
import urllib.parse
from importlib.metadata import distribution
import aiohttp
import termcolor
PYRIGHT_CONFIG = "pyrightconfig.stricter.json"
def search_pip_freeze_output(project: str, output: str) -> tuple[str, str] | None:
# Look for lines such as "typed-ast==1.4.2". '-' matches '_' and
# '_' matches '-' in project name, so that "typed_ast" matches
# "typed-ast", and vice versa.
regex = "^(" + re.sub(r"[-_]", "[-_]", project) + ")==(.*)"
m = re.search(regex, output, flags=re.IGNORECASE | re.MULTILINE)
if not m:
return None
return m.group(1), m.group(2)
def get_installed_package_info(project: str) -> tuple[str, str] | None:
"""Find package information from pip freeze output.
Match project name somewhat fuzzily (case sensitive; '-' matches '_', and
vice versa).
Return (normalized project name, installed version) if successful.
"""
r = subprocess.run(["pip", "freeze"], capture_output=True, text=True, check=True)
return search_pip_freeze_output(project, r.stdout)
def run_stubgen(package: str, output: str) -> None:
print(f"Running stubgen: stubgen -o {output} -p {package}")
subprocess.run(["stubgen", "-o", output, "-p", package, "--export-less"], check=True)
def run_stubdefaulter(stub_dir: str) -> None:
print(f"Running stubdefaulter: stubdefaulter --packages {stub_dir}")
subprocess.run(["stubdefaulter", "--packages", stub_dir])
def run_black(stub_dir: str) -> None:
print(f"Running Black: black {stub_dir}")
subprocess.run(["pre-commit", "run", "black", "--files", *glob.iglob(f"{stub_dir}/**/*.pyi")])
def run_ruff(stub_dir: str) -> None:
print(f"Running Ruff: ruff check {stub_dir} --fix-only")
subprocess.run([sys.executable, "-m", "ruff", "check", stub_dir, "--fix-only"])
async def get_project_urls_from_pypi(project: str, session: aiohttp.ClientSession) -> dict[str, str]:
pypi_root = f"https://pypi.org/pypi/{urllib.parse.quote(project)}"
async with session.get(f"{pypi_root}/json") as response:
if response.status != 200:
return {}
j: dict[str, dict[str, dict[str, str]]]
j = await response.json()
return j["info"].get("project_urls") or {}
async def get_upstream_repo_url(project: str) -> str | None:
# aiohttp is overkill here, but it would also just be silly
# to have both requests and aiohttp in our requirements-tests.txt file.
async with aiohttp.ClientSession() as session:
project_urls = await get_project_urls_from_pypi(project, session)
if not project_urls:
return None
# Order the project URLs so that we put the ones
# that are most likely to point to the source code first
urls_to_check: list[str] = []
url_names_probably_pointing_to_source = ("Source", "Repository", "Homepage")
for url_name in url_names_probably_pointing_to_source:
if url := project_urls.get(url_name):
urls_to_check.append(url)
urls_to_check.extend(
url for url_name, url in project_urls.items() if url_name not in url_names_probably_pointing_to_source
)
for url in urls_to_check:
# Remove `www.`; replace `http://` with `https://`
url = re.sub(r"^(https?://)?(www\.)?", "https://", url)
netloc = urllib.parse.urlparse(url).netloc
if netloc in {"gitlab.com", "github.com", "bitbucket.org", "foss.heptapod.net"}:
# truncate to https://site.com/user/repo
upstream_repo_url = "/".join(url.split("/")[:5])
async with session.get(upstream_repo_url) as response:
if response.status == 200:
return upstream_repo_url
return None
def create_metadata(project: str, stub_dir: str, version: str) -> None:
"""Create a METADATA.toml file."""
match = re.match(r"[0-9]+.[0-9]+", version)
if match is None:
sys.exit(f"Error: Cannot parse version number: {version}")
filename = os.path.join(stub_dir, "METADATA.toml")
version = match.group(0)
if os.path.exists(filename):
return
metadata = f'version = "{version}.*"\n'
upstream_repo_url = asyncio.run(get_upstream_repo_url(project))
if upstream_repo_url is None:
warning = (
f"\nCould not find a URL pointing to the source code for {project!r}.\n"
f"Please add it as `upstream_repository` to `stubs/{project}/METADATA.toml`, if possible!\n"
)
print(termcolor.colored(warning, "red"))
else:
metadata += f'upstream_repository = "{upstream_repo_url}"\n'
print(f"Writing {filename}")
with open(filename, "w", encoding="UTF-8") as file:
file.write(metadata)
def add_pyright_exclusion(stub_dir: str) -> None:
"""Exclude stub_dir from strict pyright checks."""
with open(PYRIGHT_CONFIG, encoding="UTF-8") as f:
lines = f.readlines()
i = 0
while i < len(lines) and not lines[i].strip().startswith('"exclude": ['):
i += 1
assert i < len(lines), f"Error parsing {PYRIGHT_CONFIG}"
while not lines[i].strip().startswith("]"):
i += 1
end = i
# We assume that all third-party excludes must be at the end of the list.
# This helps with skipping special entries, such as "stubs/**/@tests/test_cases".
while lines[i - 1].strip().startswith('"stubs/'):
i -= 1
start = i
before_third_party_excludes = lines[:start]
third_party_excludes = lines[start:end]
after_third_party_excludes = lines[end:]
last_line = third_party_excludes[-1].rstrip()
if not last_line.endswith(","):
last_line += ","
third_party_excludes[-1] = last_line + "\n"
# Must use forward slash in the .json file
line_to_add = f' "{stub_dir}",\n'.replace("\\", "/")
if line_to_add in third_party_excludes:
print(f"{PYRIGHT_CONFIG} already up-to-date")
return
third_party_excludes.append(line_to_add)
third_party_excludes.sort(key=str.lower)
print(f"Updating {PYRIGHT_CONFIG}")
with open(PYRIGHT_CONFIG, "w", encoding="UTF-8") as f:
f.writelines(before_third_party_excludes)
f.writelines(third_party_excludes)
f.writelines(after_third_party_excludes)
def main() -> None:
parser = argparse.ArgumentParser(
description="""Generate baseline stubs automatically for an installed pip package
using stubgen. Also run Black and Ruff. If the name of
the project is different from the runtime Python package name, you may
need to use --package (example: --package yaml PyYAML)."""
)
parser.add_argument("project", help="name of PyPI project for which to generate stubs under stubs/")
parser.add_argument("--package", help="generate stubs for this Python package (default is autodetected)")
args = parser.parse_args()
project = args.project
package = args.package
if not re.match(r"[a-zA-Z0-9-_.]+$", project):
sys.exit(f"Invalid character in project name: {project!r}")
if not package:
package = project # default
# Try to find which packages are provided by the project
# Use default if that fails or if several packages are found
#
# The importlib.metadata module is used for projects whose name is different
# from the runtime Python package name (example: PyYAML/yaml)
dist = distribution(project).read_text("top_level.txt")
if dist is not None:
packages = [name for name in dist.split() if not name.startswith("_")]
if len(packages) == 1:
package = packages[0]
print(f'Using detected package "{package}" for project "{project}"', file=sys.stderr)
print("Suggestion: Try again with --package argument if that's not what you wanted", file=sys.stderr)
if not os.path.isdir("stubs") or not os.path.isdir("stdlib"):
sys.exit("Error: Current working directory must be the root of typeshed repository")
# Get normalized project name and version of installed package.
info = get_installed_package_info(project)
if info is None:
print(f'Error: "{project}" is not installed', file=sys.stderr)
print(file=sys.stderr)
print(f'Suggestion: Run "python3 -m pip install {project}" and try again', file=sys.stderr)
sys.exit(1)
project, version = info
stub_dir = os.path.join("stubs", project)
package_dir = os.path.join(stub_dir, package)
if os.path.exists(package_dir):
sys.exit(f"Error: {package_dir} already exists (delete it first)")
run_stubgen(package, stub_dir)
run_stubdefaulter(stub_dir)
run_ruff(stub_dir)
run_black(stub_dir)
create_metadata(project, stub_dir, version)
# Since the generated stubs won't have many type annotations, we
# have to exclude them from strict pyright checks.
add_pyright_exclusion(stub_dir)
print("\nDone!\n\nSuggested next steps:")
print(f" 1. Manually review the generated stubs in {stub_dir}")
print(" 2. Optionally run tests and autofixes (see tests/README.md for details)")
print(" 3. Commit the changes on a new branch and create a typeshed PR (don't force-push!)")
if __name__ == "__main__":
main()
scripts/install_all_third_party_dependencies.py
-6
View File
@@ -1,6 +0,0 @@
import subprocess
from ts_utils.requirements import get_external_stub_requirements
requirements = get_external_stub_requirements()
subprocess.check_call(("pip", "install", *[str(requirement) for requirement in requirements]))
scripts/stubsabot.py
-823
View File
@@ -1,823 +0,0 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import asyncio
import contextlib
import datetime
import enum
import functools
import io
import os
import re
import subprocess
import sys
import tarfile
import textwrap
import urllib.parse
import zipfile
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from http import HTTPStatus
from pathlib import Path
from typing import Annotated, Any, ClassVar, NamedTuple
from typing_extensions import Self, TypeAlias
import aiohttp
import packaging.version
import tomlkit
from packaging.specifiers import Specifier
from termcolor import colored
from ts_utils.metadata import StubMetadata, read_metadata, update_metadata
from ts_utils.paths import STUBS_PATH, distribution_path
TYPESHED_OWNER = "python"
TYPESHED_API_URL = f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed"
STUBSABOT_LABEL = "bot: stubsabot"
class ActionLevel(enum.IntEnum):
def __new__(cls, value: int, doc: str) -> Self:
member = int.__new__(cls, value)
member._value_ = value
member.__doc__ = doc
return member
@classmethod
def from_cmd_arg(cls, cmd_arg: str) -> ActionLevel:
try:
return cls[cmd_arg]
except KeyError:
raise argparse.ArgumentTypeError(f'Argument must be one of "{list(cls.__members__)}"') from None
nothing = 0, "make no changes"
local = 1, "make changes that affect local repo"
fork = 2, "make changes that affect remote repo, but won't open PRs against upstream"
everything = 3, "do everything, e.g. open PRs"
@dataclass
class PypiReleaseDownload:
distribution: str
url: str
packagetype: Annotated[str, "Should hopefully be either 'bdist_wheel' or 'sdist'"]
filename: str
version: packaging.version.Version
upload_date: datetime.datetime
VersionString: TypeAlias = str
ReleaseDownload: TypeAlias = dict[str, Any]
def _best_effort_version(version: VersionString) -> packaging.version.Version:
try:
return packaging.version.Version(version)
except packaging.version.InvalidVersion:
# packaging.version.Version no longer parses legacy versions
try:
return packaging.version.Version(version.replace("-", "+"))
except packaging.version.InvalidVersion:
return packaging.version.Version("0")
@dataclass
class PypiInfo:
distribution: str
pypi_root: str
releases: dict[VersionString, list[ReleaseDownload]] = field(repr=False)
info: dict[str, Any] = field(repr=False)
def get_release(self, *, version: VersionString) -> PypiReleaseDownload:
# prefer wheels, since it's what most users will get / it's pretty easy to mess up MANIFEST
release_info = sorted(self.releases[version], key=lambda x: bool(x["packagetype"] == "bdist_wheel"))[-1]
return PypiReleaseDownload(
distribution=self.distribution,
url=release_info["url"],
packagetype=release_info["packagetype"],
filename=release_info["filename"],
version=packaging.version.Version(version),
upload_date=datetime.datetime.fromisoformat(release_info["upload_time"]),
)
def get_latest_release(self) -> PypiReleaseDownload:
return self.get_release(version=self.info["version"])
def releases_in_descending_order(self) -> Iterator[PypiReleaseDownload]:
for version in sorted(self.releases, key=_best_effort_version, reverse=True):
yield self.get_release(version=version)
async def fetch_pypi_info(distribution: str, session: aiohttp.ClientSession) -> PypiInfo:
# Cf. # https://warehouse.pypa.io/api-reference/json.html#get--pypi--project_name--json
pypi_root = f"https://pypi.org/pypi/{urllib.parse.quote(distribution)}"
async with session.get(f"{pypi_root}/json") as response:
response.raise_for_status()
j = await response.json()
return PypiInfo(distribution=distribution, pypi_root=pypi_root, releases=j["releases"], info=j["info"])
@dataclass
class Update:
distribution: str
old_version_spec: Specifier
new_version_spec: Specifier
links: dict[str, str]
diff_analysis: DiffAnalysis | None
def __str__(self) -> str:
return f"Updating {self.distribution} from {self.old_version_spec!r} to {self.new_version_spec!r}"
@property
def new_version(self) -> str:
if self.new_version_spec.operator == "==":
return str(self.new_version_spec)[2:]
else:
return str(self.new_version_spec)
@dataclass
class Obsolete:
distribution: str
obsolete_since_version: str
obsolete_since_date: datetime.datetime
links: dict[str, str]
def __str__(self) -> str:
return f"Marking {self.distribution} as obsolete since {self.obsolete_since_version!r}"
@dataclass
class NoUpdate:
distribution: str
reason: str
def __str__(self) -> str:
return f"Skipping {self.distribution}: {self.reason}"
def all_py_files_in_source_are_in_py_typed_dirs(source: zipfile.ZipFile | tarfile.TarFile) -> bool:
py_typed_dirs: list[Path] = []
all_python_files: list[Path] = []
py_file_suffixes = {".py", ".pyi"}
# Obtain an iterator over all files in the zipfile/tarfile.
# Filter out empty __init__.py files: this reduces false negatives
# in cases like this:
#
# repo_root/
# ├─ src/
# | ├─ pkg/
# | | ├─ __init__.py <-- This file is empty
# | | ├─ subpkg/
# | | | ├─ __init__.py
# | | | ├─ libraryfile.py
# | | | ├─ libraryfile2.py
# | | | ├─ py.typed
if isinstance(source, zipfile.ZipFile):
path_iter = (
Path(zip_info.filename)
for zip_info in source.infolist()
if ((not zip_info.is_dir()) and not (Path(zip_info.filename).name == "__init__.py" and zip_info.file_size == 0))
)
else:
path_iter = (
Path(tar_info.path)
for tar_info in source
if (tar_info.isfile() and not (Path(tar_info.name).name == "__init__.py" and tar_info.size == 0))
)
for path in path_iter:
if path.suffix in py_file_suffixes:
all_python_files.append(path)
elif path.name == "py.typed":
py_typed_dirs.append(path.parent)
if not py_typed_dirs:
return False
if not all_python_files:
return False
for path in all_python_files:
if not any(py_typed_dir in path.parents for py_typed_dir in py_typed_dirs):
return False
return True
async def release_contains_py_typed(release_to_download: PypiReleaseDownload, *, session: aiohttp.ClientSession) -> bool:
async with session.get(release_to_download.url) as response:
body = io.BytesIO(await response.read())
packagetype = release_to_download.packagetype
if packagetype == "bdist_wheel":
assert release_to_download.filename.endswith(".whl")
with zipfile.ZipFile(body) as zf:
return all_py_files_in_source_are_in_py_typed_dirs(zf)
elif packagetype == "sdist":
# sdist defaults to `.tar.gz` on Lunix and to `.zip` on Windows:
# https://docs.python.org/3.11/distutils/sourcedist.html
if release_to_download.filename.endswith(".tar.gz"):
with tarfile.open(fileobj=body, mode="r:gz") as zf:
return all_py_files_in_source_are_in_py_typed_dirs(zf)
elif release_to_download.filename.endswith(".zip"):
with zipfile.ZipFile(body) as zf:
return all_py_files_in_source_are_in_py_typed_dirs(zf)
else:
raise AssertionError(f"Package file {release_to_download.filename!r} does not end with '.tar.gz' or '.zip'")
else:
raise AssertionError(f"Unknown package type for {release_to_download.distribution}: {packagetype!r}")
async def find_first_release_with_py_typed(pypi_info: PypiInfo, *, session: aiohttp.ClientSession) -> PypiReleaseDownload | None:
"""If the latest release is py.typed, return the first release that included a py.typed file.
If the latest release is not py.typed, return None.
"""
release_iter = (release for release in pypi_info.releases_in_descending_order() if not release.version.is_prerelease)
latest_release = next(release_iter)
# If the latest release is not py.typed, assume none are.
if not (await release_contains_py_typed(latest_release, session=session)):
return None
first_release_with_py_typed = latest_release
while await release_contains_py_typed(release := next(release_iter), session=session):
first_release_with_py_typed = release
return first_release_with_py_typed
def get_updated_version_spec(spec: Specifier, version: packaging.version.Version) -> Specifier:
"""
Given the old specifier and an updated version, returns an updated specifier that has the
specificity of the old specifier, but matches the updated version.
For example:
spec="1", version="1.2.3" -> "1.2.3"
spec="1.0.1", version="1.2.3" -> "1.2.3"
spec="1.*", version="1.2.3" -> "1.*"
spec="1.*", version="2.3.4" -> "2.*"
spec="1.1.*", version="1.2.3" -> "1.2.*"
spec="1.1.1.*", version="1.2.3" -> "1.2.3.*"
spec="~=1.0.1", version="1.0.3" -> "~=1.0.3"
spec="~=1.0.1", version="1.1.0" -> "~=1.1.0"
"""
if spec.operator == "==" and spec.version.endswith(".*"):
specificity = spec.version.count(".") if spec.version.removesuffix(".*") else 0
rounded_version = version.base_version.split(".")[:specificity]
rounded_version.extend(["0"] * (specificity - len(rounded_version)))
updated_spec = Specifier("==" + ".".join(rounded_version) + ".*")
elif spec.operator == "==":
updated_spec = Specifier(f"=={version}")
elif spec.operator == "~=":
updated_spec = Specifier(f"~={version}")
else:
raise ValueError(f"Unsupported version operator: {spec.operator}")
assert version in updated_spec, f"{version} not in {updated_spec}"
return updated_spec
@functools.cache
def get_github_api_headers() -> Mapping[str, str]:
headers = {"Accept": "application/vnd.github.v3+json"}
secret = os.environ.get("GITHUB_TOKEN")
if secret is not None:
headers["Authorization"] = f"token {secret}" if secret.startswith("ghp") else f"Bearer {secret}"
return headers
@dataclass
class GitHubInfo:
repo_path: str
tags: list[dict[str, Any]] = field(repr=False)
async def get_github_repo_info(session: aiohttp.ClientSession, stub_info: StubMetadata) -> GitHubInfo | None:
"""
If the project represented by `stub_info` is hosted on GitHub,
return information regarding the project as it exists on GitHub.
Else, return None.
"""
if stub_info.upstream_repository:
# We have various sanity checks for the upstream_repository field in ts_utils.metadata,
# so no need to repeat all of them here
split_url = urllib.parse.urlsplit(stub_info.upstream_repository)
if split_url.netloc == "github.com":
url_path = split_url.path.strip("/")
assert len(Path(url_path).parts) == 2
github_tags_info_url = f"https://api.github.com/repos/{url_path}/tags"
async with session.get(github_tags_info_url, headers=get_github_api_headers()) as response:
if response.status == 200:
tags: list[dict[str, Any]] = await response.json()
assert isinstance(tags, list)
return GitHubInfo(repo_path=url_path, tags=tags)
return None
class GitHubDiffInfo(NamedTuple):
repo_path: str
old_tag: str
new_tag: str
diff_url: str
async def get_diff_info(
session: aiohttp.ClientSession, stub_info: StubMetadata, pypi_version: packaging.version.Version
) -> GitHubDiffInfo | None:
"""Return a tuple giving info about the diff between two releases, if possible.
Return `None` if the project isn't hosted on GitHub,
or if a link pointing to the diff couldn't be found for any other reason.
"""
github_info = await get_github_repo_info(session, stub_info)
if github_info is None:
return None
versions_to_tags: dict[packaging.version.Version, str] = {}
for tag in github_info.tags:
tag_name = tag["name"]
# Some packages in typeshed have tag names
# that are invalid to be passed to the Version() constructor,
# e.g. v.1.4.2
with contextlib.suppress(packaging.version.InvalidVersion):
versions_to_tags[packaging.version.Version(tag_name)] = tag_name
try:
new_tag = versions_to_tags[pypi_version]
except KeyError:
return None
try:
old_version = max(version for version in versions_to_tags if version in stub_info.version_spec and version < pypi_version)
except ValueError:
return None
else:
old_tag = versions_to_tags[old_version]
diff_url = f"https://github.com/{github_info.repo_path}/compare/{old_tag}...{new_tag}"
return GitHubDiffInfo(repo_path=github_info.repo_path, old_tag=old_tag, new_tag=new_tag, diff_url=diff_url)
FileInfo: TypeAlias = dict[str, Any]
def _plural_s(num: int, /) -> str:
return "s" if num != 1 else ""
@dataclass(repr=False)
class DiffAnalysis:
MAXIMUM_NUMBER_OF_FILES_TO_LIST: ClassVar[int] = 7
py_files: list[FileInfo]
py_files_stubbed_in_typeshed: list[FileInfo]
@property
def runtime_definitely_has_consistent_directory_structure_with_typeshed(self) -> bool:
"""
If 0 .py files in the GitHub diff exist in typeshed's stubs,
there's a possibility that the .py files might be found
in a different directory at runtime.
For example: pyopenssl has its .py files in the `src/OpenSSL/` directory at runtime,
but in typeshed the stubs are in the `OpenSSL/` directory.
"""
return bool(self.py_files_stubbed_in_typeshed)
@functools.cached_property
def public_files_added(self) -> Sequence[str]:
def is_public(path: Path) -> bool:
return not re.match(r"_[^_]", path.name) and not path.name.startswith("test_")
return [file["filename"] for file in self.py_files if is_public(Path(file["filename"])) and file["status"] == "added"]
@functools.cached_property
def typeshed_files_deleted(self) -> Sequence[str]:
return [file["filename"] for file in self.py_files_stubbed_in_typeshed if file["status"] == "removed"]
@functools.cached_property
def typeshed_files_modified(self) -> Sequence[str]:
return [file["filename"] for file in self.py_files_stubbed_in_typeshed if file["status"] in {"modified", "renamed"}]
@property
def total_lines_added(self) -> int:
return sum(file["additions"] for file in self.py_files)
@property
def total_lines_deleted(self) -> int:
return sum(file["deletions"] for file in self.py_files)
def _describe_files(self, *, verb: str, filenames: Sequence[str]) -> str:
num_files = len(filenames)
if num_files > 1:
description = f"have been {verb}"
# Don't list the filenames if there are *loads* of files
if num_files <= self.MAXIMUM_NUMBER_OF_FILES_TO_LIST:
description += ": "
description += ", ".join(f"`{filename}`" for filename in filenames)
description += "."
return description
if num_files == 1:
return f"has been {verb}: `{filenames[0]}`."
return f"have been {verb}."
def describe_public_files_added(self) -> str:
num_files_added = len(self.public_files_added)
analysis = f"{num_files_added} public Python file{_plural_s(num_files_added)} "
analysis += self._describe_files(verb="added", filenames=self.public_files_added)
return analysis
def describe_typeshed_files_deleted(self) -> str:
num_files_deleted = len(self.typeshed_files_deleted)
analysis = f"{num_files_deleted} file{_plural_s(num_files_deleted)} included in typeshed's stubs "
analysis += self._describe_files(verb="deleted", filenames=self.typeshed_files_deleted)
return analysis
def describe_typeshed_files_modified(self) -> str:
num_files_modified = len(self.typeshed_files_modified)
analysis = f"{num_files_modified} file{_plural_s(num_files_modified)} included in typeshed's stubs "
analysis += self._describe_files(verb="modified or renamed", filenames=self.typeshed_files_modified)
return analysis
def __str__(self) -> str:
data_points: list[str] = []
if self.runtime_definitely_has_consistent_directory_structure_with_typeshed:
data_points += [
self.describe_public_files_added(),
self.describe_typeshed_files_deleted(),
self.describe_typeshed_files_modified(),
]
data_points += [
f"Total lines of Python code added: {self.total_lines_added}.",
f"Total lines of Python code deleted: {self.total_lines_deleted}.",
]
return "Stubsabot analysis of the diff between the two releases:\n - " + "\n - ".join(data_points)
async def analyze_diff(
github_repo_path: str, distribution: str, old_tag: str, new_tag: str, *, session: aiohttp.ClientSession
) -> DiffAnalysis | None:
url = f"https://api.github.com/repos/{github_repo_path}/compare/{old_tag}...{new_tag}"
async with session.get(url, headers=get_github_api_headers()) as response:
response.raise_for_status()
json_resp: dict[str, list[FileInfo]] = await response.json()
assert isinstance(json_resp, dict)
# https://docs.github.com/en/rest/commits/commits#compare-two-commits
py_files: list[FileInfo] = [file for file in json_resp["files"] if Path(file["filename"]).suffix == ".py"]
stub_path = distribution_path(distribution)
files_in_typeshed = set(stub_path.rglob("*.pyi"))
py_files_stubbed_in_typeshed = [file for file in py_files if (stub_path / f"{file['filename']}i") in files_in_typeshed]
return DiffAnalysis(py_files=py_files, py_files_stubbed_in_typeshed=py_files_stubbed_in_typeshed)
async def determine_action(distribution: str, session: aiohttp.ClientSession) -> Update | NoUpdate | Obsolete:
stub_info = read_metadata(distribution)
if stub_info.is_obsolete:
return NoUpdate(stub_info.distribution, "obsolete")
if stub_info.no_longer_updated:
return NoUpdate(stub_info.distribution, "no longer updated")
pypi_info = await fetch_pypi_info(stub_info.distribution, session)
latest_release = pypi_info.get_latest_release()
latest_version = latest_release.version
obsolete_since = await find_first_release_with_py_typed(pypi_info, session=session)
if obsolete_since is None and latest_version in stub_info.version_spec:
return NoUpdate(stub_info.distribution, "up to date")
relevant_version = obsolete_since.version if obsolete_since else latest_version
project_urls: dict[str, str] = pypi_info.info["project_urls"] or {}
maybe_links: dict[str, str | None] = {
"Release": f"{pypi_info.pypi_root}/{relevant_version}",
"Homepage": project_urls.get("Homepage"),
"Repository": stub_info.upstream_repository,
"Typeshed stubs": f"https://github.com/{TYPESHED_OWNER}/typeshed/tree/main/stubs/{stub_info.distribution}",
"Changelog": project_urls.get("Changelog") or project_urls.get("Changes") or project_urls.get("Change Log"),
}
links = {k: v for k, v in maybe_links.items() if v is not None}
diff_info = await get_diff_info(session, stub_info, relevant_version)
if diff_info is not None:
links["Diff"] = diff_info.diff_url
if obsolete_since:
return Obsolete(
stub_info.distribution,
obsolete_since_version=str(obsolete_since.version),
obsolete_since_date=obsolete_since.upload_date,
links=links,
)
if diff_info is None:
diff_analysis: DiffAnalysis | None = None
else:
diff_analysis = await analyze_diff(
github_repo_path=diff_info.repo_path,
distribution=distribution,
old_tag=diff_info.old_tag,
new_tag=diff_info.new_tag,
session=session,
)
return Update(
distribution=stub_info.distribution,
old_version_spec=stub_info.version_spec,
new_version_spec=get_updated_version_spec(stub_info.version_spec, latest_version),
links=links,
diff_analysis=diff_analysis,
)
@functools.lru_cache
def get_origin_owner() -> str:
output = subprocess.check_output(["git", "remote", "get-url", "origin"], text=True).strip()
match = re.match(r"(git@github.com:|https://github.com/)(?P<owner>[^/]+)/(?P<repo>[^/\s]+)", output)
assert match is not None, f"Couldn't identify origin's owner: {output!r}"
assert match.group("repo").removesuffix(".git") == "typeshed", f'Unexpected repo: {match.group("repo")!r}'
return match.group("owner")
async def create_or_update_pull_request(*, title: str, body: str, branch_name: str, session: aiohttp.ClientSession) -> None:
fork_owner = get_origin_owner()
async with session.post(
f"{TYPESHED_API_URL}/pulls",
json={"title": title, "body": body, "head": f"{fork_owner}:{branch_name}", "base": "main"},
headers=get_github_api_headers(),
) as response:
resp_json = await response.json()
if response.status == HTTPStatus.CREATED:
pr_number = resp_json["number"]
assert isinstance(pr_number, int)
elif response.status == HTTPStatus.UNPROCESSABLE_ENTITY and any(
"A pull request already exists" in e.get("message", "") for e in resp_json.get("errors", [])
):
pr_number = await update_existing_pull_request(title=title, body=body, branch_name=branch_name, session=session)
else:
response.raise_for_status()
raise AssertionError(f"Unexpected response: {response.status}")
await update_pull_request_label(pr_number=pr_number, session=session)
async def update_existing_pull_request(*, title: str, body: str, branch_name: str, session: aiohttp.ClientSession) -> int:
fork_owner = get_origin_owner()
# Find the existing PR
async with session.get(
f"{TYPESHED_API_URL}/pulls",
params={"state": "open", "head": f"{fork_owner}:{branch_name}", "base": "main"},
headers=get_github_api_headers(),
) as response:
response.raise_for_status()
resp_json = await response.json()
assert len(resp_json) >= 1
pr_number = resp_json[0]["number"]
assert isinstance(pr_number, int)
# Update the PR's title and body
async with session.patch(
f"{TYPESHED_API_URL}/pulls/{pr_number}", json={"title": title, "body": body}, headers=get_github_api_headers()
) as response:
response.raise_for_status()
return pr_number
async def update_pull_request_label(*, pr_number: int, session: aiohttp.ClientSession) -> None:
# There is no pulls/.../labels endpoint, which is why we need to use the issues endpoint.
async with session.post(
f"{TYPESHED_API_URL}/issues/{pr_number}/labels", json={"labels": [STUBSABOT_LABEL]}, headers=get_github_api_headers()
) as response:
response.raise_for_status()
def has_non_stubsabot_commits(branch: str) -> bool:
assert not branch.startswith("origin/")
try:
# commits on origin/branch that are not on branch or are
# patch equivalent to a commit on branch
print(
"[debugprint]",
subprocess.check_output(
["git", "log", "--right-only", "--pretty=%an %s", "--cherry-pick", f"{branch}...origin/{branch}"]
),
)
print(
"[debugprint]",
subprocess.check_output(
["git", "log", "--right-only", "--pretty=%an", "--cherry-pick", f"{branch}...origin/{branch}"]
),
)
output = subprocess.check_output(
["git", "log", "--right-only", "--pretty=%an", "--cherry-pick", f"{branch}...origin/{branch}"],
stderr=subprocess.DEVNULL,
)
return bool(set(output.splitlines()) - {b"stubsabot"})
except subprocess.CalledProcessError:
# origin/branch does not exist
return False
def latest_commit_is_different_to_last_commit_on_origin(branch: str) -> bool:
assert not branch.startswith("origin/")
try:
# https://www.git-scm.com/docs/git-range-diff
# If the number of lines is >1,
# it indicates that something about our commit is different to the last commit
# (Could be the commit "content", or the commit message).
commit_comparison = subprocess.run(
["git", "range-diff", f"origin/{branch}~1..origin/{branch}", "HEAD~1..HEAD"], check=True, capture_output=True
)
return len(commit_comparison.stdout.splitlines()) > 1
except subprocess.CalledProcessError:
# origin/branch does not exist
return True
class RemoteConflictError(Exception):
pass
def somewhat_safe_force_push(branch: str) -> None:
if has_non_stubsabot_commits(branch):
raise RemoteConflictError(f"origin/{branch} has non-stubsabot changes that are not on {branch}!")
subprocess.check_call(["git", "push", "origin", branch, "--force"])
def normalize(name: str) -> str:
# PEP 503 normalization
return re.sub(r"[-_.]+", "-", name).lower()
# lock should be unnecessary, but can't hurt to enforce mutual exclusion
_repo_lock = asyncio.Lock()
BRANCH_PREFIX = "stubsabot"
def get_update_pr_body(update: Update, metadata: Mapping[str, Any]) -> str:
body = "\n".join(f"{k}: {v}" for k, v in update.links.items())
if update.diff_analysis is not None:
body += f"\n\n{update.diff_analysis}"
stubtest_settings: dict[str, Any] = metadata.get("tool", {}).get("stubtest", {})
stubtest_will_run = not stubtest_settings.get("skip", False)
if stubtest_will_run:
body += textwrap.dedent(
"""
If stubtest fails for this PR:
- Leave this PR open (as a reminder, and to prevent stubsabot from opening another PR)
- Fix stubtest failures in another PR, then close this PR
Note that you will need to close and re-open the PR in order to trigger CI
"""
)
else:
body += textwrap.dedent(
f"""
:warning: Review this PR manually, as stubtest is skipped in CI for {update.distribution}! :warning:
"""
)
return body
async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession, action_level: ActionLevel) -> None:
if action_level <= ActionLevel.nothing:
return
title = f"[stubsabot] Bump {update.distribution} to {update.new_version}"
async with _repo_lock:
branch_name = f"{BRANCH_PREFIX}/{normalize(update.distribution)}"
subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"])
meta = update_metadata(update.distribution, version=update.new_version)
body = get_update_pr_body(update, meta)
subprocess.check_call(["git", "commit", "--all", "-m", f"{title}\n\n{body}"])
if action_level <= ActionLevel.local:
return
if not latest_commit_is_different_to_last_commit_on_origin(branch_name):
print(f"No pushing to origin required: origin/{branch_name} exists and requires no changes!")
return
somewhat_safe_force_push(branch_name)
if action_level <= ActionLevel.fork:
return
await create_or_update_pull_request(title=title, body=body, branch_name=branch_name, session=session)
async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientSession, action_level: ActionLevel) -> None:
if action_level <= ActionLevel.nothing:
return
title = f"[stubsabot] Mark {obsolete.distribution} as obsolete since {obsolete.obsolete_since_version}"
async with _repo_lock:
branch_name = f"{BRANCH_PREFIX}/{normalize(obsolete.distribution)}"
subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"])
obs_string = tomlkit.string(obsolete.obsolete_since_version)
obs_string.comment(f"Released on {obsolete.obsolete_since_date.date().isoformat()}")
update_metadata(obsolete.distribution, obsolete_since=obs_string)
body = "\n".join(f"{k}: {v}" for k, v in obsolete.links.items())
subprocess.check_call(["git", "commit", "--all", "-m", f"{title}\n\n{body}"])
if action_level <= ActionLevel.local:
return
if not latest_commit_is_different_to_last_commit_on_origin(branch_name):
print(f"No PR required: origin/{branch_name} exists and requires no changes!")
return
somewhat_safe_force_push(branch_name)
if action_level <= ActionLevel.fork:
return
await create_or_update_pull_request(title=title, body=body, branch_name=branch_name, session=session)
async def main() -> None:
assert sys.version_info >= (3, 9)
parser = argparse.ArgumentParser()
parser.add_argument(
"--action-level",
type=ActionLevel.from_cmd_arg,
default=ActionLevel.everything,
help="Limit actions performed to achieve dry runs for different levels of dryness",
)
parser.add_argument(
"--action-count-limit",
type=int,
default=None,
help="Limit number of actions performed and the remainder are logged. Useful for testing",
)
parser.add_argument("distributions", nargs="*", help="Distributions to update, default = all")
args = parser.parse_args()
if args.distributions:
dists_to_update = args.distributions
else:
dists_to_update = [path.name for path in STUBS_PATH.iterdir()]
if args.action_level > ActionLevel.nothing:
subprocess.run(["git", "update-index", "--refresh"], capture_output=True)
diff_result = subprocess.run(["git", "diff-index", "HEAD", "--name-only"], text=True, capture_output=True)
if diff_result.returncode:
print("Unexpected exception!")
print(diff_result.stdout)
print(diff_result.stderr)
sys.exit(diff_result.returncode)
if diff_result.stdout:
changed_files = ", ".join(repr(line) for line in diff_result.stdout.split("\n") if line)
print(f"Cannot run stubsabot, as uncommitted changes are present in {changed_files}!")
sys.exit(1)
if args.action_level > ActionLevel.fork:
if os.environ.get("GITHUB_TOKEN") is None:
raise ValueError("GITHUB_TOKEN environment variable must be set")
denylist = {"gdb"} # gdb is not a pypi distribution
original_branch = subprocess.run(
["git", "branch", "--show-current"], text=True, capture_output=True, check=True
).stdout.strip()
if args.action_level >= ActionLevel.local:
subprocess.check_call(["git", "fetch", "--prune", "--all"])
try:
conn = aiohttp.TCPConnector(limit_per_host=10)
async with aiohttp.ClientSession(connector=conn) as session:
tasks = [
asyncio.create_task(determine_action(distribution, session))
for distribution in dists_to_update
if distribution not in denylist
]
action_count = 0
for task in asyncio.as_completed(tasks):
update = await task
print(update)
if isinstance(update, NoUpdate):
continue
if args.action_count_limit is not None and action_count >= args.action_count_limit:
print(colored("... but we've reached action count limit", "red"))
continue
action_count += 1
try:
if isinstance(update, Update):
await suggest_typeshed_update(update, session, action_level=args.action_level)
continue
# Redundant, but keeping for extra runtime validation
if isinstance(update, Obsolete): # pyright: ignore[reportUnnecessaryIsInstance]
await suggest_typeshed_obsolete(update, session, action_level=args.action_level)
continue
except RemoteConflictError as e:
print(colored(f"... but ran into {type(e).__qualname__}: {e}", "red"))
continue
raise AssertionError
finally:
# if you need to cleanup, try:
# git branch -D $(git branch --list 'stubsabot/*')
if args.action_level >= ActionLevel.local and original_branch:
subprocess.check_call(["git", "checkout", original_branch])
if __name__ == "__main__":
asyncio.run(main())
scripts/sync_protobuf/_utils.py
-50
View File
@@ -1,50 +0,0 @@
from __future__ import annotations
import subprocess
import sys
from http.client import HTTPResponse
from typing import TYPE_CHECKING, Iterable
from urllib.request import urlopen
from zipfile import ZipFile
from mypy_protobuf.main import ( # type: ignore[import-untyped] # pyright: ignore[reportMissingTypeStubs]
__version__ as mypy_protobuf__version__,
)
if TYPE_CHECKING:
from _typeshed import StrOrBytesPath, StrPath
MYPY_PROTOBUF_VERSION = mypy_protobuf__version__
def download_file(url: str, destination: StrPath) -> None:
print(f"Downloading '{url}' to '{destination}'")
resp: HTTPResponse
with urlopen(url) as resp, open(destination, "wb") as file:
file.write(resp.read())
def extract_archive(archive_path: StrPath, destination: StrPath) -> None:
print(f"Extracting '{archive_path}' to '{destination}'")
with ZipFile(archive_path) as file_in:
file_in.extractall(destination)
def run_protoc(
proto_paths: Iterable[StrPath], mypy_out: StrPath, proto_globs: Iterable[str], cwd: StrOrBytesPath | None = None
) -> str:
"""TODO: Describe parameters and return."""
protoc_version = (
subprocess.run([sys.executable, "-m", "grpc_tools.protoc", "--version"], capture_output=True).stdout.decode().strip()
)
print()
print(protoc_version)
protoc_args = [
*[f"--proto_path={proto_path}" for proto_path in proto_paths],
"--mypy_out",
f"relax_strict_optional_primitives:{mypy_out}",
*proto_globs,
]
print("Running: protoc\n " + "\n ".join(protoc_args) + "\n")
subprocess.run((sys.executable, "-m", "grpc_tools.protoc", *protoc_args), cwd=cwd, check=True)
return protoc_version
scripts/sync_protobuf/google_protobuf.py
-97
View File
@@ -1,97 +0,0 @@
#!/usr/bin/env python3
"""
Generates the protobuf stubs for the given protobuf version using mypy-protobuf.
Generally, new minor versions are a good time to update the stubs.
"""
from __future__ import annotations
import json
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any
from _utils import MYPY_PROTOBUF_VERSION, download_file, extract_archive, run_protoc
from ts_utils.metadata import read_metadata, update_metadata
from ts_utils.paths import distribution_path
# PyPi version has an extra "major" number that the Git version doesn't have
PACKAGE_VERSION = read_metadata("protobuf").version_spec.version[2:]
STUBS_FOLDER = distribution_path("protobuf").absolute()
ARCHIVE_FILENAME = f"protobuf-{PACKAGE_VERSION}.zip"
ARCHIVE_URL = f"https://github.com/protocolbuffers/protobuf/releases/download/v{PACKAGE_VERSION}/{ARCHIVE_FILENAME}"
EXTRACTED_PACKAGE_DIR = f"protobuf-{PACKAGE_VERSION}"
VERSION_PATTERN = re.compile(r'def game_version\(\):\n return "(.+?)"')
PROTO_FILE_PATTERN = re.compile(r'"//:(.*)_proto"')
def extract_python_version(file_path: Path) -> str:
"""Extract the Python version from https://github.com/protocolbuffers/protobuf/blob/main/version.json ."""
with open(file_path) as file:
data: dict[str, Any] = json.load(file)
# The root key will be the protobuf source code version
version = next(iter(data.values()))["languages"]["python"]
assert isinstance(version, str)
return version
def extract_proto_file_paths(temp_dir: Path) -> list[str]:
"""
Roughly reproduce the subset of .proto files on the public interface
as described in py_proto_library calls in
https://github.com/protocolbuffers/protobuf/blob/main/python/dist/BUILD.bazel .
"""
with open(temp_dir / EXTRACTED_PACKAGE_DIR / "python" / "dist" / "BUILD.bazel") as file:
matched_lines = filter(None, (re.search(PROTO_FILE_PATTERN, line) for line in file))
proto_files = [
EXTRACTED_PACKAGE_DIR + "/src/google/protobuf/" + match.group(1).replace("compiler_", "compiler/") + ".proto"
for match in matched_lines
]
return proto_files
def main() -> None:
temp_dir = Path(tempfile.mkdtemp())
# Fetch s2clientprotocol (which contains all the .proto files)
archive_path = temp_dir / ARCHIVE_FILENAME
download_file(ARCHIVE_URL, archive_path)
extract_archive(archive_path, temp_dir)
# Remove existing pyi
for old_stub in STUBS_FOLDER.rglob("*_pb2.pyi"):
old_stub.unlink()
protoc_version = run_protoc(
proto_paths=(f"{EXTRACTED_PACKAGE_DIR}/src",),
mypy_out=STUBS_FOLDER,
proto_globs=extract_proto_file_paths(temp_dir),
cwd=temp_dir,
)
python_protobuf_version = extract_python_version(temp_dir / EXTRACTED_PACKAGE_DIR / "version.json")
# Cleanup after ourselves, this is a temp dir, but it can still grow fast if run multiple times
shutil.rmtree(temp_dir)
update_metadata(
"protobuf",
extra_description=f"""Partially generated using \
[mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
and {protoc_version} on \
[protobuf v{PACKAGE_VERSION}](https://github.com/protocolbuffers/protobuf/releases/tag/v{PACKAGE_VERSION}) \
(python `protobuf=={python_protobuf_version}`).""",
)
print("Updated protobuf/METADATA.toml")
# Run pre-commit to cleanup the stubs
subprocess.run((sys.executable, "-m", "pre_commit", "run", "--files", *STUBS_FOLDER.rglob("*_pb2.pyi")))
if __name__ == "__main__":
main()
scripts/sync_protobuf/s2clientprotocol.py
-76
View File
@@ -1,76 +0,0 @@
#!/usr/bin/env python3
"""
Generates the protobuf stubs for the given s2clientprotocol version using mypy-protobuf.
Generally, new minor versions are a good time to update the stubs.
"""
from __future__ import annotations
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from _utils import MYPY_PROTOBUF_VERSION, download_file, extract_archive, run_protoc
from ts_utils.metadata import update_metadata
from ts_utils.paths import distribution_path
# Whenever you update PACKAGE_VERSION here, version should be updated
# in stubs/s2clientprotocol/METADATA.toml and vice-versa.
PACKAGE_VERSION = "c04df4adbe274858a4eb8417175ee32ad02fd609"
STUBS_FOLDER = distribution_path("s2clientprotocol").absolute()
ARCHIVE_FILENAME = f"{PACKAGE_VERSION}.zip"
ARCHIVE_URL = f"https://github.com/Blizzard/s2client-proto/archive/{ARCHIVE_FILENAME}"
EXTRACTED_PACKAGE_DIR = f"s2client-proto-{PACKAGE_VERSION}"
VERSION_PATTERN = re.compile(r'def game_version\(\):\n return "(.+?)"')
def extract_python_version(file_path: Path) -> str:
"""Extract Python version from s2clientprotocol's build file."""
match = re.search(VERSION_PATTERN, file_path.read_text())
assert match
return match.group(1)
def main() -> None:
temp_dir = Path(tempfile.mkdtemp())
# Fetch s2clientprotocol (which contains all the .proto files)
archive_path = temp_dir / ARCHIVE_FILENAME
download_file(ARCHIVE_URL, archive_path)
extract_archive(archive_path, temp_dir)
# Remove existing pyi
for old_stub in STUBS_FOLDER.rglob("*_pb2.pyi"):
old_stub.unlink()
protoc_version = run_protoc(
proto_paths=(EXTRACTED_PACKAGE_DIR,),
mypy_out=STUBS_FOLDER,
proto_globs=(f"{EXTRACTED_PACKAGE_DIR}/s2clientprotocol/*.proto",),
cwd=temp_dir,
)
python_s2_client_proto_version = extract_python_version(temp_dir / EXTRACTED_PACKAGE_DIR / "s2clientprotocol" / "build.py")
# Cleanup after ourselves, this is a temp dir, but it can still grow fast if run multiple times
shutil.rmtree(temp_dir)
update_metadata(
"s2clientprotocol",
extra_description=f"""Partially generated using \
[mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
and {protoc_version} on \
[s2client-proto {python_s2_client_proto_version}](https://github.com/Blizzard/s2client-proto/tree/{PACKAGE_VERSION}).""",
)
print("Updated s2clientprotocol/METADATA.toml")
# Run pre-commit to cleanup the stubs
subprocess.run((sys.executable, "-m", "pre_commit", "run", "--files", *STUBS_FOLDER.rglob("*_pb2.pyi")))
if __name__ == "__main__":
main()
scripts/sync_protobuf/tensorflow.py
-144
View File
@@ -1,144 +0,0 @@
#!/usr/bin/env python3
"""
Generates the protobuf stubs for the given tensorflow version using mypy-protobuf.
Generally, new minor versions are a good time to update the stubs.
"""
from __future__ import annotations
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from _utils import MYPY_PROTOBUF_VERSION, download_file, extract_archive, run_protoc
from ts_utils.metadata import read_metadata, update_metadata
from ts_utils.paths import distribution_path
PACKAGE_VERSION = read_metadata("tensorflow").version_spec.version
STUBS_FOLDER = distribution_path("tensorflow").absolute()
ARCHIVE_FILENAME = f"v{PACKAGE_VERSION}.zip"
ARCHIVE_URL = f"https://github.com/tensorflow/tensorflow/archive/refs/tags/{ARCHIVE_FILENAME}"
EXTRACTED_PACKAGE_DIR = f"tensorflow-{PACKAGE_VERSION}"
PROTOS_TO_REMOVE = (
"compiler/xla/autotune_results_pb2.pyi",
"compiler/xla/autotuning_pb2.pyi",
"compiler/xla/service/buffer_assignment_pb2.pyi",
"compiler/xla/service/hlo_execution_profile_data_pb2.pyi",
"core/protobuf/autotuning_pb2.pyi",
"core/protobuf/conv_autotuning_pb2.pyi",
"core/protobuf/critical_section_pb2.pyi",
"core/protobuf/eager_service_pb2.pyi",
"core/protobuf/master_pb2.pyi",
"core/protobuf/master_service_pb2.pyi",
"core/protobuf/replay_log_pb2.pyi",
"core/protobuf/tpu/compile_metadata_pb2.pyi",
"core/protobuf/worker_pb2.pyi",
"core/protobuf/worker_service_pb2.pyi",
"core/util/example_proto_fast_parsing_test_pb2.pyi",
)
"""
These protos exist in a folder with protos used in python,
but are not included in the python wheel.
They are likely only used for other language builds.
stubtest was used to identify them by looking for ModuleNotFoundError.
(comment out ".*_pb2.*" from the allowlist)
"""
TSL_IMPORT_PATTERN = re.compile(r"(\[|\s)tsl\.")
XLA_IMPORT_PATTERN = re.compile(r"(\[|\s)xla\.")
def move_tree(source: Path, destination: Path) -> None:
"""Move directory and merge if destination already exists.
Can't use shutil.move because it can't merge existing directories.
"""
print(f"Moving '{source}' to '{destination}'")
shutil.copytree(source, destination, dirs_exist_ok=True)
shutil.rmtree(source)
def post_creation() -> None:
"""Move third-party and fix imports."""
print()
move_tree(STUBS_FOLDER / "tsl", STUBS_FOLDER / "tensorflow" / "tsl")
move_tree(STUBS_FOLDER / "xla", STUBS_FOLDER / "tensorflow" / "compiler" / "xla")
for path in STUBS_FOLDER.rglob("*_pb2.pyi"):
print(f"Fixing imports in '{path}'")
with open(path) as file:
filedata = file.read()
# Replace the target string
filedata = re.sub(TSL_IMPORT_PATTERN, "\\1tensorflow.tsl.", filedata)
filedata = re.sub(XLA_IMPORT_PATTERN, "\\1tensorflow.compiler.xla.", filedata)
# Write the file out again
with open(path, "w") as file:
file.write(filedata)
print()
for to_remove in PROTOS_TO_REMOVE:
file_path = STUBS_FOLDER / "tensorflow" / to_remove
os.remove(file_path)
print(f"Removed '{file_path}'")
def main() -> None:
temp_dir = Path(tempfile.mkdtemp())
# Fetch tensorflow (which contains all the .proto files)
archive_path = temp_dir / ARCHIVE_FILENAME
download_file(ARCHIVE_URL, archive_path)
extract_archive(archive_path, temp_dir)
# Remove existing pyi
for old_stub in STUBS_FOLDER.rglob("*_pb2.pyi"):
old_stub.unlink()
protoc_version = run_protoc(
proto_paths=(
f"{EXTRACTED_PACKAGE_DIR}/third_party/xla/third_party/tsl",
f"{EXTRACTED_PACKAGE_DIR}/third_party/xla",
f"{EXTRACTED_PACKAGE_DIR}",
),
mypy_out=STUBS_FOLDER,
proto_globs=(
f"{EXTRACTED_PACKAGE_DIR}/third_party/xla/xla/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/third_party/xla/xla/service/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/third_party/xla/xla/tsl/protobuf/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/tensorflow/core/example/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/tensorflow/core/framework/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/tensorflow/core/protobuf/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/tensorflow/core/protobuf/tpu/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/tensorflow/core/util/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/tensorflow/python/keras/protobuf/*.proto",
f"{EXTRACTED_PACKAGE_DIR}/third_party/xla/third_party/tsl/tsl/protobuf/*.proto",
),
cwd=temp_dir,
)
# Cleanup after ourselves, this is a temp dir, but it can still grow fast if run multiple times
shutil.rmtree(temp_dir)
post_creation()
update_metadata(
"tensorflow",
extra_description=f"""Partially generated using \
[mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
and {protoc_version} on `tensorflow=={PACKAGE_VERSION}`.""",
)
print("Updated tensorflow/METADATA.toml")
# Run pre-commit to cleanup the stubs
subprocess.run((sys.executable, "-m", "pre_commit", "run", "--files", *STUBS_FOLDER.rglob("*_pb2.pyi")))
if __name__ == "__main__":
main()
stdlib/2/BaseHTTPServer.pyi
+47
View File
@@ -0,0 +1,47 @@
# Stubs for BaseHTTPServer (Python 2.7)
from typing import Any, BinaryIO, Callable, Mapping, Optional, Tuple, Union
import SocketServer
import mimetools
class HTTPServer(SocketServer.TCPServer):
server_name: str
server_port: int
def __init__(self, server_address: Tuple[str, int],
RequestHandlerClass: Callable[..., BaseHTTPRequestHandler]) -> None: ...
class BaseHTTPRequestHandler(SocketServer.StreamRequestHandler):
client_address: Tuple[str, int]
server: SocketServer.BaseServer
close_connection: bool
command: str
path: str
request_version: str
headers: mimetools.Message
rfile: BinaryIO
wfile: BinaryIO
server_version: str
sys_version: str
error_message_format: str
error_content_type: str
protocol_version: str
MessageClass: type
responses: Mapping[int, Tuple[str, str]]
def __init__(self, request: bytes, client_address: Tuple[str, int],
server: SocketServer.BaseServer) -> None: ...
def handle(self) -> None: ...
def handle_one_request(self) -> None: ...
def send_error(self, code: int, message: Optional[str] = ...) -> None: ...
def send_response(self, code: int,
message: Optional[str] = ...) -> None: ...
def send_header(self, keyword: str, value: str) -> None: ...
def end_headers(self) -> None: ...
def flush_headers(self) -> None: ...
def log_request(self, code: Union[int, str] = ...,
size: Union[int, str] = ...) -> None: ...
def log_error(self, format: str, *args: Any) -> None: ...
def log_message(self, format: str, *args: Any) -> None: ...
def version_string(self) -> str: ...
def date_time_string(self, timestamp: Optional[int] = ...) -> str: ...
def log_date_time_string(self) -> str: ...
def address_string(self) -> str: ...
stdlib/2/CGIHTTPServer.pyi
+8
View File
@@ -0,0 +1,8 @@
# Stubs for CGIHTTPServer (Python 2.7)
from typing import List
import SimpleHTTPServer
class CGIHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
cgi_directories: List[str]
def do_POST(self) -> None: ...
stdlib/2/ConfigParser.pyi
+99
View File
@@ -0,0 +1,99 @@
from typing import Any, IO, Sequence, Tuple, Union, List, Dict, Protocol, Optional
DEFAULTSECT: str
MAX_INTERPOLATION_DEPTH: int
class Error(Exception):
message: Any
def __init__(self, msg: str = ...) -> None: ...
def _get_message(self) -> None: ...
def _set_message(self, value: str) -> None: ...
def __repr__(self) -> str: ...
def __str__(self) -> str: ...
class NoSectionError(Error):
section: str
def __init__(self, section: str) -> None: ...
class DuplicateSectionError(Error):
section: str
def __init__(self, section: str) -> None: ...
class NoOptionError(Error):
section: str
option: str
def __init__(self, option: str, section: str) -> None: ...
class InterpolationError(Error):
section: str
option: str
msg: str
def __init__(self, option: str, section: str, msg: str) -> None: ...
class InterpolationMissingOptionError(InterpolationError):
reference: str
def __init__(self, option: str, section: str, rawval: str, reference: str) -> None: ...
class InterpolationSyntaxError(InterpolationError): ...
class InterpolationDepthError(InterpolationError):
def __init__(self, option: str, section: str, rawval: str) -> None: ...
class ParsingError(Error):
filename: str
errors: List[Tuple[Any, Any]]
def __init__(self, filename: str) -> None: ...
def append(self, lineno: Any, line: Any) -> None: ...
class MissingSectionHeaderError(ParsingError):
lineno: Any
line: Any
def __init__(self, filename: str, lineno: Any, line: Any) -> None: ...
class _Readable(Protocol):
def readline(self) -> str: ...
class RawConfigParser:
_dict: Any
_sections: Dict[Any, Any]
_defaults: Dict[Any, Any]
_optcre: Any
SECTCRE: Any
OPTCRE: Any
OPTCRE_NV: Any
def __init__(self, defaults: Dict[Any, Any] = ..., dict_type: Any = ..., allow_no_value: bool = ...) -> None: ...
def defaults(self) -> Dict[Any, Any]: ...
def sections(self) -> List[str]: ...
def add_section(self, section: str) -> None: ...
def has_section(self, section: str) -> bool: ...
def options(self, section: str) -> List[str]: ...
def read(self, filenames: Union[str, Sequence[str]]) -> List[str]: ...
def readfp(self, fp: _Readable, filename: str = ...) -> None: ...
def get(self, section: str, option: str) -> str: ...
def items(self, section: str) -> List[Tuple[Any, Any]]: ...
def _get(self, section: str, conv: type, option: str) -> Any: ...
def getint(self, section: str, option: str) -> int: ...
def getfloat(self, section: str, option: str) -> float: ...
_boolean_states: Dict[str, bool]
def getboolean(self, section: str, option: str) -> bool: ...
def optionxform(self, optionstr: str) -> str: ...
def has_option(self, section: str, option: str) -> bool: ...
def set(self, section: str, option: str, value: Any = ...) -> None: ...
def write(self, fp: IO[str]) -> None: ...
def remove_option(self, section: str, option: Any) -> bool: ...
def remove_section(self, section: str) -> bool: ...
def _read(self, fp: IO[str], fpname: str) -> None: ...
class ConfigParser(RawConfigParser):
_KEYCRE: Any
def get(self, section: str, option: str, raw: bool = ..., vars: Optional[Dict[Any, Any]] = ...) -> Any: ...
def items(self, section: str, raw: bool = ..., vars: Optional[Dict[Any, Any]] = ...) -> List[Tuple[str, Any]]: ...
def _interpolate(self, section: str, option: str, rawval: Any, vars: Any) -> str: ...
def _interpolation_replace(self, match: Any) -> str: ...
class SafeConfigParser(ConfigParser):
_interpvar_re: Any
def _interpolate(self, section: str, option: str, rawval: Any, vars: Any) -> str: ...
def _interpolate_some(
self, option: str, accum: List[Any], rest: str, section: str, map: Dict[Any, Any], depth: int,
) -> None: ...
stdlib/2/Cookie.pyi
+40
View File
@@ -0,0 +1,40 @@
from typing import Any, Dict, Optional
class CookieError(Exception): ...
class Morsel(Dict[Any, Any]):
key: Any
def __init__(self): ...
def __setitem__(self, K, V): ...
def isReservedKey(self, K): ...
value: Any
coded_value: Any
def set(self, key, val, coded_val, LegalChars=..., idmap=..., translate=...): ...
def output(self, attrs: Optional[Any] = ..., header=...): ...
def js_output(self, attrs: Optional[Any] = ...): ...
def OutputString(self, attrs: Optional[Any] = ...): ...
class BaseCookie(Dict[Any, Any]):
def value_decode(self, val): ...
def value_encode(self, val): ...
def __init__(self, input: Optional[Any] = ...): ...
def __setitem__(self, key, value): ...
def output(self, attrs: Optional[Any] = ..., header=..., sep=...): ...
def js_output(self, attrs: Optional[Any] = ...): ...
def load(self, rawdata): ...
class SimpleCookie(BaseCookie):
def value_decode(self, val): ...
def value_encode(self, val): ...
class SerialCookie(BaseCookie):
def __init__(self, input: Optional[Any] = ...): ...
def value_decode(self, val): ...
def value_encode(self, val): ...
class SmartCookie(BaseCookie):
def __init__(self, input: Optional[Any] = ...): ...
def value_decode(self, val): ...
def value_encode(self, val): ...
Cookie: Any
stdlib/2/HTMLParser.pyi
+31
View File
@@ -0,0 +1,31 @@
from typing import List, Tuple, AnyStr
from markupbase import ParserBase
class HTMLParser(ParserBase):
def __init__(self) -> None: ...
def feed(self, feed: AnyStr) -> None: ...
def close(self) -> None: ...
def reset(self) -> None: ...
def get_starttag_text(self) -> AnyStr: ...
def set_cdata_mode(self, AnyStr) -> None: ...
def clear_cdata_mode(self) -> None: ...
def handle_startendtag(self, tag: AnyStr, attrs: List[Tuple[AnyStr, AnyStr]]): ...
def handle_starttag(self, tag: AnyStr, attrs: List[Tuple[AnyStr, AnyStr]]): ...
def handle_endtag(self, tag: AnyStr): ...
def handle_charref(self, name: AnyStr): ...
def handle_entityref(self, name: AnyStr): ...
def handle_data(self, data: AnyStr): ...
def handle_comment(self, data: AnyStr): ...
def handle_decl(self, decl: AnyStr): ...
def handle_pi(self, data: AnyStr): ...
def unknown_decl(self, data: AnyStr): ...
def unescape(self, s: AnyStr) -> AnyStr: ...
class HTMLParseError(Exception):
msg: str
lineno: int
offset: int
stdlib/2/Queue.pyi
+31
View File
@@ -0,0 +1,31 @@
# Stubs for Queue (Python 2)
from collections import deque
from typing import Any, Deque, TypeVar, Generic, Optional
_T = TypeVar('_T')
class Empty(Exception): ...
class Full(Exception): ...
class Queue(Generic[_T]):
maxsize: Any
mutex: Any
not_empty: Any
not_full: Any
all_tasks_done: Any
unfinished_tasks: Any
queue: Deque[Any] # undocumented
def __init__(self, maxsize: int = ...) -> None: ...
def task_done(self) -> None: ...
def join(self) -> None: ...
def qsize(self) -> int: ...
def empty(self) -> bool: ...
def full(self) -> bool: ...
def put(self, item: _T, block: bool = ..., timeout: Optional[float] = ...) -> None: ...
def put_nowait(self, item: _T) -> None: ...
def get(self, block: bool = ..., timeout: Optional[float] = ...) -> _T: ...
def get_nowait(self) -> _T: ...
class PriorityQueue(Queue[_T]): ...
class LifoQueue(Queue[_T]): ...
stdlib/2/SimpleHTTPServer.pyi
+16
View File
@@ -0,0 +1,16 @@
# Stubs for SimpleHTTPServer (Python 2)
from typing import Any, AnyStr, IO, Mapping, Optional, Union
import BaseHTTPServer
from StringIO import StringIO
class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
server_version: str
def do_GET(self) -> None: ...
def do_HEAD(self) -> None: ...
def send_head(self) -> Optional[IO[str]]: ...
def list_directory(self, path: Union[str, unicode]) -> Optional[StringIO[Any]]: ...
def translate_path(self, path: AnyStr) -> AnyStr: ...
def copyfile(self, source: IO[AnyStr], outputfile: IO[AnyStr]): ...
def guess_type(self, path: Union[str, unicode]) -> str: ...
extensions_map: Mapping[str, str]
stdlib/2/SocketServer.pyi
+99
View File
@@ -0,0 +1,99 @@
# NB: SocketServer.pyi and socketserver.pyi must remain consistent!
# Stubs for socketserver
from typing import Any, BinaryIO, Callable, Optional, Tuple, Type, Text, Union
from socket import SocketType
import sys
import types
class BaseServer:
address_family: int
RequestHandlerClass: Callable[..., BaseRequestHandler]
server_address: Tuple[str, int]
socket: SocketType
allow_reuse_address: bool
request_queue_size: int
socket_type: int
timeout: Optional[float]
def __init__(self, server_address: Any,
RequestHandlerClass: Callable[..., BaseRequestHandler]) -> None: ...
def fileno(self) -> int: ...
def handle_request(self) -> None: ...
def serve_forever(self, poll_interval: float = ...) -> None: ...
def shutdown(self) -> None: ...
def server_close(self) -> None: ...
def finish_request(self, request: bytes,
client_address: Tuple[str, int]) -> None: ...
def get_request(self) -> None: ...
def handle_error(self, request: bytes,
client_address: Tuple[str, int]) -> None: ...
def handle_timeout(self) -> None: ...
def process_request(self, request: bytes,
client_address: Tuple[str, int]) -> None: ...
def server_activate(self) -> None: ...
def server_bind(self) -> None: ...
def verify_request(self, request: bytes,
client_address: Tuple[str, int]) -> bool: ...
if sys.version_info >= (3, 6):
def __enter__(self) -> BaseServer: ...
def __exit__(self, exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[types.TracebackType]) -> None: ...
if sys.version_info >= (3, 3):
def service_actions(self) -> None: ...
class TCPServer(BaseServer):
def __init__(self, server_address: Tuple[str, int],
RequestHandlerClass: Callable[..., BaseRequestHandler],
bind_and_activate: bool = ...) -> None: ...
class UDPServer(BaseServer):
def __init__(self, server_address: Tuple[str, int],
RequestHandlerClass: Callable[..., BaseRequestHandler],
bind_and_activate: bool = ...) -> None: ...
if sys.platform != 'win32':
class UnixStreamServer(BaseServer):
def __init__(self, server_address: Union[Text, bytes],
RequestHandlerClass: Callable[..., BaseRequestHandler],
bind_and_activate: bool = ...) -> None: ...
class UnixDatagramServer(BaseServer):
def __init__(self, server_address: Union[Text, bytes],
RequestHandlerClass: Callable[..., BaseRequestHandler],
bind_and_activate: bool = ...) -> None: ...
class ForkingMixIn: ...
class ThreadingMixIn: ...
class ForkingTCPServer(ForkingMixIn, TCPServer): ...
class ForkingUDPServer(ForkingMixIn, UDPServer): ...
class ThreadingTCPServer(ThreadingMixIn, TCPServer): ...
class ThreadingUDPServer(ThreadingMixIn, UDPServer): ...
if sys.platform != 'win32':
class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): ...
class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): ...
class BaseRequestHandler:
# Those are technically of types, respectively:
# * Union[SocketType, Tuple[bytes, SocketType]]
# * Union[Tuple[str, int], str]
# But there are some concerns that having unions here would cause
# too much inconvenience to people using it (see
# https://github.com/python/typeshed/pull/384#issuecomment-234649696)
request: Any
client_address: Any
server: BaseServer
def setup(self) -> None: ...
def handle(self) -> None: ...
def finish(self) -> None: ...
class StreamRequestHandler(BaseRequestHandler):
rfile: BinaryIO
wfile: BinaryIO
class DatagramRequestHandler(BaseRequestHandler):
rfile: BinaryIO
wfile: BinaryIO
stdlib/2/StringIO.pyi
+30
View File
@@ -0,0 +1,30 @@
# Stubs for StringIO (Python 2)
from typing import Any, IO, AnyStr, Iterator, Iterable, Generic, List, Optional
class StringIO(IO[AnyStr], Generic[AnyStr]):
closed: bool
softspace: int
len: int
name: str
def __init__(self, buf: AnyStr = ...) -> None: ...
def __iter__(self) -> Iterator[AnyStr]: ...
def next(self) -> AnyStr: ...
def close(self) -> None: ...
def isatty(self) -> bool: ...
def seek(self, pos: int, mode: int = ...) -> int: ...
def tell(self) -> int: ...
def read(self, n: int = ...) -> AnyStr: ...
def readline(self, length: int = ...) -> AnyStr: ...
def readlines(self, sizehint: int = ...) -> List[AnyStr]: ...
def truncate(self, size: Optional[int] = ...) -> int: ...
def write(self, s: AnyStr) -> int: ...
def writelines(self, iterable: Iterable[AnyStr]) -> None: ...
def flush(self) -> None: ...
def getvalue(self) -> AnyStr: ...
def __enter__(self) -> Any: ...
def __exit__(self, type: Any, value: Any, traceback: Any) -> Any: ...
def fileno(self) -> int: ...
def readable(self) -> bool: ...
def seekable(self) -> bool: ...
def writable(self) -> bool: ...
stdlib/2/UserDict.pyi
+44
View File
@@ -0,0 +1,44 @@
from typing import (Any, Container, Dict, Generic, Iterable, Iterator, List,
Mapping, Optional, Sized, Tuple, TypeVar, Union, overload)
_KT = TypeVar('_KT')
_VT = TypeVar('_VT')
_T = TypeVar('_T')
class UserDict(Dict[_KT, _VT], Generic[_KT, _VT]):
data: Dict[_KT, _VT]
def __init__(self, initialdata: Mapping[_KT, _VT] = ...) -> None: ...
# TODO: __iter__ is not available for UserDict
class IterableUserDict(UserDict[_KT, _VT], Generic[_KT, _VT]):
...
class DictMixin(Iterable[_KT], Container[_KT], Sized, Generic[_KT, _VT]):
def has_key(self, key: _KT) -> bool: ...
def __len__(self) -> int: ...
def __iter__(self) -> Iterator[_KT]: ...
# From typing.Mapping[_KT, _VT]
# (can't inherit because of keys())
@overload
def get(self, k: _KT) -> Optional[_VT]: ...
@overload
def get(self, k: _KT, default: Union[_VT, _T]) -> Union[_VT, _T]: ...
def values(self) -> List[_VT]: ...
def items(self) -> List[Tuple[_KT, _VT]]: ...
def iterkeys(self) -> Iterator[_KT]: ...
def itervalues(self) -> Iterator[_VT]: ...
def iteritems(self) -> Iterator[Tuple[_KT, _VT]]: ...
def __contains__(self, o: Any) -> bool: ...
# From typing.MutableMapping[_KT, _VT]
def clear(self) -> None: ...
def pop(self, k: _KT, default: _VT = ...) -> _VT: ...
def popitem(self) -> Tuple[_KT, _VT]: ...
def setdefault(self, k: _KT, default: _VT = ...) -> _VT: ...
@overload
def update(self, m: Mapping[_KT, _VT], **kwargs: _VT) -> None: ...
@overload
def update(self, m: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ...
stdlib/2/UserList.pyi
+19
View File
@@ -0,0 +1,19 @@
from typing import Iterable, MutableSequence, TypeVar, Union, overload, List
_T = TypeVar("_T")
_S = TypeVar("_S")
class UserList(MutableSequence[_T]):
data: List[_T]
def insert(self, index: int, object: _T) -> None: ...
@overload
def __setitem__(self, i: int, o: _T) -> None: ...
@overload
def __setitem__(self, s: slice, o: Iterable[_T]) -> None: ...
def __delitem__(self, i: Union[int, slice]) -> None: ...
def __len__(self) -> int: ...
@overload
def __getitem__(self, i: int) -> _T: ...
@overload
def __getitem__(self: _S, s: slice) -> _S: ...
def sort(self) -> None: ...
stdlib/2/UserString.pyi
+75
View File
@@ -0,0 +1,75 @@
import collections
from typing import Any, Iterable, List, MutableSequence, Sequence, Optional, overload, Text, TypeVar, Tuple, Union
_UST = TypeVar("_UST", bound=UserString)
_MST = TypeVar("_MST", bound=MutableString)
class UserString(Sequence[UserString]):
data: unicode
def __init__(self, seq: object) -> None: ...
def __int__(self) -> int: ...
def __long__(self) -> long: ...
def __float__(self) -> float: ...
def __complex__(self) -> complex: ...
def __hash__(self) -> int: ...
def __len__(self) -> int: ...
@overload
def __getitem__(self: _UST, i: int) -> _UST: ...
@overload
def __getitem__(self: _UST, s: slice) -> _UST: ...
def __add__(self: _UST, other: Any) -> _UST: ...
def __radd__(self: _UST, other: Any) -> _UST: ...
def __mul__(self: _UST, other: int) -> _UST: ...
def __rmul__(self: _UST, other: int) -> _UST: ...
def __mod__(self: _UST, args: Any) -> _UST: ...
def capitalize(self: _UST) -> _UST: ...
def center(self: _UST, width: int, *args: Any) -> _UST: ...
def count(self, sub: int, start: int = ..., end: int = ...) -> int: ...
def decode(self: _UST, encoding: Optional[str] = ..., errors: Optional[str] = ...) -> _UST: ...
def encode(self: _UST, encoding: Optional[str] = ..., errors: Optional[str] = ...) -> _UST: ...
def endswith(self, suffix: Text, start: int = ..., end: int = ...) -> bool: ...
def expandtabs(self: _UST, tabsize: int = ...) -> _UST: ...
def find(self, sub: Text, start: int = ..., end: int = ...) -> int: ...
def index(self, sub: Text, start: int = ..., end: int = ...) -> int: ...
def isalpha(self) -> bool: ...
def isalnum(self) -> bool: ...
def isdecimal(self) -> bool: ...
def isdigit(self) -> bool: ...
def islower(self) -> bool: ...
def isnumeric(self) -> bool: ...
def isspace(self) -> bool: ...
def istitle(self) -> bool: ...
def isupper(self) -> bool: ...
def join(self, seq: Iterable[Text]) -> Text: ...
def ljust(self: _UST, width: int, *args: Any) -> _UST: ...
def lower(self: _UST) -> _UST: ...
def lstrip(self: _UST, chars: Optional[Text] = ...) -> _UST: ...
def partition(self, sep: Text) -> Tuple[Text, Text, Text]: ...
def replace(self: _UST, old: Text, new: Text, maxsplit: int = ...) -> _UST: ...
def rfind(self, sub: Text, start: int = ..., end: int = ...) -> int: ...
def rindex(self, sub: Text, start: int = ..., end: int = ...) -> int: ...
def rjust(self: _UST, width: int, *args: Any) -> _UST: ...
def rpartition(self, sep: Text) -> Tuple[Text, Text, Text]: ...
def rstrip(self: _UST, chars: Optional[Text] = ...) -> _UST: ...
def split(self, sep: Optional[Text] = ..., maxsplit: int = ...) -> List[Text]: ...
def rsplit(self, sep: Optional[Text] = ..., maxsplit: int = ...) -> List[Text]: ...
def splitlines(self, keepends: int = ...) -> List[Text]: ...
def startswith(self, suffix: Text, start: int = ..., end: int = ...) -> bool: ...
def strip(self: _UST, chars: Optional[Text] = ...) -> _UST: ...
def swapcase(self: _UST) -> _UST: ...
def title(self: _UST) -> _UST: ...
def translate(self: _UST, *args: Any) -> _UST: ...
def upper(self: _UST) -> _UST: ...
def zfill(self: _UST, width: int) -> _UST: ...
class MutableString(UserString, MutableSequence[MutableString]):
@overload
def __getitem__(self: _MST, i: int) -> _MST: ...
@overload
def __getitem__(self: _MST, s: slice) -> _MST: ...
def __setitem__(self, index: Union[int, slice], sub: Any) -> None: ...
def __delitem__(self, index: Union[int, slice]) -> None: ...
def immutable(self) -> UserString: ...
def __iadd__(self: _MST, other: Any) -> _MST: ...
def __imul__(self, n: int) -> _MST: ...
def insert(self, index: int, value: Any) -> None: ...
stdlib/2/__builtin__.pyi
+1658
View File
File diff suppressed because it is too large Load Diff
stdlib/2/_ast.pyi
+330
View File
@@ -0,0 +1,330 @@
import typing
from typing import Optional
__version__: str
PyCF_ONLY_AST: int
_identifier = str
class AST:
_attributes: typing.Tuple[str, ...]
_fields: typing.Tuple[str, ...]
def __init__(self, *args, **kwargs) -> None: ...
class mod(AST):
...
class Module(mod):
body: typing.List[stmt]
class Interactive(mod):
body: typing.List[stmt]
class Expression(mod):
body: expr
class Suite(mod):
body: typing.List[stmt]
class stmt(AST):
lineno: int
col_offset: int
class FunctionDef(stmt):
name: _identifier
args: arguments
body: typing.List[stmt]
decorator_list: typing.List[expr]
class ClassDef(stmt):
name: _identifier
bases: typing.List[expr]
body: typing.List[stmt]
decorator_list: typing.List[expr]
class Return(stmt):
value: Optional[expr]
class Delete(stmt):
targets: typing.List[expr]
class Assign(stmt):
targets: typing.List[expr]
value: expr
class AugAssign(stmt):
target: expr
op: operator
value: expr
class Print(stmt):
dest: Optional[expr]
values: typing.List[expr]
nl: bool
class For(stmt):
target: expr
iter: expr
body: typing.List[stmt]
orelse: typing.List[stmt]
class While(stmt):
test: expr
body: typing.List[stmt]
orelse: typing.List[stmt]
class If(stmt):
test: expr
body: typing.List[stmt]
orelse: typing.List[stmt]
class With(stmt):
context_expr: expr
optional_vars: Optional[expr]
body: typing.List[stmt]
class Raise(stmt):
type: Optional[expr]
inst: Optional[expr]
tback: Optional[expr]
class TryExcept(stmt):
body: typing.List[stmt]
handlers: typing.List[ExceptHandler]
orelse: typing.List[stmt]
class TryFinally(stmt):
body: typing.List[stmt]
finalbody: typing.List[stmt]
class Assert(stmt):
test: expr
msg: Optional[expr]
class Import(stmt):
names: typing.List[alias]
class ImportFrom(stmt):
module: Optional[_identifier]
names: typing.List[alias]
level: Optional[int]
class Exec(stmt):
body: expr
globals: Optional[expr]
locals: Optional[expr]
class Global(stmt):
names: typing.List[_identifier]
class Expr(stmt):
value: expr
class Pass(stmt): ...
class Break(stmt): ...
class Continue(stmt): ...
class slice(AST):
...
_slice = slice # this lets us type the variable named 'slice' below
class Slice(slice):
lower: Optional[expr]
upper: Optional[expr]
step: Optional[expr]
class ExtSlice(slice):
dims: typing.List[slice]
class Index(slice):
value: expr
class Ellipsis(slice): ...
class expr(AST):
lineno: int
col_offset: int
class BoolOp(expr):
op: boolop
values: typing.List[expr]
class BinOp(expr):
left: expr
op: operator
right: expr
class UnaryOp(expr):
op: unaryop
operand: expr
class Lambda(expr):
args: arguments
body: expr
class IfExp(expr):
test: expr
body: expr
orelse: expr
class Dict(expr):
keys: typing.List[expr]
values: typing.List[expr]
class Set(expr):
elts: typing.List[expr]
class ListComp(expr):
elt: expr
generators: typing.List[comprehension]
class SetComp(expr):
elt: expr
generators: typing.List[comprehension]
class DictComp(expr):
key: expr
value: expr
generators: typing.List[comprehension]
class GeneratorExp(expr):
elt: expr
generators: typing.List[comprehension]
class Yield(expr):
value: Optional[expr]
class Compare(expr):
left: expr
ops: typing.List[cmpop]
comparators: typing.List[expr]
class Call(expr):
func: expr
args: typing.List[expr]
keywords: typing.List[keyword]
starargs: Optional[expr]
kwargs: Optional[expr]
class Repr(expr):
value: expr
class Num(expr):
n: float
class Str(expr):
s: str
class Attribute(expr):
value: expr
attr: _identifier
ctx: expr_context
class Subscript(expr):
value: expr
slice: _slice
ctx: expr_context
class Name(expr):
id: _identifier
ctx: expr_context
class List(expr):
elts: typing.List[expr]
ctx: expr_context
class Tuple(expr):
elts: typing.List[expr]
ctx: expr_context
class expr_context(AST):
...
class AugLoad(expr_context): ...
class AugStore(expr_context): ...
class Del(expr_context): ...
class Load(expr_context): ...
class Param(expr_context): ...
class Store(expr_context): ...
class boolop(AST):
...
class And(boolop): ...
class Or(boolop): ...
class operator(AST):
...
class Add(operator): ...
class BitAnd(operator): ...
class BitOr(operator): ...
class BitXor(operator): ...
class Div(operator): ...
class FloorDiv(operator): ...
class LShift(operator): ...
class Mod(operator): ...
class Mult(operator): ...
class Pow(operator): ...
class RShift(operator): ...
class Sub(operator): ...
class unaryop(AST):
...
class Invert(unaryop): ...
class Not(unaryop): ...
class UAdd(unaryop): ...
class USub(unaryop): ...
class cmpop(AST):
...
class Eq(cmpop): ...
class Gt(cmpop): ...
class GtE(cmpop): ...
class In(cmpop): ...
class Is(cmpop): ...
class IsNot(cmpop): ...
class Lt(cmpop): ...
class LtE(cmpop): ...
class NotEq(cmpop): ...
class NotIn(cmpop): ...
class comprehension(AST):
target: expr
iter: expr
ifs: typing.List[expr]
class excepthandler(AST):
...
class ExceptHandler(excepthandler):
type: Optional[expr]
name: Optional[expr]
body: typing.List[stmt]
lineno: int
col_offset: int
class arguments(AST):
args: typing.List[expr]
vararg: Optional[_identifier]
kwarg: Optional[_identifier]
defaults: typing.List[expr]
class keyword(AST):
arg: _identifier
value: expr
class alias(AST):
name: _identifier
asname: Optional[_identifier]
stdlib/2/_collections.pyi
+38
View File
@@ -0,0 +1,38 @@
"""Stub file for the '_collections' module."""
from typing import Any, Callable, Dict, Generic, Iterator, TypeVar, Optional, Union
_K = TypeVar("_K")
_V = TypeVar("_V")
_T = TypeVar('_T')
_T2 = TypeVar('_T2')
class defaultdict(Dict[_K, _V]):
default_factory: None
def __init__(self, __default_factory: Callable[[], _V] = ..., init: Any = ...) -> None: ...
def __missing__(self, key: _K) -> _V: ...
def __copy__(self: _T) -> _T: ...
def copy(self: _T) -> _T: ...
class deque(Generic[_T]):
maxlen: Optional[int]
def __init__(self, iterable: Iterator[_T] = ..., maxlen: int = ...) -> None: ...
def append(self, x: _T) -> None: ...
def appendleft(self, x: _T) -> None: ...
def clear(self) -> None: ...
def count(self, x: Any) -> int: ...
def extend(self, iterable: Iterator[_T]) -> None: ...
def extendleft(self, iterable: Iterator[_T]) -> None: ...
def pop(self) -> _T: ...
def popleft(self) -> _T: ...
def remove(self, value: _T) -> None: ...
def reverse(self) -> None: ...
def rotate(self, n: int = ...) -> None: ...
def __contains__(self, o: Any) -> bool: ...
def __copy__(self) -> deque[_T]: ...
def __getitem__(self, i: int) -> _T: ...
def __iadd__(self, other: deque[_T2]) -> deque[Union[_T, _T2]]: ...
def __iter__(self) -> Iterator[_T]: ...
def __len__(self) -> int: ...
def __reversed__(self) -> Iterator[_T]: ...
def __setitem__(self, i: int, x: _T) -> None: ...
stdlib/2/_functools.pyi
+20
View File
@@ -0,0 +1,20 @@
"""Stub file for the '_functools' module."""
from typing import Any, Callable, Dict, Iterable, Optional, TypeVar, Tuple, overload
_T = TypeVar("_T")
_S = TypeVar("_S")
@overload
def reduce(function: Callable[[_T, _T], _T],
sequence: Iterable[_T]) -> _T: ...
@overload
def reduce(function: Callable[[_T, _S], _T],
sequence: Iterable[_S], initial: _T) -> _T: ...
class partial(object):
func: Callable[..., Any]
args: Tuple[Any, ...]
keywords: Dict[str, Any]
def __init__(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> None: ...
def __call__(self, *args: Any, **kwargs: Any) -> Any: ...
stdlib/2/_hotshot.pyi
+24
View File
@@ -0,0 +1,24 @@
"""Stub file for the '_hotshot' module."""
# This is an autogenerated file. It serves as a starting point
# for a more precise manual annotation of this module.
# Feel free to edit the source below, but remove this header when you do.
from typing import Any, List, Tuple, Dict, Generic
def coverage(a: str) -> Any: ...
def logreader(a: str) -> LogReaderType: ...
def profiler(a: str, *args, **kwargs) -> Any: ...
def resolution() -> Tuple[Any, ...]: ...
class LogReaderType(object):
def close(self) -> None: ...
def fileno(self) -> int: ...
class ProfilerType(object):
def addinfo(self, a: str, b: str) -> None: ...
def close(self) -> None: ...
def fileno(self) -> int: ...
def runcall(self, *args, **kwargs) -> Any: ...
def runcode(self, a, b, *args, **kwargs) -> Any: ...
def start(self) -> None: ...
def stop(self) -> None: ...
stdlib/2/_io.pyi
+186
View File
@@ -0,0 +1,186 @@
from typing import Any, AnyStr, BinaryIO, IO, Text, TextIO, Iterable, Iterator, List, Optional, Type, Tuple, TypeVar, Union
from mmap import mmap
from types import TracebackType
_bytearray_like = Union[bytearray, mmap]
DEFAULT_BUFFER_SIZE: int
class BlockingIOError(IOError):
characters_written: int
class UnsupportedOperation(ValueError, IOError): ...
_T = TypeVar("_T")
class _IOBase(BinaryIO):
@property
def closed(self) -> bool: ...
def _checkClosed(self, msg: Optional[str] = ...) -> None: ... # undocumented
def _checkReadable(self) -> None: ...
def _checkSeekable(self) -> None: ...
def _checkWritable(self) -> None: ...
# All these methods are concrete here (you can instantiate this)
def close(self) -> None: ...
def fileno(self) -> int: ...
def flush(self) -> None: ...
def isatty(self) -> bool: ...
def readable(self) -> bool: ...
def seek(self, offset: int, whence: int = ...) -> int: ...
def seekable(self) -> bool: ...
def tell(self) -> int: ...
def truncate(self, size: Optional[int] = ...) -> int: ...
def writable(self) -> bool: ...
def __enter__(self: _T) -> _T: ...
def __exit__(self, t: Optional[Type[BaseException]], value: Optional[BaseException], traceback: Optional[Any]) -> Optional[bool]: ...
def __iter__(self: _T) -> _T: ...
# The parameter type of writelines[s]() is determined by that of write():
def writelines(self, lines: Iterable[bytes]) -> None: ...
# The return type of readline[s]() and next() is determined by that of read():
def readline(self, limit: int = ...) -> bytes: ...
def readlines(self, hint: int = ...) -> List[bytes]: ...
def next(self) -> bytes: ...
# These don't actually exist but we need to pretend that it does
# so that this class is concrete.
def write(self, s: bytes) -> int: ...
def read(self, n: int = ...) -> bytes: ...
class _BufferedIOBase(_IOBase):
def read1(self, n: int) -> bytes: ...
def read(self, size: int = ...) -> bytes: ...
def readinto(self, buffer: _bytearray_like) -> int: ...
def write(self, s: bytes) -> int: ...
def detach(self) -> _IOBase: ...
class BufferedRWPair(_BufferedIOBase):
def __init__(self, reader: _RawIOBase, writer: _RawIOBase,
buffer_size: int = ..., max_buffer_size: int = ...) -> None: ...
def peek(self, n: int = ...) -> bytes: ...
def __enter__(self) -> BufferedRWPair: ...
class BufferedRandom(_BufferedIOBase):
mode: str
name: str
raw: _IOBase
def __init__(self, raw: _IOBase,
buffer_size: int = ...,
max_buffer_size: int = ...) -> None: ...
def peek(self, n: int = ...) -> bytes: ...
class BufferedReader(_BufferedIOBase):
mode: str
name: str
raw: _IOBase
def __init__(self, raw: _IOBase, buffer_size: int = ...) -> None: ...
def peek(self, n: int = ...) -> bytes: ...
class BufferedWriter(_BufferedIOBase):
name: str
raw: _IOBase
mode: str
def __init__(self, raw: _IOBase,
buffer_size: int = ...,
max_buffer_size: int = ...) -> None: ...
class BytesIO(_BufferedIOBase):
def __init__(self, initial_bytes: bytes = ...) -> None: ...
def __setstate__(self, state: Tuple[Any, ...]) -> None: ...
def __getstate__(self) -> Tuple[Any, ...]: ...
# BytesIO does not contain a "name" field. This workaround is necessary
# to allow BytesIO sub-classes to add this field, as it is defined
# as a read-only property on IO[].
name: Any
def getvalue(self) -> bytes: ...
def write(self, s: bytes) -> int: ...
def writelines(self, lines: Iterable[bytes]) -> None: ...
def read1(self, size: int) -> bytes: ...
def next(self) -> bytes: ...
class _RawIOBase(_IOBase):
def readall(self) -> str: ...
def read(self, n: int = ...) -> str: ...
class FileIO(_RawIOBase, BytesIO):
mode: str
closefd: bool
def __init__(self, file: Union[str, int], mode: str = ..., closefd: bool = ...) -> None: ...
def readinto(self, buffer: _bytearray_like) -> int: ...
def write(self, pbuf: str) -> int: ...
class IncrementalNewlineDecoder(object):
newlines: Union[str, unicode]
def __init__(self, decoder, translate, z=...) -> None: ...
def decode(self, input, final) -> Any: ...
def getstate(self) -> Tuple[Any, int]: ...
def setstate(self, state: Tuple[Any, int]) -> None: ...
def reset(self) -> None: ...
# Note: In the actual _io.py, _TextIOBase inherits from _IOBase.
class _TextIOBase(TextIO):
errors: Optional[str]
# TODO: On _TextIOBase, this is always None. But it's unicode/bytes in subclasses.
newlines: Union[None, unicode, bytes]
encoding: str
@property
def closed(self) -> bool: ...
def _checkClosed(self) -> None: ...
def _checkReadable(self) -> None: ...
def _checkSeekable(self) -> None: ...
def _checkWritable(self) -> None: ...
def close(self) -> None: ...
def detach(self) -> IO[Any]: ...
def fileno(self) -> int: ...
def flush(self) -> None: ...
def isatty(self) -> bool: ...
def next(self) -> unicode: ...
def read(self, size: int = ...) -> unicode: ...
def readable(self) -> bool: ...
def readline(self, limit: int = ...) -> unicode: ...
def readlines(self, hint: int = ...) -> list[unicode]: ...
def seek(self, offset: int, whence: int = ...) -> int: ...
def seekable(self) -> bool: ...
def tell(self) -> int: ...
def truncate(self, size: Optional[int] = ...) -> int: ...
def writable(self) -> bool: ...
def write(self, pbuf: unicode) -> int: ...
def writelines(self, lines: Iterable[unicode]) -> None: ...
def __enter__(self: _T) -> _T: ...
def __exit__(self, t: Optional[Type[BaseException]], value: Optional[BaseException], traceback: Optional[Any]) -> Optional[bool]: ...
def __iter__(self: _T) -> _T: ...
class StringIO(_TextIOBase):
line_buffering: bool
def __init__(self,
initial_value: Optional[unicode] = ...,
newline: Optional[unicode] = ...) -> None: ...
def __setstate__(self, state: Tuple[Any, ...]) -> None: ...
def __getstate__(self) -> Tuple[Any, ...]: ...
# StringIO does not contain a "name" field. This workaround is necessary
# to allow StringIO sub-classes to add this field, as it is defined
# as a read-only property on IO[].
name: Any
def getvalue(self) -> unicode: ...
class TextIOWrapper(_TextIOBase):
name: str
line_buffering: bool
buffer: BinaryIO
_CHUNK_SIZE: int
def __init__(
self,
buffer: IO[Any],
encoding: Optional[Text] = ...,
errors: Optional[Text] = ...,
newline: Optional[Text] = ...,
line_buffering: bool = ...,
write_through: bool = ...,
) -> None: ...
def open(file: Union[str, unicode, int],
mode: Text = ...,
buffering: int = ...,
encoding: Optional[Text] = ...,
errors: Optional[Text] = ...,
newline: Optional[Text] = ...,
closefd: bool = ...) -> IO[Any]: ...
stdlib/2/_json.pyi
+7
View File
@@ -0,0 +1,7 @@
from typing import Any, List, Tuple, Dict, Generic, Tuple
def encode_basestring_ascii(*args, **kwargs) -> str: ...
def scanstring(a, b, *args, **kwargs) -> Tuple[Any, ...]: ...
class Encoder(object): ...
class Scanner(object): ...
stdlib/2/_md5.pyi
+13
View File
@@ -0,0 +1,13 @@
blocksize: int
digest_size: int
class MD5Type(object):
name: str
block_size: int
digest_size: int
def copy(self) -> MD5Type: ...
def digest(self) -> str: ...
def hexdigest(self) -> str: ...
def update(self, arg: str) -> None: ...
def new(arg: str = ...) -> MD5Type: ...
stdlib/2/_sha.pyi
+15
View File
@@ -0,0 +1,15 @@
blocksize: int
block_size: int
digest_size: int
class sha(object): # not actually exposed
name: str
block_size: int
digest_size: int
digestsize: int
def copy(self) -> sha: ...
def digest(self) -> str: ...
def hexdigest(self) -> str: ...
def update(self, arg: str) -> None: ...
def new(arg: str = ...) -> sha: ...
stdlib/2/_sha256.pyi
+23
View File
@@ -0,0 +1,23 @@
from typing import Optional
class sha224(object):
name: str
block_size: int
digest_size: int
digestsize: int
def __init__(self, init: Optional[str]) -> None: ...
def copy(self) -> sha224: ...
def digest(self) -> str: ...
def hexdigest(self) -> str: ...
def update(self, arg: str) -> None: ...
class sha256(object):
name: str
block_size: int
digest_size: int
digestsize: int
def __init__(self, init: Optional[str]) -> None: ...
def copy(self) -> sha256: ...
def digest(self) -> str: ...
def hexdigest(self) -> str: ...
def update(self, arg: str) -> None: ...
stdlib/2/_sha512.pyi
+23
View File
@@ -0,0 +1,23 @@
from typing import Optional
class sha384(object):
name: str
block_size: int
digest_size: int
digestsize: int
def __init__(self, init: Optional[str]) -> None: ...
def copy(self) -> sha384: ...
def digest(self) -> str: ...
def hexdigest(self) -> str: ...
def update(self, arg: str) -> None: ...
class sha512(object):
name: str
block_size: int
digest_size: int
digestsize: int
def __init__(self, init: Optional[str]) -> None: ...
def copy(self) -> sha512: ...
def digest(self) -> str: ...
def hexdigest(self) -> str: ...
def update(self, arg: str) -> None: ...
stdlib/2/_socket.pyi
+283
View File
@@ -0,0 +1,283 @@
from typing import Tuple, Union, IO, Any, Optional, overload
AF_APPLETALK: int
AF_ASH: int
AF_ATMPVC: int
AF_ATMSVC: int
AF_AX25: int
AF_BLUETOOTH: int
AF_BRIDGE: int
AF_DECnet: int
AF_ECONET: int
AF_INET: int
AF_INET6: int
AF_IPX: int
AF_IRDA: int
AF_KEY: int
AF_LLC: int
AF_NETBEUI: int
AF_NETLINK: int
AF_NETROM: int
AF_PACKET: int
AF_PPPOX: int
AF_ROSE: int
AF_ROUTE: int
AF_SECURITY: int
AF_SNA: int
AF_TIPC: int
AF_UNIX: int
AF_UNSPEC: int
AF_WANPIPE: int
AF_X25: int
AI_ADDRCONFIG: int
AI_ALL: int
AI_CANONNAME: int
AI_NUMERICHOST: int
AI_NUMERICSERV: int
AI_PASSIVE: int
AI_V4MAPPED: int
BDADDR_ANY: str
BDADDR_LOCAL: str
BTPROTO_HCI: int
BTPROTO_L2CAP: int
BTPROTO_RFCOMM: int
BTPROTO_SCO: int
EAI_ADDRFAMILY: int
EAI_AGAIN: int
EAI_BADFLAGS: int
EAI_FAIL: int
EAI_FAMILY: int
EAI_MEMORY: int
EAI_NODATA: int
EAI_NONAME: int
EAI_OVERFLOW: int
EAI_SERVICE: int
EAI_SOCKTYPE: int
EAI_SYSTEM: int
EBADF: int
EINTR: int
HCI_DATA_DIR: int
HCI_FILTER: int
HCI_TIME_STAMP: int
INADDR_ALLHOSTS_GROUP: int
INADDR_ANY: int
INADDR_BROADCAST: int
INADDR_LOOPBACK: int
INADDR_MAX_LOCAL_GROUP: int
INADDR_NONE: int
INADDR_UNSPEC_GROUP: int
IPPORT_RESERVED: int
IPPORT_USERRESERVED: int
IPPROTO_AH: int
IPPROTO_DSTOPTS: int
IPPROTO_EGP: int
IPPROTO_ESP: int
IPPROTO_FRAGMENT: int
IPPROTO_GRE: int
IPPROTO_HOPOPTS: int
IPPROTO_ICMP: int
IPPROTO_ICMPV6: int
IPPROTO_IDP: int
IPPROTO_IGMP: int
IPPROTO_IP: int
IPPROTO_IPIP: int
IPPROTO_IPV6: int
IPPROTO_NONE: int
IPPROTO_PIM: int
IPPROTO_PUP: int
IPPROTO_RAW: int
IPPROTO_ROUTING: int
IPPROTO_RSVP: int
IPPROTO_TCP: int
IPPROTO_TP: int
IPPROTO_UDP: int
IPV6_CHECKSUM: int
IPV6_DSTOPTS: int
IPV6_HOPLIMIT: int
IPV6_HOPOPTS: int
IPV6_JOIN_GROUP: int
IPV6_LEAVE_GROUP: int
IPV6_MULTICAST_HOPS: int
IPV6_MULTICAST_IF: int
IPV6_MULTICAST_LOOP: int
IPV6_NEXTHOP: int
IPV6_PKTINFO: int
IPV6_RECVDSTOPTS: int
IPV6_RECVHOPLIMIT: int
IPV6_RECVHOPOPTS: int
IPV6_RECVPKTINFO: int
IPV6_RECVRTHDR: int
IPV6_RECVTCLASS: int
IPV6_RTHDR: int
IPV6_RTHDRDSTOPTS: int
IPV6_RTHDR_TYPE_0: int
IPV6_TCLASS: int
IPV6_UNICAST_HOPS: int
IPV6_V6ONLY: int
IP_ADD_MEMBERSHIP: int
IP_DEFAULT_MULTICAST_LOOP: int
IP_DEFAULT_MULTICAST_TTL: int
IP_DROP_MEMBERSHIP: int
IP_HDRINCL: int
IP_MAX_MEMBERSHIPS: int
IP_MULTICAST_IF: int
IP_MULTICAST_LOOP: int
IP_MULTICAST_TTL: int
IP_OPTIONS: int
IP_RECVOPTS: int
IP_RECVRETOPTS: int
IP_RETOPTS: int
IP_TOS: int
IP_TTL: int
MSG_CTRUNC: int
MSG_DONTROUTE: int
MSG_DONTWAIT: int
MSG_EOR: int
MSG_OOB: int
MSG_PEEK: int
MSG_TRUNC: int
MSG_WAITALL: int
MethodType: type
NETLINK_DNRTMSG: int
NETLINK_FIREWALL: int
NETLINK_IP6_FW: int
NETLINK_NFLOG: int
NETLINK_ROUTE: int
NETLINK_USERSOCK: int
NETLINK_XFRM: int
NI_DGRAM: int
NI_MAXHOST: int
NI_MAXSERV: int
NI_NAMEREQD: int
NI_NOFQDN: int
NI_NUMERICHOST: int
NI_NUMERICSERV: int
PACKET_BROADCAST: int
PACKET_FASTROUTE: int
PACKET_HOST: int
PACKET_LOOPBACK: int
PACKET_MULTICAST: int
PACKET_OTHERHOST: int
PACKET_OUTGOING: int
PF_PACKET: int
SHUT_RD: int
SHUT_RDWR: int
SHUT_WR: int
SOCK_DGRAM: int
SOCK_RAW: int
SOCK_RDM: int
SOCK_SEQPACKET: int
SOCK_STREAM: int
SOL_HCI: int
SOL_IP: int
SOL_SOCKET: int
SOL_TCP: int
SOL_TIPC: int
SOL_UDP: int
SOMAXCONN: int
SO_ACCEPTCONN: int
SO_BROADCAST: int
SO_DEBUG: int
SO_DONTROUTE: int
SO_ERROR: int
SO_KEEPALIVE: int
SO_LINGER: int
SO_OOBINLINE: int
SO_RCVBUF: int
SO_RCVLOWAT: int
SO_RCVTIMEO: int
SO_REUSEADDR: int
SO_REUSEPORT: int
SO_SNDBUF: int
SO_SNDLOWAT: int
SO_SNDTIMEO: int
SO_TYPE: int
SSL_ERROR_EOF: int
SSL_ERROR_INVALID_ERROR_CODE: int
SSL_ERROR_SSL: int
SSL_ERROR_SYSCALL: int
SSL_ERROR_WANT_CONNECT: int
SSL_ERROR_WANT_READ: int
SSL_ERROR_WANT_WRITE: int
SSL_ERROR_WANT_X509_LOOKUP: int
SSL_ERROR_ZERO_RETURN: int
TCP_CORK: int
TCP_DEFER_ACCEPT: int
TCP_INFO: int
TCP_KEEPCNT: int
TCP_KEEPIDLE: int
TCP_KEEPINTVL: int
TCP_LINGER2: int
TCP_MAXSEG: int
TCP_NODELAY: int
TCP_QUICKACK: int
TCP_SYNCNT: int
TCP_WINDOW_CLAMP: int
TIPC_ADDR_ID: int
TIPC_ADDR_NAME: int
TIPC_ADDR_NAMESEQ: int
TIPC_CFG_SRV: int
TIPC_CLUSTER_SCOPE: int
TIPC_CONN_TIMEOUT: int
TIPC_CRITICAL_IMPORTANCE: int
TIPC_DEST_DROPPABLE: int
TIPC_HIGH_IMPORTANCE: int
TIPC_IMPORTANCE: int
TIPC_LOW_IMPORTANCE: int
TIPC_MEDIUM_IMPORTANCE: int
TIPC_NODE_SCOPE: int
TIPC_PUBLISHED: int
TIPC_SRC_DROPPABLE: int
TIPC_SUBSCR_TIMEOUT: int
TIPC_SUB_CANCEL: int
TIPC_SUB_PORTS: int
TIPC_SUB_SERVICE: int
TIPC_TOP_SRV: int
TIPC_WAIT_FOREVER: int
TIPC_WITHDRAWN: int
TIPC_ZONE_SCOPE: int
# PyCapsule
CAPI: Any
has_ipv6: bool
class error(IOError): ...
class gaierror(error): ...
class timeout(error): ...
class SocketType(object):
family: int
type: int
proto: int
timeout: float
def __init__(self, family: int = ..., type: int = ..., proto: int = ...) -> None: ...
def accept(self) -> Tuple[SocketType, Tuple[Any, ...]]: ...
def bind(self, address: Tuple[Any, ...]) -> None: ...
def close(self) -> None: ...
def connect(self, address: Tuple[Any, ...]) -> None: ...
def connect_ex(self, address: Tuple[Any, ...]) -> int: ...
def dup(self) -> SocketType: ...
def fileno(self) -> int: ...
def getpeername(self) -> Tuple[Any, ...]: ...
def getsockname(self) -> Tuple[Any, ...]: ...
def getsockopt(self, level: int, option: int, buffersize: int = ...) -> str: ...
def gettimeout(self) -> float: ...
def listen(self, backlog: int) -> None: ...
def makefile(self, mode: str = ..., buffersize: int = ...) -> IO[Any]: ...
def recv(self, buffersize: int, flags: int = ...) -> str: ...
def recv_into(self, buffer: bytearray, nbytes: int = ..., flags: int = ...) -> int: ...
def recvfrom(self, buffersize: int, flags: int = ...) -> Tuple[Any, ...]: ...
def recvfrom_into(self, buffer: bytearray, nbytes: int = ...,
flags: int = ...) -> int: ...
def send(self, data: str, flags: int = ...) -> int: ...
def sendall(self, data: str, flags: int = ...) -> None: ...
@overload
def sendto(self, data: str, address: Tuple[Any, ...]) -> int: ...
@overload
def sendto(self, data: str, flags: int, address: Tuple[Any, ...]) -> int: ...
def setblocking(self, flag: bool) -> None: ...
def setsockopt(self, level: int, option: int, value: Union[int, str]) -> None: ...
def settimeout(self, value: Optional[float]) -> None: ...
def shutdown(self, flag: int) -> None: ...
stdlib/2/_sre.pyi
+55
View File
@@ -0,0 +1,55 @@
"""Stub file for the '_sre' module."""
from typing import Any, Union, Iterable, Optional, Mapping, Sequence, Dict, List, Tuple, overload
CODESIZE: int
MAGIC: int
MAXREPEAT: long
copyright: str
class SRE_Match(object):
def start(self, group: int = ...) -> int: ...
def end(self, group: int = ...) -> int: ...
def expand(self, s: str) -> Any: ...
@overload
def group(self) -> str: ...
@overload
def group(self, group: int = ...) -> Optional[str]: ...
def groupdict(self) -> Dict[int, Optional[str]]: ...
def groups(self) -> Tuple[Optional[str], ...]: ...
def span(self) -> Tuple[int, int]: ...
@property
def regs(self) -> Tuple[Tuple[int, int], ...]: ... # undocumented
class SRE_Scanner(object):
pattern: str
def match(self) -> SRE_Match: ...
def search(self) -> SRE_Match: ...
class SRE_Pattern(object):
pattern: str
flags: int
groups: int
groupindex: Mapping[str, int]
indexgroup: Sequence[int]
def findall(self, source: str, pos: int = ..., endpos: int = ...) -> List[Union[Tuple[Any, ...], str]]: ...
def finditer(self, source: str, pos: int = ..., endpos: int = ...) -> Iterable[Union[Tuple[Any, ...], str]]: ...
def match(self, pattern, pos: int = ..., endpos: int = ...) -> SRE_Match: ...
def scanner(self, s: str, start: int = ..., end: int = ...) -> SRE_Scanner: ...
def search(self, pattern, pos: int = ..., endpos: int = ...) -> SRE_Match: ...
def split(self, source: str, maxsplit: int = ...) -> List[Optional[str]]: ...
def sub(self, repl: str, string: str, count: int = ...) -> Tuple[Any, ...]: ...
def subn(self, repl: str, string: str, count: int = ...) -> Tuple[Any, ...]: ...
def compile(
pattern: str,
flags: int,
code: List[int],
groups: int = ...,
groupindex: Mapping[str, int] = ...,
indexgroup: Sequence[int] = ...,
) -> SRE_Pattern: ...
def getcodesize() -> int: ...
def getlower(a: int, b: int) -> int: ...
stdlib/2/_struct.pyi
+22
View File
@@ -0,0 +1,22 @@
"""Stub file for the '_struct' module."""
from typing import Any, AnyStr, Tuple
class error(Exception): ...
class Struct(object):
size: int
format: str
def __init__(self, fmt: str) -> None: ...
def pack_into(self, buffer: bytearray, offset: int, obj: Any) -> None: ...
def pack(self, *args) -> str: ...
def unpack(self, s: str) -> Tuple[Any, ...]: ...
def unpack_from(self, buffer: bytearray, offset: int = ...) -> Tuple[Any, ...]: ...
def _clearcache() -> None: ...
def calcsize(fmt: str) -> int: ...
def pack(fmt: AnyStr, obj: Any) -> str: ...
def pack_into(fmt: AnyStr, buffer: bytearray, offset: int, obj: Any) -> None: ...
def unpack(fmt: AnyStr, data: str) -> Tuple[Any, ...]: ...
def unpack_from(fmt: AnyStr, buffer: bytearray, offset: int = ...) -> Tuple[Any, ...]: ...
stdlib/2/_symtable.pyi
+39
View File
@@ -0,0 +1,39 @@
from typing import List, Dict
CELL: int
DEF_BOUND: int
DEF_FREE: int
DEF_FREE_CLASS: int
DEF_GLOBAL: int
DEF_IMPORT: int
DEF_LOCAL: int
DEF_PARAM: int
FREE: int
GLOBAL_EXPLICIT: int
GLOBAL_IMPLICIT: int
LOCAL: int
OPT_BARE_EXEC: int
OPT_EXEC: int
OPT_IMPORT_STAR: int
SCOPE_MASK: int
SCOPE_OFF: int
TYPE_CLASS: int
TYPE_FUNCTION: int
TYPE_MODULE: int
USE: int
class _symtable_entry(object):
...
class symtable(object):
children: List[_symtable_entry]
id: int
lineno: int
name: str
nested: int
optimized: int
symbols: Dict[str, int]
type: int
varnames: List[str]
def __init__(self, src: str, filename: str, startstr: str) -> None: ...
stdlib/2/_threading_local.pyi
+12
View File
@@ -0,0 +1,12 @@
# Source: https://hg.python.org/cpython/file/2.7/Lib/_threading_local.py
from typing import Any
class _localbase(object): ...
class local(_localbase):
def __getattribute__(self, name: str) -> Any: ...
def __setattr__(self, name: str, value: Any) -> None: ...
def __delattr__(self, name: str) -> None: ...
def __del__(self) -> None: ...
def _patch(self: local) -> None: ...
stdlib/2/abc.pyi
+31
View File
@@ -0,0 +1,31 @@
from typing import Any, Callable, Dict, Set, Tuple, Type, TypeVar
import _weakrefset
_FuncT = TypeVar('_FuncT', bound=Callable[..., Any])
# NOTE: mypy has special processing for ABCMeta and abstractmethod.
def abstractmethod(funcobj: _FuncT) -> _FuncT: ...
class ABCMeta(type):
# TODO: FrozenSet
__abstractmethods__: Set[Any]
_abc_cache: _weakrefset.WeakSet[Any]
_abc_invalidation_counter: int
_abc_negative_cache: _weakrefset.WeakSet[Any]
_abc_negative_cache_version: int
_abc_registry: _weakrefset.WeakSet[Any]
def __init__(self, name: str, bases: Tuple[type, ...], namespace: Dict[Any, Any]) -> None: ...
def __instancecheck__(cls: ABCMeta, instance: Any) -> Any: ...
def __subclasscheck__(cls: ABCMeta, subclass: Any) -> Any: ...
def _dump_registry(cls: ABCMeta, *args: Any, **kwargs: Any) -> None: ...
def register(cls: ABCMeta, subclass: Type[Any]) -> None: ...
# TODO: The real abc.abstractproperty inherits from "property".
class abstractproperty(object):
def __new__(cls, func: Any) -> Any: ...
__isabstractmethod__: bool
doc: Any
fdel: Any
fget: Any
fset: Any
stdlib/2/ast.pyi
+28
View File
@@ -0,0 +1,28 @@
# Python 2.7 ast
# Rename typing to _typing, as not to conflict with typing imported
# from _ast below when loaded in an unorthodox way by the Dropbox
# internal Bazel integration.
import typing as _typing
from typing import Any, Iterator, Optional, Union
from _ast import *
from _ast import AST, Module
def parse(source: Union[str, unicode], filename: Union[str, unicode] = ..., mode: Union[str, unicode] = ...) -> Module: ...
def copy_location(new_node: AST, old_node: AST) -> AST: ...
def dump(node: AST, annotate_fields: bool = ..., include_attributes: bool = ...) -> str: ...
def fix_missing_locations(node: AST) -> AST: ...
def get_docstring(node: AST, clean: bool = ...) -> str: ...
def increment_lineno(node: AST, n: int = ...) -> AST: ...
def iter_child_nodes(node: AST) -> Iterator[AST]: ...
def iter_fields(node: AST) -> Iterator[_typing.Tuple[str, Any]]: ...
def literal_eval(node_or_string: Union[str, unicode, AST]) -> Any: ...
def walk(node: AST) -> Iterator[AST]: ...
class NodeVisitor():
def visit(self, node: AST) -> Any: ...
def generic_visit(self, node: AST) -> Any: ...
class NodeTransformer(NodeVisitor):
def generic_visit(self, node: AST) -> Optional[AST]: ...
stdlib/2/atexit.pyi
+5
View File
@@ -0,0 +1,5 @@
from typing import TypeVar, Any
_FT = TypeVar('_FT')
def register(func: _FT, *args: Any, **kargs: Any) -> _FT: ...
stdlib/2/cPickle.pyi
+32
View File
@@ -0,0 +1,32 @@
from typing import Any, IO, List
HIGHEST_PROTOCOL: int
compatible_formats: List[str]
format_version: str
class Pickler:
def __init__(self, file: IO[str], protocol: int = ...) -> None: ...
def dump(self, obj: Any) -> None: ...
def clear_memo(self) -> None: ...
class Unpickler:
def __init__(self, file: IO[str]) -> None: ...
def load(self) -> Any: ...
def noload(self) -> Any: ...
def dump(obj: Any, file: IO[str], protocol: int = ...) -> None: ...
def dumps(obj: Any, protocol: int = ...) -> str: ...
def load(file: IO[str]) -> Any: ...
def loads(str: str) -> Any: ...
class PickleError(Exception): ...
class UnpicklingError(PickleError): ...
class BadPickleGet(UnpicklingError): ...
class PicklingError(PickleError): ...
class UnpickleableError(PicklingError): ...
stdlib/2/cStringIO.pyi
+54
View File
@@ -0,0 +1,54 @@
# Stubs for cStringIO (Python 2.7)
# See https://docs.python.org/2/library/stringio.html
from abc import ABCMeta
from typing import overload, IO, List, Iterable, Iterator, Optional, Union
from types import TracebackType
# TODO the typing.IO[] generics should be split into input and output.
# This class isn't actually abstract, but you can't instantiate it
# directly, so we might as well treat it as abstract in the stub.
class InputType(IO[str], Iterator[str], metaclass=ABCMeta):
def getvalue(self) -> str: ...
def close(self) -> None: ...
@property
def closed(self) -> bool: ...
def flush(self) -> None: ...
def isatty(self) -> bool: ...
def read(self, size: int = ...) -> str: ...
def readline(self, size: int = ...) -> str: ...
def readlines(self, hint: int = ...) -> List[str]: ...
def seek(self, offset: int, whence: int = ...) -> int: ...
def tell(self) -> int: ...
def truncate(self, size: Optional[int] = ...) -> int: ...
def __iter__(self) -> InputType: ...
def next(self) -> str: ...
def reset(self) -> None: ...
class OutputType(IO[str], Iterator[str], metaclass=ABCMeta):
@property
def softspace(self) -> int: ...
def getvalue(self) -> str: ...
def close(self) -> None: ...
@property
def closed(self) -> bool: ...
def flush(self) -> None: ...
def isatty(self) -> bool: ...
def read(self, size: int = ...) -> str: ...
def readline(self, size: int = ...) -> str: ...
def readlines(self, hint: int = ...) -> List[str]: ...
def seek(self, offset: int, whence: int = ...) -> int: ...
def tell(self) -> int: ...
def truncate(self, size: Optional[int] = ...) -> int: ...
def __iter__(self) -> OutputType: ...
def next(self) -> str: ...
def reset(self) -> None: ...
def write(self, b: Union[str, unicode]) -> int: ...
def writelines(self, lines: Iterable[Union[str, unicode]]) -> None: ...
@overload
def StringIO() -> OutputType: ...
@overload
def StringIO(s: str) -> InputType: ...
stdlib/2/collections.pyi
+120
View File
@@ -0,0 +1,120 @@
# These are not exported.
from typing import Any, Dict, Generic, TypeVar, Tuple, overload, Type, Optional, List, Union, Reversible
# These are exported.
from typing import (
Callable as Callable,
Container as Container,
Hashable as Hashable,
ItemsView as ItemsView,
Iterable as Iterable,
Iterator as Iterator,
KeysView as KeysView,
Mapping as Mapping,
MappingView as MappingView,
MutableMapping as MutableMapping,
MutableSequence as MutableSequence,
MutableSet as MutableSet,
Sequence as Sequence,
AbstractSet as Set,
Sized as Sized,
ValuesView as ValuesView,
)
_S = TypeVar('_S')
_T = TypeVar('_T')
_KT = TypeVar('_KT')
_VT = TypeVar('_VT')
# namedtuple is special-cased in the type checker; the initializer is ignored.
def namedtuple(typename: Union[str, unicode], field_names: Union[str, unicode, Iterable[Union[str, unicode]]],
verbose: bool = ..., rename: bool = ...) -> Type[Tuple[Any, ...]]: ...
class deque(Sized, Iterable[_T], Reversible[_T], Generic[_T]):
def __init__(self, iterable: Iterable[_T] = ...,
maxlen: int = ...) -> None: ...
@property
def maxlen(self) -> Optional[int]: ...
def append(self, x: _T) -> None: ...
def appendleft(self, x: _T) -> None: ...
def clear(self) -> None: ...
def count(self, x: _T) -> int: ...
def extend(self, iterable: Iterable[_T]) -> None: ...
def extendleft(self, iterable: Iterable[_T]) -> None: ...
def pop(self) -> _T: ...
def popleft(self) -> _T: ...
def remove(self, value: _T) -> None: ...
def reverse(self) -> None: ...
def rotate(self, n: int) -> None: ...
def __len__(self) -> int: ...
def __iter__(self) -> Iterator[_T]: ...
def __str__(self) -> str: ...
def __hash__(self) -> int: ...
def __getitem__(self, i: int) -> _T: ...
def __setitem__(self, i: int, x: _T) -> None: ...
def __contains__(self, o: _T) -> bool: ...
def __reversed__(self) -> Iterator[_T]: ...
def __iadd__(self: _S, iterable: Iterable[_T]) -> _S: ...
class Counter(Dict[_T, int], Generic[_T]):
@overload
def __init__(self, **kwargs: int) -> None: ...
@overload
def __init__(self, mapping: Mapping[_T, int]) -> None: ...
@overload
def __init__(self, iterable: Iterable[_T]) -> None: ...
def copy(self: _S) -> _S: ...
def elements(self) -> Iterator[_T]: ...
def most_common(self, n: Optional[int] = ...) -> List[Tuple[_T, int]]: ...
@overload
def subtract(self, __mapping: Mapping[_T, int]) -> None: ...
@overload
def subtract(self, iterable: Iterable[_T]) -> None: ...
# The Iterable[Tuple[...]] argument type is not actually desirable
# (the tuples will be added as keys, breaking type safety) but
# it's included so that the signature is compatible with
# Dict.update. Not sure if we should use '# type: ignore' instead
# and omit the type from the union.
@overload
def update(self, __m: Mapping[_T, int], **kwargs: int) -> None: ...
@overload
def update(self, __m: Union[Iterable[_T], Iterable[Tuple[_T, int]]], **kwargs: int) -> None: ...
@overload
def update(self, **kwargs: int) -> None: ...
def __add__(self, other: Counter[_T]) -> Counter[_T]: ...
def __sub__(self, other: Counter[_T]) -> Counter[_T]: ...
def __and__(self, other: Counter[_T]) -> Counter[_T]: ...
def __or__(self, other: Counter[_T]) -> Counter[_T]: ...
def __iadd__(self, other: Counter[_T]) -> Counter[_T]: ...
def __isub__(self, other: Counter[_T]) -> Counter[_T]: ...
def __iand__(self, other: Counter[_T]) -> Counter[_T]: ...
def __ior__(self, other: Counter[_T]) -> Counter[_T]: ...
class OrderedDict(Dict[_KT, _VT], Reversible[_KT], Generic[_KT, _VT]):
def popitem(self, last: bool = ...) -> Tuple[_KT, _VT]: ...
def copy(self: _S) -> _S: ...
def __reversed__(self) -> Iterator[_KT]: ...
class defaultdict(Dict[_KT, _VT], Generic[_KT, _VT]):
default_factory: Callable[[], _VT]
@overload
def __init__(self, **kwargs: _VT) -> None: ...
@overload
def __init__(self, default_factory: Optional[Callable[[], _VT]]) -> None: ...
@overload
def __init__(self, default_factory: Optional[Callable[[], _VT]], **kwargs: _VT) -> None: ...
@overload
def __init__(self, default_factory: Optional[Callable[[], _VT]],
map: Mapping[_KT, _VT]) -> None: ...
@overload
def __init__(self, default_factory: Optional[Callable[[], _VT]],
map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ...
@overload
def __init__(self, default_factory: Optional[Callable[[], _VT]],
iterable: Iterable[Tuple[_KT, _VT]]) -> None: ...
@overload
def __init__(self, default_factory: Optional[Callable[[], _VT]],
iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ...
def __missing__(self, key: _KT) -> _VT: ...
def copy(self: _S) -> _S: ...
stdlib/2/commands.pyi
+12
View File
@@ -0,0 +1,12 @@
from typing import overload, AnyStr, Text, Tuple
def getstatus(file: Text) -> str: ...
def getoutput(cmd: Text) -> str: ...
def getstatusoutput(cmd: Text) -> Tuple[int, str]: ...
@overload
def mk2arg(head: bytes, x: bytes) -> bytes: ...
@overload
def mk2arg(head: Text, x: Text) -> Text: ...
def mkarg(x: AnyStr) -> AnyStr: ...
stdlib/2/compileall.pyi
+19
View File
@@ -0,0 +1,19 @@
# Stubs for compileall (Python 2)
from typing import Any, Optional, Pattern, Union
_Path = Union[str, bytes]
# rx can be any object with a 'search' method; once we have Protocols we can change the type
def compile_dir(
dir: _Path,
maxlevels: int = ...,
ddir: Optional[_Path] = ...,
force: bool = ...,
rx: Optional[Pattern[Any]] = ...,
quiet: int = ...,
) -> int: ...
def compile_file(
fullname: _Path, ddir: Optional[_Path] = ..., force: bool = ..., rx: Optional[Pattern[Any]] = ..., quiet: int = ...,
) -> int: ...
def compile_path(skip_curdir: bool = ..., maxlevels: int = ..., force: bool = ..., quiet: int = ...) -> int: ...
stdlib/2/cookielib.pyi
+112
View File
@@ -0,0 +1,112 @@
from typing import Any, Optional
class Cookie:
version: Any
name: Any
value: Any
port: Any
port_specified: Any
domain: Any
domain_specified: Any
domain_initial_dot: Any
path: Any
path_specified: Any
secure: Any
expires: Any
discard: Any
comment: Any
comment_url: Any
rfc2109: Any
def __init__(self, version, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path,
path_specified, secure, expires, discard, comment, comment_url, rest, rfc2109: bool = ...): ...
def has_nonstandard_attr(self, name): ...
def get_nonstandard_attr(self, name, default: Optional[Any] = ...): ...
def set_nonstandard_attr(self, name, value): ...
def is_expired(self, now: Optional[Any] = ...): ...
class CookiePolicy:
def set_ok(self, cookie, request): ...
def return_ok(self, cookie, request): ...
def domain_return_ok(self, domain, request): ...
def path_return_ok(self, path, request): ...
class DefaultCookiePolicy(CookiePolicy):
DomainStrictNoDots: Any
DomainStrictNonDomain: Any
DomainRFC2965Match: Any
DomainLiberal: Any
DomainStrict: Any
netscape: Any
rfc2965: Any
rfc2109_as_netscape: Any
hide_cookie2: Any
strict_domain: Any
strict_rfc2965_unverifiable: Any
strict_ns_unverifiable: Any
strict_ns_domain: Any
strict_ns_set_initial_dollar: Any
strict_ns_set_path: Any
def __init__(self, blocked_domains: Optional[Any] = ..., allowed_domains: Optional[Any] = ..., netscape: bool = ...,
rfc2965: bool = ..., rfc2109_as_netscape: Optional[Any] = ..., hide_cookie2: bool = ...,
strict_domain: bool = ..., strict_rfc2965_unverifiable: bool = ..., strict_ns_unverifiable: bool = ...,
strict_ns_domain=..., strict_ns_set_initial_dollar: bool = ..., strict_ns_set_path: bool = ...): ...
def blocked_domains(self): ...
def set_blocked_domains(self, blocked_domains): ...
def is_blocked(self, domain): ...
def allowed_domains(self): ...
def set_allowed_domains(self, allowed_domains): ...
def is_not_allowed(self, domain): ...
def set_ok(self, cookie, request): ...
def set_ok_version(self, cookie, request): ...
def set_ok_verifiability(self, cookie, request): ...
def set_ok_name(self, cookie, request): ...
def set_ok_path(self, cookie, request): ...
def set_ok_domain(self, cookie, request): ...
def set_ok_port(self, cookie, request): ...
def return_ok(self, cookie, request): ...
def return_ok_version(self, cookie, request): ...
def return_ok_verifiability(self, cookie, request): ...
def return_ok_secure(self, cookie, request): ...
def return_ok_expires(self, cookie, request): ...
def return_ok_port(self, cookie, request): ...
def return_ok_domain(self, cookie, request): ...
def domain_return_ok(self, domain, request): ...
def path_return_ok(self, path, request): ...
class Absent: ...
class CookieJar:
non_word_re: Any
quote_re: Any
strict_domain_re: Any
domain_re: Any
dots_re: Any
magic_re: Any
def __init__(self, policy: Optional[Any] = ...): ...
def set_policy(self, policy): ...
def add_cookie_header(self, request): ...
def make_cookies(self, response, request): ...
def set_cookie_if_ok(self, cookie, request): ...
def set_cookie(self, cookie): ...
def extract_cookies(self, response, request): ...
def clear(self, domain: Optional[Any] = ..., path: Optional[Any] = ..., name: Optional[Any] = ...): ...
def clear_session_cookies(self): ...
def clear_expired_cookies(self): ...
def __iter__(self): ...
def __len__(self): ...
class LoadError(IOError): ...
class FileCookieJar(CookieJar):
filename: Any
delayload: Any
def __init__(self, filename: Optional[Any] = ..., delayload: bool = ..., policy: Optional[Any] = ...): ...
def save(self, filename: Optional[Any] = ..., ignore_discard: bool = ..., ignore_expires: bool = ...): ...
def load(self, filename: Optional[Any] = ..., ignore_discard: bool = ..., ignore_expires: bool = ...): ...
def revert(self, filename: Optional[Any] = ..., ignore_discard: bool = ..., ignore_expires: bool = ...): ...
class LWPCookieJar(FileCookieJar):
def as_lwp_str(self, ignore_discard: bool = ..., ignore_expires: bool = ...) -> str: ... # undocumented
MozillaCookieJar = FileCookieJar
def lwp_cookie_str(cookie: Cookie) -> str: ...
stdlib/2/copy_reg.pyi
+14
View File
@@ -0,0 +1,14 @@
from typing import TypeVar, Callable, Union, Tuple, Any, Optional, SupportsInt, Hashable, List
_Type = TypeVar("_Type", bound=type)
_Reduce = Union[Tuple[Callable[..., _Type], Tuple[Any, ...]], Tuple[Callable[..., _Type], Tuple[Any, ...], Optional[Any]]]
__all__: List[str]
def pickle(ob_type: _Type, pickle_function: Callable[[_Type], Union[str, _Reduce[_Type]]], constructor_ob: Optional[Callable[[_Reduce[_Type]], _Type]] = ...) -> None: ...
def constructor(object: Callable[[_Reduce[_Type]], _Type]) -> None: ...
def add_extension(module: Hashable, name: Hashable, code: SupportsInt) -> None: ...
def remove_extension(module: Hashable, name: Hashable, code: int) -> None: ...
def clear_extension_cache() -> None: ...
stdlib/2/dircache.pyi
+10
View File
@@ -0,0 +1,10 @@
# Source: https://hg.python.org/cpython/file/2.7/Lib/dircache.py
from typing import List, MutableSequence, Text, Union
def reset() -> None: ...
def listdir(path: Text) -> List[str]: ...
opendir = listdir
def annotate(head: Text, list: Union[MutableSequence[str], MutableSequence[Text], MutableSequence[Union[str, Text]]]) -> None: ...
stdlib/concurrent/__init__.pyi → stdlib/2/distutils/__init__.pyi
View File
stdlib/2/distutils/emxccompiler.pyi
+5
View File
@@ -0,0 +1,5 @@
# Stubs for emxccompiler
from distutils.unixccompiler import UnixCCompiler
class EMXCCompiler(UnixCCompiler): ...
stdlib/2/dummy_thread.pyi
+21
View File
@@ -0,0 +1,21 @@
from typing import Any, Callable, Dict, NoReturn, Optional, Tuple
class error(Exception):
def __init__(self, *args: Any) -> None: ...
def start_new_thread(function: Callable[..., Any], args: Tuple[Any, ...], kwargs: Dict[str, Any] = ...) -> None: ...
def exit() -> NoReturn: ...
def get_ident() -> int: ...
def allocate_lock() -> LockType: ...
def stack_size(size: Optional[int] = ...) -> int: ...
class LockType(object):
locked_status: bool
def __init__(self) -> None: ...
def acquire(self, waitflag: Optional[bool] = ...) -> bool: ...
def __enter__(self, waitflag: Optional[bool] = ...) -> bool: ...
def __exit__(self, typ: Any, val: Any, tb: Any) -> None: ...
def release(self) -> bool: ...
def locked(self) -> bool: ...
def interrupt_main() -> None: ...
stdlib/2/email/MIMEText.pyi
+4
View File
@@ -0,0 +1,4 @@
from email.mime.nonmultipart import MIMENonMultipart
class MIMEText(MIMENonMultipart):
def __init__(self, _text, _subtype=..., _charset=...) -> None: ...
stdlib/2/email/__init__.pyi
+6
View File
@@ -0,0 +1,6 @@
from typing import IO, Any, AnyStr
def message_from_string(s: AnyStr, *args, **kwargs): ...
def message_from_bytes(s: str, *args, **kwargs): ...
def message_from_file(fp: IO[AnyStr], *args, **kwargs): ...
def message_from_binary_file(fp: IO[str], *args, **kwargs): ...
stdlib/2/email/_parseaddr.pyi
+40
View File
@@ -0,0 +1,40 @@
from typing import Any, Optional
def parsedate_tz(data): ...
def parsedate(data): ...
def mktime_tz(data): ...
def quote(str): ...
class AddrlistClass:
specials: Any
pos: Any
LWS: Any
CR: Any
FWS: Any
atomends: Any
phraseends: Any
field: Any
commentlist: Any
def __init__(self, field): ...
def gotonext(self): ...
def getaddrlist(self): ...
def getaddress(self): ...
def getrouteaddr(self): ...
def getaddrspec(self): ...
def getdomain(self): ...
def getdelimited(self, beginchar, endchars, allowcomments: bool = ...): ...
def getquote(self): ...
def getcomment(self): ...
def getdomainliteral(self): ...
def getatom(self, atomends: Optional[Any] = ...): ...
def getphraselist(self): ...
class AddressList(AddrlistClass):
addresslist: Any
def __init__(self, field): ...
def __len__(self): ...
def __add__(self, other): ...
def __iadd__(self, other): ...
def __sub__(self, other): ...
def __isub__(self, other): ...
def __getitem__(self, index): ...
stdlib/2/email/base64mime.pyi
+8
View File
@@ -0,0 +1,8 @@
def base64_len(s: bytes) -> int: ...
def header_encode(header, charset=..., keep_eols=..., maxlinelen=..., eol=...): ...
def encode(s, binary=..., maxlinelen=..., eol=...): ...
body_encode = encode
encodestring = encode
def decode(s, convert_eols=...): ...
body_decode = decode
decodestring = decode
stdlib/2/email/charset.pyi
+26
View File
@@ -0,0 +1,26 @@
def add_charset(charset, header_enc=..., body_enc=..., output_charset=...) -> None: ...
def add_alias(alias, canonical) -> None: ...
def add_codec(charset, codecname) -> None: ...
QP: int # undocumented
BASE64: int # undocumented
SHORTEST: int # undocumented
class Charset:
input_charset = ...
header_encoding = ...
body_encoding = ...
output_charset = ...
input_codec = ...
output_codec = ...
def __init__(self, input_charset=...) -> None: ...
def __eq__(self, other): ...
def __ne__(self, other): ...
def get_body_encoding(self): ...
def convert(self, s): ...
def to_splittable(self, s): ...
def from_splittable(self, ustr, to_output: bool = ...): ...
def get_output_charset(self): ...
def encoded_header_len(self, s): ...
def header_encode(self, s, convert: bool = ...): ...
def body_encode(self, s, convert: bool = ...): ...
stdlib/2/email/encoders.pyi
+4
View File
@@ -0,0 +1,4 @@
def encode_base64(msg) -> None: ...
def encode_quopri(msg) -> None: ...
def encode_7or8bit(msg) -> None: ...
def encode_noop(msg) -> None: ...
stdlib/2/email/feedparser.pyi
+18
View File
@@ -0,0 +1,18 @@
class BufferedSubFile:
def __init__(self) -> None: ...
def push_eof_matcher(self, pred) -> None: ...
def pop_eof_matcher(self): ...
def close(self) -> None: ...
def readline(self): ...
def unreadline(self, line) -> None: ...
def push(self, data): ...
def pushlines(self, lines) -> None: ...
def is_closed(self): ...
def __iter__(self): ...
def next(self): ...
class FeedParser:
def __init__(self, _factory=...) -> None: ...
def feed(self, data) -> None: ...
def close(self): ...
stdlib/2/email/generator.pyi
+9
View File
@@ -0,0 +1,9 @@
class Generator:
def __init__(self, outfp, mangle_from_: bool = ..., maxheaderlen: int = ...) -> None: ...
def write(self, s) -> None: ...
def flatten(self, msg, unixfrom: bool = ...) -> None: ...
def clone(self, fp): ...
class DecodedGenerator(Generator):
def __init__(self, outfp, mangle_from_: bool = ..., maxheaderlen: int = ..., fmt=...) -> None: ...
stdlib/2/email/header.pyi
+11
View File
@@ -0,0 +1,11 @@
def decode_header(header): ...
def make_header(decoded_seq, maxlinelen=..., header_name=..., continuation_ws=...): ...
class Header:
def __init__(self, s=..., charset=..., maxlinelen=..., header_name=..., continuation_ws=...,
errors=...) -> None: ...
def __unicode__(self): ...
def __eq__(self, other): ...
def __ne__(self, other): ...
def append(self, s, charset=..., errors=...) -> None: ...
def encode(self, splitchars=...): ...
stdlib/2/email/iterators.pyi
+5
View File
@@ -0,0 +1,5 @@
from typing import Any, Generator
def walk(self) -> Generator[Any, Any, Any]: ...
def body_line_iterator(msg, decode: bool = ...) -> Generator[Any, Any, Any]: ...
def typed_subpart_iterator(msg, maintype=..., subtype=...) -> Generator[Any, Any, Any]: ...
stdlib/2/email/message.pyi
+45
View File
@@ -0,0 +1,45 @@
from typing import Any, Generator
class Message:
preamble = ...
epilogue = ...
defects = ...
def __init__(self): ...
def as_string(self, unixfrom=...): ...
def is_multipart(self) -> bool: ...
def set_unixfrom(self, unixfrom) -> None: ...
def get_unixfrom(self): ...
def attach(self, payload) -> None: ...
def get_payload(self, i=..., decode: bool = ...): ...
def set_payload(self, payload, charset=...) -> None: ...
def set_charset(self, charset): ...
def get_charset(self): ...
def __len__(self): ...
def __getitem__(self, name): ...
def __setitem__(self, name, val) -> None: ...
def __delitem__(self, name) -> None: ...
def __contains__(self, name): ...
def has_key(self, name) -> bool: ...
def keys(self): ...
def values(self): ...
def items(self): ...
def get(self, name, failobj=...): ...
def get_all(self, name, failobj=...): ...
def add_header(self, _name, _value, **_params) -> None: ...
def replace_header(self, _name, _value) -> None: ...
def get_content_type(self): ...
def get_content_maintype(self): ...
def get_content_subtype(self): ...
def get_default_type(self): ...
def set_default_type(self, ctype) -> None: ...
def get_params(self, failobj=..., header=..., unquote: bool = ...): ...
def get_param(self, param, failobj=..., header=..., unquote: bool = ...): ...
def set_param(self, param, value, header=..., requote: bool = ..., charset=..., language=...) -> None: ...
def del_param(self, param, header=..., requote: bool = ...): ...
def set_type(self, type, header=..., requote: bool = ...): ...
def get_filename(self, failobj=...): ...
def get_boundary(self, failobj=...): ...
def set_boundary(self, boundary) -> None: ...
def get_content_charset(self, failobj=...): ...
def get_charsets(self, failobj=...): ...
def walk(self) -> Generator[Any, Any, Any]: ...
stdlib/email/mime/__init__.pyi → stdlib/2/email/mime/__init__.pyi
View File
stdlib/2/email/mime/application.pyi
+11
View File
@@ -0,0 +1,11 @@
# Stubs for email.mime.application
from typing import Callable, Optional, Tuple, Union
from email.mime.nonmultipart import MIMENonMultipart
_ParamsType = Union[str, None, Tuple[str, Optional[str], str]]
class MIMEApplication(MIMENonMultipart):
def __init__(self, _data: bytes, _subtype: str = ...,
_encoder: Callable[[MIMEApplication], None] = ...,
**_params: _ParamsType) -> None: ...
stdlib/2/email/mime/audio.pyi
+5
View File
@@ -0,0 +1,5 @@
from email.mime.nonmultipart import MIMENonMultipart
class MIMEAudio(MIMENonMultipart):
def __init__(self, _audiodata, _subtype=..., _encoder=..., **_params) -> None: ...
stdlib/2/email/mime/base.pyi
+4
View File
@@ -0,0 +1,4 @@
from email import message
class MIMEBase(message.Message):
def __init__(self, _maintype, _subtype, **_params) -> None: ...
stdlib/2/email/mime/image.pyi
+5
View File
@@ -0,0 +1,5 @@
from email.mime.nonmultipart import MIMENonMultipart
class MIMEImage(MIMENonMultipart):
def __init__(self, _imagedata, _subtype=..., _encoder=..., **_params) -> None: ...
stdlib/2/email/mime/message.pyi
+5
View File
@@ -0,0 +1,5 @@
from email.mime.nonmultipart import MIMENonMultipart
class MIMEMessage(MIMENonMultipart):
def __init__(self, _msg, _subtype=...) -> None: ...

Some files were not shown because too many files have changed in this diff Show More