VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2025-12-06 20:24:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update mypy-protobuf (#10914)

Browse Source 
Co-authored-by: Avasam <samuel.06@hotmail.com>
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
This commit is contained in:
Jelle Zijlstra
2023-10-22 16:31:02 -07:00
committed by GitHub
parent b9640005eb
commit f9f30cc0f2
47 changed files with 3183 additions and 958 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/generate_proto_stubs.sh
View File

@@ -13,7 +13,7 @@ set -ex -o pipefail
# Update these two variables when rerunning script
PROTOBUF_VERSION=21.8
PYTHON_PROTOBUF_VERSION=4.21.8
MYPY_PROTOBUF_VERSION=v3.4.0
MYPY_PROTOBUF_VERSION=3.5.0
if uname -a | grep Darwin; then
# brew install coreutils wget
@@ -48,7 +48,7 @@ source "$VENV/bin/activate"
pip install -r "$REPO_ROOT/requirements-tests.txt" # for Black and isort
# Install mypy-protobuf
pip install "git+https://github.com/dropbox/mypy-protobuf@$MYPY_PROTOBUF_VERSION"
pip install mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
# Remove existing pyi
find "$REPO_ROOT/stubs/protobuf/" -name '*_pb2.pyi' -delete
@@ -76,5 +76,6 @@ protoc_install/bin/protoc --proto_path="$PYTHON_PROTOBUF_DIR/src" --mypy_out="re
isort "$REPO_ROOT/stubs/protobuf"
black "$REPO_ROOT/stubs/protobuf"
sed -i "" "s/mypy-protobuf [^\"]*/mypy-protobuf ${MYPY_PROTOBUF_VERSION}/" "$REPO_ROOT/stubs/protobuf/METADATA.toml"
sed -i "" "s/version = .*$/version = \"$(echo ${PYTHON_PROTOBUF_VERSION} | cut -d. -f1-2)\.\*\"/" "$REPO_ROOT/stubs/protobuf/METADATA.toml"
sed --in-place="" \
"s/extra_description = .*$/extra_description = \"Generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on protobuf==$PYTHON_PROTOBUF_VERSION\"/" \
"$REPO_ROOT/stubs/protobuf/METADATA.toml"

51
scripts/sync_tensorflow_protobuf_stubs.sh
View File

@@ -3,54 +3,52 @@ set -euxo pipefail
# Partly based on scripts/generate_proto_stubs.sh.
# Generates the protobuf stubs for the given tensorflow version using
# mypy-protobuf. Should be run like ./sync_tensorflow_protobuf_stubs.sh
# Generates the protobuf stubs for the given tensorflow version using mypy-protobuf.
# Generally, new minor versions are a good time to update the stubs.
cd "$(dirname "$0")" > /dev/null
cd ../stubs/tensorflow
REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
# This version should be consistent with the version in tensorflow's METADATA.toml.
TENSORFLOW_VERSION=2.11.0
TENSORFLOW_VERSION=2.12.1
# Latest mypy-protobuf has dependency on protobuf >4, which is incompatible at runtime
# with tensorflow. However, the stubs produced do still work with tensorflow. So after
# installing mypy-protobuf, before running stubtest on tensorflow you should downgrade
# protobuf<4.
MYPY_PROTOBUF_VERSION=3.4.0
MYPY_PROTOBUF_VERSION=3.5.0
pip install mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
mkdir repository
cd "$(dirname "$0")" > /dev/null
cd ../stubs/tensorflow
mkdir -p repository
pushd repository &> /dev/null
git clone https://github.com/tensorflow/tensorflow.git
# If the script fails halfway, it's nice to be able to re-run it immediately
if [ ! -d "tensorflow" ] ; then
git clone --depth 1 --branch v"$TENSORFLOW_VERSION" https://github.com/tensorflow/tensorflow.git
fi
pushd tensorflow &> /dev/null
git checkout v"$TENSORFLOW_VERSION"
# Folders here cover the more commonly used protobufs externally and
# their dependencies. Tensorflow has more protobufs and can be added if requested.
protoc --mypy_out "relax_strict_optional_primitives:$REPO_ROOT/stubs/tensorflow" \
tensorflow/compiler/xla/*.proto \
tensorflow/compiler/xla/service/*.proto \
tensorflow/core/example/*.proto \
tensorflow/core/framework/*.proto \
tensorflow/core/protobuf/*.proto \
tensorflow/core/protobuf/tpu/*.proto \
tensorflow/core/framework/*.proto \
tensorflow/core/util/*.proto \
tensorflow/core/example/*.proto \
tensorflow/python/keras/protobuf/*.proto \
tensorflow/tsl/protobuf/*.proto \
tensorflow/compiler/xla/*.proto \
tensorflow/compiler/xla/service/*.proto
tensorflow/tsl/protobuf/*.proto
popd &> /dev/null
popd &> /dev/null
rm -rf repository/
# These protos exist in a folder with protos used in python, but are not
# included in the python wheel. They are likely only used for other
# language builds. stubtest was used to identify them by looking for
# ModuleNotFoundError.
rm tensorflow/core/protobuf/coordination_service_pb2.pyi \
tensorflow/compiler/xla/service/hlo_execution_profile_data_pb2.pyi \
rm tensorflow/compiler/xla/service/hlo_execution_profile_data_pb2.pyi \
tensorflow/compiler/xla/service/hlo_profile_printer_data_pb2.pyi \
tensorflow/compiler/xla/service/test_compilation_environment_pb2.pyi \
tensorflow/compiler/xla/xla_pb2.pyi \
tensorflow/core/protobuf/autotuning_pb2.pyi \
tensorflow/core/protobuf/conv_autotuning_pb2.pyi \
tensorflow/core/protobuf/critical_section_pb2.pyi \
@@ -58,8 +56,17 @@ rm tensorflow/core/protobuf/coordination_service_pb2.pyi \
tensorflow/core/protobuf/master_pb2.pyi \
tensorflow/core/protobuf/master_service_pb2.pyi \
tensorflow/core/protobuf/replay_log_pb2.pyi \
tensorflow/core/protobuf/tpu/compile_metadata_pb2.pyi \
tensorflow/core/protobuf/worker_pb2.pyi \
tensorflow/core/protobuf/worker_service_pb2.pyi \
tensorflow/core/protobuf/tpu/compile_metadata_pb2.pyi \
tensorflow/core/util/example_proto_fast_parsing_test_pb2.pyi \
tensorflow/compiler/xla/xla_pb2.pyi
tensorflow/core/util/example_proto_fast_parsing_test_pb2.pyi
isort "$REPO_ROOT/stubs/tensorflow/tensorflow"
black "$REPO_ROOT/stubs/tensorflow/tensorflow"
sed --in-place="" \
"s/extra_description = .*$/extra_description = \"Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on tensorflow==$TENSORFLOW_VERSION\"/" \
"$REPO_ROOT/stubs/tensorflow/METADATA.toml"
# Cleanup last. If the script fails halfway, it's nice to be able to re-run it immediately
rm -rf repository/