VimPlug/typeshed
1
0
Fork 0
mirror of https://github.com/davidhalter/typeshed.git synced 2025-12-06 20:24:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Simplify protoc install in protobuf generation scripts (#11785)

Browse Source
This commit is contained in:
Avasam
2024-04-21 13:33:31 -04:00
committed by GitHub
parent 4872c30a98
commit 1017c525f8
4 changed files with 26 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
scripts/generate_proto_stubs.sh
View File

@@ -15,38 +15,25 @@ set -ex -o pipefail
PROTOBUF_VERSION=26.1
MYPY_PROTOBUF_VERSION=3.6.0
if uname -a | grep Darwin; then
# brew install coreutils wget
PLAT=osx
else
# sudo apt install -y unzip
PLAT=linux
fi
# brew install coreutils wget
# sudo apt install -y unzip
REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
TMP_DIR="$(mktemp -d)"
PYTHON_PROTOBUF_FILENAME="protobuf-$PROTOBUF_VERSION.zip"
PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip"
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME"
PYTHON_PROTOBUF_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PYTHON_PROTOBUF_FILENAME"
PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION"
cd "$TMP_DIR"
echo "Working in $TMP_DIR"
# Install protoc
wget "$PROTOC_URL"
mkdir protoc_install
unzip "$PROTOC_FILENAME" -d protoc_install
protoc_install/bin/protoc --version
# Fetch protoc-python (which contains all the .proto files)
wget "$PYTHON_PROTOBUF_URL"
unzip "$PYTHON_PROTOBUF_FILENAME"
PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION"
# Prepare virtualenv
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
# Remove existing pyi
find "$REPO_ROOT/stubs/protobuf/" -name '*_pb2.pyi' -delete
@@ -62,8 +49,9 @@ PROTO_FILES=$(grep '"//:.*_proto"' $PYTHON_PROTOBUF_DIR/python/dist/BUILD.bazel
)
# And regenerate!
# shellcheck disable=SC2086
protoc_install/bin/protoc \
PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version)
echo $PROTOC_VERSION
python3 -m grpc_tools.protoc \
--proto_path="$PYTHON_PROTOBUF_DIR/src" \
--mypy_out="relax_strict_optional_primitives:$REPO_ROOT/stubs/protobuf" \
$PROTO_FILES
@@ -76,7 +64,11 @@ rm -rf "$TMP_DIR"
cd "$REPO_ROOT"
sed --in-place="" \
"s/extra_description = .*$/extra_description = \"Generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) (python protobuf==$PYTHON_PROTOBUF_VERSION)\"/" \
"s/extra_description = .*$/extra_description = \"\
Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \
and $PROTOC_VERSION \
on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) \
(python protobuf==$PYTHON_PROTOBUF_VERSION).\"/" \
stubs/protobuf/METADATA.toml
# use `|| true` so the script still continues even if a pre-commit hook

33
scripts/sync_tensorflow_protobuf_stubs.sh
View File

@@ -5,51 +5,39 @@
set -euxo pipefail
# Need protoc >= 3.15 for explicit optional
PROTOBUF_VERSION=25.3 # 4.25.3
# Whenever you update TENSORFLOW_VERSION here, version should be updated
# in stubs/tensorflow/METADATA.toml and vice-versa.
TENSORFLOW_VERSION=2.12.1
MYPY_PROTOBUF_VERSION=3.6.0
if uname -a | grep Darwin; then
# brew install coreutils wget
PLAT=osx
else
PLAT=linux
fi
# brew install coreutils wget
# sudo apt install -y unzip
REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
TMP_DIR="$(mktemp -d)"
TENSORFLOW_FILENAME="v$TENSORFLOW_VERSION.zip"
PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip"
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME"
TENSORFLOW_URL="https://github.com/tensorflow/tensorflow/archive/refs/tags/$TENSORFLOW_FILENAME"
TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION"
cd "$TMP_DIR"
echo "Working in $TMP_DIR"
# Install protoc
wget "$PROTOC_URL"
mkdir protoc_install
unzip "$PROTOC_FILENAME" -d protoc_install
protoc_install/bin/protoc --version
# Fetch tensorflow (which contains all the .proto files)
wget "$TENSORFLOW_URL"
unzip "$TENSORFLOW_FILENAME"
TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION"
# Prepare virtualenv
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
# Remove existing pyi
find "$REPO_ROOT/stubs/tensorflow/" -name "*_pb2.pyi" -delete
# Folders here cover the more commonly used protobufs externally and
# their dependencies. Tensorflow has more protobufs and can be added if requested.
protoc_install/bin/protoc \
PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version)
echo $PROTOC_VERSION
python3 -m grpc_tools.protoc \
--proto_path="$TENSORFLOW_DIR" \
--mypy_out "relax_strict_optional_primitives:$REPO_ROOT/stubs/tensorflow" \
$TENSORFLOW_DIR/tensorflow/compiler/xla/*.proto \
@@ -89,8 +77,11 @@ rm \
stubs/tensorflow/tensorflow/core/util/example_proto_fast_parsing_test_pb2.pyi \
sed --in-place="" \
"s/extra_description = .*$/extra_description = \"Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on tensorflow==$TENSORFLOW_VERSION\"/" \
stubs/tensorflow/METADATA.toml
"s/extra_description = .*$/extra_description = \"\
Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \
and $PROTOC_VERSION \
on tensorflow==$TENSORFLOW_VERSION.\"/" \
stubs/tensorflow/METADATA.toml
# use `|| true` so the script still continues even if a pre-commit hook
# applies autofixes (which will result in a nonzero exit code)