Skip to content

Commit b17d8a1

Browse files
JoeWang1127JoeWang1127
authored
feat: add gapic options as inputs to generate_library.sh (#2121)
* feat: add gapic options as inputs * fix unit tests * fix showcase generation * make sure gapic option exists in `proto_path` * add docs * change log format * add two libraries in integration test * add a warning if gapic option does not exist * change grpc_service_config to service_config * search for gapic options in `proto_path` if they are empty * format gapic options * fix unit tests
1 parent 9683111 commit b17d8a1

8 files changed

+160
-18
lines changed

‎library_generation/README.md

+18
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,24 @@ The default value is `true`.
122122

123123
Use `--rest_numeric_enums` to specify the value.
124124

125+
### gapic_yaml (optional)
126+
One of GAPIC options passed to the generator.
127+
The default value is an empty string.
128+
129+
Use `--gapic_yaml` to specify the value.
130+
131+
### service_config (optional)
132+
One of GAPIC options passed to the generator.
133+
The default value is an empty string.
134+
135+
Use `--service_config` to specify the value.
136+
137+
### service_yaml (optional)
138+
One of GAPIC options passed to the generator.
139+
The default value is an empty string.
140+
141+
Use `--service_yaml` to specify the value.
142+
125143
### include_samples (optional)
126144
Whether generates code samples. The value is either `true` or `false`.
127145
The default value is `true`.

‎library_generation/generate_library.sh

+25-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,18 @@ case $key in
4545
rest_numeric_enums="$2"
4646
shift
4747
;;
48+
--gapic_yaml)
49+
gapic_yaml="$2"
50+
shift
51+
;;
52+
--service_config)
53+
service_config="$2"
54+
shift
55+
;;
56+
--service_yaml)
57+
service_yaml="$2"
58+
shift
59+
;;
4860
--include_samples)
4961
include_samples="$2"
5062
shift
@@ -89,6 +101,18 @@ if [ -z "${rest_numeric_enums}" ]; then
89101
rest_numeric_enums="true"
90102
fi
91103

104+
if [ -z "${gapic_yaml}" ]; then
105+
gapic_yaml=""
106+
fi
107+
108+
if [ -z "${service_config}" ]; then
109+
service_config=""
110+
fi
111+
112+
if [ -z "${service_yaml}" ]; then
113+
service_yaml=""
114+
fi
115+
92116
if [ -z "${include_samples}" ]; then
93117
include_samples="true"
94118
fi
@@ -166,7 +190,7 @@ if [[ "${proto_only}" == "false" ]]; then
166190
"$protoc_path"/protoc --experimental_allow_proto3_optional \
167191
"--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \
168192
"--java_gapic_out=metadata:${destination_path}/java_gapic_srcjar_raw.srcjar.zip" \
169-
"--java_gapic_opt=$(get_gapic_opts)" \
193+
"--java_gapic_opt=$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${gapic_yaml}" "${service_config}" "${service_yaml}")" \
170194
${proto_files} ${gapic_additional_protos}
171195

172196
unzip -o -q "${destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${destination_path}"

‎library_generation/test/generate_library_integration_test.sh

+13-1
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,18 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do
6868
gapic_additional_protos=$(get_gapic_additional_protos_from_BUILD "${proto_build_file_path}")
6969
transport=$(get_transport_from_BUILD "${proto_build_file_path}")
7070
rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}")
71+
gapic_yaml=$(get_gapic_yaml_from_BUILD "${proto_build_file_path}")
72+
service_config=$(get_service_config_from_BUILD "${proto_build_file_path}")
73+
service_yaml=$(get_service_yaml_from_BUILD "${proto_build_file_path}")
7174
include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}")
7275
popd # output_folder
73-
echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}."
76+
echo "GAPIC options are
77+
transport=${transport},
78+
rest_numeric_enums=${rest_numeric_enums},
79+
gapic_yaml=${gapic_yaml},
80+
service_config=${service_config},
81+
service_yaml=${service_yaml},
82+
include_samples=${include_samples}."
7483
# generate GAPIC client library
7584
echo "Generating library from ${proto_path}, to ${destination_path}..."
7685
"${library_generation_dir}"/generate_library.sh \
@@ -82,6 +91,9 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do
8291
--gapic_additional_protos "${gapic_additional_protos}" \
8392
--transport "${transport}" \
8493
--rest_numeric_enums "${rest_numeric_enums}" \
94+
--gapic_yaml "${gapic_yaml}" \
95+
--service_config "${service_config}" \
96+
--service_yaml "${service_yaml}" \
8597
--include_samples "${include_samples}"
8698
echo "Generate library finished."
8799
echo "Checking out googleapis-gen repository..."

‎library_generation/test/generate_library_unit_tests.sh

+15-3
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ get_gapic_opts_with_rest_test() {
4646
local transport="grpc"
4747
local rest_numeric_enums="true"
4848
local gapic_opts
49-
gapic_opts="$(get_gapic_opts)"
49+
gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")"
5050
assertEquals \
5151
"transport=grpc,rest-numeric-enums,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
5252
"${gapic_opts}"
@@ -57,9 +57,20 @@ get_gapic_opts_without_rest_test() {
5757
local transport="grpc"
5858
local rest_numeric_enums="false"
5959
local gapic_opts
60-
gapic_opts="$(get_gapic_opts)"
60+
gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")"
6161
assertEquals \
62-
"transport=grpc,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
62+
"transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
63+
"$gapic_opts"
64+
}
65+
66+
get_gapic_opts_with_non_default_test() {
67+
local proto_path="${script_dir}/resources/gapic_options"
68+
local transport="grpc"
69+
local rest_numeric_enums="false"
70+
local gapic_opts
71+
gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${proto_path}/example_gapic.yaml" "${proto_path}/example_grpc_service_config.json" "${proto_path}/example.yaml")"
72+
assertEquals \
73+
"transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
6374
"$gapic_opts"
6475
}
6576

@@ -304,6 +315,7 @@ test_list=(
304315
get_protobuf_version_failed_with_invalid_generator_version_test
305316
get_gapic_opts_with_rest_test
306317
get_gapic_opts_without_rest_test
318+
get_gapic_opts_with_non_default_test
307319
remove_grpc_version_test
308320
download_generator_success_with_valid_version_test
309321
download_generator_failed_with_invalid_version_test

‎library_generation/test/resources/proto_path_list.txt

+2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ google/cloud/apigeeconnect/v1 google-cloud-apigeeconnect-v1-java
66
google/cloud/asset/v1 google-cloud-asset-v1-java
77
google/cloud/compute/v1 google-cloud-compute-v1-java
88
google/cloud/kms/v1 google-cloud-kms-v1-java
9+
google/cloud/optimization/v1 google-cloud-optimization-v1-java
910
google/cloud/redis/v1 google-cloud-redis-v1-java
11+
google/cloud/videointelligence/v1p3beta1 google-cloud-videointelligence-v1p3beta1-java
1012
google/example/library/v1 google-cloud-example-library-v1-java
1113
google/devtools/containeranalysis/v1 google-cloud-devtools-containeranalysis-v1-java
1214
google/firestore/bundle google-cloud-firestore-bundle-v1-java

‎library_generation/test/test_utilities.sh

+61
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,46 @@ __get_config_from_BUILD() {
4444
echo "${result}"
4545
}
4646

47+
__get_gapic_option_from_BUILD() {
48+
local build_file=$1
49+
local pattern=$2
50+
local gapic_option
51+
local file_path
52+
gapic_option=$(grep "${pattern}" "${build_file}" |\
53+
head -1 |\
54+
sed 's/.*\"\([^]]*\)\".*/\1/g' |\
55+
sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
56+
)
57+
if [ -z "${gapic_option}" ] || [[ "${gapic_option}" == *"None"* ]]; then
58+
echo ""
59+
return
60+
fi
61+
62+
if [[ "${gapic_option}" == ":"* ]] || [[ "${gapic_option}" == "*"* ]]; then
63+
# if gapic_option starts with : or *, remove the first character.
64+
gapic_option="${gapic_option:1}"
65+
elif [[ "${gapic_option}" == "//"* ]]; then
66+
# gapic option is a bazel target, use the file path and name directly.
67+
# remove the leading "//".
68+
gapic_option="${gapic_option:2}"
69+
# replace ":" with "/"
70+
gapic_option="${gapic_option//://}"
71+
echo "${gapic_option}"
72+
return
73+
fi
74+
75+
file_path="${build_file%/*}"
76+
# Make sure gapic option (*.yaml or *.json) exists in proto_path; otherwise
77+
# reset gapic option to empty string.
78+
if [ -f "${file_path}/${gapic_option}" ]; then
79+
gapic_option="${file_path}/${gapic_option}"
80+
else
81+
echo "WARNING: file ${file_path}/${gapic_option} does not exist, reset gapic option to empty string." >&2
82+
gapic_option=""
83+
fi
84+
echo "${gapic_option}"
85+
}
86+
4787
__get_iam_policy_from_BUILD() {
4888
local build_file=$1
4989
local contains_iam_policy
@@ -190,6 +230,27 @@ get_rest_numeric_enums_from_BUILD() {
190230
echo "${rest_numeric_enums}"
191231
}
192232

233+
get_gapic_yaml_from_BUILD() {
234+
local build_file=$1
235+
local gapic_yaml
236+
gapic_yaml=$(__get_gapic_option_from_BUILD "${build_file}" "gapic_yaml = ")
237+
echo "${gapic_yaml}"
238+
}
239+
240+
get_service_config_from_BUILD() {
241+
local build_file=$1
242+
local service_config
243+
service_config=$(__get_gapic_option_from_BUILD "${build_file}" "grpc_service_config = ")
244+
echo "${service_config}"
245+
}
246+
247+
get_service_yaml_from_BUILD() {
248+
local build_file=$1
249+
local service_yaml
250+
service_yaml=$(__get_gapic_option_from_BUILD "${build_file}" "service_yaml")
251+
echo "${service_yaml}"
252+
}
253+
193254
get_include_samples_from_BUILD() {
194255
local build_file=$1
195256
local include_samples

‎library_generation/utilities.sh

+20-13
Original file line numberDiff line numberDiff line change
@@ -56,23 +56,30 @@ unzip_src_files() {
5656

5757
# get gapic options from .yaml and .json files from proto_path.
5858
get_gapic_opts() {
59-
local gapic_config
60-
local grpc_service_config
61-
local api_service_config
62-
gapic_config=$(find "${proto_path}" -type f -name "*gapic.yaml")
63-
if [ -z "${gapic_config}" ]; then
64-
gapic_config=""
65-
else
66-
gapic_config="gapic-config=${gapic_config},"
67-
fi
68-
grpc_service_config=$(find "${proto_path}" -type f -name "*service_config.json")
69-
api_service_config=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \))
59+
local transport=$1
60+
local rest_numeric_enums=$2
61+
local gapic_yaml=$3
62+
local service_config=$4
63+
local service_yaml=$5
7064
if [ "${rest_numeric_enums}" == "true" ]; then
71-
rest_numeric_enums="rest-numeric-enums,"
65+
rest_numeric_enums="rest-numeric-enums"
7266
else
7367
rest_numeric_enums=""
7468
fi
75-
echo "transport=${transport},${rest_numeric_enums}grpc-service-config=${grpc_service_config},${gapic_config}api-service-config=${api_service_config}"
69+
# If any of the gapic options is empty (default value), try to search for
70+
# it in proto_path.
71+
if [[ "${gapic_yaml}" == "" ]]; then
72+
gapic_yaml=$(find "${proto_path}" -type f -name "*gapic.yaml")
73+
fi
74+
75+
if [[ "${service_config}" == "" ]]; then
76+
service_config=$(find "${proto_path}" -type f -name "*service_config.json")
77+
fi
78+
79+
if [[ "${service_yaml}" == "" ]]; then
80+
service_yaml=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \))
81+
fi
82+
echo "transport=${transport},${rest_numeric_enums},grpc-service-config=${service_config},gapic-config=${gapic_yaml},api-service-config=${service_yaml}"
7683
}
7784

7885
remove_grpc_version() {

‎showcase/scripts/generate_showcase.sh

+6
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-
4949
gapic_additional_protos="google/iam/v1/iam_policy.proto google/cloud/location/locations.proto"
5050
rest_numeric_enums="false"
5151
transport="grpc+rest"
52+
gapic_yaml=""
53+
service_config="schema/google/showcase/v1beta1/showcase_grpc_service_config.json"
54+
service_yaml="schema/google/showcase/v1beta1/showcase_v1beta1.yaml"
5255
include_samples="false"
5356
rm -rdf output/showcase-output
5457
mkdir output/showcase-output
@@ -59,6 +62,9 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \
5962
--gapic_generator_version "${ggj_version}" \
6063
--gapic_additional_protos "${gapic_additional_protos}" \
6164
--rest_numeric_enums "${rest_numeric_enums}" \
65+
--gapic_yaml "${gapic_yaml}" \
66+
--service_config "${service_config}" \
67+
--service_yaml "${service_yaml}" \
6268
--include_samples "${include_samples}" \
6369
--transport "${transport}"
6470

0 commit comments

Comments
 (0)