From bb0d49e6bea454f07744eff4cb8c865b3b929a68 Mon Sep 17 00:00:00 2001 From: Garrett Jones Date: Tue, 3 Jul 2018 16:14:02 -0700 Subject: [PATCH 1/2] Creating batch_generate_apis.py --- .gitignore | 4 ++ utilities/batch_generate_apis.py | 77 ++++++++++++++++++++++++++++++++ utilities/generate_api.py | 55 +++++++++++++++-------- 3 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 utilities/batch_generate_apis.py diff --git a/.gitignore b/.gitignore index cd502da93be4..19f0ce05fe3e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ src/test/resources/gcd-v1beta2-rev1-2.1.1.zip # API key file containing value of GOOGLE_API_KEY for integration tests api_key + +# Python utilities +*.pyc +artman-genfiles diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py new file mode 100644 index 000000000000..7b271b9373d7 --- /dev/null +++ b/utilities/batch_generate_apis.py @@ -0,0 +1,77 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import io +import os +import shutil +import subprocess + +import generate_api + + +def run(googleapis): + def generate(artman_yaml): + generate_api.run_generate_api(os.path.join(googleapis, artman_yaml)) + + # TODO Needs to have java_proto called instead of java_grpc + #generate('google/datastore/artman_datastore.yaml') + + generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') + generate('google/bigtable/artman_bigtable.yaml') + generate('google/bigtable/admin/artman_bigtableadmin.yaml') + generate('google/container/artman_container.yaml') + generate('google/cloud/dataproc/artman_dataproc_v1.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') + generate('google/privacy/dlp/artman_dlp_v2.yaml') + generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') + generate('google/firestore/artman_firestore.yaml') + generate('google/cloud/language/artman_language_v1.yaml') + generate('google/cloud/language/artman_language_v1beta2.yaml') + generate('google/logging/artman_logging.yaml') + generate('google/monitoring/artman_monitoring.yaml') + generate('google/pubsub/artman_pubsub.yaml') + generate('google/cloud/oslogin/artman_oslogin_v1.yaml') + generate('google/cloud/redis/artman_redis_v1beta1.yaml') + generate('google/spanner/artman_spanner.yaml') + generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') + generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') + generate('google/cloud/speech/artman_speech_v1.yaml') + generate('google/cloud/speech/artman_speech_v1beta1.yaml') + generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') + generate('google/cloud/vision/artman_vision_v1.yaml') + generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') + generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') + generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + + +def main(): + # TODO Make the docker image the default, add --local option + parser = argparse.ArgumentParser(description='Batch generate all APIs.') + parser.add_argument('googleapis', help='The path to the googleapis repo') + args = parser.parse_args() + + run(args.googleapis) + +if __name__ == '__main__': + main() diff --git a/utilities/generate_api.py b/utilities/generate_api.py index 4e040afbcafc..e69aa4e91594 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -21,14 +21,28 @@ from distutils import dir_util from ruamel import yaml -def run(config_path): - api_dir_index = config_path.find('/google/') - if api_dir_index == -1: - raise ValueError('Didn\'t find /google/ in config file path; need absolute path to the artman config file.') - root_dir = config_path[0:api_dir_index] - api_dir = config_path[api_dir_index+1:] - subprocess.check_call(['artman', '--config', api_dir, '--local', '-v', '--root-dir', root_dir, 'generate', 'java_gapic']) +dir_overrides = { + 'bigtable-admin': 'google-cloud-bigtable', + 'error-reporting': 'google-cloud-errorreporting', + 'spanner-admin-instance': 'google-cloud-spanner', + 'spanner-admin-database': 'google-cloud-spanner' +} + + +def run_generate_api(config_path, noisy=False): + googleapis_index = config_path.find('/googleapis/') + if googleapis_index == -1: + raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') + api_dir_index = googleapis_index + len('/googleapis/') + root_dir = config_path[0:api_dir_index-1] + api_dir = config_path[api_dir_index:] + + extra_options = [] + if noisy: + extra_options = ['-v'] + + subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir, 'generate', 'java_gapic'] + extra_options) with io.open(config_path, encoding='UTF-8') as config_file: artman_config_data = yaml.load(config_file, Loader=yaml.Loader) @@ -44,9 +58,6 @@ def run(config_path): proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname) - print(proto_dir) - print(grpc_dir) - print(gapic_dir) if not os.path.exists(proto_dir): raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) if not os.path.exists(grpc_dir): @@ -78,24 +89,30 @@ def run(config_path): subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')]) api_unversioned_name = '{}-{}'.format(org_name, api_name) + if api_name in dir_overrides: + api_unversioned_name = dir_overrides[api_name] + target_gapic_dir = os.path.join('google-cloud-clients', api_unversioned_name) dir_util.copy_tree(os.path.join(gapic_dir, 'src'), os.path.join(target_gapic_dir, 'src')) - print('**** REMAINING MANUAL WORK: *****') - print('This script doesn\'t set up new clients. If this is a new client, you need to:') - print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml') - print('3. Copy an existing client pom.xml to the new client directory, update its text') - print('4. Copy an existing client README.md to the new client directory, update its text') - print('5. Add the new proto, grpc, and client modules into versions.txt') - print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml') - print('7. Run `utilities/replace_versions.py` to update the module versions') + if noisy: + print('**** REMAINING MANUAL WORK: *****') + print('This script doesn\'t set up new clients. If this is a new client, you need to:') + print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml') + print('3. Copy an existing client pom.xml to the new client directory, update its text') + print('4. Copy an existing client README.md to the new client directory, update its text') + print('5. Add the new proto, grpc, and client modules into versions.txt') + print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml') + print('7. Run `utilities/replace_versions.py` to update the module versions') def main(): parser = argparse.ArgumentParser(description='Regenerate a single API.') parser.add_argument('config_file', help='The artman config file for the API') + parser.add_argument('--quiet', action="store_true", default=False, + help='Don\'t print informational instructions') args = parser.parse_args() - run(args.config_file) + run_generate_api(args.config_file, not args.quiet) if __name__ == '__main__': main() From e8c3af1c9358bb7cfc379419241b226ab49b97cf Mon Sep 17 00:00:00 2001 From: Garrett Jones Date: Mon, 9 Jul 2018 14:23:19 -0700 Subject: [PATCH 2/2] Adding instructions, other fixes --- utilities/batch_generate_apis.py | 14 +++++++++++--- utilities/generate_api.py | 15 ++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index 7b271b9373d7..a7ca2c031089 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -12,11 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Instructions: +# +# Check out the googleapis repo somewhere locally (e.g. from +# https://github.com/googleapis/googleapis): +# +# $ git checkout https://github.com/googleapis/googleapis.git +# +# Run this script: +# +# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS + import argparse -import io import os -import shutil -import subprocess import generate_api diff --git a/utilities/generate_api.py b/utilities/generate_api.py index e69aa4e91594..ff0f810c058b 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Instructions: +# +# Find the artman config file the describes the API you want to generate a client for. +# +# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE + import argparse import io import os @@ -31,18 +37,17 @@ def run_generate_api(config_path, noisy=False): - googleapis_index = config_path.find('/googleapis/') + googleapis_index = config_path.rfind('/google/') if googleapis_index == -1: raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') - api_dir_index = googleapis_index + len('/googleapis/') - root_dir = config_path[0:api_dir_index-1] - api_dir = config_path[api_dir_index:] + root_dir = config_path[0:googleapis_index] + api_dir = config_path[googleapis_index+1:] extra_options = [] if noisy: extra_options = ['-v'] - subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir, 'generate', 'java_gapic'] + extra_options) + subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', 'java_gapic']) with io.open(config_path, encoding='UTF-8') as config_file: artman_config_data = yaml.load(config_file, Loader=yaml.Loader)