Creating batch_generate_apis.py (#3428)

Author: garrettjonesgoogle

.gitignore

@@ -37,3 +37,7 @@ src/test/resources/gcd-v1beta2-rev1-2.1.1.zip
 
 # API key file containing value of GOOGLE_API_KEY for integration tests
 api_key
+
+# Python utilities
+*.pyc
+artman-genfiles

utilities/batch_generate_apis.py

@@ -0,0 +1,85 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Instructions:
+#
+# Check out the googleapis repo somewhere locally (e.g. from
+# https://github.com/googleapis/googleapis):
+#
+# $ git checkout https://github.com/googleapis/googleapis.git
+#
+# Run this script:
+#
+# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS
+
+import argparse
+import os
+
+import generate_api
+
+
+def run(googleapis):
+    def generate(artman_yaml):
+        generate_api.run_generate_api(os.path.join(googleapis, artman_yaml))
+
+    # TODO Needs to have java_proto called instead of java_grpc
+    #generate('google/datastore/artman_datastore.yaml')
+
+    generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml')
+    generate('google/bigtable/artman_bigtable.yaml')
+    generate('google/bigtable/admin/artman_bigtableadmin.yaml')
+    generate('google/container/artman_container.yaml')
+    generate('google/cloud/dataproc/artman_dataproc_v1.yaml')
+    generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml')
+    generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml')
+    generate('google/privacy/dlp/artman_dlp_v2.yaml')
+    generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml')
+    generate('google/firestore/artman_firestore.yaml')
+    generate('google/cloud/language/artman_language_v1.yaml')
+    generate('google/cloud/language/artman_language_v1beta2.yaml')
+    generate('google/logging/artman_logging.yaml')
+    generate('google/monitoring/artman_monitoring.yaml')
+    generate('google/pubsub/artman_pubsub.yaml')
+    generate('google/cloud/oslogin/artman_oslogin_v1.yaml')
+    generate('google/cloud/redis/artman_redis_v1beta1.yaml')
+    generate('google/spanner/artman_spanner.yaml')
+    generate('google/spanner/admin/database/artman_spanner_admin_database.yaml')
+    generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml')
+    generate('google/cloud/speech/artman_speech_v1.yaml')
+    generate('google/cloud/speech/artman_speech_v1beta1.yaml')
+    generate('google/cloud/speech/artman_speech_v1p1beta1.yaml')
+    generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml')
+    generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml')
+    generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml')
+    generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml')
+    generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml')
+    generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml')
+    generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml')
+    generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml')
+    generate('google/cloud/vision/artman_vision_v1.yaml')
+    generate('google/cloud/vision/artman_vision_v1p1beta1.yaml')
+    generate('google/cloud/vision/artman_vision_v1p2beta1.yaml')
+    generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml')
+
+
+def main():
+    # TODO Make the docker image the default, add --local option
+    parser = argparse.ArgumentParser(description='Batch generate all APIs.')
+    parser.add_argument('googleapis', help='The path to the googleapis repo')
+    args = parser.parse_args()
+
+    run(args.googleapis)
+
+if __name__ == '__main__':
+    main()

utilities/generate_api.py

@@ -12,6 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Instructions:
+#
+# Find the artman config file the describes the API you want to generate a client for.
+#
+# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE
+
 import argparse
 import io
 import os
@@ -21,14 +27,27 @@
 from distutils import dir_util
 from ruamel import yaml
 
-def run(config_path):
-    api_dir_index = config_path.find('/google/')
-    if api_dir_index == -1:
-        raise ValueError('Didn\'t find /google/ in config file path; need absolute path to the artman config file.')
-    root_dir = config_path[0:api_dir_index]
-    api_dir = config_path[api_dir_index+1:]
 
-    subprocess.check_call(['artman', '--config', api_dir, '--local', '-v', '--root-dir', root_dir, 'generate', 'java_gapic'])
+dir_overrides = {
+    'bigtable-admin': 'google-cloud-bigtable',
+    'error-reporting': 'google-cloud-errorreporting',
+    'spanner-admin-instance': 'google-cloud-spanner',
+    'spanner-admin-database': 'google-cloud-spanner'
+}
+
+
+def run_generate_api(config_path, noisy=False):
+    googleapis_index = config_path.rfind('/google/')
+    if googleapis_index == -1:
+        raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.')
+    root_dir = config_path[0:googleapis_index]
+    api_dir = config_path[googleapis_index+1:]
+
+    extra_options = []
+    if noisy:
+        extra_options = ['-v']
+
+    subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', 'java_gapic'])
 
     with io.open(config_path, encoding='UTF-8') as config_file:
         artman_config_data = yaml.load(config_file, Loader=yaml.Loader)
@@ -44,9 +63,6 @@ def run(config_path):
     proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname)
     grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname)
     gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname)
-    print(proto_dir)
-    print(grpc_dir)
-    print(gapic_dir)
     if not os.path.exists(proto_dir):
         raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir))
     if not os.path.exists(grpc_dir):
@@ -78,24 +94,30 @@ def run(config_path):
     subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')])
 
     api_unversioned_name = '{}-{}'.format(org_name, api_name)
+    if api_name in dir_overrides:
+        api_unversioned_name = dir_overrides[api_name]
+
     target_gapic_dir = os.path.join('google-cloud-clients', api_unversioned_name)
     dir_util.copy_tree(os.path.join(gapic_dir, 'src'), os.path.join(target_gapic_dir, 'src'))
 
-    print('**** REMAINING MANUAL WORK: *****')
-    print('This script doesn\'t set up new clients. If this is a new client, you need to:')
-    print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml')
-    print('3. Copy an existing client pom.xml to the new client directory, update its text')
-    print('4. Copy an existing client README.md to the new client directory, update its text')
-    print('5. Add the new proto, grpc, and client modules into versions.txt')
-    print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml')
-    print('7. Run `utilities/replace_versions.py` to update the module versions')
+    if noisy:
+        print('**** REMAINING MANUAL WORK: *****')
+        print('This script doesn\'t set up new clients. If this is a new client, you need to:')
+        print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml')
+        print('3. Copy an existing client pom.xml to the new client directory, update its text')
+        print('4. Copy an existing client README.md to the new client directory, update its text')
+        print('5. Add the new proto, grpc, and client modules into versions.txt')
+        print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml')
+        print('7. Run `utilities/replace_versions.py` to update the module versions')
 
 def main():
     parser = argparse.ArgumentParser(description='Regenerate a single API.')
     parser.add_argument('config_file', help='The artman config file for the API')
+    parser.add_argument('--quiet', action="store_true", default=False,
+                        help='Don\'t print informational instructions')
     args = parser.parse_args()
 
-    run(args.config_file)
+    run_generate_api(args.config_file, not args.quiet)
 
 if __name__ == '__main__':
     main()