|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Copyright 2024 Google LLC |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
1 | 16 | """ |
2 | 17 | This script allows generation of libraries that are composed of more than one |
3 | 18 | service version. It is achieved by calling `generate_library.sh` without |
|
12 | 27 | - A "grafeas" folder found in the googleapis/googleapis repository |
13 | 28 | Note: googleapis repo is found in https://github.com/googleapis/googleapis. |
14 | 29 | """ |
15 | | - |
16 | | -import click |
17 | | -import utilities as util |
18 | 30 | import os |
19 | | -import sys |
20 | | -import subprocess |
21 | | -import json |
22 | | -from model.GenerationConfig import GenerationConfig |
23 | | -from model.LibraryConfig import LibraryConfig |
24 | | -from model.ClientInputs import parse as parse_build_file |
| 31 | +from pathlib import Path |
| 32 | +from typing import List |
| 33 | +import library_generation.utilities as util |
| 34 | +from library_generation.model.generation_config import GenerationConfig |
| 35 | +from library_generation.model.gapic_config import GapicConfig |
| 36 | +from library_generation.model.gapic_inputs import GapicInputs |
| 37 | +from library_generation.model.library_config import LibraryConfig |
| 38 | +from library_generation.model.gapic_inputs import parse as parse_build_file |
25 | 39 |
|
26 | 40 | script_dir = os.path.dirname(os.path.realpath(__file__)) |
27 | 41 |
|
28 | | -""" |
29 | | -Main function in charge of generating libraries composed of more than one |
30 | | -service or service version. |
31 | | -Arguments |
32 | | - - config: a GenerationConfig object representing a parsed configuration |
33 | | - yaml |
34 | | - - library: a LibraryConfig object contained inside config, passed here for |
35 | | - convenience and to prevent all libraries to be processed |
36 | | - - enable_postprocessing: true if postprocessing should be done on the generated |
37 | | - libraries |
38 | | - - repository_path: path to the repository where the generated files will be |
39 | | - sent. If not specified, it will default to the one defined in the configuration yaml |
40 | | - and will be downloaded. The versions file will be inferred from this folder |
41 | | -""" |
| 42 | + |
42 | 43 | def generate_composed_library( |
43 | 44 | config: GenerationConfig, |
| 45 | + library_path: str, |
44 | 46 | library: LibraryConfig, |
45 | | - repository_path: str, |
46 | | - enable_postprocessing: bool = True, |
| 47 | + output_folder: str, |
| 48 | + versions_file: str, |
47 | 49 | ) -> None: |
48 | | - output_folder = util.sh_util('get_output_folder') |
49 | | - |
50 | | - print(f'output_folder: {output_folder}') |
51 | | - print('library: ', library) |
52 | | - os.makedirs(output_folder, exist_ok=True) |
53 | | - |
54 | | - googleapis_commitish = config.googleapis_commitish |
55 | | - if library.googleapis_commitish is not None: |
56 | | - googleapis_commitish = library.googleapis_commitish |
57 | | - print('using library-specific googleapis commitish: ' + googleapis_commitish) |
58 | | - else: |
59 | | - print('using common googleapis_commitish') |
60 | | - |
61 | | - print('removing old googleapis folders and files') |
62 | | - util.delete_if_exists(f'{output_folder}/google') |
63 | | - util.delete_if_exists(f'{output_folder}/grafeas') |
64 | | - |
65 | | - print('downloading googleapis') |
66 | | - util.sh_util(f'download_googleapis_files_and_folders "{output_folder}" "{googleapis_commitish}"') |
67 | | - |
68 | | - is_monorepo = len(config.libraries) > 1 |
| 50 | + """ |
| 51 | + Generate libraries composed of more than one service or service version |
| 52 | + :param config: a GenerationConfig object representing a parsed configuration |
| 53 | + yaml |
| 54 | + :param library_path: the path to which the generated file goes |
| 55 | + :param library: a LibraryConfig object contained inside config, passed here |
| 56 | + for convenience and to prevent all libraries to be processed |
| 57 | + :param output_folder: |
| 58 | + :param versions_file: |
| 59 | + :return None |
| 60 | + """ |
| 61 | + util.pull_api_definition( |
| 62 | + config=config, library=library, output_folder=output_folder |
| 63 | + ) |
| 64 | + |
| 65 | + is_monorepo = util.check_monorepo(config=config) |
| 66 | + base_arguments = __construct_tooling_arg(config=config) |
| 67 | + owlbot_cli_source_folder = util.sh_util("mktemp -d") |
| 68 | + os.makedirs(f"{library_path}", exist_ok=True) |
| 69 | + for gapic in library.gapic_configs: |
| 70 | + build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve() |
| 71 | + print(f"build_file_folder: {build_file_folder}") |
| 72 | + gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path) |
| 73 | + # generate prerequisite files (.repo-metadata.json, .OwlBot.yaml, |
| 74 | + # owlbot.py) here because transport is parsed from BUILD.bazel, |
| 75 | + # which lives in a versioned proto_path. |
| 76 | + util.generate_prerequisite_files( |
| 77 | + library=library, |
| 78 | + proto_path=util.remove_version_from(gapic.proto_path), |
| 79 | + transport=gapic_inputs.transport, |
| 80 | + library_path=library_path, |
| 81 | + ) |
| 82 | + service_version = gapic.proto_path.split("/")[-1] |
| 83 | + temp_destination_path = f"java-{library.api_shortname}-{service_version}" |
| 84 | + effective_arguments = __construct_effective_arg( |
| 85 | + base_arguments=base_arguments, |
| 86 | + gapic=gapic, |
| 87 | + gapic_inputs=gapic_inputs, |
| 88 | + temp_destination_path=temp_destination_path, |
| 89 | + ) |
| 90 | + print("arguments: ") |
| 91 | + print(effective_arguments) |
| 92 | + print(f"Generating library from {gapic.proto_path} to {library_path}") |
| 93 | + util.run_process_and_print_output( |
| 94 | + ["bash", f"{script_dir}/generate_library.sh", *effective_arguments], |
| 95 | + "Library generation", |
| 96 | + ) |
| 97 | + |
| 98 | + util.sh_util( |
| 99 | + f'build_owlbot_cli_source_folder "{library_path}"' |
| 100 | + + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"' |
| 101 | + + f' "{gapic.proto_path}"', |
| 102 | + cwd=output_folder, |
| 103 | + ) |
69 | 104 |
|
70 | | - base_arguments = [] |
71 | | - base_arguments += util.create_argument('gapic_generator_version', config) |
72 | | - base_arguments += util.create_argument('grpc_version', config) |
73 | | - base_arguments += util.create_argument('protobuf_version', config) |
74 | | - |
75 | | - library_name = f'java-{library.api_shortname}' |
76 | | - library_path = None |
77 | | - |
78 | | - versions_file = '' |
79 | | - if is_monorepo: |
80 | | - print('this is a monorepo library') |
81 | | - destination_path = config.destination_path + '/' + library_name |
82 | | - library_folder = destination_path.split('/')[-1] |
83 | | - if repository_path is None: |
84 | | - print(f'sparse_cloning monorepo with {library_name}') |
85 | | - repository_path = f'{output_folder}/{config.destination_path}' |
86 | | - clone_out = util.sh_util(f'sparse_clone "https://github.com/googleapis/{MONOREPO_NAME}.git" "{library_folder} google-cloud-pom-parent google-cloud-jar-parent versions.txt .github"', cwd=output_folder) |
87 | | - print(clone_out) |
88 | | - library_path = f'{repository_path}/{library_name}' |
89 | | - versions_file = f'{repository_path}/versions.txt' |
90 | | - else: |
91 | | - print('this is a HW library') |
92 | | - destination_path = library_name |
93 | | - if repository_path is None: |
94 | | - repository_path = f'{output_folder}/{destination_path}' |
95 | | - util.delete_if_exists(f'{output_folder}/{destination_path}') |
96 | | - clone_out = util.sh_util(f'git clone "https://github.com/googleapis/{destination_path}.git"', cwd=output_folder) |
97 | | - print(clone_out) |
98 | | - library_path = f'{repository_path}' |
99 | | - versions_file = f'{repository_path}/versions.txt' |
100 | | - |
101 | | - owlbot_cli_source_folder = util.sh_util('mktemp -d') |
102 | | - for gapic in library.gapic_configs: |
103 | | - |
104 | | - effective_arguments = list(base_arguments) |
105 | | - effective_arguments += util.create_argument('proto_path', gapic) |
106 | | - |
107 | | - build_file_folder = f'{output_folder}/{gapic.proto_path}' |
108 | | - print(f'build_file_folder: {build_file_folder}') |
109 | | - client_inputs = parse_build_file(build_file_folder, gapic.proto_path) |
110 | | - effective_arguments += [ |
111 | | - '--proto_only', client_inputs.proto_only, |
112 | | - '--gapic_additional_protos', client_inputs.additional_protos, |
113 | | - '--transport', client_inputs.transport, |
114 | | - '--rest_numeric_enums', client_inputs.rest_numeric_enum, |
115 | | - '--gapic_yaml', client_inputs.gapic_yaml, |
116 | | - '--service_config', client_inputs.service_config, |
117 | | - '--service_yaml', client_inputs.service_yaml, |
118 | | - '--include_samples', client_inputs.include_samples, |
119 | | - ] |
120 | | - service_version = gapic.proto_path.split('/')[-1] |
121 | | - temp_destination_path = f'java-{library.api_shortname}-{service_version}' |
122 | | - effective_arguments += [ '--destination_path', temp_destination_path ] |
123 | | - print('arguments: ') |
124 | | - print(effective_arguments) |
125 | | - print(f'Generating library from {gapic.proto_path} to {destination_path}...') |
126 | | - util.run_process_and_print_output(['bash', '-x', f'{script_dir}/generate_library.sh', |
127 | | - *effective_arguments], 'Library generation') |
128 | | - |
129 | | - |
130 | | - if enable_postprocessing: |
131 | | - util.sh_util(f'build_owlbot_cli_source_folder "{library_path}"' |
132 | | - + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"' |
133 | | - + f' "{gapic.proto_path}"', |
134 | | - cwd=output_folder) |
135 | | - |
136 | | - if enable_postprocessing: |
137 | 105 | # call postprocess library |
138 | | - util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh', |
139 | | - f'{library_path}', '', versions_file, owlbot_cli_source_folder, |
140 | | - config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower()], 'Library postprocessing') |
| 106 | + util.run_process_and_print_output( |
| 107 | + [ |
| 108 | + f"{script_dir}/postprocess_library.sh", |
| 109 | + f"{library_path}", |
| 110 | + "", |
| 111 | + versions_file, |
| 112 | + owlbot_cli_source_folder, |
| 113 | + config.owlbot_cli_image, |
| 114 | + config.synthtool_commitish, |
| 115 | + str(is_monorepo).lower(), |
| 116 | + ], |
| 117 | + "Library postprocessing", |
| 118 | + ) |
| 119 | + |
| 120 | + |
| 121 | +def __construct_tooling_arg(config: GenerationConfig) -> List[str]: |
| 122 | + """ |
| 123 | + Construct arguments of tooling versions used in generate_library.sh |
| 124 | + :param config: the generation config |
| 125 | + :return: arguments containing tooling versions |
| 126 | + """ |
| 127 | + arguments = [] |
| 128 | + arguments += util.create_argument("gapic_generator_version", config) |
| 129 | + arguments += util.create_argument("grpc_version", config) |
| 130 | + arguments += util.create_argument("protobuf_version", config) |
| 131 | + |
| 132 | + return arguments |
| 133 | + |
| 134 | + |
| 135 | +def __construct_effective_arg( |
| 136 | + base_arguments: List[str], |
| 137 | + gapic: GapicConfig, |
| 138 | + gapic_inputs: GapicInputs, |
| 139 | + temp_destination_path: str, |
| 140 | +) -> List[str]: |
| 141 | + """ |
| 142 | + Construct arguments consist attributes of a GAPIC library which used in |
| 143 | + generate_library.sh |
| 144 | + :param base_arguments: arguments consist of tooling versions |
| 145 | + :param gapic: an object of GapicConfig |
| 146 | + :param gapic_inputs: an object of GapicInput |
| 147 | + :param temp_destination_path: the path to which the generated library goes |
| 148 | + :return: arguments containing attributes to generate a GAPIC library |
| 149 | + """ |
| 150 | + arguments = list(base_arguments) |
| 151 | + arguments += util.create_argument("proto_path", gapic) |
| 152 | + arguments += [ |
| 153 | + "--proto_only", |
| 154 | + gapic_inputs.proto_only, |
| 155 | + "--gapic_additional_protos", |
| 156 | + gapic_inputs.additional_protos, |
| 157 | + "--transport", |
| 158 | + gapic_inputs.transport, |
| 159 | + "--rest_numeric_enums", |
| 160 | + gapic_inputs.rest_numeric_enum, |
| 161 | + "--gapic_yaml", |
| 162 | + gapic_inputs.gapic_yaml, |
| 163 | + "--service_config", |
| 164 | + gapic_inputs.service_config, |
| 165 | + "--service_yaml", |
| 166 | + gapic_inputs.service_yaml, |
| 167 | + "--include_samples", |
| 168 | + gapic_inputs.include_samples, |
| 169 | + ] |
| 170 | + arguments += ["--destination_path", temp_destination_path] |
141 | 171 |
|
| 172 | + return arguments |
0 commit comments