feat: move synthtool templates to library_generation/owlbot

.github/snippet-bot.yml

@@ -4,3 +4,4 @@ ignoreFiles:
   - src/test/**
   - test/**
   - showcase/**
+  - library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml

.github/workflows/verify_library_generation.yaml

@@ -42,6 +42,7 @@ jobs:
         set -ex
         pushd library_generation
         pip install -r requirements.in
+        pip install .
         popd
     - name: Run integration tests
       shell: bash
@@ -74,6 +75,21 @@ jobs:
         pushd library_generation
         pip install -r requirements.in
         popd
+    - name: install synthtool
+      shell: bash
+      run: |
+        set -ex
+        mkdir -p /tmp/synthtool
+        pushd /tmp/synthtool
+        if [ ! -d "synthtool" ]; then
+          git clone https://github.com/googleapis/synthtool.git
+        fi
+        pushd "synthtool"
+
+        git reset --hard origin/no-java-templates
+
+        python3 -m pip install -e .
+        python3 -m pip install -r requirements.in
     - name: Run shell unit tests
       run: |
         set -x
@@ -108,4 +124,4 @@ jobs:
       run: |
         # exclude generated golden files
         # exclude owlbot until further refaction
-        black --check library_generation --exclude "(library_generation/owlbot)|(library_generation/test/resources/goldens)"
+        black --check library_generation --exclude "(library_generation/test/resources/goldens)"

library_generation/generate_composed_library.py

@@ -113,6 +113,7 @@ def generate_composed_library(
             config.owlbot_cli_image,
             config.synthtool_commitish,
             str(is_monorepo).lower(),
+            config.path_to_yaml,
         ],
         "Library postprocessing",
     )

library_generation/model/generation_config.py

@@ -31,6 +31,8 @@ def __init__(
         googleapis_commitish: str,
         owlbot_cli_image: str,
         synthtool_commitish: str,
+        template_excludes: str,
+        path_to_yaml: str,
         libraries: List[LibraryConfig],
         grpc_version: Optional[str] = None,
         protobuf_version: Optional[str] = None,
@@ -39,6 +41,8 @@ def __init__(
         self.googleapis_commitish = googleapis_commitish
         self.owlbot_cli_image = owlbot_cli_image
         self.synthtool_commitish = synthtool_commitish
+        self.template_excludes = template_excludes
+        self.path_to_yaml = path_to_yaml
         self.libraries = libraries
         self.grpc_version = grpc_version
         self.protobuf_version = protobuf_version
@@ -94,6 +98,8 @@ def from_yaml(path_to_yaml: str):
         googleapis_commitish=__required(config, "googleapis_commitish"),
         owlbot_cli_image=__required(config, "owlbot_cli_image"),
         synthtool_commitish=__required(config, "synthtool_commitish"),
+        template_excludes=__required(config, "template_excludes"),
+        path_to_yaml=path_to_yaml,
         libraries=parsed_libraries,
     )
 

library_generation/owlbot/bin/entrypoint.sh

@@ -26,50 +26,14 @@
 set -ex
 scripts_root=$1
 versions_file=$2
-
-# Runs template and etc in current working directory
-function processModule() {
-  # templates as well as retrieving files from owl-bot-staging
-  echo "Generating templates and retrieving files from owl-bot-staging directory..."
-  if [ -f "owlbot.py" ]
-  then
-    # defaults to run owlbot.py
-    python3 owlbot.py
-  fi
-  echo "...done"
-
-  # write or restore pom.xml files
-  echo "Generating missing pom.xml..."
-  python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
-  echo "...done"
-
-  # write or restore clirr-ignored-differences.xml
-  echo "Generating clirr-ignored-differences.xml..."
-  ${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
-  echo "...done"
-
-  # fix license headers
-  echo "Fixing missing license headers..."
-  python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
-  echo "...done"
-
-  # TODO: re-enable this once we resolve thrashing
-  # restore license headers years
-  # echo "Restoring copyright years..."
-  # /owlbot/bin/restore_license_headers.sh
-  # echo "...done"
-
-  # ensure formatting on all .java files in the repository
-  echo "Reformatting source..."
-  mvn fmt:format -q
-  echo "...done"
-}
+configuration_yaml=$3
 
 # This script can be used to process HW libraries and monorepo
 # (google-cloud-java) libraries, which require a slightly different treatment
 # monorepo folders have an .OwlBot.yaml file in the module folder (e.g.
 # java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in
 # `.github/.OwlBot.yaml`
+monorepo="false"
 if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then
   monorepo="true"
 fi
@@ -80,4 +44,48 @@ if [[ "${monorepo}" == "true" ]]; then
   mv temp owl-bot-staging
 fi
 
-processModule
+
+# Runs template and etc in current working directory
+monorepo=$1
+
+# apply repo templates
+echo "Rendering templates"
+python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}"
+
+# templates as well as retrieving files from owl-bot-staging
+echo "Retrieving files from owl-bot-staging directory..."
+if [ -f "owlbot.py" ]
+then
+  # defaults to run owlbot.py
+  python3 owlbot.py
+fi
+echo "...done"
+
+# write or restore pom.xml files
+echo "Generating missing pom.xml..."
+python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
+echo "...done"
+
+# write or restore clirr-ignored-differences.xml
+echo "Generating clirr-ignored-differences.xml..."
+${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
+echo "...done"
+
+# fix license headers
+echo "Fixing missing license headers..."
+python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
+echo "...done"
+
+# TODO: re-enable this once we resolve thrashing
+# restore license headers years
+# echo "Restoring copyright years..."
+# /owlbot/bin/restore_license_headers.sh
+# echo "...done"
+
+# ensure formatting on all .java files in the repository
+echo "Reformatting source..."
+mvn fmt:format
+echo "...done"
+
+
+

library_generation/owlbot/src/apply_repo_templates.py

@@ -0,0 +1,42 @@
+"""
+This script parses an owlbot.py file, specifically the call to `java.common_templates` in
+order to extract the excluded files so it can be called with a custom template path
+pointing to the templates hosted in `sdk-platform-java/library_generation/owlbot/templates`.
+Briefly, this wraps the call to synthtool's common templates using a custom template folder.
+"""
+
+import os
+import sys
+from collections.abc import Sequence
+from synthtool.languages.java import common_templates
+from pathlib import Path
+from library_generation.model.generation_config import from_yaml
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+repo_templates_path = os.path.join(script_dir, "..", "templates", "java_library")
+
+
+def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None:
+    config = from_yaml(configuration_yaml_path)
+    print(f"repo_templates_path: {repo_templates_path}")
+    print(f"excludes: {config.template_excludes}")
+    common_templates(
+        excludes=config.template_excludes,
+        template_path=Path(repo_templates_path),
+        monorepo=monorepo,
+    )
+
+
+def main(argv: Sequence[str]) -> None:
+    if len(argv) != 3:
+        raise ValueError(
+            "Usage: python apply-repo-templates.py configuration_yaml_path monorepo"
+        )
+
+    configuration_yaml_path = argv[1]
+    monorepo = argv[2]
+    apply_repo_templates(configuration_yaml_path, monorepo.lower() == "true")
+
+
+if __name__ == "__main__":
+    main(sys.argv)

library_generation/owlbot/src/fix-poms.py

@@ -285,9 +285,11 @@ def update_bom_pom(filename: str, modules: List[module.Module]):
 # https://github.com/googleapis/google-cloud-java/issues/9304
 def __proto_group_id(main_artifact_group_id: str) -> str:
     prefix = "com.google"
-    list_of_group_id = ["com.google.cloud",
-                        "com.google.area120",
-                        "com.google.analytics"]
+    list_of_group_id = [
+        "com.google.cloud",
+        "com.google.area120",
+        "com.google.analytics",
+    ]
     if main_artifact_group_id not in list_of_group_id:
         prefix = main_artifact_group_id
     return f"{prefix}.api.grpc"
@@ -343,11 +345,17 @@ def main(versions_file, monorepo):
     main_module = existing_modules[artifact_id]
 
     # Artifact ID is part of distribution name field in .repo-metadata.json
-    if artifact_id in ["grafeas", "google-cloud-dns",
-                       "google-cloud-notification", "google-iam-policy"]:
+    if artifact_id in [
+        "grafeas",
+        "google-cloud-dns",
+        "google-cloud-notification",
+        "google-iam-policy",
+    ]:
         # There are special libraries that are not automatically generated
-        print(f"Skipping a special case library {artifact_id} that do not have "
-              " the standard module structure.")
+        print(
+            f"Skipping a special case library {artifact_id} that do not have "
+            " the standard module structure."
+        )
         return
 
     parent_artifact_id = f"{artifact_id}-parent"
@@ -383,8 +391,10 @@ def main(versions_file, monorepo):
                 version=main_module.version,
                 release_version=main_module.release_version,
             )
-            if path not in excluded_dependencies_list \
-                    and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -401,8 +411,10 @@ def main(versions_file, monorepo):
                 main_module=main_module,
                 monorepo=monorepo,
             )
-            if path not in excluded_dependencies_list \
-                and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -418,8 +430,10 @@ def main(versions_file, monorepo):
                 version=main_module.version,
                 release_version=main_module.release_version,
             )
-            if path not in excluded_dependencies_list \
-                and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -439,8 +453,10 @@ def main(versions_file, monorepo):
                 proto_module=existing_modules[proto_artifact_id],
                 monorepo=monorepo,
             )
-            if path not in excluded_dependencies_list \
-                and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -451,13 +467,13 @@ def main(versions_file, monorepo):
         module
         for module in required_dependencies.values()
         if module.artifact_id.startswith("proto-")
-           and module.artifact_id not in parent_artifact_id
+        and module.artifact_id not in parent_artifact_id
     ]
     grpc_modules = [
         module
         for module in required_dependencies.values()
-        if module.artifact_id.startswith("grpc-") \
-           and module.artifact_id not in parent_artifact_id
+        if module.artifact_id.startswith("grpc-")
+        and module.artifact_id not in parent_artifact_id
     ]
     if main_module in grpc_modules or main_module in proto_modules:
         modules = grpc_modules + proto_modules
@@ -489,12 +505,11 @@ def main(versions_file, monorepo):
 
     if os.path.isfile(f"{artifact_id}-bom/pom.xml"):
         print("updating modules in bom pom.xml")
-        if artifact_id+"-bom" not in excluded_poms_list:
+        if artifact_id + "-bom" not in excluded_poms_list:
             update_bom_pom(f"{artifact_id}-bom/pom.xml", modules)
-    elif artifact_id+"-bom" not in excluded_poms_list:
+    elif artifact_id + "-bom" not in excluded_poms_list:
         print("creating missing bom pom.xml")
-        monorepo_version = __get_monorepo_version(versions_file) \
-            if monorepo else ""
+        monorepo_version = __get_monorepo_version(versions_file) if monorepo else ""
         templates.render(
             template_name="bom_pom.xml.j2",
             output_name=f"{artifact_id}-bom/pom.xml",
@@ -503,16 +518,15 @@ def main(versions_file, monorepo):
             modules=modules,
             main_module=main_module,
             monorepo=monorepo,
-            monorepo_version=monorepo_version
+            monorepo_version=monorepo_version,
         )
 
     if os.path.isfile("pom.xml"):
         print("updating modules in parent pom.xml")
         update_parent_pom("pom.xml", modules)
     else:
         print("creating missing parent pom.xml")
-        monorepo_version = __get_monorepo_version(versions_file) \
-            if monorepo else ""
+        monorepo_version = __get_monorepo_version(versions_file) if monorepo else ""
         templates.render(
             template_name="parent_pom.xml.j2",
             output_name="./pom.xml",
@@ -521,7 +535,7 @@ def main(versions_file, monorepo):
             main_module=main_module,
             name=name,
             monorepo=monorepo,
-            monorepo_version=monorepo_version
+            monorepo_version=monorepo_version,
         )
 
     print(f"updating modules in {versions_file}")
@@ -537,13 +551,15 @@ def main(versions_file, monorepo):
                 release_version=main_module.release_version,
             )
     templates.render(
-        template_name="versions.txt.j2", output_name=versions_file, modules=existing_modules.values(),
+        template_name="versions.txt.j2",
+        output_name=versions_file,
+        modules=existing_modules.values(),
     )
 
 
 if __name__ == "__main__":
     versions_file = sys.argv[1]
     monorepo = sys.argv[2]
-    if monorepo == 'true':
+    if monorepo == "true":
         monorepo = True
     main(versions_file, monorepo)