feat: write samples to file (pt3) (#980)

* feat: write to files

* test: update test scripts

* test: goldens

* feat: sample src jar

* feat: package tar with samples

* refactor: sample class and file names should match

* test: unit goldens

* test: golden IT

* refactor: keep samples seperate from gapic jar

* test: ComposerTest

* formatting

* fix test

* refactor: updateSample naming

* refactor: include cause with GapicWriterException

* ignore test files in snippetbot check

* update composeSamples name

* include javadoc comments

* formatting

Author: eaball35

.github/snippet-bot.yml

@@ -2,3 +2,4 @@ aggregateChecks: false
 alwaysCreateStatusCheck: false
 ignoreFiles:
   - src/test/**
+  - test/**

rules_java_gapic/java_gapic.bzl

@@ -21,6 +21,7 @@ def _java_gapic_postprocess_srcjar_impl(ctx):
     output_srcjar_name = ctx.label.name
     output_main = ctx.outputs.main
     output_test = ctx.outputs.test
+    output_samples = ctx.outputs.samples
     output_resource_name = ctx.outputs.resource_name
     formatter = ctx.executable.formatter
 
@@ -58,11 +59,16 @@ def _java_gapic_postprocess_srcjar_impl(ctx):
     cd $WORKING_DIR/{output_dir_path}/src/test/java
     zip -r $WORKING_DIR/{output_srcjar_name}-tests.srcjar ./
 
+    # Sample source files.
+    cd $WORKING_DIR/{output_dir_path}/samples/snippets/generated/src/main/java
+    zip -r $WORKING_DIR/{output_srcjar_name}-samples.srcjar ./
+
     cd $WORKING_DIR
 
     mv {output_srcjar_name}.srcjar {output_main}
     mv {output_srcjar_name}-resource-name.srcjar {output_resource_name}
     mv {output_srcjar_name}-tests.srcjar {output_test}
+    mv {output_srcjar_name}-samples.srcjar {output_samples}
     """.format(
         gapic_srcjar = gapic_srcjar.path,
         output_srcjar_name = output_srcjar_name,
@@ -72,13 +78,14 @@ def _java_gapic_postprocess_srcjar_impl(ctx):
         output_main = output_main.path,
         output_resource_name = output_resource_name.path,
         output_test = output_test.path,
+        output_samples = output_samples.path,
     )
 
     ctx.actions.run_shell(
         inputs = [gapic_srcjar],
         tools = [formatter],
         command = script,
-        outputs = [output_main, output_resource_name, output_test],
+        outputs = [output_main, output_resource_name, output_test, output_samples],
     )
 
 _java_gapic_postprocess_srcjar = rule(
@@ -94,10 +101,66 @@ _java_gapic_postprocess_srcjar = rule(
         "main": "%{name}.srcjar",
         "resource_name": "%{name}-resource-name.srcjar",
         "test": "%{name}-test.srcjar",
+        "samples": "%{name}-samples.srcjar",
     },
     implementation = _java_gapic_postprocess_srcjar_impl,
 )
 
+def _java_gapic_samples_srcjar_impl(ctx):
+    gapic_srcjar = ctx.file.gapic_srcjar
+    output_srcjar_name = ctx.label.name
+    output_samples = ctx.outputs.samples
+    formatter = ctx.executable.formatter
+
+    output_dir_name = ctx.label.name
+    output_dir_path = "%s/%s" % (output_samples.dirname, output_dir_name)
+
+    script = """
+    unzip -q {gapic_srcjar}
+    # Sync'd to the output file name in Writer.java.
+    unzip -q temp-codegen.srcjar -d {output_dir_path}
+    # This may fail if there are spaces and/or too many files (exceed max length of command length).
+    {formatter} --replace $(find {output_dir_path} -type f -printf "%p ")
+    WORKING_DIR=`pwd`
+
+    # Sample source files.
+    cd $WORKING_DIR/{output_dir_path}/samples/snippets/generated/src/main/java
+    zip -r $WORKING_DIR/{output_srcjar_name}-samples.srcjar ./
+
+    cd $WORKING_DIR
+
+    mv {output_srcjar_name}-samples.srcjar {output_samples}
+    """.format(
+        gapic_srcjar = gapic_srcjar.path,
+        output_srcjar_name = output_srcjar_name,
+        formatter = formatter,
+        output_dir_name = output_dir_name,
+        output_dir_path = output_dir_path,
+        output_samples = output_samples.path,
+    )
+
+    ctx.actions.run_shell(
+        inputs = [gapic_srcjar],
+        tools = [formatter],
+        command = script,
+        outputs = [output_samples],
+    )
+
+_java_gapic_samples_srcjar = rule(
+    attrs = {
+        "gapic_srcjar": attr.label(mandatory = True, allow_single_file = True),
+        "formatter": attr.label(
+            default = Label("//:google_java_format_binary"),
+            executable = True,
+            cfg = "host",
+        ),
+    },
+    outputs = {
+        "samples": "%{name}-samples.srcjar",
+    },
+    implementation = _java_gapic_samples_srcjar_impl,
+)
+
 def _extract_common_proto_dep(dep):
     return dep[dep.index("/"):] if "//google" in dep else dep
 
@@ -198,6 +261,12 @@ def java_gapic_library(
         **kwargs
     )
 
+    _java_gapic_samples_srcjar(
+        name = "%s_samples" % name,
+        gapic_srcjar = "%s.srcjar" % raw_srcjar_name,
+        **kwargs
+    )
+
     resource_name_name = "%s_resource_name" % name
     resource_name_deps = [resource_name_name]
     native.java_library(

rules_java_gapic/java_gapic_pkg.bzl

@@ -63,18 +63,29 @@ def _gapic_pkg_tar_impl(ctx):
         for f in dep.files.to_list():
             deps.append(f)
 
+    samples =[]
+    for s in ctx.attr.samples:
+        for f in s.files.to_list():
+            samples.append(f)
+
     paths = _construct_package_dir_paths(
         ctx.attr.package_dir,
         ctx.outputs.pkg,
         ctx.label.name,
     )
 
     script = """
+    for s in {samples}; do
+        mkdir -p {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/
+        unzip -q ./$s -d {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/
+    done
+
     mkdir -p {package_dir_path}
     for dep in {deps}; do
         tar -xzpf $dep -C {package_dir_path}
     done
     cd {package_dir_path}/{tar_cd_suffix}
+
     tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/*
     cd -
     mv {package_dir_path}/{package_dir}.tar.gz {pkg}
@@ -83,13 +94,14 @@ def _gapic_pkg_tar_impl(ctx):
         deps = " ".join(["'%s'" % d.path for d in deps]),
         package_dir_path = paths.package_dir_path,
         package_dir = paths.package_dir,
+        samples = " ".join(["'%s'" % s.path for s in samples]),
         pkg = ctx.outputs.pkg.path,
         tar_cd_suffix = paths.tar_cd_suffix,
         tar_prefix = paths.tar_prefix,
     )
 
     ctx.actions.run_shell(
-        inputs = deps,
+        inputs = deps + samples,
         command = script,
         outputs = [ctx.outputs.pkg],
     )
@@ -101,6 +113,7 @@ def _gapic_pkg_tar_impl(ctx):
 gapic_pkg_tar = rule(
     attrs = {
         "deps": attr.label_list(mandatory = True),
+        "samples": attr.label_list(mandatory = False),
         "package_dir": attr.string(mandatory = False, default = ""),
         "extension": attr.string(mandatory = False, default = "tar.gz"),
     },
@@ -256,6 +269,10 @@ def _java_gapic_srcs_pkg_impl(ctx):
         # Remove empty files. If there are no resource names, one such file might have
         # been created. See java_gapic.bzl.
         rm $(find {package_dir_path}/src/main/java -size 0)
+
+        if [ -d {package_dir_path}/src/main/java/samples ]; then
+            mv {package_dir_path}/src/main/java/samples {package_dir_path}
+        fi
     done
     for proto_src in {proto_srcs}; do
         mkdir -p {package_dir_path}/src/main/proto
@@ -319,11 +336,14 @@ def java_gapic_assembly_gradle_pkg(
     client_test_deps = []
     grpc_deps = []
     proto_deps = []
+    samples = []
 
     processed_deps = {}  #there is no proper Set in Starlark
     for dep in deps:
         # Use contains instead of endswith since microgenerator testing may use differently-named targets.
-        if "_java_gapic" in dep:
+        if "samples" in dep:
+            samples.append(dep)
+        elif "_java_gapic" in dep:
             _put_dep_in_a_bucket(dep, client_deps, processed_deps)
             _put_dep_in_a_bucket("%s_test" % dep, client_test_deps, processed_deps)
             _put_dep_in_a_bucket("%s_resource_name" % dep, proto_deps, processed_deps)
@@ -371,6 +391,7 @@ def java_gapic_assembly_gradle_pkg(
         name = name,
         assembly_name = package_dir,
         deps = proto_target_dep + grpc_target_dep + client_target_dep,
+        samples = samples,
     )
 
 def _java_gapic_gradle_pkg(
@@ -416,7 +437,7 @@ def _java_gapic_gradle_pkg(
         **kwargs
     )
 
-def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, visibility = None):
+def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, samples = None, visibility = None):
     resource_target_name = "%s-resources" % assembly_name
     java_gapic_build_configs_pkg(
         name = resource_target_name,
@@ -434,6 +455,7 @@ def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, visibility = None
             Label("//rules_java_gapic:gradlew"),
             resource_target_name,
         ] + deps,
+        samples = samples,
         package_dir = assembly_name,
         visibility = visibility,
     )

scripts/diff_gen_and_golden.sh

@@ -8,6 +8,10 @@ RAW_SRCJAR=$( find . -name '*_java_gapic_srcjar_raw.srcjar' )
 mkdir unpacked src
 cd unpacked
 unzip -q -c "../${RAW_SRCJAR}" temp-codegen.srcjar | jar x
+
+mkdir -p ../samples/snippets/generated/
+cp -r samples/snippets/generated/**/* ../samples/snippets/generated
+
 cp -r src/main/java/* ../src
 cp -r src/test/java/* ../src
 [ -d proto ] && cp -r proto/src/main/java/* ../src
@@ -17,7 +21,7 @@ cd ..
 find src -type f ! -name '*.java' -a ! -name '*gapic_metadata.json' -delete
 find src -type f -name 'PlaceholderFile.java' -delete
 find src -type d -empty -delete
-
 # This will not print diff_output to the console unless `--test_output=all` option
 # is enabled, it only emits the comparison results to the test.log.
-diff -ru src test/integration/goldens/${API_NAME}
+diff -ru src test/integration/goldens/${API_NAME}/src
+diff -ru samples test/integration/goldens/${API_NAME}/samples

scripts/update_golden.sh

@@ -17,9 +17,13 @@ cd ${BUILD_WORKSPACE_DIRECTORY}/test/integration/goldens/${API_NAME}
 find . -name '*.java' -delete
 find . -name 'gapic_metadata.json' -delete
 
-cp -r ${UNPACK_DIR}/src/main/java/* .
-cp -r ${UNPACK_DIR}/src/test/java/* .
-[ -d ${UNPACK_DIR}/proto ] && cp -r ${UNPACK_DIR}/proto/src/main/java/* .
+mkdir -p ./src
+cp -r ${UNPACK_DIR}/src/main/java/* ./src
+cp -r ${UNPACK_DIR}/src/test/java/* ./src
+[ -d ${UNPACK_DIR}/proto ] && cp -r ${UNPACK_DIR}/proto/src/main/java/* ./src
+
+mkdir -p ./samples/snippets/generated
+cp -r ${UNPACK_DIR}/samples/snippets/generated/**/* ./samples/snippets/generated
 
 find . -name 'PlaceholderFile.java' -delete
 find . -type d -empty -delete

src/main/java/com/google/api/generator/gapic/composer/Composer.java

@@ -32,10 +32,12 @@
 import com.google.api.generator.gapic.model.GapicClass;
 import com.google.api.generator.gapic.model.GapicContext;
 import com.google.api.generator.gapic.model.GapicPackageInfo;
+import com.google.api.generator.gapic.model.Sample;
 import com.google.api.generator.gapic.model.Service;
 import com.google.api.generator.gapic.model.Transport;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -45,7 +47,8 @@ public static List<GapicClass> composeServiceClasses(GapicContext context) {
     clazzes.addAll(generateServiceClasses(context));
     clazzes.addAll(generateMockClasses(context, context.mixinServices()));
     clazzes.addAll(generateResourceNameHelperClasses(context));
-    return addApacheLicense(clazzes);
+    return addApacheLicense(
+        prepareExecutableSamples(clazzes, context.gapicMetadata().getProtoPackage()));
   }
 
   public static GapicPackageInfo composePackageInfo(GapicContext context) {
@@ -186,6 +189,42 @@ public static List<GapicClass> generateTestClasses(GapicContext context) {
     return clazzes;
   }
 
+  @VisibleForTesting
+  static List<GapicClass> prepareExecutableSamples(List<GapicClass> clazzes, String protoPackage) {
+    //  parse protoPackage for apiVersion and apiShortName
+    String[] pakkage = protoPackage.split("\\.");
+    String apiVersion;
+    String apiShortName;
+    //  e.g. v1, v2, v1beta1
+    if (pakkage[pakkage.length - 1].matches("v[0-9].*")) {
+      apiVersion = pakkage[pakkage.length - 1];
+      apiShortName = pakkage[pakkage.length - 2];
+    } else {
+      apiVersion = "";
+      apiShortName = pakkage[pakkage.length - 1];
+    }
+    //  Include license header, apiShortName, and apiVersion
+    return clazzes.stream()
+        .map(
+            gapicClass -> {
+              List<Sample> samples =
+                  gapicClass.samples().stream()
+                      .map(
+                          sample -> addRegionTagAndHeaderToSample(sample, apiShortName, apiVersion))
+                      .collect(Collectors.toList());
+              return gapicClass.withSamples(samples);
+            })
+        .collect(Collectors.toList());
+  }
+
+  private static Sample addRegionTagAndHeaderToSample(
+      Sample sample, String apiShortName, String apiVersion) {
+    return sample
+        .withHeader(Arrays.asList(CommentComposer.APACHE_LICENSE_COMMENT))
+        .withRegionTag(
+            sample.regionTag().withApiVersion(apiVersion).withApiShortName(apiShortName));
+  }
+
   @VisibleForTesting
   protected static List<GapicClass> addApacheLicense(List<GapicClass> gapicClassList) {
     return gapicClassList.stream()
@@ -197,7 +236,7 @@ protected static List<GapicClass> addApacheLicense(List<GapicClass> gapicClassLi
                       .toBuilder()
                       .setFileHeader(CommentComposer.APACHE_LICENSE_COMMENT)
                       .build();
-              return GapicClass.create(gapicClass.kind(), classWithHeader);
+              return GapicClass.create(gapicClass.kind(), classWithHeader, gapicClass.samples());
             })
         .collect(Collectors.toList());
   }