feat: standing up the GRPC service and running conformance tests with kokoro and Github actions

Author: cy-yun

.github/workflows/bigtable-conformance.yml

@@ -0,0 +1,42 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Github action job to test core java library features on
+# downstream client libraries before they are released.
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+name: bigtable-conformance
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: actions/checkout@v4
+      with:
+        repository: googleapis/cloud-bigtable-clients-test
+        ref: main
+        path: cloud-bigtable-clients-test
+    - uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: 6.0.x
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '>=1.20.2'
+    - run: dotnet --version
+    - run: go version
+    - run: bash .kokoro/bigtable-conformance.sh

.kokoro/bigtable-conformance.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+## Get the directory of the build script
+scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+## cd to the parent directory, i.e. the root of the git repo
+cd ${scriptDir}/..
+
+set +e
+
+# Build the proxy
+pushd .
+cd apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests
+dotnet build
+# Start the proxy in a separate process
+dotnet run &
+
+# Run the conformance test
+echo "Testing the client with all optional features enabled..."
+configFlag="--enable_features_all"
+popd
+
+pushd .
+cd cloud-bigtable-clients-test/tests
+eval "go test -v -proxy_addr=:7238 ${configFlag} -skip Generic_DeadlineExceeded\|_NoRetry_TransientError\|Generic_CloseClient\|Retry_WithRetryInfo\|Retry_WithRoutingCookie\|TestFeatureGap\|TestReadRows\|TestExecuteQuery_EmptyResponse\|TestExecuteQuery_SingleSimpleRow\|TestMutateRows_NoRetry_NonTransientErrors"
+returnCode=$?
+popd
+
+# Stop the proxy
+kill ${proxyPID}
+
+if [[ ${returnCode} -gt 0 ]]
+then
+  echo "Conformance test failed for config: ${config}"
+  RETURN_CODE=${returnCode}
+else
+  echo "Conformance test passed for config: ${config}"
+fi
+
+exit ${RETURN_CODE}

apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Google.Cloud.Bigtable.V2.ConformanceTests.csproj

@@ -1,12 +1,18 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
   <PropertyGroup>
-    <TargetFrameworks>net6.0;net462</TargetFrameworks>
-    <TargetFrameworks Condition=" '$(OS)' != 'Windows_NT' ">net6.0</TargetFrameworks>
-    <IsPackable>false</IsPackable>
-    <NoWarn>1701;1702;1705;xUnit2004;xUnit2013</NoWarn>
+    <TargetFramework>net6.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\Google.Cloud.Bigtable.V2\Google.Cloud.Bigtable.V2.csproj" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Grpc.AspNetCore" VersionOverride="2.40.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Core" VersionOverride="2.3.0" />
+  </ItemGroup>
+
 </Project>

apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Program.cs

@@ -0,0 +1,24 @@
+using google.bigtable.testproxy;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+
+var builder = WebApplication.CreateBuilder(args);
+
+builder.WebHost.ConfigureKestrel(options =>
+{
+  // Setup a HTTP/2 endpoint without TLS.
+  // options.ListenLocalhost(7238, o => o.Protocols =
+  //     HttpProtocols.Http2);
+});
+// Additional configuration is required to successfully run gRPC on macOS.
+// For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682
+
+// Add services to the container.
+builder.Services.AddGrpc();
+
+var app = builder.Build();
+
+// Configure the HTTP request pipeline.
+app.MapGrpcService<CloudBigtableV2TestProxyImpl>();
+app.MapGet("/", () => "Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909");
+
+app.Run();

apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Protos/test_proxy.proto

@@ -14,7 +14,7 @@
 
 syntax = "proto3";
 
-package google.cloud.bigtable.v2.conformanceTests;
+package google.bigtable.testproxy;
 
 import "google/api/client.proto";
 import "google/bigtable/v2/bigtable.proto";
@@ -23,8 +23,7 @@ import "google/protobuf/duration.proto";
 import "google/rpc/status.proto";
 
 option go_package = "./testproxypb";
-option java_multiple_files = true;
-option java_package = "com.google.cloud.bigtable.testproxy";
+option csharp_namespace = "google.bigtable.testproxy";
 
 // A config flag that dictates how the optional features should be enabled
 // during the client creation. The optional features customize how the client

apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/CloudBigtableV2TestProxyImpl.cs renamed to apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Services/CloudBigtableV2TestProxyImpl.cs

@@ -16,25 +16,29 @@
 using Google.Api.Gax.Grpc;
 using Google.Apis.Auth.OAuth2;
 using Google.Cloud.Bigtable.Common.V2;
+using Google.Cloud.Bigtable.V2;
 using Grpc.Auth;
 using Grpc.Core;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
-namespace Google.Cloud.Bigtable.V2.ConformanceTests;
+namespace google.bigtable.testproxy;
 
 public sealed class CloudBigtableV2TestProxyImpl : CloudBigtableV2TestProxy.CloudBigtableV2TestProxyBase
 {
     private class CbtClient
     {
-        public BigtableClient Client { get; set; }
-        public ChannelBase LastCreatedChannel { get; set; }
-        public InstanceName InstanceName { get; set; }
+        public BigtableClient Client { get; private set; }
+        public ChannelBase LastCreatedChannel { get; private set; }
+        public InstanceName InstanceName { get; private set; }
+
+        public CbtClient(BigtableClient bigtableClient, ChannelBase channelBase, InstanceName instanceName)
+        {
+            Client = bigtableClient;
+            LastCreatedChannel = channelBase;
+            InstanceName = instanceName;
+        }
     }
 
-    private readonly Dictionary<string, CbtClient> _idClientMap;
+    private static readonly Dictionary<string, CbtClient> _idClientMap = new Dictionary<string, CbtClient>();
 
     public override async Task<CreateClientResponse> CreateClient(CreateClientRequest request, ServerCallContext context)
     {
@@ -48,15 +52,17 @@ public override async Task<CreateClientResponse> CreateClient(CreateClientReques
         GaxPreconditions.CheckArgument(projectId is not ("" or null), "ProjectId", "project id must be provided");
         GaxPreconditions.CheckArgument(instanceId is not ("" or null), "InstanceId", "instance id must be provided");
         GaxPreconditions.CheckArgument(dataTarget is not ("" or null), "DataTarget", "data target must be provided");
-        GaxPreconditions.CheckArgument(!securityOptions.UseSsl
-            || (securityOptions.SslRootCertsPem is not ("" or null)), "SecurityOptions",
+        GaxPreconditions.CheckArgument(securityOptions is null || (!securityOptions.UseSsl
+            || (securityOptions.SslRootCertsPem is not ("" or null))), "SecurityOptions",
             "security_options.ssl_root_certs_pem must be provided if security_options.use_ssl is true");
 
+#pragma warning disable CS8604 // Possible null reference argument.
         if (_idClientMap.ContainsKey(clientId))
         {
             context.Status = new Status(StatusCode.AlreadyExists, $"Client {clientId} already exists");
             throw new RpcException(context.Status);
         }
+#pragma warning restore CS8604 // Possible null reference argument.
 
         try
         {
@@ -74,27 +80,25 @@ public override async Task<CreateClientResponse> CreateClient(CreateClientReques
             {
                 Settings = settings
             };
-            if (dataTarget != "emulator")
+            if (dataTarget != "emulator" && securityOptions is not null)
             {
                 builder.Endpoint = dataTarget;
+#pragma warning disable CS8604 // Possible null reference argument.
                 builder.ChannelCredentials = GetChannelCredentials(securityOptions.UseSsl, securityOptions.SslRootCertsPem, securityOptions.AccessToken);
+#pragma warning restore CS8604 // Possible null reference argument.
                 builder.GrpcChannelOptions = (securityOptions.UseSsl && securityOptions.SslEndpointOverride is not null)
                     ? GrpcChannelOptions.Empty.WithCustomOption("grpc.ssl_target_name_override", securityOptions.SslEndpointOverride)
                     : GrpcChannelOptions.Empty;
             }
             else
             {
+                Environment.SetEnvironmentVariable("BIGTABLE_EMULATOR_HOST", dataTarget);
                 builder.EmulatorDetection = EmulatorDetection.EmulatorOnly;
             }
             InstanceName instanceName = new InstanceName(projectId, instanceId);
             BigtableServiceApiClient apiClient = await builder.BuildAsync();
 
-            CbtClient cbtClient = new CbtClient
-            {
-                Client = BigtableClient.Create(apiClient),
-                LastCreatedChannel = builder.LastCreatedChannel,
-                InstanceName = instanceName
-            };
+            CbtClient cbtClient = new CbtClient(BigtableClient.Create(apiClient), builder.LastCreatedChannel, instanceName);
             _idClientMap[clientId] = cbtClient;
         }
         catch (Exception e)
@@ -111,7 +115,10 @@ public override async Task<CloseClientResponse> CloseClient(CloseClientRequest r
         CbtClient cbtClient = GetClient(request.ClientId, context);
         try
         {
-            await cbtClient.LastCreatedChannel.ShutdownAsync();
+            if (cbtClient.LastCreatedChannel is not null)
+            {
+                await cbtClient.LastCreatedChannel.ShutdownAsync();
+            }
         }
         catch (Exception e)
         {
@@ -126,7 +133,9 @@ public override Task<RemoveClientResponse> RemoveClient(RemoveClientRequest requ
     {
         string clientId = request.ClientId;
         GaxPreconditions.CheckArgument(clientId is not ("" or null), "ClientId", "client id must be provided", context);
+#pragma warning disable CS8604 // Possible null reference argument.
         bool removed = _idClientMap.Remove(clientId);
+#pragma warning restore CS8604 // Possible null reference argument.
         if (!removed)
         {
             context.Status = new Status(StatusCode.NotFound, $"Client {clientId} not found.");
@@ -145,9 +154,9 @@ public override async Task<RowResult> ReadRow(ReadRowRequest request, ServerCall
             context.Status = new Status(StatusCode.InvalidArgument, "Invalid TableName");
             return new RowResult
             {
-                Status = new Rpc.Status()
+                Status = new Google.Rpc.Status()
                 {
-                    Code = (int) Rpc.Code.InvalidArgument,
+                    Code = (int) Google.Rpc.Code.InvalidArgument,
                     Message = "Invalid TableName"
                 }
             };
@@ -182,9 +191,13 @@ public override async Task<RowsResult> ReadRows(ReadRowsRequest request, ServerC
             ReadRowsStream stream = cbtClient.Client.ReadRows(request.Request);
             IAsyncEnumerator<Row> enumerator = stream.GetAsyncEnumerator(new System.Threading.CancellationToken(false));
             RowsResult rowsResult = new RowsResult();
-            while (await enumerator.MoveNextAsync())
+            while (enumerator.Current is not null)
             {
                 rowsResult.Rows.Add(enumerator.Current);
+                if (!await enumerator.MoveNextAsync())
+                {
+                    break;
+                }
             }
             string message = rowsResult.Rows.Count == 0 ? $"ReadRows didn't find rows" : "ReadRows succeeded";
             rowsResult.Status = SetSuccessStatus(message, context);
@@ -258,10 +271,12 @@ public override async Task<CheckAndMutateRowResult> CheckAndMutateRow(CheckAndMu
         }
         catch (Exception e)
         {
-            return new CheckAndMutateRowResult
+            CheckAndMutateRowResult res = new CheckAndMutateRowResult
             {
+                Result = new CheckAndMutateRowResponse(),
                 Status = SetExceptionStatus(e, context)
             };
+            return res;
         }
     }
 
@@ -320,7 +335,7 @@ public override async Task<ExecuteQueryResult> ExecuteQuery(ExecuteQueryRequest
                 Metadata = new ResultSetMetadata()
             };
             IEnumerable<byte> bytes = Enumerable.Empty<byte>();
-            while (await enumerator.MoveNextAsync())
+            while (enumerator.Current is not null)
             {
                 ExecuteQueryResponse response = enumerator.Current;
                 if (response.ResponseCase == ExecuteQueryResponse.ResponseOneofCase.Metadata)
@@ -340,6 +355,10 @@ public override async Task<ExecuteQueryResult> ExecuteQuery(ExecuteQueryRequest
                 {
                     break;
                 }
+                if (!await enumerator.MoveNextAsync())
+                {
+                    break;
+                }
             }
             result.Status = SetSuccessStatus("ExecuteQuery succeeded", context);
             return result;
@@ -355,17 +374,17 @@ public override async Task<ExecuteQueryResult> ExecuteQuery(ExecuteQueryRequest
 
     public static CloudBigtableV2TestProxyImpl Create() => new();
 
-    private CloudBigtableV2TestProxyImpl() => _idClientMap = new();
-
     private CbtClient GetClient(string clientId, ServerCallContext context)
     {
         GaxPreconditions.CheckArgument(clientId is not ("" or null), "ClientId", "client id must be provided", context);
 
+#pragma warning disable CS8604 // Possible null reference argument.
         if (!_idClientMap.ContainsKey(clientId))
         {
             context.Status = new Status(StatusCode.NotFound, $"Client {clientId} not found.");
             throw new RpcException(context.Status);
         }
+#pragma warning restore CS8604 // Possible null reference argument.
         return _idClientMap[clientId];
     }
 
@@ -394,22 +413,22 @@ private static ChannelCredentials GetChannelCredentials(bool encrypted, string r
         return encrypted ? new SslCredentials(rootCertsPem) : ChannelCredentials.Insecure;
     }
 
-    private static Rpc.Status SetExceptionStatus(Exception e, ServerCallContext context)
+    private static Google.Rpc.Status SetExceptionStatus(Exception e, ServerCallContext context)
     {
         context.Status = new Status(StatusCode.Internal, e.Message, e);
-        return new Rpc.Status()
+        return new Google.Rpc.Status()
         {
-            Code = (int) Rpc.Code.Internal,
+            Code = (int) Google.Rpc.Code.Internal,
             Message = e.Message
         };
     }
 
-    private static Rpc.Status SetSuccessStatus(string message, ServerCallContext context)
+    private static Google.Rpc.Status SetSuccessStatus(string message, ServerCallContext context)
     {
         context.Status = new Status(StatusCode.OK, message);
-        return new Rpc.Status()
+        return new Google.Rpc.Status()
         {
-            Code = (int) Rpc.Code.Ok,
+            Code = (int) Google.Rpc.Code.Ok,
             Message = message
         };
     }