Skip to content

Commit 005eb5a

Browse files
cy-yuncy-yun
committed
feat: standing up the GRPC service and running conformance tests with kokoro and Github actions
1 parent 91718ea commit 005eb5a

11 files changed

+499
-341
lines changed

‎.github/workflows/bigtable-conformance.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# Github action job to test core java library features on
15+
# downstream client libraries before they are released.
16+
on:
17+
push:
18+
branches:
19+
- main
20+
pull_request:
21+
name: bigtable-conformance
22+
jobs:
23+
conformance:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- uses: actions/checkout@v4
27+
with:
28+
submodules: true
29+
- uses: actions/checkout@v4
30+
with:
31+
repository: googleapis/cloud-bigtable-clients-test
32+
ref: main
33+
path: cloud-bigtable-clients-test
34+
- uses: actions/setup-dotnet@v4
35+
with:
36+
dotnet-version: 6.0.x
37+
- uses: actions/setup-go@v5
38+
with:
39+
go-version: '>=1.20.2'
40+
- run: dotnet --version
41+
- run: go version
42+
- run: bash .kokoro/bigtable-conformance.sh

‎.kokoro/bigtable-conformance.sh

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
# Copyright 2025 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
## Get the directory of the build script
19+
scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
20+
## cd to the parent directory, i.e. the root of the git repo
21+
cd ${scriptDir}/..
22+
23+
set +e
24+
25+
# Build the proxy
26+
pushd .
27+
cd apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests
28+
dotnet build
29+
# Start the proxy in a separate process
30+
dotnet run &
31+
32+
# Run the conformance test
33+
echo "Testing the client with all optional features enabled..."
34+
configFlag="--enable_features_all"
35+
popd
36+
37+
pushd .
38+
cd cloud-bigtable-clients-test/tests
39+
eval "go test -v -proxy_addr=:7238 ${configFlag} -skip Generic_DeadlineExceeded\|NoRetry_TransientError\|Generic_CloseClient\|Retry_WithRetryInfo\|Retry_WithRoutingCookie\|TestFeatureGap\|TestReadRows\|TestExecuteQuery_EmptyResponse\|TestExecuteQuery_SingleSimpleRow\|TestMutateRows_NoRetry_NonTransientErrors"
40+
returnCode=$?
41+
popd
42+
43+
# Stop the proxy
44+
kill ${proxyPID}
45+
46+
if [[ ${returnCode} -gt 0 ]]
47+
then
48+
echo "Conformance test failed for config: ${config}"
49+
RETURN_CODE=${returnCode}
50+
else
51+
echo "Conformance test passed for config: ${config}"
52+
fi
53+
54+
exit ${RETURN_CODE}

‎apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Google.Cloud.Bigtable.V2.ConformanceTests.csproj

+12-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
1-
<?xml version="1.0" encoding="utf-8"?>
2-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
33
<PropertyGroup>
4-
<TargetFrameworks>net6.0;net462</TargetFrameworks>
5-
<TargetFrameworks Condition=" '$(OS)' != 'Windows_NT' ">net6.0</TargetFrameworks>
6-
<IsPackable>false</IsPackable>
7-
<NoWarn>1701;1702;1705;xUnit2004;xUnit2013</NoWarn>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<Nullable>enable</Nullable>
6+
<ImplicitUsings>enable</ImplicitUsings>
87
</PropertyGroup>
8+
99
<ItemGroup>
1010
<ProjectReference Include="..\Google.Cloud.Bigtable.V2\Google.Cloud.Bigtable.V2.csproj" />
1111
</ItemGroup>
12+
13+
<ItemGroup>
14+
<PackageReference Include="Grpc.AspNetCore" VersionOverride="2.40.0" />
15+
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Core" VersionOverride="2.3.0" />
16+
</ItemGroup>
17+
1218
</Project>

‎apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Program.cs

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
using google.bigtable.testproxy;
2+
using Microsoft.AspNetCore.Server.Kestrel.Core;
3+
4+
var builder = WebApplication.CreateBuilder(args);
5+
6+
builder.WebHost.ConfigureKestrel(options =>
7+
{
8+
// Setup a HTTP/2 endpoint without TLS.
9+
// options.ListenLocalhost(7238, o => o.Protocols =
10+
// HttpProtocols.Http2);
11+
});
12+
// Additional configuration is required to successfully run gRPC on macOS.
13+
// For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682
14+
15+
// Add services to the container.
16+
builder.Services.AddGrpc();
17+
18+
var app = builder.Build();
19+
20+
// Configure the HTTP request pipeline.
21+
app.MapGrpcService<CloudBigtableV2TestProxyImpl>();
22+
app.MapGet("/", () => "Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909");
23+
24+
app.Run();

‎apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Protos/test_proxy.proto

+2-3
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
syntax = "proto3";
1616

17-
package google.cloud.bigtable.v2.conformanceTests;
17+
package google.bigtable.testproxy;
1818

1919
import "google/api/client.proto";
2020
import "google/bigtable/v2/bigtable.proto";
@@ -23,8 +23,7 @@ import "google/protobuf/duration.proto";
2323
import "google/rpc/status.proto";
2424

2525
option go_package = "./testproxypb";
26-
option java_multiple_files = true;
27-
option java_package = "com.google.cloud.bigtable.testproxy";
26+
option csharp_namespace = "google.bigtable.testproxy";
2827

2928
// A config flag that dictates how the optional features should be enabled
3029
// during the client creation. The optional features customize how the client

‎apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/CloudBigtableV2TestProxyImpl.cs renamed to ‎apis/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.ConformanceTests/Services/CloudBigtableV2TestProxyImpl.cs

+51-32
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,29 @@
1616
using Google.Api.Gax.Grpc;
1717
using Google.Apis.Auth.OAuth2;
1818
using Google.Cloud.Bigtable.Common.V2;
19+
using Google.Cloud.Bigtable.V2;
1920
using Grpc.Auth;
2021
using Grpc.Core;
21-
using System;
22-
using System.Collections.Generic;
23-
using System.Linq;
24-
using System.Threading.Tasks;
2522

26-
namespace Google.Cloud.Bigtable.V2.ConformanceTests;
23+
namespace google.bigtable.testproxy;
2724

2825
public sealed class CloudBigtableV2TestProxyImpl : CloudBigtableV2TestProxy.CloudBigtableV2TestProxyBase
2926
{
3027
private class CbtClient
3128
{
32-
public BigtableClient Client { get; set; }
33-
public ChannelBase LastCreatedChannel { get; set; }
34-
public InstanceName InstanceName { get; set; }
29+
public BigtableClient Client { get; private set; }
30+
public ChannelBase LastCreatedChannel { get; private set; }
31+
public InstanceName InstanceName { get; private set; }
32+
33+
public CbtClient(BigtableClient bigtableClient, ChannelBase channelBase, InstanceName instanceName)
34+
{
35+
Client = bigtableClient;
36+
LastCreatedChannel = channelBase;
37+
InstanceName = instanceName;
38+
}
3539
}
3640

37-
private readonly Dictionary<string, CbtClient> _idClientMap;
41+
private static readonly Dictionary<string, CbtClient> _idClientMap = new Dictionary<string, CbtClient>();
3842

3943
public override async Task<CreateClientResponse> CreateClient(CreateClientRequest request, ServerCallContext context)
4044
{
@@ -48,15 +52,17 @@ public override async Task<CreateClientResponse> CreateClient(CreateClientReques
4852
GaxPreconditions.CheckArgument(projectId is not ("" or null), "ProjectId", "project id must be provided");
4953
GaxPreconditions.CheckArgument(instanceId is not ("" or null), "InstanceId", "instance id must be provided");
5054
GaxPreconditions.CheckArgument(dataTarget is not ("" or null), "DataTarget", "data target must be provided");
51-
GaxPreconditions.CheckArgument(!securityOptions.UseSsl
52-
|| (securityOptions.SslRootCertsPem is not ("" or null)), "SecurityOptions",
55+
GaxPreconditions.CheckArgument(securityOptions is null || (!securityOptions.UseSsl
56+
|| (securityOptions.SslRootCertsPem is not ("" or null))), "SecurityOptions",
5357
"security_options.ssl_root_certs_pem must be provided if security_options.use_ssl is true");
5458

59+
#pragma warning disable CS8604 // Possible null reference argument.
5560
if (_idClientMap.ContainsKey(clientId))
5661
{
5762
context.Status = new Status(StatusCode.AlreadyExists, $"Client {clientId} already exists");
5863
throw new RpcException(context.Status);
5964
}
65+
#pragma warning restore CS8604 // Possible null reference argument.
6066

6167
try
6268
{
@@ -74,27 +80,25 @@ public override async Task<CreateClientResponse> CreateClient(CreateClientReques
7480
{
7581
Settings = settings
7682
};
77-
if (dataTarget != "emulator")
83+
if (dataTarget != "emulator" && securityOptions is not null)
7884
{
7985
builder.Endpoint = dataTarget;
86+
#pragma warning disable CS8604 // Possible null reference argument.
8087
builder.ChannelCredentials = GetChannelCredentials(securityOptions.UseSsl, securityOptions.SslRootCertsPem, securityOptions.AccessToken);
88+
#pragma warning restore CS8604 // Possible null reference argument.
8189
builder.GrpcChannelOptions = (securityOptions.UseSsl && securityOptions.SslEndpointOverride is not null)
8290
? GrpcChannelOptions.Empty.WithCustomOption("grpc.ssl_target_name_override", securityOptions.SslEndpointOverride)
8391
: GrpcChannelOptions.Empty;
8492
}
8593
else
8694
{
95+
Environment.SetEnvironmentVariable("BIGTABLE_EMULATOR_HOST", dataTarget);
8796
builder.EmulatorDetection = EmulatorDetection.EmulatorOnly;
8897
}
8998
InstanceName instanceName = new InstanceName(projectId, instanceId);
9099
BigtableServiceApiClient apiClient = await builder.BuildAsync();
91100

92-
CbtClient cbtClient = new CbtClient
93-
{
94-
Client = BigtableClient.Create(apiClient),
95-
LastCreatedChannel = builder.LastCreatedChannel,
96-
InstanceName = instanceName
97-
};
101+
CbtClient cbtClient = new CbtClient(BigtableClient.Create(apiClient), builder.LastCreatedChannel, instanceName);
98102
_idClientMap[clientId] = cbtClient;
99103
}
100104
catch (Exception e)
@@ -111,7 +115,10 @@ public override async Task<CloseClientResponse> CloseClient(CloseClientRequest r
111115
CbtClient cbtClient = GetClient(request.ClientId, context);
112116
try
113117
{
114-
await cbtClient.LastCreatedChannel.ShutdownAsync();
118+
if (cbtClient.LastCreatedChannel is not null)
119+
{
120+
await cbtClient.LastCreatedChannel.ShutdownAsync();
121+
}
115122
}
116123
catch (Exception e)
117124
{
@@ -126,7 +133,9 @@ public override Task<RemoveClientResponse> RemoveClient(RemoveClientRequest requ
126133
{
127134
string clientId = request.ClientId;
128135
GaxPreconditions.CheckArgument(clientId is not ("" or null), "ClientId", "client id must be provided", context);
136+
#pragma warning disable CS8604 // Possible null reference argument.
129137
bool removed = _idClientMap.Remove(clientId);
138+
#pragma warning restore CS8604 // Possible null reference argument.
130139
if (!removed)
131140
{
132141
context.Status = new Status(StatusCode.NotFound, $"Client {clientId} not found.");
@@ -145,9 +154,9 @@ public override async Task<RowResult> ReadRow(ReadRowRequest request, ServerCall
145154
context.Status = new Status(StatusCode.InvalidArgument, "Invalid TableName");
146155
return new RowResult
147156
{
148-
Status = new Rpc.Status()
157+
Status = new Google.Rpc.Status()
149158
{
150-
Code = (int) Rpc.Code.InvalidArgument,
159+
Code = (int) Google.Rpc.Code.InvalidArgument,
151160
Message = "Invalid TableName"
152161
}
153162
};
@@ -182,9 +191,13 @@ public override async Task<RowsResult> ReadRows(ReadRowsRequest request, ServerC
182191
ReadRowsStream stream = cbtClient.Client.ReadRows(request.Request);
183192
IAsyncEnumerator<Row> enumerator = stream.GetAsyncEnumerator(new System.Threading.CancellationToken(false));
184193
RowsResult rowsResult = new RowsResult();
185-
while (await enumerator.MoveNextAsync())
194+
while (enumerator.Current is not null)
186195
{
187196
rowsResult.Rows.Add(enumerator.Current);
197+
if (!await enumerator.MoveNextAsync())
198+
{
199+
break;
200+
}
188201
}
189202
string message = rowsResult.Rows.Count == 0 ? $"ReadRows didn't find rows" : "ReadRows succeeded";
190203
rowsResult.Status = SetSuccessStatus(message, context);
@@ -258,10 +271,12 @@ public override async Task<CheckAndMutateRowResult> CheckAndMutateRow(CheckAndMu
258271
}
259272
catch (Exception e)
260273
{
261-
return new CheckAndMutateRowResult
274+
CheckAndMutateRowResult res = new CheckAndMutateRowResult
262275
{
276+
Result = new CheckAndMutateRowResponse(),
263277
Status = SetExceptionStatus(e, context)
264278
};
279+
return res;
265280
}
266281
}
267282

@@ -320,7 +335,7 @@ public override async Task<ExecuteQueryResult> ExecuteQuery(ExecuteQueryRequest
320335
Metadata = new ResultSetMetadata()
321336
};
322337
IEnumerable<byte> bytes = Enumerable.Empty<byte>();
323-
while (await enumerator.MoveNextAsync())
338+
while (enumerator.Current is not null)
324339
{
325340
ExecuteQueryResponse response = enumerator.Current;
326341
if (response.ResponseCase == ExecuteQueryResponse.ResponseOneofCase.Metadata)
@@ -340,6 +355,10 @@ public override async Task<ExecuteQueryResult> ExecuteQuery(ExecuteQueryRequest
340355
{
341356
break;
342357
}
358+
if (!await enumerator.MoveNextAsync())
359+
{
360+
break;
361+
}
343362
}
344363
result.Status = SetSuccessStatus("ExecuteQuery succeeded", context);
345364
return result;
@@ -355,17 +374,17 @@ public override async Task<ExecuteQueryResult> ExecuteQuery(ExecuteQueryRequest
355374

356375
public static CloudBigtableV2TestProxyImpl Create() => new();
357376

358-
private CloudBigtableV2TestProxyImpl() => _idClientMap = new();
359-
360377
private CbtClient GetClient(string clientId, ServerCallContext context)
361378
{
362379
GaxPreconditions.CheckArgument(clientId is not ("" or null), "ClientId", "client id must be provided", context);
363380

381+
#pragma warning disable CS8604 // Possible null reference argument.
364382
if (!_idClientMap.ContainsKey(clientId))
365383
{
366384
context.Status = new Status(StatusCode.NotFound, $"Client {clientId} not found.");
367385
throw new RpcException(context.Status);
368386
}
387+
#pragma warning restore CS8604 // Possible null reference argument.
369388
return _idClientMap[clientId];
370389
}
371390

@@ -394,22 +413,22 @@ private static ChannelCredentials GetChannelCredentials(bool encrypted, string r
394413
return encrypted ? new SslCredentials(rootCertsPem) : ChannelCredentials.Insecure;
395414
}
396415

397-
private static Rpc.Status SetExceptionStatus(Exception e, ServerCallContext context)
416+
private static Google.Rpc.Status SetExceptionStatus(Exception e, ServerCallContext context)
398417
{
399418
context.Status = new Status(StatusCode.Internal, e.Message, e);
400-
return new Rpc.Status()
419+
return new Google.Rpc.Status()
401420
{
402-
Code = (int) Rpc.Code.Internal,
421+
Code = (int) Google.Rpc.Code.Internal,
403422
Message = e.Message
404423
};
405424
}
406425

407-
private static Rpc.Status SetSuccessStatus(string message, ServerCallContext context)
426+
private static Google.Rpc.Status SetSuccessStatus(string message, ServerCallContext context)
408427
{
409428
context.Status = new Status(StatusCode.OK, message);
410-
return new Rpc.Status()
429+
return new Google.Rpc.Status()
411430
{
412-
Code = (int) Rpc.Code.Ok,
431+
Code = (int) Google.Rpc.Code.Ok,
413432
Message = message
414433
};
415434
}

0 commit comments

Comments
 (0)