Skip to content

Commit 5cd30ae

Browse files
murgatroid99murgatroid99
authored
Merge pull request #2868 from murgatroid99/grpc-js_error_propagation
grpc-js: Propagate error messages through LB policy tree
2 parents 614e5f9 + fad797f commit 5cd30ae

14 files changed

+137
-77
lines changed

‎packages/grpc-js-xds/interop/xds-interop-client.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,11 @@ class RpcBehaviorLoadBalancer implements LoadBalancer {
9191
private latestConfig: RpcBehaviorLoadBalancingConfig | null = null;
9292
constructor(channelControlHelper: ChannelControlHelper) {
9393
const childChannelControlHelper = createChildChannelControlHelper(channelControlHelper, {
94-
updateState: (connectivityState, picker) => {
94+
updateState: (connectivityState, picker, errorMessage) => {
9595
if (connectivityState === grpc.connectivityState.READY && this.latestConfig) {
9696
picker = new RpcBehaviorPicker(picker, this.latestConfig.getRpcBehavior());
9797
}
98-
channelControlHelper.updateState(connectivityState, picker);
98+
channelControlHelper.updateState(connectivityState, picker, errorMessage);
9999
}
100100
});
101101
this.child = new ChildLoadBalancerHandler(childChannelControlHelper);

‎packages/grpc-js-xds/src/load-balancer-cds.ts

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ export class CdsLoadBalancer implements LoadBalancer {
254254
}
255255
if (!maybeClusterConfig.success) {
256256
this.childBalancer.destroy();
257-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error));
257+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details);
258258
return;
259259
}
260260
const clusterConfig = maybeClusterConfig.value;
@@ -265,7 +265,8 @@ export class CdsLoadBalancer implements LoadBalancer {
265265
leafClusters = getLeafClusters(xdsConfig, clusterName);
266266
} catch (e) {
267267
trace('xDS config parsing failed with error ' + (e as Error).message);
268-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `xDS config parsing failed with error ${(e as Error).message}`}));
268+
const errorMessage = `xDS config parsing failed with error ${(e as Error).message}`;
269+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
269270
return;
270271
}
271272
const priorityChildren: {[name: string]: PriorityChildRaw} = {};
@@ -290,14 +291,16 @@ export class CdsLoadBalancer implements LoadBalancer {
290291
typedChildConfig = parseLoadBalancingConfig(childConfig);
291292
} catch (e) {
292293
trace('LB policy config parsing failed with error ' + (e as Error).message);
293-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`}));
294+
const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`;
295+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
294296
return;
295297
}
296298
this.childBalancer.updateAddressList(endpointList, typedChildConfig, {...options, [ROOT_CLUSTER_KEY]: clusterName});
297299
} else {
298300
if (!clusterConfig.children.endpoints) {
299301
trace('Received update with no resolved endpoints for cluster ' + clusterName);
300-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`}));
302+
const errorMessage = `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`;
303+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
301304
return;
302305
}
303306
const newPriorityNames: string[] = [];
@@ -402,7 +405,8 @@ export class CdsLoadBalancer implements LoadBalancer {
402405
typedChildConfig = parseLoadBalancingConfig(childConfig);
403406
} catch (e) {
404407
trace('LB policy config parsing failed with error ' + (e as Error).message);
405-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`}));
408+
const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`;
409+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
406410
return;
407411
}
408412
const childOptions: ChannelOptions = {...options};
@@ -411,13 +415,15 @@ export class CdsLoadBalancer implements LoadBalancer {
411415
const xdsClient = options[XDS_CLIENT_KEY] as XdsClient;
412416
const caCertProvider = xdsClient.getCertificateProvider(securityUpdate.caCertificateProviderInstance);
413417
if (!caCertProvider) {
414-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`}));
418+
const errorMessage = `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`;
419+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
415420
return;
416421
}
417422
if (securityUpdate.identityCertificateProviderInstance) {
418423
const identityCertProvider = xdsClient.getCertificateProvider(securityUpdate.identityCertificateProviderInstance);
419424
if (!identityCertProvider) {
420-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`}));
425+
const errorMessage = `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`;
426+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
421427
return;
422428
}
423429
childOptions[IDENTITY_CERT_PROVIDER_KEY] = identityCertProvider;

‎packages/grpc-js-xds/src/load-balancer-priority.ts

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ interface PriorityChildBalancer {
166166
isFailoverTimerPending(): boolean;
167167
getConnectivityState(): ConnectivityState;
168168
getPicker(): Picker;
169+
getErrorMessage(): string | null;
169170
getName(): string;
170171
destroy(): void;
171172
}
@@ -183,14 +184,15 @@ export class PriorityLoadBalancer implements LoadBalancer {
183184
private PriorityChildImpl = class implements PriorityChildBalancer {
184185
private connectivityState: ConnectivityState = ConnectivityState.IDLE;
185186
private picker: Picker;
187+
private errorMessage: string | null = null;
186188
private childBalancer: ChildLoadBalancerHandler;
187189
private failoverTimer: NodeJS.Timeout | null = null;
188190
private deactivationTimer: NodeJS.Timeout | null = null;
189191
private seenReadyOrIdleSinceTransientFailure = false;
190192
constructor(private parent: PriorityLoadBalancer, private name: string, ignoreReresolutionRequests: boolean) {
191193
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
192-
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
193-
this.updateState(connectivityState, picker);
194+
updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => {
195+
this.updateState(connectivityState, picker, errorMessage);
194196
},
195197
requestReresolution: () => {
196198
if (!ignoreReresolutionRequests) {
@@ -202,10 +204,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
202204
this.startFailoverTimer();
203205
}
204206

205-
private updateState(connectivityState: ConnectivityState, picker: Picker) {
207+
private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) {
206208
trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
207209
this.connectivityState = connectivityState;
208210
this.picker = picker;
211+
this.errorMessage = errorMessage;
209212
if (connectivityState === ConnectivityState.CONNECTING) {
210213
if (this.seenReadyOrIdleSinceTransientFailure && this.failoverTimer === null) {
211214
this.startFailoverTimer();
@@ -226,9 +229,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
226229
this.failoverTimer = setTimeout(() => {
227230
trace('Failover timer triggered for child ' + this.name);
228231
this.failoverTimer = null;
232+
const errorMessage = `No connection established. Last error: ${this.errorMessage}`;
229233
this.updateState(
230234
ConnectivityState.TRANSIENT_FAILURE,
231-
new UnavailablePicker()
235+
new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}),
236+
errorMessage
232237
);
233238
}, DEFAULT_FAILOVER_TIME_MS);
234239
}
@@ -285,6 +290,10 @@ export class PriorityLoadBalancer implements LoadBalancer {
285290
return this.picker;
286291
}
287292

293+
getErrorMessage() {
294+
return this.errorMessage;
295+
}
296+
288297
getName() {
289298
return this.name;
290299
}
@@ -325,7 +334,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
325334

326335
constructor(private channelControlHelper: ChannelControlHelper) {}
327336

328-
private updateState(state: ConnectivityState, picker: Picker) {
337+
private updateState(state: ConnectivityState, picker: Picker, errorMessage: string | null) {
329338
trace(
330339
'Transitioning to ' +
331340
ConnectivityState[state]
@@ -336,7 +345,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
336345
if (state === ConnectivityState.IDLE) {
337346
picker = new QueuePicker(this, picker);
338347
}
339-
this.channelControlHelper.updateState(state, picker);
348+
this.channelControlHelper.updateState(state, picker, errorMessage);
340349
}
341350

342351
private onChildStateChange(child: PriorityChildBalancer) {
@@ -363,7 +372,8 @@ export class PriorityLoadBalancer implements LoadBalancer {
363372
const chosenChild = this.children.get(this.priorities[priority])!;
364373
this.updateState(
365374
chosenChild.getConnectivityState(),
366-
chosenChild.getPicker()
375+
chosenChild.getPicker(),
376+
chosenChild.getErrorMessage()
367377
);
368378
if (deactivateLowerPriorities) {
369379
for (let i = priority + 1; i < this.priorities.length; i++) {
@@ -374,7 +384,8 @@ export class PriorityLoadBalancer implements LoadBalancer {
374384

375385
private choosePriority() {
376386
if (this.priorities.length === 0) {
377-
this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: 'priority policy has empty priority list', metadata: new Metadata()}));
387+
const errorMessage = 'priority policy has empty priority list';
388+
this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}), errorMessage);
378389
return;
379390
}
380391

‎packages/grpc-js-xds/src/load-balancer-ring-hash.ts

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -225,11 +225,15 @@ class RingHashLoadBalancer implements LoadBalancer {
225225
private updatesPaused = false;
226226
private currentState: connectivityState = connectivityState.IDLE;
227227
private ring: RingEntry[] = [];
228+
private latestErrorMessage: string | null = null;
228229
constructor(private channelControlHelper: ChannelControlHelper) {
229230
this.childChannelControlHelper = createChildChannelControlHelper(
230231
channelControlHelper,
231232
{
232-
updateState: (state, picker) => {
233+
updateState: (state, picker, errorMessage) => {
234+
if (errorMessage) {
235+
this.latestErrorMessage = errorMessage;
236+
}
233237
this.calculateAndUpdateState();
234238
/* If this LB policy is in the TRANSIENT_FAILURE state, requests will
235239
* not trigger new connections, so we need to explicitly try connecting
@@ -270,44 +274,50 @@ class RingHashLoadBalancer implements LoadBalancer {
270274
stateCounts[leaf.getConnectivityState()] += 1;
271275
}
272276
if (stateCounts[connectivityState.READY] > 0) {
273-
this.updateState(connectivityState.READY, new RingHashPicker(this.ring));
277+
this.updateState(connectivityState.READY, new RingHashPicker(this.ring), null);
274278
// REPORT READY
275279
} else if (stateCounts[connectivityState.TRANSIENT_FAILURE] > 1) {
280+
const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`;
276281
this.updateState(
277282
connectivityState.TRANSIENT_FAILURE,
278-
new UnavailablePicker()
283+
new UnavailablePicker({details: errorMessage}),
284+
errorMessage
279285
);
280286
} else if (stateCounts[connectivityState.CONNECTING] > 0) {
281287
this.updateState(
282288
connectivityState.CONNECTING,
283-
new RingHashPicker(this.ring)
289+
new RingHashPicker(this.ring),
290+
null
284291
);
285292
} else if (
286293
stateCounts[connectivityState.TRANSIENT_FAILURE] > 0 &&
287294
this.leafMap.size > 1
288295
) {
289296
this.updateState(
290297
connectivityState.CONNECTING,
291-
new RingHashPicker(this.ring)
298+
new RingHashPicker(this.ring),
299+
null
292300
);
293301
} else if (stateCounts[connectivityState.IDLE] > 0) {
294-
this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring));
302+
this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring), null);
295303
} else {
304+
const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`;
296305
this.updateState(
297306
connectivityState.TRANSIENT_FAILURE,
298-
new UnavailablePicker()
307+
new UnavailablePicker({details: errorMessage}),
308+
errorMessage
299309
);
300310
}
301311
}
302312

303-
private updateState(newState: connectivityState, picker: Picker) {
313+
private updateState(newState: connectivityState, picker: Picker, errorMessage: string | null) {
304314
trace(
305315
connectivityState[this.currentState] +
306316
' -> ' +
307317
connectivityState[newState]
308318
);
309319
this.currentState = newState;
310-
this.channelControlHelper.updateState(newState, picker);
320+
this.channelControlHelper.updateState(newState, picker, errorMessage);
311321
}
312322

313323
private constructRing(

‎packages/grpc-js-xds/src/load-balancer-weighted-target.ts

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -175,18 +175,21 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
175175

176176
constructor(private parent: WeightedTargetLoadBalancer, private name: string) {
177177
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
178-
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
179-
this.updateState(connectivityState, picker);
178+
updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => {
179+
this.updateState(connectivityState, picker, errorMessage);
180180
},
181181
}));
182182

183183
this.picker = new QueuePicker(this.childBalancer);
184184
}
185185

186-
private updateState(connectivityState: ConnectivityState, picker: Picker) {
186+
private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) {
187187
trace('Target ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
188188
this.connectivityState = connectivityState;
189189
this.picker = picker;
190+
if (errorMessage) {
191+
this.parent.latestChildErrorMessage = errorMessage;
192+
}
190193
this.parent.maybeUpdateState();
191194
}
192195

@@ -242,6 +245,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
242245
*/
243246
private targetList: string[] = [];
244247
private updatesPaused = false;
248+
private latestChildErrorMessage: string | null = null;
245249

246250
constructor(private channelControlHelper: ChannelControlHelper) {}
247251

@@ -297,6 +301,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
297301
}
298302

299303
let picker: Picker;
304+
let errorMessage: string | null = null;
300305
switch (connectivityState) {
301306
case ConnectivityState.READY:
302307
picker = new WeightedTargetPicker(pickerList);
@@ -306,17 +311,18 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
306311
picker = new QueuePicker(this);
307312
break;
308313
default:
314+
const errorMessage = `weighted_target: all children report state TRANSIENT_FAILURE. Latest error: ${this.latestChildErrorMessage}`;
309315
picker = new UnavailablePicker({
310316
code: Status.UNAVAILABLE,
311-
details: 'weighted_target: all children report state TRANSIENT_FAILURE',
317+
details: errorMessage,
312318
metadata: new Metadata()
313319
});
314320
}
315321
trace(
316322
'Transitioning to ' +
317323
ConnectivityState[connectivityState]
318324
);
319-
this.channelControlHelper.updateState(connectivityState, picker);
325+
this.channelControlHelper.updateState(connectivityState, picker, errorMessage);
320326
}
321327

322328
updateAddressList(addressList: Endpoint[], lbConfig: TypedLoadBalancingConfig, options: ChannelOptions): void {

‎packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -241,12 +241,12 @@ class XdsClusterImplBalancer implements LoadBalancer {
241241
}
242242
return new LocalitySubchannelWrapper(wrapperChild, statsObj);
243243
},
244-
updateState: (connectivityState, originalPicker) => {
244+
updateState: (connectivityState, originalPicker, errorMessage) => {
245245
if (this.latestConfig === null || this.latestClusterConfig === null || this.latestClusterConfig.children.type === 'aggregate' || !this.latestClusterConfig.children.endpoints) {
246-
channelControlHelper.updateState(connectivityState, originalPicker);
246+
channelControlHelper.updateState(connectivityState, originalPicker, errorMessage);
247247
} else {
248248
const picker = new XdsClusterImplPicker(originalPicker, getCallCounterMapKey(this.latestConfig.getCluster(), this.latestClusterConfig.cluster.edsServiceName), this.latestClusterConfig.cluster.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS, this.latestClusterConfig.children.endpoints.dropCategories, this.clusterDropStats);
249-
channelControlHelper.updateState(connectivityState, picker);
249+
channelControlHelper.updateState(connectivityState, picker, errorMessage);
250250
}
251251
}
252252
}));
@@ -266,7 +266,7 @@ class XdsClusterImplBalancer implements LoadBalancer {
266266
if (!maybeClusterConfig.success) {
267267
this.latestClusterConfig = null;
268268
this.childBalancer.destroy();
269-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error));
269+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details);
270270
return;
271271
}
272272
const clusterConfig = maybeClusterConfig.value;
@@ -276,7 +276,7 @@ class XdsClusterImplBalancer implements LoadBalancer {
276276
}
277277
if (!clusterConfig.children.endpoints) {
278278
this.childBalancer.destroy();
279-
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}));
279+
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}), clusterConfig.children.resolutionNote ?? null);
280280

281281
}
282282
this.lastestEndpointList = endpointList;

0 commit comments

Comments
 (0)