Skip to content

grpc-js: Propagate error messages through LB policy tree #2868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 10, 2024
Merged

grpc-js: Propagate error messages through LB policy tree #2868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a4cd42
grpc-js: Propagate error messages through LB policy tree
murgatroid99 Dec 10, 2024
fad797f
Improve round robin error message
murgatroid99 Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/grpc-js-xds/interop/xds-interop-client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ class RpcBehaviorLoadBalancer implements LoadBalancer {
private latestConfig: RpcBehaviorLoadBalancingConfig | null = null;
constructor(channelControlHelper: ChannelControlHelper) {
const childChannelControlHelper = createChildChannelControlHelper(channelControlHelper, {
updateState: (connectivityState, picker) => {
updateState: (connectivityState, picker, errorMessage) => {
if (connectivityState === grpc.connectivityState.READY && this.latestConfig) {
picker = new RpcBehaviorPicker(picker, this.latestConfig.getRpcBehavior());
}
channelControlHelper.updateState(connectivityState, picker);
channelControlHelper.updateState(connectivityState, picker, errorMessage);
}
});
this.child = new ChildLoadBalancerHandler(childChannelControlHelper);
Expand Down
20 changes: 13 additions & 7 deletions packages/grpc-js-xds/src/load-balancer-cds.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ export class CdsLoadBalancer implements LoadBalancer {
}
if (!maybeClusterConfig.success) {
this.childBalancer.destroy();
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error));
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details);
return;
}
const clusterConfig = maybeClusterConfig.value;
Expand All @@ -265,7 +265,8 @@ export class CdsLoadBalancer implements LoadBalancer {
leafClusters = getLeafClusters(xdsConfig, clusterName);
} catch (e) {
trace('xDS config parsing failed with error ' + (e as Error).message);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `xDS config parsing failed with error ${(e as Error).message}`}));
const errorMessage = `xDS config parsing failed with error ${(e as Error).message}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
const priorityChildren: {[name: string]: PriorityChildRaw} = {};
Expand All @@ -290,14 +291,16 @@ export class CdsLoadBalancer implements LoadBalancer {
typedChildConfig = parseLoadBalancingConfig(childConfig);
} catch (e) {
trace('LB policy config parsing failed with error ' + (e as Error).message);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`}));
const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
this.childBalancer.updateAddressList(endpointList, typedChildConfig, {...options, [ROOT_CLUSTER_KEY]: clusterName});
} else {
if (!clusterConfig.children.endpoints) {
trace('Received update with no resolved endpoints for cluster ' + clusterName);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`}));
const errorMessage = `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
const newPriorityNames: string[] = [];
Expand Down Expand Up @@ -402,7 +405,8 @@ export class CdsLoadBalancer implements LoadBalancer {
typedChildConfig = parseLoadBalancingConfig(childConfig);
} catch (e) {
trace('LB policy config parsing failed with error ' + (e as Error).message);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`}));
const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
const childOptions: ChannelOptions = {...options};
Expand All @@ -411,13 +415,15 @@ export class CdsLoadBalancer implements LoadBalancer {
const xdsClient = options[XDS_CLIENT_KEY] as XdsClient;
const caCertProvider = xdsClient.getCertificateProvider(securityUpdate.caCertificateProviderInstance);
if (!caCertProvider) {
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`}));
const errorMessage = `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
if (securityUpdate.identityCertificateProviderInstance) {
const identityCertProvider = xdsClient.getCertificateProvider(securityUpdate.identityCertificateProviderInstance);
if (!identityCertProvider) {
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`}));
const errorMessage = `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
childOptions[IDENTITY_CERT_PROVIDER_KEY] = identityCertProvider;
Expand Down
27 changes: 19 additions & 8 deletions packages/grpc-js-xds/src/load-balancer-priority.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ interface PriorityChildBalancer {
isFailoverTimerPending(): boolean;
getConnectivityState(): ConnectivityState;
getPicker(): Picker;
getErrorMessage(): string | null;
getName(): string;
destroy(): void;
}
Expand All @@ -183,14 +184,15 @@ export class PriorityLoadBalancer implements LoadBalancer {
private PriorityChildImpl = class implements PriorityChildBalancer {
private connectivityState: ConnectivityState = ConnectivityState.IDLE;
private picker: Picker;
private errorMessage: string | null = null;
private childBalancer: ChildLoadBalancerHandler;
private failoverTimer: NodeJS.Timeout | null = null;
private deactivationTimer: NodeJS.Timeout | null = null;
private seenReadyOrIdleSinceTransientFailure = false;
constructor(private parent: PriorityLoadBalancer, private name: string, ignoreReresolutionRequests: boolean) {
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
this.updateState(connectivityState, picker);
updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => {
this.updateState(connectivityState, picker, errorMessage);
},
requestReresolution: () => {
if (!ignoreReresolutionRequests) {
Expand All @@ -202,10 +204,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.startFailoverTimer();
}

private updateState(connectivityState: ConnectivityState, picker: Picker) {
private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) {
trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
this.connectivityState = connectivityState;
this.picker = picker;
this.errorMessage = errorMessage;
if (connectivityState === ConnectivityState.CONNECTING) {
if (this.seenReadyOrIdleSinceTransientFailure && this.failoverTimer === null) {
this.startFailoverTimer();
Expand All @@ -226,9 +229,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.failoverTimer = setTimeout(() => {
trace('Failover timer triggered for child ' + this.name);
this.failoverTimer = null;
const errorMessage = `No connection established. Last error: ${this.errorMessage}`;
this.updateState(
ConnectivityState.TRANSIENT_FAILURE,
new UnavailablePicker()
new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}),
errorMessage
);
}, DEFAULT_FAILOVER_TIME_MS);
}
Expand Down Expand Up @@ -285,6 +290,10 @@ export class PriorityLoadBalancer implements LoadBalancer {
return this.picker;
}

getErrorMessage() {
return this.errorMessage;
}

getName() {
return this.name;
}
Expand Down Expand Up @@ -325,7 +334,7 @@ export class PriorityLoadBalancer implements LoadBalancer {

constructor(private channelControlHelper: ChannelControlHelper) {}

private updateState(state: ConnectivityState, picker: Picker) {
private updateState(state: ConnectivityState, picker: Picker, errorMessage: string | null) {
trace(
'Transitioning to ' +
ConnectivityState[state]
Expand All @@ -336,7 +345,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
if (state === ConnectivityState.IDLE) {
picker = new QueuePicker(this, picker);
}
this.channelControlHelper.updateState(state, picker);
this.channelControlHelper.updateState(state, picker, errorMessage);
}

private onChildStateChange(child: PriorityChildBalancer) {
Expand All @@ -363,7 +372,8 @@ export class PriorityLoadBalancer implements LoadBalancer {
const chosenChild = this.children.get(this.priorities[priority])!;
this.updateState(
chosenChild.getConnectivityState(),
chosenChild.getPicker()
chosenChild.getPicker(),
chosenChild.getErrorMessage()
);
if (deactivateLowerPriorities) {
for (let i = priority + 1; i < this.priorities.length; i++) {
Expand All @@ -374,7 +384,8 @@ export class PriorityLoadBalancer implements LoadBalancer {

private choosePriority() {
if (this.priorities.length === 0) {
this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: 'priority policy has empty priority list', metadata: new Metadata()}));
const errorMessage = 'priority policy has empty priority list';
this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}

Expand Down
28 changes: 19 additions & 9 deletions packages/grpc-js-xds/src/load-balancer-ring-hash.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,11 +225,15 @@ class RingHashLoadBalancer implements LoadBalancer {
private updatesPaused = false;
private currentState: connectivityState = connectivityState.IDLE;
private ring: RingEntry[] = [];
private latestErrorMessage: string | null = null;
constructor(private channelControlHelper: ChannelControlHelper) {
this.childChannelControlHelper = createChildChannelControlHelper(
channelControlHelper,
{
updateState: (state, picker) => {
updateState: (state, picker, errorMessage) => {
if (errorMessage) {
this.latestErrorMessage = errorMessage;
}
this.calculateAndUpdateState();
/* If this LB policy is in the TRANSIENT_FAILURE state, requests will
* not trigger new connections, so we need to explicitly try connecting
Expand Down Expand Up @@ -270,44 +274,50 @@ class RingHashLoadBalancer implements LoadBalancer {
stateCounts[leaf.getConnectivityState()] += 1;
}
if (stateCounts[connectivityState.READY] > 0) {
this.updateState(connectivityState.READY, new RingHashPicker(this.ring));
this.updateState(connectivityState.READY, new RingHashPicker(this.ring), null);
// REPORT READY
} else if (stateCounts[connectivityState.TRANSIENT_FAILURE] > 1) {
const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`;
this.updateState(
connectivityState.TRANSIENT_FAILURE,
new UnavailablePicker()
new UnavailablePicker({details: errorMessage}),
errorMessage
);
} else if (stateCounts[connectivityState.CONNECTING] > 0) {
this.updateState(
connectivityState.CONNECTING,
new RingHashPicker(this.ring)
new RingHashPicker(this.ring),
null
);
} else if (
stateCounts[connectivityState.TRANSIENT_FAILURE] > 0 &&
this.leafMap.size > 1
) {
this.updateState(
connectivityState.CONNECTING,
new RingHashPicker(this.ring)
new RingHashPicker(this.ring),
null
);
} else if (stateCounts[connectivityState.IDLE] > 0) {
this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring));
this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring), null);
} else {
const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`;
this.updateState(
connectivityState.TRANSIENT_FAILURE,
new UnavailablePicker()
new UnavailablePicker({details: errorMessage}),
errorMessage
);
}
}

private updateState(newState: connectivityState, picker: Picker) {
private updateState(newState: connectivityState, picker: Picker, errorMessage: string | null) {
trace(
connectivityState[this.currentState] +
' -> ' +
connectivityState[newState]
);
this.currentState = newState;
this.channelControlHelper.updateState(newState, picker);
this.channelControlHelper.updateState(newState, picker, errorMessage);
}

private constructRing(
Expand Down
16 changes: 11 additions & 5 deletions packages/grpc-js-xds/src/load-balancer-weighted-target.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,18 +175,21 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {

constructor(private parent: WeightedTargetLoadBalancer, private name: string) {
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
this.updateState(connectivityState, picker);
updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => {
this.updateState(connectivityState, picker, errorMessage);
},
}));

this.picker = new QueuePicker(this.childBalancer);
}

private updateState(connectivityState: ConnectivityState, picker: Picker) {
private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) {
trace('Target ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
this.connectivityState = connectivityState;
this.picker = picker;
if (errorMessage) {
this.parent.latestChildErrorMessage = errorMessage;
}
this.parent.maybeUpdateState();
}

Expand Down Expand Up @@ -242,6 +245,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
*/
private targetList: string[] = [];
private updatesPaused = false;
private latestChildErrorMessage: string | null = null;

constructor(private channelControlHelper: ChannelControlHelper) {}

Expand Down Expand Up @@ -297,6 +301,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
}

let picker: Picker;
let errorMessage: string | null = null;
switch (connectivityState) {
case ConnectivityState.READY:
picker = new WeightedTargetPicker(pickerList);
Expand All @@ -306,17 +311,18 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
picker = new QueuePicker(this);
break;
default:
const errorMessage = `weighted_target: all children report state TRANSIENT_FAILURE. Latest error: ${this.latestChildErrorMessage}`;
picker = new UnavailablePicker({
code: Status.UNAVAILABLE,
details: 'weighted_target: all children report state TRANSIENT_FAILURE',
details: errorMessage,
metadata: new Metadata()
});
}
trace(
'Transitioning to ' +
ConnectivityState[connectivityState]
);
this.channelControlHelper.updateState(connectivityState, picker);
this.channelControlHelper.updateState(connectivityState, picker, errorMessage);
}

updateAddressList(addressList: Endpoint[], lbConfig: TypedLoadBalancingConfig, options: ChannelOptions): void {
Expand Down
10 changes: 5 additions & 5 deletions packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,12 +241,12 @@ class XdsClusterImplBalancer implements LoadBalancer {
}
return new LocalitySubchannelWrapper(wrapperChild, statsObj);
},
updateState: (connectivityState, originalPicker) => {
updateState: (connectivityState, originalPicker, errorMessage) => {
if (this.latestConfig === null || this.latestClusterConfig === null || this.latestClusterConfig.children.type === 'aggregate' || !this.latestClusterConfig.children.endpoints) {
channelControlHelper.updateState(connectivityState, originalPicker);
channelControlHelper.updateState(connectivityState, originalPicker, errorMessage);
} else {
const picker = new XdsClusterImplPicker(originalPicker, getCallCounterMapKey(this.latestConfig.getCluster(), this.latestClusterConfig.cluster.edsServiceName), this.latestClusterConfig.cluster.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS, this.latestClusterConfig.children.endpoints.dropCategories, this.clusterDropStats);
channelControlHelper.updateState(connectivityState, picker);
channelControlHelper.updateState(connectivityState, picker, errorMessage);
}
}
}));
Expand All @@ -266,7 +266,7 @@ class XdsClusterImplBalancer implements LoadBalancer {
if (!maybeClusterConfig.success) {
this.latestClusterConfig = null;
this.childBalancer.destroy();
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error));
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details);
return;
}
const clusterConfig = maybeClusterConfig.value;
Expand All @@ -276,7 +276,7 @@ class XdsClusterImplBalancer implements LoadBalancer {
}
if (!clusterConfig.children.endpoints) {
this.childBalancer.destroy();
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}));
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}), clusterConfig.children.resolutionNote ?? null);

}
this.lastestEndpointList = endpointList;
Expand Down
Loading
Loading