Skip to content

Commit 4cdd771

Browse files
lauritheyamstrask
authored
Fix http.server.active_requests metric with async requests (#11638)
Co-authored-by: Helen <[email protected]> Co-authored-by: Trask Stalnaker <[email protected]>
1 parent 746ef01 commit 4cdd771

File tree

16 files changed

+57
-85
lines changed

16 files changed

+57
-85
lines changed

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/Instrumenter.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import io.opentelemetry.api.trace.SpanKind;
1212
import io.opentelemetry.api.trace.Tracer;
1313
import io.opentelemetry.context.Context;
14+
import io.opentelemetry.context.ContextKey;
1415
import io.opentelemetry.instrumentation.api.internal.HttpRouteState;
1516
import io.opentelemetry.instrumentation.api.internal.InstrumenterAccess;
1617
import io.opentelemetry.instrumentation.api.internal.InstrumenterUtil;
@@ -41,6 +42,9 @@
4142
*/
4243
public class Instrumenter<REQUEST, RESPONSE> {
4344

45+
private static final ContextKey<OperationListener[]> START_OPERATION_LISTENERS =
46+
ContextKey.named("instrumenter-start-operation-listeners");
47+
4448
/**
4549
* Returns a new {@link InstrumenterBuilder}.
4650
*
@@ -74,6 +78,7 @@ public static <REQUEST, RESPONSE> InstrumenterBuilder<REQUEST, RESPONSE> builder
7478
private final ContextCustomizer<? super REQUEST>[] contextCustomizers;
7579
private final OperationListener[] operationListeners;
7680
private final ErrorCauseExtractor errorCauseExtractor;
81+
private final boolean propagateOperationListenersToOnEnd;
7782
private final boolean enabled;
7883
private final SpanSuppressor spanSuppressor;
7984

@@ -89,6 +94,7 @@ public static <REQUEST, RESPONSE> InstrumenterBuilder<REQUEST, RESPONSE> builder
8994
this.contextCustomizers = builder.contextCustomizers.toArray(new ContextCustomizer[0]);
9095
this.operationListeners = builder.buildOperationListeners().toArray(new OperationListener[0]);
9196
this.errorCauseExtractor = builder.errorCauseExtractor;
97+
this.propagateOperationListenersToOnEnd = builder.propagateOperationListenersToOnEnd;
9298
this.enabled = builder.enabled;
9399
this.spanSuppressor = builder.buildSpanSuppressor();
94100
}
@@ -198,6 +204,15 @@ private Context doStart(Context parentContext, REQUEST request, @Nullable Instan
198204
context = operationListeners[i].onStart(context, attributes, startNanos);
199205
}
200206
}
207+
if (propagateOperationListenersToOnEnd || context.get(START_OPERATION_LISTENERS) != null) {
208+
// when start and end are not called on the same instrumenter we need to use the operation
209+
// listeners that were used during start in end to correctly handle metrics like
210+
// http.server.active_requests that is recorded both in start and end
211+
//
212+
// need to also add when there is already START_OPERATION_LISTENERS, otherwise this
213+
// instrumenter will call its parent's operation listeners in doEnd
214+
context = context.with(START_OPERATION_LISTENERS, operationListeners);
215+
}
201216

202217
if (localRoot) {
203218
context = LocalRootSpan.store(context, span);
@@ -228,6 +243,10 @@ private void doEnd(
228243
}
229244
span.setAllAttributes(attributes);
230245

246+
OperationListener[] operationListeners = context.get(START_OPERATION_LISTENERS);
247+
if (operationListeners == null) {
248+
operationListeners = this.operationListeners;
249+
}
231250
if (operationListeners.length != 0) {
232251
long endNanos = getNanos(endTime);
233252
for (int i = operationListeners.length - 1; i >= 0; i--) {

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/InstrumenterBuilder.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ public final class InstrumenterBuilder<REQUEST, RESPONSE> {
6666
SpanStatusExtractor<? super REQUEST, ? super RESPONSE> spanStatusExtractor =
6767
SpanStatusExtractor.getDefault();
6868
ErrorCauseExtractor errorCauseExtractor = ErrorCauseExtractor.getDefault();
69+
boolean propagateOperationListenersToOnEnd = false;
6970
boolean enabled = true;
7071

7172
InstrumenterBuilder(
@@ -370,6 +371,10 @@ private Set<SpanKey> getSpanKeysFromAttributesExtractors() {
370371
.collect(Collectors.toSet());
371372
}
372373

374+
private void propagateOperationListenersToOnEnd() {
375+
propagateOperationListenersToOnEnd = true;
376+
}
377+
373378
private interface InstrumenterConstructor<RQ, RS> {
374379
Instrumenter<RQ, RS> create(InstrumenterBuilder<RQ, RS> builder);
375380

@@ -406,6 +411,12 @@ public <RQ, RS> Instrumenter<RQ, RS> buildDownstreamInstrumenter(
406411
SpanKindExtractor<RQ> spanKindExtractor) {
407412
return builder.buildDownstreamInstrumenter(setter, spanKindExtractor);
408413
}
414+
415+
@Override
416+
public <RQ, RS> void propagateOperationListenersToOnEnd(
417+
InstrumenterBuilder<RQ, RS> builder) {
418+
builder.propagateOperationListenersToOnEnd();
419+
}
409420
});
410421
}
411422
}

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/internal/InstrumenterBuilderAccess.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,7 @@ <REQUEST, RESPONSE> Instrumenter<REQUEST, RESPONSE> buildDownstreamInstrumenter(
2626
InstrumenterBuilder<REQUEST, RESPONSE> builder,
2727
TextMapSetter<REQUEST> setter,
2828
SpanKindExtractor<REQUEST> spanKindExtractor);
29+
30+
<REQUEST, RESPONSE> void propagateOperationListenersToOnEnd(
31+
InstrumenterBuilder<REQUEST, RESPONSE> builder);
2932
}

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/internal/InstrumenterUtil.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,5 +67,11 @@ public static <REQUEST, RESPONSE> Instrumenter<REQUEST, RESPONSE> buildDownstrea
6767
builder, setter, spanKindExtractor);
6868
}
6969

70+
public static <REQUEST, RESPONSE> void propagateOperationListenersToOnEnd(
71+
InstrumenterBuilder<REQUEST, RESPONSE> builder) {
72+
// instrumenterBuilderAccess is guaranteed to be non-null here
73+
instrumenterBuilderAccess.propagateOperationListenersToOnEnd(builder);
74+
}
75+
7076
private InstrumenterUtil() {}
7177
}

‎instrumentation/jetty/jetty-11.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/jetty/v11_0/Jetty11Singletons.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public final class Jetty11Singletons {
2424
ServletInstrumenterBuilder.<HttpServletRequest, HttpServletResponse>create()
2525
.addContextCustomizer(
2626
(context, request, attributes) -> new AppServerBridge.Builder().init(context))
27+
.propagateOperationListenersToOnEnd()
2728
.build(INSTRUMENTATION_NAME, Servlet5Accessor.INSTANCE);
2829

2930
private static final JettyHelper<HttpServletRequest, HttpServletResponse> HELPER =

‎instrumentation/jetty/jetty-8.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/jetty/v8_0/Jetty8Singletons.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public final class Jetty8Singletons {
2424
ServletInstrumenterBuilder.<HttpServletRequest, HttpServletResponse>create()
2525
.addContextCustomizer(
2626
(context, request, attributes) -> new AppServerBridge.Builder().init(context))
27+
.propagateOperationListenersToOnEnd()
2728
.build(INSTRUMENTATION_NAME, Servlet3Accessor.INSTANCE);
2829

2930
private static final JettyHelper<HttpServletRequest, HttpServletResponse> HELPER =

‎instrumentation/liberty/liberty-20.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/liberty/LibertySingletons.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public final class LibertySingletons {
2424
.addContextCustomizer(
2525
(context, request, attributes) ->
2626
new AppServerBridge.Builder().recordException().init(context))
27+
.propagateOperationListenersToOnEnd()
2728
.build(INSTRUMENTATION_NAME, Servlet3Accessor.INSTANCE);
2829

2930
private static final LibertyHelper<HttpServletRequest, HttpServletResponse> HELPER =

‎instrumentation/servlet/servlet-3.0/javaagent/src/test/groovy/JettyServlet3Test.groovy

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,6 @@ class JettyServlet3TestAsync extends JettyServlet3Test {
149149
boolean isAsyncTest() {
150150
true
151151
}
152-
153-
@Override
154-
String getMetricsInstrumentationName() {
155-
// with async requests the span is started in one instrumentation (server instrumentation)
156-
// but ended from another (servlet instrumentation)
157-
"io.opentelemetry.servlet-3.0"
158-
}
159152
}
160153

161154
class JettyServlet3TestFakeAsync extends JettyServlet3Test {
@@ -164,13 +157,6 @@ class JettyServlet3TestFakeAsync extends JettyServlet3Test {
164157
Class<Servlet> servlet() {
165158
TestServlet3.FakeAsync
166159
}
167-
168-
@Override
169-
String getMetricsInstrumentationName() {
170-
// with async requests the span is started in one instrumentation (server instrumentation)
171-
// but ended from another (servlet instrumentation)
172-
"io.opentelemetry.servlet-3.0"
173-
}
174160
}
175161

176162
class JettyServlet3TestForward extends JettyDispatchTest {
@@ -247,13 +233,6 @@ class JettyServlet3TestDispatchImmediate extends JettyDispatchTest {
247233
true
248234
}
249235

250-
@Override
251-
String getMetricsInstrumentationName() {
252-
// with async requests the span is started in one instrumentation (server instrumentation)
253-
// but ended from another (servlet instrumentation)
254-
"io.opentelemetry.servlet-3.0"
255-
}
256-
257236
@Override
258237
protected void setupServlets(ServletContextHandler context) {
259238
super.setupServlets(context)
@@ -283,13 +262,6 @@ class JettyServlet3TestDispatchAsync extends JettyDispatchTest {
283262
true
284263
}
285264

286-
@Override
287-
String getMetricsInstrumentationName() {
288-
// with async requests the span is started in one instrumentation (server instrumentation)
289-
// but ended from another (servlet instrumentation)
290-
"io.opentelemetry.servlet-3.0"
291-
}
292-
293265
@Override
294266
protected void setupServlets(ServletContextHandler context) {
295267
super.setupServlets(context)

‎instrumentation/servlet/servlet-5.0/javaagent/src/test/groovy/JettyServlet5Test.groovy

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -123,13 +123,6 @@ class JettyServlet5TestAsync extends JettyServlet5Test {
123123
boolean errorEndpointUsesSendError() {
124124
false
125125
}
126-
127-
@Override
128-
String getMetricsInstrumentationName() {
129-
// with async requests the span is started in one instrumentation (server instrumentation)
130-
// but ended from another (servlet instrumentation)
131-
"io.opentelemetry.servlet-5.0"
132-
}
133126
}
134127

135128
@IgnoreIf({ !jvm.java11Compatible })
@@ -139,13 +132,6 @@ class JettyServlet5TestFakeAsync extends JettyServlet5Test {
139132
Class<Servlet> servlet() {
140133
TestServlet5.FakeAsync
141134
}
142-
143-
@Override
144-
String getMetricsInstrumentationName() {
145-
// with async requests the span is started in one instrumentation (server instrumentation)
146-
// but ended from another (servlet instrumentation)
147-
"io.opentelemetry.servlet-5.0"
148-
}
149135
}
150136

151137
@IgnoreIf({ !jvm.java11Compatible })
@@ -237,13 +223,6 @@ class JettyServlet5TestDispatchImmediate extends JettyDispatchTest {
237223
addServlet(context, "/dispatch" + INDEXED_CHILD.path, TestServlet5.DispatchImmediate)
238224
addServlet(context, "/dispatch/recursive", TestServlet5.DispatchRecursive)
239225
}
240-
241-
@Override
242-
String getMetricsInstrumentationName() {
243-
// with async requests the span is started in one instrumentation (server instrumentation)
244-
// but ended from another (servlet instrumentation)
245-
"io.opentelemetry.servlet-5.0"
246-
}
247226
}
248227

249228
@IgnoreIf({ !jvm.java11Compatible })
@@ -275,13 +254,6 @@ class JettyServlet5TestDispatchAsync extends JettyDispatchTest {
275254
boolean errorEndpointUsesSendError() {
276255
false
277256
}
278-
279-
@Override
280-
String getMetricsInstrumentationName() {
281-
// with async requests the span is started in one instrumentation (server instrumentation)
282-
// but ended from another (servlet instrumentation)
283-
"io.opentelemetry.servlet-5.0"
284-
}
285257
}
286258

287259
abstract class JettyDispatchTest extends JettyServlet5Test {

‎instrumentation/servlet/servlet-common/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/servlet/ServletInstrumenterBuilder.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
1515
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
1616
import io.opentelemetry.instrumentation.api.instrumenter.SpanNameExtractor;
17+
import io.opentelemetry.instrumentation.api.internal.InstrumenterUtil;
1718
import io.opentelemetry.instrumentation.api.semconv.http.HttpServerAttributesExtractor;
1819
import io.opentelemetry.instrumentation.api.semconv.http.HttpServerAttributesGetter;
1920
import io.opentelemetry.instrumentation.api.semconv.http.HttpServerMetrics;
@@ -31,6 +32,8 @@ private ServletInstrumenterBuilder() {}
3132
private final List<ContextCustomizer<? super ServletRequestContext<REQUEST>>> contextCustomizers =
3233
new ArrayList<>();
3334

35+
private boolean propagateOperationListenersToOnEnd;
36+
3437
public static <REQUEST, RESPONSE> ServletInstrumenterBuilder<REQUEST, RESPONSE> create() {
3538
return new ServletInstrumenterBuilder<>();
3639
}
@@ -42,6 +45,12 @@ public ServletInstrumenterBuilder<REQUEST, RESPONSE> addContextCustomizer(
4245
return this;
4346
}
4447

48+
@CanIgnoreReturnValue
49+
public ServletInstrumenterBuilder<REQUEST, RESPONSE> propagateOperationListenersToOnEnd() {
50+
propagateOperationListenersToOnEnd = true;
51+
return this;
52+
}
53+
4554
public Instrumenter<ServletRequestContext<REQUEST>, ServletResponseContext<RESPONSE>> build(
4655
String instrumentationName,
4756
ServletAccessor<REQUEST, RESPONSE> accessor,
@@ -85,6 +94,9 @@ public Instrumenter<ServletRequestContext<REQUEST>, ServletResponseContext<RESPO
8594
.addAttributesExtractor(HttpExperimentalAttributesExtractor.create(httpAttributesGetter))
8695
.addOperationMetrics(HttpServerExperimentalMetrics.get());
8796
}
97+
if (propagateOperationListenersToOnEnd) {
98+
InstrumenterUtil.propagateOperationListenersToOnEnd(builder);
99+
}
88100
return builder.buildServerInstrumenter(new ServletRequestGetter<>(accessor));
89101
}
90102

‎instrumentation/tomcat/tomcat-10.0/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/tomcat/v10_0/TomcatAsyncTest.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,6 @@ protected void configure(HttpServerTestOptions options) {
113113
options.setExpectedException(new ServletException(EXCEPTION.getBody()));
114114

115115
options.setHasResponseSpan(endpoint -> endpoint == NOT_FOUND || endpoint == REDIRECT);
116-
117-
// with async requests the span is started in one instrumentation (server instrumentation)
118-
// but ended from another (servlet instrumentation)
119-
options.setMetricsInstrumentationName(() -> "io.opentelemetry.servlet-5.0");
120116
}
121117

122118
@Override

‎instrumentation/tomcat/tomcat-7.0/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/tomcat/v7_0/TomcatAsyncTest.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,6 @@ protected void configure(HttpServerTestOptions options) {
108108
options.setExpectedException(new ServletException(EXCEPTION.getBody()));
109109

110110
options.setHasResponseSpan(endpoint -> endpoint == NOT_FOUND || endpoint == REDIRECT);
111-
112-
// with async requests the span is started in one instrumentation (server instrumentation)
113-
// but ended from another (servlet instrumentation)
114-
options.setMetricsInstrumentationName(() -> "io.opentelemetry.servlet-3.0");
115111
}
116112

117113
@Override

‎instrumentation/tomcat/tomcat-common/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/tomcat/common/TomcatInstrumenterFactory.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpServerExperimentalMetrics;
1111
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
1212
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
13+
import io.opentelemetry.instrumentation.api.internal.InstrumenterUtil;
1314
import io.opentelemetry.instrumentation.api.semconv.http.HttpServerAttributesExtractor;
1415
import io.opentelemetry.instrumentation.api.semconv.http.HttpServerMetrics;
1516
import io.opentelemetry.instrumentation.api.semconv.http.HttpServerRoute;
@@ -61,6 +62,7 @@ public static <REQUEST, RESPONSE> Instrumenter<Request, Response> create(
6162
.addAttributesExtractor(HttpExperimentalAttributesExtractor.create(httpAttributesGetter))
6263
.addOperationMetrics(HttpServerExperimentalMetrics.get());
6364
}
65+
InstrumenterUtil.propagateOperationListenersToOnEnd(builder);
6466
return builder.buildServerInstrumenter(TomcatRequestGetter.INSTANCE);
6567
}
6668
}

‎testing-common/src/main/groovy/io/opentelemetry/instrumentation/test/base/HttpServerTest.groovy

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -173,10 +173,6 @@ abstract class HttpServerTest<SERVER> extends InstrumentationSpecification imple
173173
return true
174174
}
175175

176-
String getMetricsInstrumentationName() {
177-
null
178-
}
179-
180176
/** A list of additional HTTP server span attributes extracted by the instrumentation per URI. */
181177
Set<AttributeKey<?>> httpAttributes(ServerEndpoint endpoint) {
182178
[
@@ -237,9 +233,6 @@ abstract class HttpServerTest<SERVER> extends InstrumentationSpecification imple
237233
options.sockPeerAddr = { endpoint ->
238234
HttpServerTest.this.sockPeerAddr(endpoint)
239235
}
240-
options.metricsInstrumentationName = {
241-
HttpServerTest.this.getMetricsInstrumentationName()
242-
}
243236
options.responseCodeOnNonStandardHttpMethod = getResponseCodeOnNonStandardHttpMethod()
244237

245238
options.testRedirect = testRedirect()

‎testing-common/src/main/java/io/opentelemetry/instrumentation/testing/junit/http/AbstractHttpServerTest.java

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -353,12 +353,8 @@ void httpServerMetrics() {
353353
spanData -> assertServerSpan(assertThat(spanData), method, SUCCESS, SUCCESS.status));
354354
});
355355

356-
String metricsInstrumentationName = options.metricsInstrumentationName.get();
357-
if (metricsInstrumentationName == null) {
358-
metricsInstrumentationName = instrumentationName.get();
359-
}
360356
testing.waitAndAssertMetrics(
361-
metricsInstrumentationName,
357+
instrumentationName.get(),
362358
"http.server.request.duration",
363359
metrics ->
364360
metrics.anySatisfy(

‎testing-common/src/main/java/io/opentelemetry/instrumentation/testing/junit/http/HttpServerTestOptions.java

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import java.util.function.BiFunction;
2020
import java.util.function.Function;
2121
import java.util.function.Predicate;
22-
import java.util.function.Supplier;
2322
import javax.annotation.Nullable;
2423

2524
public final class HttpServerTestOptions {
@@ -42,7 +41,6 @@ public final class HttpServerTestOptions {
4241
Function<ServerEndpoint, String> sockPeerAddr = unused -> "127.0.0.1";
4342
String contextPath = "";
4443
Throwable expectedException = new Exception(EXCEPTION.body);
45-
Supplier<String> metricsInstrumentationName = () -> null;
4644
// we're calling /success in the test, and most servers respond with 200 anyway
4745
int responseCodeOnNonStandardHttpMethod = ServerEndpoint.SUCCESS.status;
4846

@@ -108,13 +106,6 @@ public HttpServerTestOptions setExpectedException(Throwable expectedException) {
108106
return this;
109107
}
110108

111-
@CanIgnoreReturnValue
112-
public HttpServerTestOptions setMetricsInstrumentationName(
113-
Supplier<String> metricsInstrumentationName) {
114-
this.metricsInstrumentationName = metricsInstrumentationName;
115-
return this;
116-
}
117-
118109
@CanIgnoreReturnValue
119110
public HttpServerTestOptions setResponseCodeOnNonStandardHttpMethod(
120111
int responseCodeOnNonStandardHttpMethod) {

0 commit comments

Comments
 (0)