Skip to content

Data race in v0.53.0 instrumentation/net/http/otelhttp.(*bodyWrapper).Read() #5914

New issue
New issue
Closed
#5916
Closed
Data race in v0.53.0 instrumentation/net/http/otelhttp.(*bodyWrapper).Read()#5914
#5916
Assignees
dmathieu
Labels
bugSomething isn't working
Milestone
v1.29.0
@enj

Description

@enj
enj
opened on Jul 15, 2024

Description

Seen during local runs of Kubernetes integration tests.

==================
WARNING: DATA RACE
Write at 0x00c012172560 by goroutine 448906:
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*bodyWrapper).Read()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/wrap.go:31 +0x8c
  io.(*LimitedReader).Read()
      HOME/go/pkg/mod/golang.org/[email protected]/src/io/io.go:479 +0xbc
  io.ReadAll()
      HOME/go/pkg/mod/golang.org/[email protected]/src/io/io.go:712 +0x84
  io/ioutil.ReadAll()
      HOME/go/pkg/mod/golang.org/[email protected]/src/io/ioutil/ioutil.go:27 +0x1fc
  k8s.io/apiserver/pkg/endpoints/handlers.limitedReadBody()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go:382 +0x1ec
  k8s.io/apiserver/pkg/endpoints/handlers.limitedReadBodyWithRecordMetric()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go:393 +0x48
  k8s.io/apiextensions-apiserver/pkg/apiserver.(*crdHandler).serveResource.CreateResource.createHandler.func9()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go:93 +0x4c0
  k8s.io/apiextensions-apiserver/pkg/apiserver.(*crdHandler).ServeHTTP.InstrumentHandlerFunc.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go:617 +0x184
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiextensions-apiserver/pkg/apiserver.(*crdHandler).ServeHTTP.WithWaitGroup.withWaitGroup.func2()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/waitgroup.go:86 +0x170
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiextensions-apiserver/pkg/apiserver.(*crdHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go:355 +0x1890
  k8s.io/apiserver/pkg/server/mux.(*pathHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go:251 +0x4f0
  k8s.io/apiserver/pkg/server/mux.(*PathRecorderMux).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go:237 +0x60
  k8s.io/apiserver/pkg/server.director.ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/handler.go:154 +0x844
  k8s.io/apiserver/pkg/server.(*director).ServeHTTP()
      <autogenerated>:1 +0x74
  k8s.io/apiserver/pkg/server/mux.(*pathHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go:257 +0x68c
  k8s.io/apiserver/pkg/server/mux.(*PathRecorderMux).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go:237 +0x60
  k8s.io/apiserver/pkg/server.director.ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/handler.go:154 +0x844
  k8s.io/apiserver/pkg/server.(*director).ServeHTTP()
      <autogenerated>:1 +0x74
  k8s.io/kube-aggregator/pkg/apiserver.(*proxyHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go:118 +0x224
  k8s.io/apiserver/pkg/server/mux.(*pathHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go:251 +0x4f0
  k8s.io/apiserver/pkg/server/mux.(*PathRecorderMux).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go:237 +0x60
  k8s.io/apiserver/pkg/server.director.ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/handler.go:154 +0x844
  k8s.io/apiserver/pkg/server.(*director).ServeHTTP()
      <autogenerated>:1 +0x74
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func22.deferwrap1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:103 +0x6c
  runtime.deferreturn()
      HOME/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:602 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filters.withAuthorization.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authorization.go:78 +0x5a8
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x1c0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func23.deferwrap1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:103 +0x6c
  runtime.deferreturn()
      HOME/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:602 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server/filters.(*priorityAndFairnessHandler).Handle.func9()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go:292 +0x108
  k8s.io/apiserver/pkg/util/flowcontrol.(*configController).Handle.func2()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go:192 +0x2ac
  k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset.(*request).Finish.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go:391 +0x88
  k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset.(*request).Finish()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go:392 +0x44
  k8s.io/apiserver/pkg/util/flowcontrol.(*configController).Handle()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go:179 +0x9f8
  k8s.io/apiserver/pkg/server/filters.(*priorityAndFairnessHandler).Handle.func10()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go:298 +0xe0
  k8s.io/apiserver/pkg/server/filters.(*priorityAndFairnessHandler).Handle()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go:299 +0xff4
  k8s.io/apiserver/pkg/server/filters.(*priorityAndFairnessHandler).Handle-fm()
      <autogenerated>:1 +0x4c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x1c0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func24.deferwrap1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:103 +0x6c
  runtime.deferreturn()
      HOME/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:602 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithImpersonation.func4()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go:50 +0x178
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x1c0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func25.deferwrap1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:103 +0x6c
  runtime.deferreturn()
      HOME/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:602 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithAudit.func6()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/audit.go:55 +0x3d4
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x1c0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func27.deferwrap1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:103 +0x6c
  runtime.deferreturn()
      HOME/go/pkg/mod/golang.org/[email protected]/src/runtime/panic.go:602 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filters.withAuthentication.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go:120 +0x894
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:94 +0x3a8
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithWarningRecorder.func11()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/warning.go:35 +0x10c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP.func1()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go:115 +0xb8

Previous read at 0x00c012172560 by goroutine 448900:
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*middleware).serveHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/handler.go:224 +0x14b8
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.NewMiddleware.func1.1()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/handler.go:74 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithLatencyTrackers.func16()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/webhook_duration.go:56 +0x194
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRoutine.func17()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/routine/routine.go:65 +0x94
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRequestInfo.func18()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/requestinfo.go:39 +0x138
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRequestReceivedTimestamp.withRequestReceivedTimestampWithClock.func32()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/request_received_time.go:38 +0xb0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithMuxAndDiscoveryComplete.func19()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/mux_discovery_complete.go:52 +0xb8
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithPanicRecovery.withPanicRecovery.func33()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/wrap.go:73 +0x17c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithAuditInit.withAuditInit.func34()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/audit_init.go:63 +0x124
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.(*APIServerHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/handler.go:189 +0x5c
  net/http.serverHandler.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:3137 +0x2a8
  net/http.initALPNRequest.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:3745 +0x268
  net/http.(*initALPNRequest).ServeHTTP()
      <autogenerated>:1 +0x74
  net/http.Handler.ServeHTTP-fm()
      <autogenerated>:1 +0x60
  golang.org/x/net/http2.(*serverConn).runHandler()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:2414 +0x194
  golang.org/x/net/http2.(*serverConn).scheduleHandler.gowrap1()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:2348 +0x58

Goroutine 448906 (running) created at:
  k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go:101 +0x250
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRequestDeadline.withRequestDeadline.func28()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/request_deadline.go:100 +0x1b0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithWaitGroup.withWaitGroup.func29()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/waitgroup.go:86 +0x170
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithCacheControl.func14()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/cachecontrol.go:31 +0xbc
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithHTTPLogging.WithLogging.withLogging.func35()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/httplog/httplog.go:139 +0x1e8
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/endpoints/filters.WithTracing.func2()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/traces.go:51 +0x1d0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*middleware).serveHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/handler.go:218 +0x141c
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.NewMiddleware.func1.1()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/handler.go:74 +0x5c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithLatencyTrackers.func16()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/webhook_duration.go:56 +0x194
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRoutine.func17()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/routine/routine.go:65 +0x94
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRequestInfo.func18()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/requestinfo.go:39 +0x138
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithRequestReceivedTimestamp.withRequestReceivedTimestampWithClock.func32()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/request_received_time.go:38 +0xb0
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithMuxAndDiscoveryComplete.func19()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/mux_discovery_complete.go:52 +0xb8
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithPanicRecovery.withPanicRecovery.func33()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/filters/wrap.go:73 +0x17c
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithAuditInit.withAuditInit.func34()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/filters/audit_init.go:63 +0x124
  net/http.HandlerFunc.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:2166 +0x48
  k8s.io/apiserver/pkg/server.(*APIServerHandler).ServeHTTP()
      ROOT/src/github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver/pkg/server/handler.go:189 +0x5c
  net/http.serverHandler.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:3137 +0x2a8
  net/http.initALPNRequest.ServeHTTP()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:3745 +0x268
  net/http.(*initALPNRequest).ServeHTTP()
      <autogenerated>:1 +0x74
  net/http.Handler.ServeHTTP-fm()
      <autogenerated>:1 +0x60
  golang.org/x/net/http2.(*serverConn).runHandler()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:2414 +0x194
  golang.org/x/net/http2.(*serverConn).scheduleHandler.gowrap1()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:2348 +0x58

Goroutine 448900 (finished) created at:
  golang.org/x/net/http2.(*serverConn).scheduleHandler()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:2348 +0x1f8
  golang.org/x/net/http2.(*serverConn).processHeaders()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:2070 +0x980
  golang.org/x/net/http2.(*serverConn).processFrame()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:1571 +0x200
  golang.org/x/net/http2.(*serverConn).processFrameFromReader()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:1509 +0x2a0
  golang.org/x/net/http2.(*serverConn).serve()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:998 +0x1130
  golang.org/x/net/http2.(*Server).serveConn()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:572 +0x13b0
  golang.org/x/net/http2.(*Server).ServeConn()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:436 +0x154
  golang.org/x/net/http2.ConfigureServer.func1()
      ROOT/src/github.com/kubernetes/kubernetes/vendor/golang.org/x/net/http2/server.go:355 +0xc0
  net/http.(*conn).serve()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:1947 +0xa60
  net/http.(*Server).Serve.gowrap3()
      HOME/go/pkg/mod/golang.org/[email protected]/src/net/http/server.go:3285 +0x4c
==================

Environment

  • OS: macOS
  • Architecture: arm64
  • Go Version: 1.22.3
  • go.opentelemetry.io/contrib version: v0.53.0

Steps To Reproduce

  1. Happens rarely during intergration test runs of Kube

Expected behavior

No data race. The following line likely needs synchronization:

opentelemetry-go-contrib/instrumentation/net/http/otelhttp/wrap.go

Line 31 in e11fd07

w.err = err

Metadata

Metadata

Assignees

Labels

bugSomething isn't working

Type

No type

Projects

Go: Triage

Status

Closed

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions