Fixing commit tree:

Refactor code, so that OTel tracer provider is initialized only once
Added resource cmd option, testig
Added docs

Author: oandreeva-nv

docs/user_guide/trace.md

@@ -427,11 +427,41 @@ The meaning of the trace timestamps is:
 
   * BACKEND_OUTPUT: The tensor in the response of a backend.
 
+## Tracing for BLS models
+
+Triton does not collect traces for child models, invoked from
+[BLS](https://github.com/triton-inference-server/python_backend/tree/main#business-logic-scripting) models.
+
+To include child models into collected traces, user needs to provide `trace`
+argument (as shown in the example), when constructing an InferenceRequest object.
+
+```python
+
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+  ...
+    def execute(self, requests):
+      ...
+      for request in requests:
+        ...
+        # Create an InferenceRequest object. `model_name`,
+        # `requested_output_names`, and `inputs` are the required arguments and
+        # must be provided when constructing an InferenceRequest object. Make
+        # sure to replace `inputs` argument with a list of `pb_utils.Tensor`
+        # objects.
+        inference_request = pb_utils.InferenceRequest(
+            model_name='model_name',
+            requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'],
+            inputs=[<pb_utils.Tensor object>], trace = request.trace())
+
+```
+
 ## OpenTelemetry trace support
 
-Triton provides an option to generate and export traces
-for standalone and ensemble models
-using [OpenTelemetry APIs and SDKs](https://opentelemetry.io/).
+Triton provides an option to generate and export traces using
+[OpenTelemetry APIs and SDKs](https://opentelemetry.io/).
 
 To specify OpenTelemetry mode for tracing, specify the `--trace-config`
 flag as follows:
@@ -477,16 +507,28 @@ The following table shows available OpenTelemetry trace APIs settings for
       trace data.
     </td>
     </tr>
+    <tr>
+    <td><code>resource</code></td>
+    <td><code>Empty</code></td>
+    <td>
+      Key-value pairs to be used as resource attributes. <br/>
+      Should be specified as follows:<br/>
+      <code>--trace-config opentelemetry,resource=service.name=triton</code><br/>
+      <code>--trace-config opentelemetry,resource=service.version=1</code><br/>
+      Alternatively, key-vaue attributes can be specified through <br/>
+      <a href="https://opentelemetry.io/docs/concepts/sdk-configuration/general-sdk-configuration/#otel_resource_attributes">
+      OTEL_RESOURCE_ATTRIBUTES</a>
+      environment variable.
+    </td>
+    </tr>
   </tbody>
 </table>
 
+
 ### Limitations
 
 - OpenTelemetry trace mode is not supported on Windows systems.
 
-- Tracing [BLS](https://github.com/triton-inference-server/python_backend/tree/main#business-logic-scripting)
-models is not supported.
-
 - Triton supports only
 [OTLP/HTTP Exporter](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp)
 and allows specification of only url for this exporter through

qa/L0_trace/opentelemetry_unittest.py

@@ -36,7 +36,7 @@
 import tritonclient.grpc as grpcclient
 import tritonclient.http as httpclient
 
-EXPECTED_NUM_SPANS = 10
+EXPECTED_NUM_SPANS = 16
 
 
 class OpenTelemetryTest(tu.TestResultCollector):
@@ -50,68 +50,66 @@ def setUp(self):
                 else:
                     break
 
-        data = data.split('\n')
+        data = data.split("\n")
         full_spans = [
-            entry.split('POST')[0]
-            for entry in data
-            if "resource_spans" in entry
+            entry.split("POST")[0] for entry in data if "resource_spans" in entry
         ]
         self.spans = []
+        self.resource_attributes = []
         for span in full_spans:
             span = json.loads(span)
-            self.spans.append(
-                span["resource_spans"][0]['scope_spans'][0]['spans'][0])
+            self.spans.append(span["resource_spans"][0]["scope_spans"][0]["spans"][0])
+            self.resource_attributes.append(
+                span["resource_spans"][0]["resource"]["attributes"]
+            )
 
         self.simple_model_name = "simple"
         self.ensemble_model_name = "ensemble_add_sub_int32_int32_int32"
+        self.bls_model_name = "bls_simple"
         self.root_span = "InferRequest"
 
     def _check_events(self, span_name, events):
-        root_events_http =\
-              ["HTTP_RECV_START",
-               "HTTP_RECV_END",
-               "INFER_RESPONSE_COMPLETE",
-               "HTTP_SEND_START",
-               "HTTP_SEND_END"]
-        root_events_grpc =\
-              ["GRPC_WAITREAD_START",
-               "GRPC_WAITREAD_END",
-               "INFER_RESPONSE_COMPLETE",
-               "GRPC_SEND_START",
-               "GRPC_SEND_END"]
-        request_events =\
-              ["REQUEST_START",
-               "QUEUE_START",
-               "REQUEST_END"]
-        compute_events =\
-              ["COMPUTE_START",
-               "COMPUTE_INPUT_END",
-               "COMPUTE_OUTPUT_START",
-               "COMPUTE_END"]
+        root_events_http = [
+            "HTTP_RECV_START",
+            "HTTP_RECV_END",
+            "INFER_RESPONSE_COMPLETE",
+            "HTTP_SEND_START",
+            "HTTP_SEND_END",
+        ]
+        root_events_grpc = [
+            "GRPC_WAITREAD_START",
+            "GRPC_WAITREAD_END",
+            "INFER_RESPONSE_COMPLETE",
+            "GRPC_SEND_START",
+            "GRPC_SEND_END",
+        ]
+        request_events = ["REQUEST_START", "QUEUE_START", "REQUEST_END"]
+        compute_events = [
+            "COMPUTE_START",
+            "COMPUTE_INPUT_END",
+            "COMPUTE_OUTPUT_START",
+            "COMPUTE_END",
+        ]
 
         if span_name == "compute":
             # Check that all compute related events (and only them)
             # are recorded in compute span
             self.assertTrue(all(entry in events for entry in compute_events))
             self.assertFalse(all(entry in events for entry in request_events))
             self.assertFalse(
-                all(entry in events
-                    for entry in root_events_http + root_events_grpc))
+                all(entry in events for entry in root_events_http + root_events_grpc)
+            )
 
         elif span_name == self.root_span:
             # Check that root span has INFER_RESPONSE_COMPLETE, _RECV/_WAITREAD
             # and _SEND events (and only them)
             if "HTTP" in events:
-                self.assertTrue(
-                    all(entry in events for entry in root_events_http))
-                self.assertFalse(
-                    all(entry in events for entry in root_events_grpc))
+                self.assertTrue(all(entry in events for entry in root_events_http))
+                self.assertFalse(all(entry in events for entry in root_events_grpc))
 
             elif "GRPC" in events:
-                self.assertTrue(
-                    all(entry in events for entry in root_events_grpc))
-                self.assertFalse(
-                    all(entry in events for entry in root_events_http))
+                self.assertTrue(all(entry in events for entry in root_events_grpc))
+                self.assertFalse(all(entry in events for entry in root_events_http))
             self.assertFalse(all(entry in events for entry in request_events))
             self.assertFalse(all(entry in events for entry in compute_events))
 
@@ -120,17 +118,20 @@ def _check_events(self, span_name, events):
             # are recorded in request span
             self.assertTrue(all(entry in events for entry in request_events))
             self.assertFalse(
-                all(entry in events
-                    for entry in root_events_http + root_events_grpc))
+                all(entry in events for entry in root_events_http + root_events_grpc)
+            )
             self.assertFalse(all(entry in events for entry in compute_events))
 
     def _check_parent(self, child_span, parent_span):
         # Check that child and parent span have the same trace_id
         # and child's `parent_span_id` is the same as parent's `span_id`
-        self.assertEqual(child_span['trace_id'], parent_span['trace_id'])
-        self.assertIn('parent_span_id', child_span,
-                      "child span does not have parent span id specified")
-        self.assertEqual(child_span['parent_span_id'], parent_span['span_id'])
+        self.assertEqual(child_span["trace_id"], parent_span["trace_id"])
+        self.assertIn(
+            "parent_span_id",
+            child_span,
+            "child span does not have parent span id specified",
+        )
+        self.assertEqual(child_span["parent_span_id"], parent_span["span_id"])
 
     def test_spans(self):
         parsed_spans = []
@@ -141,16 +142,18 @@ def test_spans(self):
             self._check_events(span_name, json.dumps(span["events"]))
             parsed_spans.append(span_name)
 
-        # There should be 6 spans in total:
-        # 3 for http request, 3 for grpc request, 4 for ensemble
-        self.assertEqual(len(self.spans), 10)
-        # We should have 3 compute spans
-        self.assertEqual(parsed_spans.count("compute"), 3)
-        # 4 request spans (3 named simple - same as our model name, 1 ensemble)
-        self.assertEqual(parsed_spans.count(self.simple_model_name), 3)
-        self.assertEqual(parsed_spans.count(self.ensemble_model_name), 1)
-        # 3 root spans
-        self.assertEqual(parsed_spans.count(self.root_span), 3)
+        # There should be 16 spans in total:
+        # 3 for http request, 3 for grpc request, 4 for ensemble, 6 for bls
+        self.assertEqual(len(self.spans), EXPECTED_NUM_SPANS)
+        # We should have 5 compute spans
+        self.assertEqual(parsed_spans.count("compute"), 5)
+        # 7 request spans
+        # (4 named simple - same as our model name, 2 ensemble, 1 bls)
+        self.assertEqual(parsed_spans.count(self.simple_model_name), 4)
+        self.assertEqual(parsed_spans.count(self.ensemble_model_name), 2)
+        self.assertEqual(parsed_spans.count(self.bls_model_name), 1)
+        # 4 root spans
+        self.assertEqual(parsed_spans.count(self.root_span), 4)
 
     def test_nested_spans(self):
         # First 3 spans in `self.spans` belong to HTTP request
@@ -162,27 +165,41 @@ def test_nested_spans(self):
             self._check_parent(child, parent)
 
         # root_span should not have `parent_span_id` field
-        self.assertNotIn('parent_span_id', self.spans[2],
-                         "root span has a parent_span_id specified")
+        self.assertNotIn(
+            "parent_span_id", self.spans[2], "root span has a parent_span_id specified"
+        )
 
         # Next 3 spans in `self.spans` belong to GRPC request
         # Order of spans and their relationship described earlier
         for child, parent in zip(self.spans[3:6], self.spans[4:6]):
             self._check_parent(child, parent)
 
         # root_span should not have `parent_span_id` field
-        self.assertNotIn('parent_span_id', self.spans[5],
-                         "root span has a parent_span_id specified")
+        self.assertNotIn(
+            "parent_span_id", self.spans[5], "root span has a parent_span_id specified"
+        )
 
         # Final 4 spans in `self.spans` belong to ensemble request
         # Order of spans: compute span - request span - request span - root span
         for child, parent in zip(self.spans[6:10], self.spans[7:10]):
             self._check_parent(child, parent)
 
         # root_span should not have `parent_span_id` field
-        self.assertNotIn('parent_span_id', self.spans[9],
-                         "root span has a parent_span_id specified")
-
+        self.assertNotIn(
+            "parent_span_id", self.spans[9], "root span has a parent_span_id specified"
+        )
+
+    def test_resource_attributes(self):
+        test_attribute_entry = "{{'key': {k}, 'value': {{'string_value': {v}}}}}"
+        for attribute in self.resource_attributes:
+            self.assertIn(
+                test_attribute_entry.format(k="'test.key'", v="'test.value'"),
+                str(attribute),
+            )
+            self.assertIn(
+                test_attribute_entry.format(k="'service.name'", v="'test_triton'"),
+                str(attribute),
+            )
 
 
 def prepare_data(client):
@@ -201,11 +218,12 @@ def prepare_data(client):
 
 
 def prepare_traces():
-
-    triton_client_http = httpclient.InferenceServerClient("localhost:8000",
-                                                          verbose=True)
-    triton_client_grpc = grpcclient.InferenceServerClient("localhost:8001",
-                                                          verbose=True)
+    triton_client_http = httpclient.InferenceServerClient(
+        "localhost:8000", verbose=True
+    )
+    triton_client_grpc = grpcclient.InferenceServerClient(
+        "localhost:8001", verbose=True
+    )
     inputs = prepare_data(httpclient)
     triton_client_http.infer("simple", inputs)
 
@@ -215,16 +233,16 @@ def prepare_traces():
     inputs = prepare_data(httpclient)
     triton_client_http.infer("ensemble_add_sub_int32_int32_int32", inputs)
 
+    send_bls_request(model_name="ensemble_add_sub_int32_int32_int32")
 
-def send_bls_request():
 
+def send_bls_request(model_name="simple"):
     with httpclient.InferenceServerClient("localhost:8000") as client:
-        
         inputs = prepare_data(httpclient)
         inputs.append(httpclient.InferInput("MODEL_NAME", [1], "BYTES"))
-        inputs[2].set_data_from_numpy(np.array(["simple"], dtype=np.object_))
+        inputs[2].set_data_from_numpy(np.array([model_name], dtype=np.object_))
         client.infer("bls_simple", inputs)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()

qa/L0_trace/test.sh

@@ -659,7 +659,7 @@ set +e
 # Send bls requests to make sure simple model is traced
 for p in {1..4}; do
     python -c 'import opentelemetry_unittest; \
-        opentelemetry_unittest.send_bls_request()'  >> client_update.log 2>&1
+        opentelemetry_unittest.send_bls_request(model_name="ensemble_add_sub_int32_int32_int32")'  >> client_update.log 2>&1
 done
 
 set -e
@@ -695,18 +695,18 @@ fi
 
 OPENTELEMETRY_TEST=opentelemetry_unittest.py
 OPENTELEMETRY_LOG="opentelemetry_unittest.log"
-EXPECTED_NUM_TESTS="2"
+EXPECTED_NUM_TESTS="3"
 
 SERVER_ARGS="--trace-config=level=TIMESTAMPS --trace-config=rate=1 \
                 --trace-config=count=100 --trace-config=mode=opentelemetry \
                 --trace-config=opentelemetry,url=localhost:$OTLP_PORT \
+                --trace-config=opentelemetry,resource=test.key=test.value \
+                --trace-config=opentelemetry,resource=service.name=test_triton \
                 --model-repository=$MODELSDIR"
 SERVER_LOG="./inference_server_trace_config.log"
 
-# Increasing OTLP timeout, since we don't use a valid OTLP collector
-# and don't send a proper signal back.
-export OTEL_EXPORTER_OTLP_TIMEOUT=50000
-export OTEL_EXPORTER_OTLP_TRACES_TIMEOUT=50000
+export OTEL_EXPORTER_OTLP_TIMEOUT=5
+export OTEL_EXPORTER_OTLP_TRACES_TIMEOUT=5
 
 run_server
 if [ "$SERVER_PID" == "0" ]; then