Skip to content

Commit a413091

Browse files
jaffinitojaffinito
authored
chore: Add supportability metrics for LLM vendors and models. (#3006)
1 parent 8eba1bf commit a413091

File tree

8 files changed

+283
-4
lines changed

8 files changed

+283
-4
lines changed

‎src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Llm/SupportabilityHelpers.cs

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Copyright 2020 New Relic, Inc. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
using System;
5+
using System.Collections.Concurrent;
6+
using System.Text.RegularExpressions;
7+
using NewRelic.Agent.Api;
8+
9+
namespace NewRelic.Agent.Extensions.Llm
10+
{
11+
public static class SupportabilityHelpers
12+
{
13+
private const string OpenAiDateRemovalPattern = @"-\d{4}-\d{2}-\d{2}";
14+
15+
private static readonly ConcurrentDictionary<string, object> _seenModels = new();
16+
17+
public static void CreateModelIdSupportabilityMetricsForOpenAi(string model, IAgent agent)
18+
{
19+
if (string.IsNullOrWhiteSpace(model))
20+
{
21+
return;
22+
}
23+
24+
// Only want to send this metric once-ish per model
25+
if (!_seenModels.TryAdd(model, null))
26+
{
27+
return;
28+
}
29+
30+
try
31+
{
32+
// Example openai: o1
33+
// Example openai: gpt-4o-2024-11-20
34+
var noDateModel = Regex.Replace(model, OpenAiDateRemovalPattern, string.Empty);
35+
var modelIdDetails = noDateModel.Split('-');
36+
if (modelIdDetails.Length == 1)
37+
{
38+
agent.RecordSupportabilityMetric("DotNet/LLM/openai/" + modelIdDetails[0]);
39+
return;
40+
}
41+
42+
agent.RecordSupportabilityMetric("DotNet/LLM/openai/" + modelIdDetails[0] + "-" + modelIdDetails[1]);
43+
}
44+
catch (Exception ex) // if there is a problem, this will also only happen once-ish per model
45+
{
46+
agent.Logger.Finest($"Error creating model supportability metric for {model}: {ex.Message}");
47+
}
48+
}
49+
50+
public static void CreateModelIdSupportabilityMetricsForBedrock(string model, IAgent agent)
51+
{
52+
if (string.IsNullOrWhiteSpace(model))
53+
{
54+
return;
55+
}
56+
57+
// Only want to send this metric once-ish per model
58+
if (!_seenModels.TryAdd(model, null))
59+
{
60+
return;
61+
}
62+
63+
try
64+
{
65+
// Example foundation bedrock: anthropic.claude-3-5-sonnet-20241022-v2:0
66+
// Example inference bedrock: us.anthropic.claude-3-5-sonnet-20241022-v2:0
67+
// Example bedrock marketplace: deepseek-llm-r1
68+
var modelDetails = model.Split('.');
69+
if (modelDetails.Length == 1) // bedrock marketplace
70+
{
71+
// Format the bedrock marketplace model id into one that can be used by the standard logic.
72+
var marketplaceDetails = modelDetails[0].Split('-');
73+
modelDetails =
74+
[
75+
marketplaceDetails[0],
76+
string.Join("-", marketplaceDetails, 1, marketplaceDetails.Length - 1)
77+
];
78+
}
79+
80+
if (modelDetails.Length != 2 && modelDetails.Length != 3)
81+
{
82+
return;
83+
}
84+
85+
// if there is a region, it will be the first part of the model id
86+
var vendorIndex = modelDetails.Length == 2 ? 0 : 1;
87+
var vendor = modelDetails[vendorIndex];
88+
89+
var modelIdDetails = modelDetails[vendorIndex + 1].Split(':')[0].Split('-');
90+
if (modelIdDetails[0] == "nova" || modelIdDetails[0] == "titan" || modelIdDetails[0] == "claude") // first 2 - capture some extra details to narrow down support
91+
{
92+
agent.RecordSupportabilityMetric("DotNet/LLM/" + vendor + "/" + modelIdDetails[0] + "-" + modelIdDetails[1]);
93+
}
94+
else // first only - any model that doesn't need the above extra details
95+
{
96+
agent.RecordSupportabilityMetric("DotNet/LLM/" + vendor + "/" + modelIdDetails[0]);
97+
}
98+
}
99+
catch (Exception ex) // if there is a problem, this will also only happen once-ish per model
100+
{
101+
agent.Logger.Finest($"Error creating model supportability metric for {model}: {ex.Message}");
102+
}
103+
}
104+
}
105+
}

‎src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/Bedrock/ConverseAsyncWrapper.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins
4242

4343
dynamic converseRequest = instrumentedMethodCall.MethodCall.MethodArguments[0];
4444
string modelId = converseRequest.ModelId.ToLower();
45-
45+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(modelId, agent);
4646
var operationType = "completion"; // Converse doesn't support embedding
4747
var segment = transaction.StartCustomSegment(instrumentedMethodCall.MethodCall, $"Llm/{operationType}/{VendorName}/{instrumentedMethodCall.MethodCall.Method.MethodName}");
4848

‎src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/Bedrock/InvokeModelAsyncWrapper.cs

+1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins
4545
}
4646

4747
dynamic invokeModelRequest = instrumentedMethodCall.MethodCall.MethodArguments[0];
48+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock((string)invokeModelRequest.ModelId, agent);
4849
var operationType = invokeModelRequest.ModelId.Contains("embed") ? "embedding" : "completion";
4950
var segment = transaction.StartCustomSegment(
5051
instrumentedMethodCall.MethodCall,

‎src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/OpenAI/OpenAIChatWrapper.cs

+1
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins
9797
agent.RecordSupportabilityMetric($"DotNet/ML/{GetVendorName()}/{version}");
9898

9999
string model = _modelFieldAccessor(instrumentedMethodCall.MethodCall.InvocationTarget);
100+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(model, agent); // prepend vendor name to model id
100101

101102
if (isAsync)
102103
{

‎tests/Agent/IntegrationTests/IntegrationTests/LLM/BedrockConverseTests.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,8 @@ public void ConverseTest()
102102
var expectedMetrics = new List<Assertions.ExpectedMetric>
103103
{
104104
new() { metricName = @"Custom/Llm/completion/Bedrock/ConverseAsync", CallCountAllHarvests = 2 },
105-
new() { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true}
105+
new() { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true},
106+
new() { metricName = @"Supportability/DotNet/LLM/.*/.*", IsRegexName = true} // Supportability/DotNet/LLM/{vendor}/{model}
106107
};
107108

108109
var customEventsSuccess = _fixture.AgentLog.GetCustomEvents().Where(ce => !ce.Attributes.Keys.Contains("error")).ToList();

‎tests/Agent/IntegrationTests/IntegrationTests/LLM/BedrockInvokeTests.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ public void BedrockTest()
111111
{
112112
new Assertions.ExpectedMetric { metricName = @"Custom/Llm/completion/Bedrock/InvokeModelAsync", CallCountAllHarvests = _bedrockModelsToTest.Count - 1 },
113113
new Assertions.ExpectedMetric { metricName = @"Custom/Llm/embedding/Bedrock/InvokeModelAsync", CallCountAllHarvests = 1 },
114-
new Assertions.ExpectedMetric { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true}
114+
new Assertions.ExpectedMetric { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true},
115+
new Assertions.ExpectedMetric { metricName = @"Supportability/DotNet/LLM/.*/.*", IsRegexName = true} // Supportability/DotNet/LLM/{vendor}/{model}
115116
};
116117

117118
var customEvents = _fixture.AgentLog.GetCustomEvents().ToList();

‎tests/Agent/IntegrationTests/IntegrationTests/LLM/OpenAITests.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,8 @@ public void OpenAITest()
117117
new() { metricName = @"Custom/Llm/completion/openai/CompleteChatAsync", metricScope = "OtherTransaction/Custom/MultiFunctionApplicationHelpers.NetStandardLibraries.LLM.OpenAIExerciser/CompleteChatAsync"},
118118
new() { metricName = @"Custom/Llm/completion/openai/CompleteChat" },
119119
new() { metricName = @"Custom/Llm/completion/openai/CompleteChat", metricScope = "OtherTransaction/Custom/MultiFunctionApplicationHelpers.NetStandardLibraries.LLM.OpenAIExerciser/CompleteChat"},
120-
new() { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true}
120+
new() { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true},
121+
new() { metricName = @"Supportability/DotNet/LLM/.*/.*", IsRegexName = true} // Supportability/DotNet/LLM/{vendor}/{model}
121122
};
122123

123124
var customEventsSuccess = _fixture.AgentLog.GetCustomEvents().Where(ce => !ce.Attributes.Keys.Contains("error")).ToList();

‎tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Llm/SupportabilityHelpersTests.cs

+169
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
// Copyright 2020 New Relic, Inc. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using NewRelic.Agent.Api;
7+
using NewRelic.Agent.Extensions.Llm;
8+
using NUnit.Framework;
9+
using Telerik.JustMock;
10+
11+
namespace Agent.Extensions.Tests.Llm
12+
{
13+
// When creating tests, make sure to not use duplicate model name since CreateModelIdSupportabilityMetricsForXXXX only creates the metric once.
14+
[TestFixture]
15+
public class SupportabilityHelpersTests
16+
{
17+
private IAgent _agent;
18+
19+
[SetUp]
20+
public void Setup()
21+
{
22+
_agent = Mock.Create<IAgent>();
23+
}
24+
25+
[TestCase("anthropic.claude-3-sonnet-20240229-v1:0", "anthropic", "claude-3")]
26+
[TestCase("us.anthropic.claude-3-sonnet-20240229-v1:0", "anthropic", "claude-3")]
27+
[TestCase("apac.anthropic.claude-3-sonnet-20240229-v1:0", "anthropic", "claude-3")]
28+
[TestCase("meta.llama3-2-3b-instruct-v1:0", "meta", "llama3")]
29+
[TestCase("us.meta.llama3-2-3b-instruct-v1:0", "meta", "llama3")]
30+
[TestCase("amazon.nova-lite-v1:0", "amazon", "nova-lite")]
31+
[TestCase("eu.amazon.nova-lite-v1:0", "amazon", "nova-lite")]
32+
[TestCase("amazon.titan-embed-text-v1", "amazon", "titan-embed")]
33+
[TestCase("us.amazon.titan-embed-text-v1", "amazon", "titan-embed")]
34+
[TestCase("ai21.jamba-1-5-large-v1:0", "ai21", "jamba")]
35+
[TestCase("apac.ai21.jamba-1-5-large-v1:0", "ai21", "jamba")]
36+
[TestCase("writer-palmyra-med-70b-32k", "writer", "palmyra")]
37+
public void Bedrock_ModelFormatsTests(string fullModel, string vendor, string model)
38+
{
39+
// Supportability/DotNet/LLM/{vendor}/{model}
40+
var expectedMetric = $"Supportability/DotNet/LLM/{vendor}/{model}";
41+
var actualMetric = string.Empty;
42+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
43+
.DoInstead((string m, long c) => actualMetric = $"Supportability/{m}");
44+
45+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(fullModel, _agent);
46+
47+
Assert.That(actualMetric == expectedMetric, $"Model: '{fullModel}', Actual: '{actualMetric}', Expected: '{expectedMetric}'");
48+
}
49+
50+
[TestCase("o1", "openai", "o1")]
51+
[TestCase("o3-mini", "openai", "o3-mini")]
52+
[TestCase("gpt-4o-2024-11-20", "openai", "gpt-4o")]
53+
public void OpenAi_ModelFormatsTests(string fullModel, string vendor, string model)
54+
{
55+
// Supportability/DotNet/LLM/{vendor}/{model}
56+
var expectedMetric = $"Supportability/DotNet/LLM/{vendor}/{model}";
57+
var actualMetric = string.Empty;
58+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
59+
.DoInstead((string m, long c) => actualMetric = $"Supportability/{m}");
60+
61+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(fullModel, _agent);
62+
63+
Assert.That(actualMetric == expectedMetric, $"Model: '{fullModel}', Actual: '{actualMetric}', Expected: '{expectedMetric}'");
64+
}
65+
66+
[TestCase("bedrock", "")]
67+
[TestCase("bedrock", "bedrock.bad.model.more.than.four.sections")]
68+
[TestCase("openai", "")]
69+
public void BadModel_NoMetricTest(string source, string model)
70+
{
71+
// Supportability/DotNet/LLM/{vendor}/{model}
72+
var actualMetric = string.Empty;
73+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
74+
.DoInstead((string m, long c) => actualMetric = $"Supportability/{m}"); // Will not get called
75+
76+
// Model is always stored so we want to check different values
77+
if (source == "bedrock")
78+
{
79+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(model, _agent);
80+
}
81+
else if (source == "openai")
82+
{
83+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(model, _agent);
84+
}
85+
86+
Assert.That(actualMetric == string.Empty);
87+
}
88+
89+
[Test]
90+
public void Bedrock_DuplicateModels_OnlyOneMetricTest()
91+
{
92+
var fullModel = "luma.ray-v2:0";
93+
94+
// Supportability/DotNet/LLM/{vendor}/{model}
95+
var expectedMetric = $"Supportability/DotNet/LLM/luma/ray";
96+
var actualMetrics = new List<string>();
97+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
98+
.DoInstead((string m, long c) => actualMetrics.Add($"Supportability/{m}"));
99+
100+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(fullModel, _agent);
101+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(fullModel, _agent);
102+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(fullModel, _agent);
103+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(fullModel, _agent);
104+
105+
Assert.That(actualMetrics.Count == 1);
106+
Assert.That(actualMetrics[0] == expectedMetric, $"Model: '{fullModel}', Actual: '{actualMetrics[0]}', Expected: '{expectedMetric}'");
107+
}
108+
109+
[Test]
110+
public void OpenAi_DuplicateModels_OnlyOneMetricTest()
111+
{
112+
var fullModel = "gpt-4.5";
113+
114+
// Supportability/DotNet/LLM/{vendor}/{model}
115+
var expectedMetric = $"Supportability/DotNet/LLM/openai/gpt-4.5";
116+
var actualMetrics = new List<string>();
117+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
118+
.DoInstead((string m, long c) => actualMetrics.Add($"Supportability/{m}"));
119+
120+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(fullModel, _agent);
121+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(fullModel, _agent);
122+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(fullModel, _agent);
123+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(fullModel, _agent);
124+
125+
Assert.That(actualMetrics.Count == 1);
126+
Assert.That(actualMetrics[0] == expectedMetric, $"Model: '{fullModel}', Actual: '{actualMetrics[0]}', Expected: '{expectedMetric}'");
127+
}
128+
129+
[Test]
130+
public void Bedrock_Exception_Test()
131+
{
132+
var fullModel = "meta.llama3-1-70b-instruct-v1:0";
133+
var exception = new Exception("Test exception");
134+
var expectedExceptionMessage = $"Error creating model supportability metric for {fullModel}: {exception.Message}";
135+
136+
// Supportability/DotNet/LLM/{vendor}/{model}
137+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
138+
.Throws(exception);
139+
140+
var exceptionMessage = string.Empty;
141+
Mock.Arrange(() => _agent.Logger.Finest(Arg.AnyString))
142+
.DoInstead((string m) => exceptionMessage = m);
143+
144+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForBedrock(fullModel, _agent);
145+
146+
Assert.That(exceptionMessage == expectedExceptionMessage, message: exceptionMessage);
147+
}
148+
149+
[Test]
150+
public void OpenAi_Exception_Test()
151+
{
152+
var fullModel = "chatgpt-4o";
153+
var exception = new Exception("Test exception");
154+
var expectedExceptionMessage = $"Error creating model supportability metric for {fullModel}: {exception.Message}";
155+
156+
// Supportability/DotNet/LLM/{vendor}/{model}
157+
Mock.Arrange(() => _agent.RecordSupportabilityMetric(Arg.AnyString, Arg.AnyLong))
158+
.Throws(exception);
159+
160+
var exceptionMessage = string.Empty;
161+
Mock.Arrange(() => _agent.Logger.Finest(Arg.AnyString))
162+
.DoInstead((string m) => exceptionMessage = m);
163+
164+
SupportabilityHelpers.CreateModelIdSupportabilityMetricsForOpenAi(fullModel, _agent);
165+
166+
Assert.That(exceptionMessage == expectedExceptionMessage, message: exceptionMessage);
167+
}
168+
}
169+
}

0 commit comments

Comments
 (0)