Skip to content

Commit 30d12bb

Browse files
authored
fix: Fix incorrectly tagging AIM data when AIM is disabled (#2408)
1 parent f89fda5 commit 30d12bb

File tree

3 files changed

+90
-1
lines changed

3 files changed

+90
-1
lines changed

src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/Bedrock/InvokeModelAsyncWrapper.cs

+6
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ public CanWrapResponse CanWrap(InstrumentedMethodInfo methodInfo)
3131

3232
public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall instrumentedMethodCall, IAgent agent, ITransaction transaction)
3333
{
34+
// Don't do anything, including sending the version Supportability metric, if we're disabled
35+
if (!agent.Configuration.AiMonitoringEnabled)
36+
{
37+
return Delegates.NoOp;
38+
}
39+
3440
if (instrumentedMethodCall.IsAsync)
3541
{
3642
transaction.AttachToAsync();

tests/Agent/IntegrationTests/IntegrationTests/LLM/BedrockTests.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ public void BedrockTest()
114114
var expectedMetrics = new List<Assertions.ExpectedMetric>
115115
{
116116
new Assertions.ExpectedMetric { metricName = @"Custom/Llm/completion/Bedrock/InvokeModelAsync", CallCountAllHarvests = 5 },
117-
new Assertions.ExpectedMetric { metricName = @"Custom/Llm/embedding/Bedrock/InvokeModelAsync", CallCountAllHarvests = 1 }
117+
new Assertions.ExpectedMetric { metricName = @"Custom/Llm/embedding/Bedrock/InvokeModelAsync", CallCountAllHarvests = 1 },
118+
new Assertions.ExpectedMetric { metricName = @"Supportability/DotNet/ML/.*", IsRegexName = true}
118119
};
119120

120121
var customEvents = _fixture.AgentLog.GetCustomEvents().ToList();
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright 2020 New Relic, Inc. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using NewRelic.Agent.IntegrationTestHelpers;
8+
using NewRelic.Agent.IntegrationTestHelpers.RemoteServiceFixtures;
9+
using Xunit;
10+
using Xunit.Abstractions;
11+
12+
namespace NewRelic.Agent.IntegrationTests.LLM
13+
{
14+
public abstract class LlmDisabledTestsBase<TFixture> : NewRelicIntegrationTest<TFixture>
15+
where TFixture : ConsoleDynamicMethodFixture
16+
{
17+
private readonly TFixture _fixture;
18+
private const string _model = "meta13";
19+
private string _prompt = "In one sentence, what is a large-language model?";
20+
21+
public LlmDisabledTestsBase(TFixture fixture, ITestOutputHelper output) : base(fixture)
22+
{
23+
_fixture = fixture;
24+
_fixture.SetTimeout(TimeSpan.FromMinutes(2));
25+
_fixture.TestLogger = output;
26+
_fixture.AddActions(
27+
setupConfiguration: () =>
28+
{
29+
new NewRelicConfigModifier(fixture.DestinationNewRelicConfigFilePath)
30+
.ForceTransactionTraces()
31+
.EnableAiMonitoring(false)
32+
.SetLogLevel("finest");
33+
},
34+
exerciseApplication: () =>
35+
{
36+
_fixture.AgentLog.WaitForLogLines(AgentLogBase.MetricDataLogLineRegex, TimeSpan.FromMinutes(2), 2);
37+
}
38+
);
39+
40+
_fixture.AddCommand($"LLMExerciser InvokeModel {_model} {LLMHelpers.ConvertToBase64(_prompt)}");
41+
42+
_fixture.Initialize();
43+
}
44+
45+
[Fact]
46+
public void BedrockDisabledTest()
47+
{
48+
// Make sure it actually got called
49+
var transactionEvent = _fixture.AgentLog.TryGetTransactionEvent($"OtherTransaction/Custom/MultiFunctionApplicationHelpers.NetStandardLibraries.LLM.LLMExerciser/InvokeModel");
50+
Assert.NotNull(transactionEvent);
51+
52+
var unexpectedMetrics = new List<Assertions.ExpectedMetric>
53+
{
54+
new Assertions.ExpectedMetric { metricName = @"Supportability/DotNet/ML/Bedrock/.*", IsRegexName = true },
55+
new Assertions.ExpectedMetric { metricName = @"Custom/Llm/.*", IsRegexName = true },
56+
};
57+
58+
var metrics = _fixture.AgentLog.GetMetrics().ToList();
59+
Assertions.MetricsDoNotExist(unexpectedMetrics, metrics);
60+
61+
62+
}
63+
}
64+
[NetCoreTest]
65+
public class LlmDisabledTest_CoreLatest : LlmDisabledTestsBase<ConsoleDynamicMethodFixtureCoreLatest>
66+
{
67+
public LlmDisabledTest_CoreLatest(ConsoleDynamicMethodFixtureCoreLatest fixture, ITestOutputHelper output)
68+
: base(fixture, output)
69+
{
70+
}
71+
}
72+
73+
[NetFrameworkTest]
74+
public class LlmDisabledTest_FWLatest : LlmDisabledTestsBase<ConsoleDynamicMethodFixtureFWLatest>
75+
{
76+
public LlmDisabledTest_FWLatest(ConsoleDynamicMethodFixtureFWLatest fixture, ITestOutputHelper output)
77+
: base(fixture, output)
78+
{
79+
}
80+
}
81+
82+
}

0 commit comments

Comments
 (0)