|
| 1 | +// Copyright 2020 New Relic, Inc. All rights reserved. |
| 2 | +// SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +using System; |
| 5 | +using System.Collections.Generic; |
| 6 | +using System.Linq; |
| 7 | +using NewRelic.Agent.IntegrationTestHelpers; |
| 8 | +using NewRelic.Agent.IntegrationTestHelpers.RemoteServiceFixtures; |
| 9 | +using Xunit; |
| 10 | +using Xunit.Abstractions; |
| 11 | + |
| 12 | +namespace NewRelic.Agent.IntegrationTests.LLM |
| 13 | +{ |
| 14 | + public abstract class LlmDisabledTestsBase<TFixture> : NewRelicIntegrationTest<TFixture> |
| 15 | +where TFixture : ConsoleDynamicMethodFixture |
| 16 | + { |
| 17 | + private readonly TFixture _fixture; |
| 18 | + private const string _model = "meta13"; |
| 19 | + private string _prompt = "In one sentence, what is a large-language model?"; |
| 20 | + |
| 21 | + public LlmDisabledTestsBase(TFixture fixture, ITestOutputHelper output) : base(fixture) |
| 22 | + { |
| 23 | + _fixture = fixture; |
| 24 | + _fixture.SetTimeout(TimeSpan.FromMinutes(2)); |
| 25 | + _fixture.TestLogger = output; |
| 26 | + _fixture.AddActions( |
| 27 | + setupConfiguration: () => |
| 28 | + { |
| 29 | + new NewRelicConfigModifier(fixture.DestinationNewRelicConfigFilePath) |
| 30 | + .ForceTransactionTraces() |
| 31 | + .EnableAiMonitoring(false) |
| 32 | + .SetLogLevel("finest"); |
| 33 | + }, |
| 34 | + exerciseApplication: () => |
| 35 | + { |
| 36 | + _fixture.AgentLog.WaitForLogLines(AgentLogBase.MetricDataLogLineRegex, TimeSpan.FromMinutes(2), 2); |
| 37 | + } |
| 38 | + ); |
| 39 | + |
| 40 | + _fixture.AddCommand($"LLMExerciser InvokeModel {_model} {LLMHelpers.ConvertToBase64(_prompt)}"); |
| 41 | + |
| 42 | + _fixture.Initialize(); |
| 43 | + } |
| 44 | + |
| 45 | + [Fact] |
| 46 | + public void BedrockDisabledTest() |
| 47 | + { |
| 48 | + // Make sure it actually got called |
| 49 | + var transactionEvent = _fixture.AgentLog.TryGetTransactionEvent($"OtherTransaction/Custom/MultiFunctionApplicationHelpers.NetStandardLibraries.LLM.LLMExerciser/InvokeModel"); |
| 50 | + Assert.NotNull(transactionEvent); |
| 51 | + |
| 52 | + var unexpectedMetrics = new List<Assertions.ExpectedMetric> |
| 53 | + { |
| 54 | + new Assertions.ExpectedMetric { metricName = @"Supportability/DotNet/ML/Bedrock/.*", IsRegexName = true }, |
| 55 | + new Assertions.ExpectedMetric { metricName = @"Custom/Llm/.*", IsRegexName = true }, |
| 56 | + }; |
| 57 | + |
| 58 | + var metrics = _fixture.AgentLog.GetMetrics().ToList(); |
| 59 | + Assertions.MetricsDoNotExist(unexpectedMetrics, metrics); |
| 60 | + |
| 61 | + |
| 62 | + } |
| 63 | + } |
| 64 | + [NetCoreTest] |
| 65 | + public class LlmDisabledTest_CoreLatest : LlmDisabledTestsBase<ConsoleDynamicMethodFixtureCoreLatest> |
| 66 | + { |
| 67 | + public LlmDisabledTest_CoreLatest(ConsoleDynamicMethodFixtureCoreLatest fixture, ITestOutputHelper output) |
| 68 | + : base(fixture, output) |
| 69 | + { |
| 70 | + } |
| 71 | + } |
| 72 | + |
| 73 | + [NetFrameworkTest] |
| 74 | + public class LlmDisabledTest_FWLatest : LlmDisabledTestsBase<ConsoleDynamicMethodFixtureFWLatest> |
| 75 | + { |
| 76 | + public LlmDisabledTest_FWLatest(ConsoleDynamicMethodFixtureFWLatest fixture, ITestOutputHelper output) |
| 77 | + : base(fixture, output) |
| 78 | + { |
| 79 | + } |
| 80 | + } |
| 81 | + |
| 82 | +} |
0 commit comments