summary: Log Level Filtering (#1770)

feat: Add support for filtering log events based on a list of log levels so that they are not forwarded to New Relic. Also adds new logging metrics to count the total number of filtered log events (Logging/denied). Refer to our [application logging configuration](https://docs.newrelic.com/docs/apm/agents/net-agent/configuration/net-agent-configuration/#application_logging) documentation for more details. (#1760) (#1761) (#1762) (#1766)

Author: tippmar-nr

src/Agent/NewRelic/Agent/Core/Agent.cs

@@ -408,7 +408,7 @@ public void RecordSupportabilityMetric(string metricName, int count)
             _agentHealthReporter.ReportSupportabilityCountMetric(metricName, count);
         }
 
-        public void RecordLogMessage(string frameworkName, object logEvent, Func<object, DateTime> getTimestamp, Func<object, object> getLevel, Func<object, string> getLogMessage, Func<object, Exception> getLogException,Func<object, Dictionary<string, object>> getContextData, string spanId, string traceId)
+        public void RecordLogMessage(string frameworkName, object logEvent, Func<object, DateTime> getTimestamp, Func<object, object> getLevel, Func<object, string> getLogMessage, Func<object, Exception> getLogException, Func<object, Dictionary<string, object>> getContextData, string spanId, string traceId)
         {
             _agentHealthReporter.ReportLogForwardingFramework(frameworkName);
 
@@ -418,6 +418,15 @@ public void RecordLogMessage(string frameworkName, object logEvent, Func<object,
             {
                 var level = getLevel(logEvent).ToString();
                 normalizedLevel = string.IsNullOrWhiteSpace(level) ? "UNKNOWN" : level.ToUpper();
+
+                // LogLevelDenyList is already uppercase, so don't need case-insensitive lookup
+                if (_configurationService.Configuration.LogLevelDenyList.Contains(normalizedLevel))
+                {
+                    if (_configurationService.Configuration.LogMetricsCollectorEnabled)
+                        _agentHealthReporter.IncrementLogDeniedCount(normalizedLevel);
+
+                    return;
+                }
             }
 
             if (_configurationService.Configuration.LogMetricsCollectorEnabled)
@@ -432,7 +441,7 @@ public void RecordLogMessage(string frameworkName, object logEvent, Func<object,
 
                 var logMessage = getLogMessage(logEvent);
                 var logException = getLogException(logEvent);
-                
+
                 // exit quickly if the message and exception are missing
                 if (string.IsNullOrWhiteSpace(logMessage) && logException is null)
                 {

src/Agent/NewRelic/Agent/Core/AgentHealth/AgentHealthReporter.cs

@@ -29,6 +29,7 @@ public class AgentHealthReporter : ConfigurationBasedService, IAgentHealthReport
         private readonly IList<RecurringLogData> _recurringLogDatas = new ConcurrentList<RecurringLogData>();
         private readonly IDictionary<AgentHealthEvent, InterlockedCounter> _agentHealthEventCounters = new Dictionary<AgentHealthEvent, InterlockedCounter>();
         private readonly ConcurrentDictionary<string, InterlockedCounter> _logLinesCountByLevel = new ConcurrentDictionary<string, InterlockedCounter>();
+        private readonly ConcurrentDictionary<string, InterlockedCounter> _logDeniedCountByLevel = new ConcurrentDictionary<string, InterlockedCounter>();
 
         private PublishMetricDelegate _publishMetricDelegate;
         private InterlockedCounter _payloadCreateSuccessCounter;
@@ -583,6 +584,22 @@ public void CollectLoggingMetrics()
                     _loggingForwardingEnabledWithFrameworksReported[kvp.Key] = true;
                 }
             }
+
+            var totalDeniedCount = 0;
+            foreach (var logLinesDeniedCounter in _logDeniedCountByLevel)
+            {
+                if (TryGetCount(logLinesDeniedCounter.Value, out var linesCount))
+                {
+                    totalDeniedCount += linesCount;
+                    TrySend(_metricBuilder.TryBuildLoggingMetricsDeniedCountBySeverityMetric(logLinesDeniedCounter.Key, linesCount));
+                }
+            }
+
+            if (totalDeniedCount > 0)
+            {
+                TrySend(_metricBuilder.TryBuildLoggingMetricsDeniedCountMetric(totalDeniedCount));
+            }
+
         }
 
         public void IncrementLogLinesCount(string level)
@@ -591,6 +608,12 @@ public void IncrementLogLinesCount(string level)
             _logLinesCountByLevel[level].Increment();
         }
 
+        public void IncrementLogDeniedCount(string level)
+        {
+            _logDeniedCountByLevel.TryAdd(level, new InterlockedCounter());
+            _logDeniedCountByLevel[level].Increment();
+        }
+
         public void ReportLoggingEventCollected() => TrySend(_metricBuilder.TryBuildSupportabilityLoggingEventsCollectedMetric());
 
         public void ReportLoggingEventsSent(int count) => TrySend(_metricBuilder.TryBuildSupportabilityLoggingEventsSentMetric(count));

src/Agent/NewRelic/Agent/Core/AgentHealth/IAgentHealthReporter.cs

@@ -142,6 +142,7 @@ public interface IAgentHealthReporter : IOutOfBandMetricSource
         void ReportSupportabilityDataUsage(string api, string apiArea, long dataSent, long dataReceived);
 
         void IncrementLogLinesCount(string logLevel);
+        void IncrementLogDeniedCount(string logLevel);
         void ReportLoggingEventCollected();
         void ReportLoggingEventsSent(int count);
         void ReportLoggingEventsDropped(int droppedCount);

src/Agent/NewRelic/Agent/Core/Config/Configuration.cs

@@ -5035,6 +5035,8 @@ public partial class configurationApplicationLoggingForwarding
 
         private int maxSamplesStoredField;
 
+        private string logLevelDenyListField;
+        
         /// <summary>
         /// configurationApplicationLoggingForwarding class constructor
         /// </summary>
@@ -5085,6 +5087,19 @@ public int maxSamplesStored
             }
         }
 
+        [System.Xml.Serialization.XmlAttributeAttribute()]
+        public string logLevelDenyList
+        {
+            get
+            {
+                return this.logLevelDenyListField;
+            }
+            set
+            {
+                this.logLevelDenyListField = value;
+            }
+        }
+        
         #region Clone method
         /// <summary>
         /// Create a clone of this configurationApplicationLoggingForwarding object

src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd

@@ -1655,6 +1655,15 @@
                                             </xs:documentation>
                                         </xs:annotation>
                                     </xs:attribute>
+
+                                    <xs:attribute name="logLevelDenyList" type="xs:string" use="optional">
+                                        <xs:annotation>
+                                            <xs:documentation>
+                                                A comma-separated, case-insensitive, list of log level names from the selected logging framework that should be ignored and not sent up to New Relic.
+                                            </xs:documentation>
+                                        </xs:annotation>
+                                    </xs:attribute>
+
                                 </xs:complexType>
                             </xs:element>
                             <xs:element name="localDecorating" minOccurs="0" maxOccurs="1">

src/Agent/NewRelic/Agent/Core/Configuration/DefaultConfiguration.cs

@@ -1990,6 +1990,31 @@ public virtual bool LogDecoratorEnabled
             }
         }
 
+        private HashSet<string> _logLevelDenyList;
+        public virtual HashSet<string> LogLevelDenyList
+        {
+            get
+            {
+                if (_logLevelDenyList == null)
+                {
+                    _logLevelDenyList = new HashSet<string>(
+                        EnvironmentOverrides(_localConfiguration.applicationLogging.forwarding.logLevelDenyList,
+                                "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LOG_LEVEL_DENYLIST")
+                            ?.Split(new[] { StringSeparators.CommaChar, ' ' }, StringSplitOptions.RemoveEmptyEntries)
+                            .Select(s => s.ToUpper())
+                        ?? Enumerable.Empty<string>());
+
+                    if (_logLevelDenyList.Count > 0)
+                    {
+                        var logLevels = string.Join(",", _logLevelDenyList);
+                        Log.Info($"Log Level Filtering is enabled for the following levels: {logLevels}");
+                    }
+                }
+
+                return _logLevelDenyList;
+            }
+        }
+
         #endregion
 
         public virtual bool AppDomainCachingDisabled

src/Agent/NewRelic/Agent/Core/Configuration/ReportedConfiguration.cs

@@ -593,6 +593,9 @@ public ReportedConfiguration(IConfiguration configuration)
         [JsonProperty("application_logging.forwarding.max_samples_stored")]
         public int LogEventsMaxSamplesStored => _configuration.LogEventsMaxSamplesStored;
 
+        [JsonProperty("application_logging.forwarding.log_level_denylist")]
+        public HashSet<string> LogLevelDenyList => _configuration.LogLevelDenyList;
+
         [JsonProperty("application_logging.harvest_cycle")]
         public TimeSpan LogEventsHarvestCycle => _configuration.LogEventsHarvestCycle;
 

src/Agent/NewRelic/Agent/Core/NewRelic.Agent.Core.Metric/MetricNames.cs

@@ -1040,6 +1040,7 @@ public static string GetPerDestinationAreaDataUsageMetricName(string destination
 
         private const string LoggingMetrics = "Logging";
         private const string LoggingMetricsDotnetLines = LoggingMetrics + PathSeparator + "lines";
+        private const string LoggingMetricsDotnetDenied = LoggingMetrics + PathSeparator + "denied";
         private const string SupportabilityLoggingEventsPs = SupportabilityPs + "Logging" + PathSeparator;
         public const string SupportabilityLoggingEventsSent = SupportabilityLoggingEventsPs + Forwarding + PathSeparator + "Sent";
         public const string SupportabilityLoggingEventsCollected = SupportabilityLoggingEventsPs + Forwarding + PathSeparator + "Seen";
@@ -1055,6 +1056,16 @@ public static string GetLoggingMetricsLinesName()
             return LoggingMetricsDotnetLines;
         }
 
+        public static string GetLoggingMetricsDeniedBySeverityName(string logLevel)
+        {
+            return LoggingMetricsDotnetDenied + PathSeparator + logLevel;
+        }
+
+        public static string GetLoggingMetricsDeniedName()
+        {
+            return LoggingMetricsDotnetDenied;
+        }
+
         private const string Enabled = "enabled";
         private const string Disabled = "disabled";
         private const string Metrics = "Metrics";

src/Agent/NewRelic/Agent/Core/WireModels/IMetricBuilder.cs

@@ -196,6 +196,10 @@ public interface IMetricBuilder
 
         MetricWireModel TryBuildLoggingMetricsLinesCountMetric(int count);
 
+        MetricWireModel TryBuildLoggingMetricsDeniedCountBySeverityMetric(string logLevel, int count);
+
+        MetricWireModel TryBuildLoggingMetricsDeniedCountMetric(int count);
+
         MetricWireModel TryBuildSupportabilityLoggingEventsCollectedMetric();
 
         MetricWireModel TryBuildSupportabilityLoggingEventsSentMetric(int loggingEventCount);

src/Agent/NewRelic/Agent/Core/WireModels/MetricWireModel.cs

@@ -955,6 +955,18 @@ public MetricWireModel TryBuildLoggingMetricsLinesCountMetric(int count)
                 return BuildMetric(_metricNameService, proposedName, null, MetricDataWireModel.BuildCountData(count));
             }
 
+            public MetricWireModel TryBuildLoggingMetricsDeniedCountBySeverityMetric(string logLevel, int count)
+            {
+                var proposedName = MetricNames.GetLoggingMetricsDeniedBySeverityName(logLevel);
+                return BuildMetric(_metricNameService, proposedName, null, MetricDataWireModel.BuildCountData(count));
+            }
+
+            public MetricWireModel TryBuildLoggingMetricsDeniedCountMetric(int count)
+            {
+                var proposedName = MetricNames.GetLoggingMetricsDeniedName();
+                return BuildMetric(_metricNameService, proposedName, null, MetricDataWireModel.BuildCountData(count));
+            }
+
             public MetricWireModel TryBuildSupportabilityLoggingEventsCollectedMetric()
             {
                 const string proposedName = MetricNames.SupportabilityLoggingEventsCollected;

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Configuration/IConfiguration.cs

@@ -190,6 +190,7 @@ public interface IConfiguration
         int LogEventsMaxSamplesStored { get; }
         TimeSpan LogEventsHarvestCycle { get; }
         bool LogDecoratorEnabled { get; }
+        HashSet<string> LogLevelDenyList { get; }
         bool ContextDataEnabled { get; }
         IEnumerable<string> ContextDataInclude { get; }
         IEnumerable<string> ContextDataExclude { get; }

tests/Agent/IntegrationTests/IntegrationTestHelpers/NewRelicConfigModifier.cs

@@ -317,6 +317,13 @@ public NewRelicConfigModifier SetLogForwardingMaxSamplesStored(int samples)
             return this;
         }
 
+        public NewRelicConfigModifier SetLogForwardingLogLevelDenyList(string logLevelDenyList)
+        {
+            CommonUtils.ModifyOrCreateXmlNodeInNewRelicConfig(_configFilePath, new[] { "configuration", "applicationLogging" }, "forwarding", string.Empty);
+            CommonUtils.ModifyOrCreateXmlAttributeInNewRelicConfig(_configFilePath, new[] { "configuration", "applicationLogging", "forwarding" }, "logLevelDenyList", logLevelDenyList);
+            return this;
+        }
+
         public NewRelicConfigModifier SetCodeLevelMetricsEnabled(bool enabled = true)
         {
             CommonUtils.ModifyOrCreateXmlNodeInNewRelicConfig(_configFilePath, new[] { "configuration" }, "codeLevelMetrics", string.Empty);