Feature Work: Refactor and Optimize LogLevelDenyList (#1764)

* ci: Update AwsLamba unit tests to .NET 7.0 (#1759)

* Converted LogLevelDenyList to HashSet<string>, updated tests as required.

* Cache the logleveldenylist evaluation

* Renamed the logLevelDenyList attribute, added integration tests

* unit test fix

* Integration test fix

Author: tippmar-nr

src/Agent/NewRelic/Agent/Core/Agent.cs

@@ -408,7 +408,7 @@ public void RecordSupportabilityMetric(string metricName, int count)
             _agentHealthReporter.ReportSupportabilityCountMetric(metricName, count);
         }
 
-        public void RecordLogMessage(string frameworkName, object logEvent, Func<object, DateTime> getTimestamp, Func<object, object> getLevel, Func<object, string> getLogMessage, Func<object, Exception> getLogException,Func<object, Dictionary<string, object>> getContextData, string spanId, string traceId)
+        public void RecordLogMessage(string frameworkName, object logEvent, Func<object, DateTime> getTimestamp, Func<object, object> getLevel, Func<object, string> getLogMessage, Func<object, Exception> getLogException, Func<object, Dictionary<string, object>> getContextData, string spanId, string traceId)
         {
             _agentHealthReporter.ReportLogForwardingFramework(frameworkName);
 
@@ -420,8 +420,8 @@ public void RecordLogMessage(string frameworkName, object logEvent, Func<object,
                 normalizedLevel = string.IsNullOrWhiteSpace(level) ? "UNKNOWN" : level.ToUpper();
             }
 
-            // we want to ignore case so that we don't have to normalize the deny-list values
-            if (_configurationService.Configuration.LogLevelDenylist.Contains(normalizedLevel, StringComparer.OrdinalIgnoreCase))
+            // LogLevelDenyList is already uppercase
+            if (normalizedLevel != string.Empty && _configurationService.Configuration.LogLevelDenyList.Contains(normalizedLevel))
             {
                 return;
             }
@@ -438,7 +438,7 @@ public void RecordLogMessage(string frameworkName, object logEvent, Func<object,
 
                 var logMessage = getLogMessage(logEvent);
                 var logException = getLogException(logEvent);
-                
+
                 // exit quickly if the message and exception are missing
                 if (string.IsNullOrWhiteSpace(logMessage) && logException is null)
                 {

src/Agent/NewRelic/Agent/Core/Config/Configuration.cs

@@ -5035,7 +5035,7 @@ public partial class configurationApplicationLoggingForwarding
 
         private int maxSamplesStoredField;
 
-        private string logLevelDenylistField;
+        private string logLevelDenyListField;
 
         /// <summary>
         /// configurationApplicationLoggingForwarding class constructor
@@ -5088,15 +5088,15 @@ public int maxSamplesStored
         }
 
         [System.Xml.Serialization.XmlAttributeAttribute()]
-        public string logLevelDenylist
+        public string logLevelDenyList
         {
             get
             {
-                return this.logLevelDenylistField;
+                return this.logLevelDenyListField;
             }
             set
             {
-                this.logLevelDenylistField = value;
+                this.logLevelDenyListField = value;
             }
         }
 

src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd

@@ -1656,7 +1656,7 @@
                                         </xs:annotation>
                                     </xs:attribute>
 
-                                    <xs:attribute name="logLevelDenylist" type="xs:string" use="optional">
+                                    <xs:attribute name="logLevelDenyList" type="xs:string" use="optional">
                                         <xs:annotation>
                                             <xs:documentation>
                                                 A comma-separated, case-insensitive, list of log levels that should be ignored and not sent up to New Relic.

src/Agent/NewRelic/Agent/Core/Configuration/DefaultConfiguration.cs

@@ -1990,13 +1990,17 @@ public virtual bool LogDecoratorEnabled
             }
         }
 
-        public virtual IEnumerable<string> LogLevelDenylist
+        private HashSet<string> _logLevelDenyList;
+        public virtual HashSet<string> LogLevelDenyList
         {
             get
             {
-                return EnvironmentOverrides(_localConfiguration.applicationLogging.forwarding.logLevelDenylist,
-                    "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LOG_LEVEL_DENYLIST")
-                    .Split(new[] { StringSeparators.CommaChar, ' ' }, StringSplitOptions.RemoveEmptyEntries);
+                return _logLevelDenyList ??= new HashSet<string>(
+                    EnvironmentOverrides(_localConfiguration.applicationLogging.forwarding.logLevelDenyList,
+                            "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LOG_LEVEL_DENYLIST")
+                        ?.Split(new[] { StringSeparators.CommaChar, ' ' }, StringSplitOptions.RemoveEmptyEntries)
+                        .Select(s => s.ToUpper())
+                    ?? Enumerable.Empty<string>());
             }
         }
 

src/Agent/NewRelic/Agent/Core/Configuration/ReportedConfiguration.cs

@@ -593,15 +593,15 @@ public ReportedConfiguration(IConfiguration configuration)
         [JsonProperty("application_logging.forwarding.max_samples_stored")]
         public int LogEventsMaxSamplesStored => _configuration.LogEventsMaxSamplesStored;
 
+        [JsonProperty("application_logging.forwarding.log_level_denylist")]
+        public HashSet<string> LogLevelDenyList => _configuration.LogLevelDenyList;
+
         [JsonProperty("application_logging.harvest_cycle")]
         public TimeSpan LogEventsHarvestCycle => _configuration.LogEventsHarvestCycle;
 
         [JsonProperty("application_logging.local_decorating.enabled")]
         public bool LogDecoratorEnabled => _configuration.LogDecoratorEnabled;
 
-        [JsonProperty("application_logging.log_level_denylist")]
-        public IEnumerable<string> LogLevelDenylist => _configuration.LogLevelDenylist;
-
         [JsonProperty("agent.app_domain_caching_disabled")]
         public bool AppDomainCachingDisabled => _configuration.AppDomainCachingDisabled;
 

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Configuration/IConfiguration.cs

@@ -190,7 +190,7 @@ public interface IConfiguration
         int LogEventsMaxSamplesStored { get; }
         TimeSpan LogEventsHarvestCycle { get; }
         bool LogDecoratorEnabled { get; }
-        IEnumerable<string> LogLevelDenylist { get; }
+        HashSet<string> LogLevelDenyList { get; }
         bool ContextDataEnabled { get; }
         IEnumerable<string> ContextDataInclude { get; }
         IEnumerable<string> ContextDataExclude { get; }

tests/Agent/IntegrationTests/IntegrationTestHelpers/NewRelicConfigModifier.cs

@@ -317,6 +317,13 @@ public NewRelicConfigModifier SetLogForwardingMaxSamplesStored(int samples)
             return this;
         }
 
+        public NewRelicConfigModifier SetLogForwardingLogLevelDenyList(string logLevelDenyList)
+        {
+            CommonUtils.ModifyOrCreateXmlNodeInNewRelicConfig(_configFilePath, new[] { "configuration", "applicationLogging" }, "forwarding", string.Empty);
+            CommonUtils.ModifyOrCreateXmlAttributeInNewRelicConfig(_configFilePath, new[] { "configuration", "applicationLogging", "forwarding" }, "logLevelDenyList", logLevelDenyList);
+            return this;
+        }
+
         public NewRelicConfigModifier SetCodeLevelMetricsEnabled(bool enabled = true)
         {
             CommonUtils.ModifyOrCreateXmlNodeInNewRelicConfig(_configFilePath, new[] { "configuration" }, "codeLevelMetrics", string.Empty);