feat: Allow the agent to run in environments with read-only filesystems. (#2085)

* Don't bail if log file can't be created

* Support logging to stdout in the Profiler (#2025)

* Agent support for read-only file systems (#2045)

* New option to disable logging
* Env var for console logging
* Supportability metric if logging fails
* Don't allow audit logging if logging is disabled

* chore: Update Profiler NuGet Package Reference to v10.18.0.15. (#2077)

Co-authored-by: nr-ahemsath <[email protected]>

* Add integration tests for file logging disabled and console logging enabled (#2076)

* Checkpoint in order to switch branches

* Refactored AgentLogFile

Moved AgentLogFile to the fixtures instead of the applications

* Shorten some names

* Fix bug caused by Visual Studio autocorrect

* WIP

* Working tests for logging disabled

* Working console logging tests

* Fix nuget package warnings-as-errors

* Fix GuidConfigurationTests

EndsWith() fails if the profiler logs the version, e.g. New Relic .NET CoreCLR Agent v10.18.0.26.  Use Contains() instead

* Fix integration test build error

* PR feedback

---------

Co-authored-by: chynesNR <[email protected]>
Co-authored-by: Chris Hynes <[email protected]>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: nr-ahemsath <[email protected]>
Co-authored-by: Marty Tippin <[email protected]>

Author: 4 people

src/Agent/NewRelic/Agent/Core/AgentHealth/AgentHealthReporter.cs

@@ -695,6 +695,7 @@ private void CollectOneTimeMetrics()
             ReportLogForwardingConfiguredValues();
             ReportIfAppDomainCachingDisabled();
             ReportInfiniteTracingOneTimeMetrics();
+            ReportIfLoggingDisabled();
         }
 
         public void CollectMetrics()
@@ -829,5 +830,17 @@ protected override void OnConfigurationUpdated(ConfigurationUpdateSource configu
             // Some one time metrics are reporting configured values, so we want to re-report them if the configuration changed
             _oneTimeMetricsCollected = false;
         }
+
+        private void ReportIfLoggingDisabled()
+        {
+            if (!_configuration.LoggingEnabled)
+            {
+                ReportSupportabilityCountMetric(MetricNames.SupportabilityLoggingDisabled);
+            }
+            if (Log.FileLoggingHasFailed)
+            {
+                ReportSupportabilityCountMetric(MetricNames.SupportabilityLoggingFatalError);
+            }
+        }
     }
 }

src/Agent/NewRelic/Agent/Core/AgentManager.cs

@@ -220,6 +220,8 @@ private void LogInitialized()
                     "NEW_RELIC_LOG",
                     "NEWRELIC_PROFILER_LOG_DIRECTORY",
                     "NEWRELIC_LOG_LEVEL",
+                    "NEW_RELIC_LOG_ENABLED",
+                    "NEW_RELIC_LOG_CONSOLE",
                     "NEW_RELIC_LABELS",
                     "NEW_RELIC_PROXY_HOST",
                     "NEW_RELIC_PROXY_URI_PATH",

src/Agent/NewRelic/Agent/Core/Config/Configuration.cs

@@ -1284,6 +1284,7 @@ public virtual configurationApplication Clone()
     [System.Xml.Serialization.XmlTypeAttribute(AnonymousType=true, Namespace="urn:newrelic-config")]
     public partial class configurationLog
     {
+        private bool enabledField;
 
         private string levelField;
 
@@ -1300,6 +1301,7 @@ public partial class configurationLog
         /// </summary>
         public configurationLog()
         {
+            this.enabledField = true;
             this.levelField = "info";
             this.consoleField = false;
             this.auditLogField = false;
@@ -1358,6 +1360,20 @@ public bool console
                 this.consoleField = value;
             }
         }
+
+        [System.Xml.Serialization.XmlAttributeAttribute()]
+        [System.ComponentModel.DefaultValueAttribute(true)]
+        public bool enabled
+        {
+            get
+            {
+                return this.enabledField;
+            }
+            set
+            {
+                this.enabledField = value;
+            }
+        }
 
         [System.Xml.Serialization.XmlAttributeAttribute()]
         [System.ComponentModel.DefaultValueAttribute(false)]

src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd

@@ -313,6 +313,14 @@
                             </xs:annotation>
                         </xs:attribute>
 
+                        <xs:attribute name="enabled" type="xs:boolean" default="true">
+                            <xs:annotation>
+                                <xs:documentation>
+                                    If this is false, no attempts will be made to write to a log file. This is primarily used for installations on read-only file systems.
+                                </xs:documentation>
+                            </xs:annotation>
+                        </xs:attribute>
+                        
                         <xs:attribute name="directory" type="xs:string" use="optional">
                             <xs:annotation>
                                 <xs:documentation>

src/Agent/NewRelic/Agent/Core/Config/ConfigurationLoader.cs

@@ -59,6 +59,7 @@ private static string InternalGetAppDomainAppPath()
 
         public static Func<string, bool> FileExists = File.Exists;
         public static Func<string, string> PathGetDirectoryName = Path.GetDirectoryName;
+        public static Func<string, string> GetEnvironmentVar = System.Environment.GetEnvironmentVariable;
 
         private static string InternalGetNewRelicHome()
         {
@@ -522,6 +523,10 @@ public string LogLevel
         {
             get
             {
+                if (!Enabled)
+                {
+                    return "off";
+                }
                 // Environment variable or log.level from config...
                 return (AgentInstallConfiguration.NewRelicLogLevel
                     ?? this.level).ToUpper();
@@ -559,7 +564,7 @@ public string GetFullLogFileName()
 
         private string GetLogFileName()
         {
-            string name = System.Environment.GetEnvironmentVariable("NEW_RELIC_LOG");
+            string name = ConfigurationLoader.GetEnvironmentVar("NEW_RELIC_LOG");
             if (name != null)
             {
                 return Strings.SafeFileName(name);
@@ -594,11 +599,46 @@ private string GetLogFileName()
             return "newrelic_agent_" + Strings.SafeFileName(name) + ".log";
         }
 
+        private bool GetOverride(string name, bool fallback)
+        {
+            var val = ConfigurationLoader.GetEnvironmentVar(name);
+
+            if (val != null)
+            {
+                val = val.ToLower();
+            }
+
+            if (bool.TryParse(val, out var parsedValue))
+            {
+                return parsedValue;
+            }
+
+            if ("0" == val)
+            {
+                return false;
+            }
+
+            if ("1" == val)
+            {
+                return true;
+            }
+
+            return fallback;
+        }
+
         public bool Console
         {
             get
             {
-                return console;
+                return GetOverride("NEW_RELIC_LOG_CONSOLE", console);
+            }
+        }
+
+        public bool Enabled
+        {
+            get
+            {
+                return GetOverride("NEW_RELIC_LOG_ENABLED", enabled);
             }
         }
 

src/Agent/NewRelic/Agent/Core/Config/ILogConfig.cs

@@ -5,6 +5,8 @@ namespace NewRelic.Agent.Core.Config
 {
     public interface ILogConfig
     {
+        bool Enabled { get; }
+
         string LogLevel { get; }
 
         string GetFullLogFileName();

src/Agent/NewRelic/Agent/Core/Configuration/ConfigurationService.cs

@@ -64,7 +64,7 @@ private void OnConfigurationDeserialized(ConfigurationDeserializedEvent configur
         private static void UpdateLogLevel(configuration localConfiguration)
         {
             Log.Info("The log level was updated to {0}", localConfiguration.LogConfig.LogLevel);
-            LoggerBootstrapper.UpdateLoggingLevel(localConfiguration.LogConfig.LogLevel);
+            LoggerBootstrapper.SetLoggingLevel(localConfiguration.LogConfig.LogLevel);
         }
 
         private void OnServerConfigurationUpdated(ServerConfigurationUpdatedEvent serverConfigurationUpdatedEvent)

src/Agent/NewRelic/Agent/Core/Configuration/DefaultConfiguration.cs

@@ -2775,6 +2775,8 @@ public bool CodeLevelMetricsEnabled
 
         #endregion
 
+        public bool LoggingEnabled => _localConfiguration.log.Enabled;
+
         private const bool CaptureTransactionTraceAttributesDefault = true;
         private const bool CaptureErrorCollectorAttributesDefault = true;
         private const bool CaptureBrowserMonitoringAttributesDefault = false;

src/Agent/NewRelic/Agent/Core/Configuration/ReportedConfiguration.cs

@@ -647,6 +647,9 @@ public ReportedConfiguration(IConfiguration configuration)
         [JsonProperty("stackexchangeredis_cleanup.cycle")]
         public TimeSpan StackExchangeRedisCleanupCycle => _configuration.StackExchangeRedisCleanupCycle;
 
+        [JsonProperty("agent.logging_enabled")]
+        public bool LoggingEnabled => _configuration.LoggingEnabled;
+
         public IReadOnlyDictionary<string, string> GetAppSettings()
         {
             return _configuration.GetAppSettings();

src/Agent/NewRelic/Agent/Core/Logging/LoggerBootstrapper.cs

@@ -31,10 +31,7 @@ public static class LoggerBootstrapper
 
         private static InMemorySink _inMemorySink = new InMemorySink();
 
-        public static void UpdateLoggingLevel(string newLogLevel)
-        {
-            _loggingLevelSwitch.MinimumLevel = newLogLevel.MapToSerilogLogLevel();
-        }
+        public static void SetLoggingLevel(string newLogLevel) => _loggingLevelSwitch.MinimumLevel = newLogLevel.MapToSerilogLogLevel();
 
         public static void Initialize()
         {
@@ -54,7 +51,7 @@ public static void Initialize()
         /// <remarks>This should only be called once, as soon as you have a valid config.</remarks>
         public static void ConfigureLogger(ILogConfig config)
         {
-            SetupLogLevel(config);
+            SetLoggingLevel(config.LogLevel);
 
             AuditLog.IsAuditLogEnabled = config.IsAuditLogEnabled;
 
@@ -78,6 +75,8 @@ public static void ConfigureLogger(ILogConfig config)
             Log.Logger = configuredLogger;
 
             NewRelic.Core.Logging.Log.Initialize(new Logger());
+
+            Log.Logger.Information("Log level set to {0}", config.LogLevel);
         }
 
         private static void EchoInMemoryLogsToConfiguredLogger(ILogger configuredLogger)
@@ -90,12 +89,6 @@ private static void EchoInMemoryLogsToConfiguredLogger(ILogger configuredLogger)
             _inMemorySink.Dispose();
         }
 
-        /// <summary>
-        /// Sets the log level for logger to either the level provided by the config or an public default.
-        /// </summary>
-        /// <param name="config">The LogConfig to look for the level setting in.</param>
-        private static void SetupLogLevel(ILogConfig config) => _loggingLevelSwitch.MinimumLevel = config.LogLevel.MapToSerilogLogLevel();
-
         private static LoggerConfiguration ConfigureInMemoryLogSink(this LoggerConfiguration loggerConfiguration)
         {
             // formatter not needed since this will be pushed to other sinks for output.
@@ -195,6 +188,10 @@ private static LoggerConfiguration ConfigureConsoleSink(this LoggerConfiguration
         /// <param name="config">The configuration for the appender.</param>
         private static LoggerConfiguration ConfigureFileSink(this LoggerConfiguration loggerConfiguration, ILogConfig config)
         {
+            if (!config.Enabled)
+            {
+                return loggerConfiguration;
+            }
             string logFileName = config.GetFullLogFileName();
 
             try
@@ -215,6 +212,7 @@ private static LoggerConfiguration ConfigureFileSink(this LoggerConfiguration lo
                 Log.Logger.Warning(ex, "Unexpected exception when configuring file sink.");
 
                 // Fallback to the event log sink if we cannot setup a file logger.
+                NewRelic.Core.Logging.Log.FileLoggingHasFailed = true;
                 Log.Logger.Warning("Falling back to EventLog sink.");
                 loggerConfiguration.ConfigureEventLogSink();
             }
@@ -227,7 +225,7 @@ private static LoggerConfiguration ConfigureFileSink(this LoggerConfiguration lo
         /// </summary>
         private static LoggerConfiguration ConfigureAuditLogSink(this LoggerConfiguration loggerConfiguration, ILogConfig config)
         {
-            if (!config.IsAuditLogEnabled) return loggerConfiguration;
+            if (!config.IsAuditLogEnabled || !config.Enabled) return loggerConfiguration;
 
             string logFileName = config.GetFullLogFileName().Replace(".log", "_audit.log");
 

src/Agent/NewRelic/Agent/Core/Metrics/MetricNames.cs

@@ -833,6 +833,9 @@ public static string GetSupportabilityInstallType(string installType)
         // AppDomain caching disabled
         public const string SupportabilityAppDomainCachingDisabled = "Supportability/DotNET/AppDomainCaching/Disabled";
 
+        public const string SupportabilityLoggingDisabled = "Supportability/DotNET/AgentLogging/Disabled";
+        public const string SupportabilityLoggingFatalError = "Supportability/DotNET/AgentLogging/DisabledDueToError";
+
         #endregion Supportability
 
         #region Distributed Trace Metrics

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Configuration/IConfiguration.cs

@@ -205,5 +205,6 @@ public interface IConfiguration
         TimeSpan SqlTracesHarvestCycle { get; }
         TimeSpan UpdateLoadedModulesCycle { get; }
         TimeSpan StackExchangeRedisCleanupCycle { get; }
+        bool LoggingEnabled { get; }
     }
 }

src/Agent/NewRelic/Home/Home.csproj

@@ -13,7 +13,7 @@
   </Target>
 
   <ItemGroup>
-    <PackageReference Include="NewRelic.Agent.Internal.Profiler" Version="10.17.0.8"/>
+    <PackageReference Include="NewRelic.Agent.Internal.Profiler" Version="10.18.0.15"/>
   </ItemGroup>
 
 </Project>

src/Agent/NewRelic/Profiler/Configuration/Configuration.h

@@ -32,6 +32,8 @@ namespace NewRelic { namespace Profiler { namespace Configuration {
             : _agentEnabled(true)
             , _agentEnabledInLocalConfig(false)
             , _logLevel(Logger::Level::LEVEL_INFO)
+            , _consoleLogging(false)
+            , _loggingEnabled(true)
             , _processes(new Processes())
             , _applicationPoolsWhiteList(new ApplicationPools())
             , _applicationPoolsBlackList(new ApplicationPools())
@@ -118,6 +120,8 @@ namespace NewRelic { namespace Profiler { namespace Configuration {
             : _agentEnabled(agentEnabled)
             , _agentEnabledInLocalConfig(false)
             , _logLevel(logLevel)
+            , _consoleLogging(false)
+            , _loggingEnabled(true)
             , _processes(processes)
             , _applicationPoolsWhiteList(whiteList)
             , _applicationPoolsBlackList(blackList)
@@ -188,10 +192,21 @@ namespace NewRelic { namespace Profiler { namespace Configuration {
             return _logLevel;
         }
 
+        bool GetConsoleLogging()
+        {
+            return _consoleLogging;
+        }
+        bool GetLoggingEnabled()
+        {
+            return _loggingEnabled;
+        }
+
     private:
         bool _agentEnabled;
         bool _agentEnabledInLocalConfig;
         Logger::Level _logLevel;
+        bool _consoleLogging;
+        bool _loggingEnabled;
         ProcessesPtr _processes;
         ApplicationPoolsPtr _applicationPoolsWhiteList;
         ApplicationPoolsPtr _applicationPoolsBlackList;
@@ -250,6 +265,18 @@ namespace NewRelic { namespace Profiler { namespace Configuration {
             if (logNode == nullptr)
                 return;
 
+            auto consoleAttribute = logNode->first_attribute(_X("console"), 0, false);
+            if (consoleAttribute != nullptr)
+            {
+                _consoleLogging = Strings::AreEqualCaseInsensitive(consoleAttribute->value(), _X("true"));
+            }
+
+            auto enabledAttribute = logNode->first_attribute(_X("enabled"), 0, false);
+            if (enabledAttribute != nullptr)
+            {
+                _loggingEnabled = Strings::AreEqualCaseInsensitive(enabledAttribute->value(), _X("true"));
+            }
+
             auto logLevelAttribute = logNode->first_attribute(_X("level"), 0, false);
             if (logLevelAttribute == nullptr)
                 return;