dlt-qnx-system: prevent message loss in high load situations (#490)

* Add wait method, which waits up to 5 seconds for buffer space
  to become available.
  This method tries to flush the buffer via tcp every 10ms.
  An error will be logged when there was not enough space.
  The error shows up in dlt to simplify debugging.
* improve cmake that more qnx compilers are detected
* log missing context mapping only once, this greatly reduces
  noise and the load of the system, as previously the
  message was logged with every log of an unmapped application

Signed-off-by: Alexander Mohr <[email protected]>

Author: alexmohr

CMakeLists.txt

@@ -82,6 +82,7 @@ option(WITH_DLT_LOG_LEVEL_APP_CONFIG "Set to ON to enable default log levels bas
 
 set(DLT_IPC "FIFO" CACHE STRING "UNIX_SOCKET,FIFO")
 set(DLT_USER "genivi" CACHE STRING "Set user for process not run as root")
+set(DLT_QNX_SLOG_ADAPTER_WAIT_BUFFER_TIMEOUT_MS "100" CACHE STRING "Timeout in milliseconds to wait before messages are dropped when input buffer is full")
 
 option(WITH_DLT_PKGCONFIG     "Set to ON to generate pkgconfig .pc files"                                        ON)
 option(WITH_DLT_CXX11_EXT     "Set to ON to build C++11 extensions"                                              OFF)
@@ -142,6 +143,7 @@ include_directories(
 add_definitions(-D_GNU_SOURCE)
 add_definitions(-DDLT_WRITEV_TIMEOUT_SEC=${DLT_WRITEV_TIMEOUT_SEC})
 add_definitions(-DDLT_WRITEV_TIMEOUT_USEC=${DLT_WRITEV_TIMEOUT_USEC})
+add_definitions(-DDLT_QNX_SLOG_ADAPTER_WAIT_BUFFER_TIMEOUT_MS=${DLT_QNX_SLOG_ADAPTER_WAIT_BUFFER_TIMEOUT_MS})
 
 if(NOT DLT_IPC STREQUAL "UNIX_SOCKET" AND NOT DLT_IPC STREQUAL "FIFO")
     message(FATAL_ERROR "${DLT_IPC} is not a valid value for DLT_IPC")
@@ -180,8 +182,8 @@ if(WITH_DLT_USE_IPv6)
     add_definitions(-DDLT_USE_IPv6)
 endif()
 
-if(WITH_DLT_QNX_SYSTEM AND NOT "${CMAKE_C_COMPILER}" MATCHES "nto-qnx|qcc")
-    message(FATAL_ERROR "Can only compile for QNX with a QNX compiler.")
+if(WITH_DLT_QNX_SYSTEM AND NOT "${CMAKE_C_COMPILER}" MATCHES "nto-qnx|qcc|ntoaarch64-gcc|ntox86_64-gcc")
+    message(FATAL_ERROR "Can only compile for QNX with a QNX compiler, but found '${CMAKE_C_COMPILER}'.")
 endif()
 
 if (WITH_DLT_FILE_LOGGING_SYSLOG_FALLBACK)

doc/dlt_build_options.md

@@ -59,6 +59,7 @@ WITH\_DLT\_DBUS                   | OFF            | Set to ON to build src/dbus
 Option | Value | Comment
 :--- | :--- | :---
 WITH\_DLT\_QNX\_SYSTEM            | OFF            | Set to ON to build QNX system binary dlt-qnx-system
+DLT\_QNX\_SLOG\_ADAPTER\_WAIT\_BUFFER\_TIMEOUT\_MS | 100 | Maximum time in milliseconds to wait for buffer space in dlt before messages from the slog will be discarded.
 
 ## Documentation Options
 

src/dlt-qnx-system/dlt-qnx-slogger2-adapter.cpp

@@ -28,9 +28,13 @@
 #include <sys/slog2.h>
 #include <sys/json.h>
 #include <slog2_parse.h>
+#include <thread>
+#include <set>
 
 #include "dlt-qnx-system.h"
 #include "dlt_cpp_extension.hpp"
+using std::chrono_literals::operator""ms;
+using std::chrono_literals::operator""s;
 
 /* Teach dlt about json_decoder_error_t */
 template<>
@@ -42,6 +46,7 @@ inline int32_t logToDlt(DltContextData &log, const json_decoder_error_t &value)
 extern DltContext dltQnxSystem;
 
 static DltContext dltQnxSlogger2Context;
+static std::set<std::string> dltWarnedMissingMappings;
 
 extern DltQnxSystemThreads g_threads;
 
@@ -121,14 +126,49 @@ static DltContext *dlt_context_from_slog2file(const char *file_name) {
 
     auto search = g_slog2file.find(name);
     if (search == g_slog2file.end()) {
-        DLT_LOG_CXX(dltQnxSlogger2Context, DLT_LOG_VERBOSE,
-                "slog2 filename not found in mapping: ", name.c_str());
+        // Only warn once about missing mapping.
+        auto it = dltWarnedMissingMappings.find(name);
+        if (it == dltWarnedMissingMappings.end()) {
+            dltWarnedMissingMappings.insert(name);
+            DLT_LOG_CXX(dltQnxSlogger2Context, DLT_LOG_INFO,
+                        "slog2 filename not found in mapping: ", name.c_str());
+        }
+
         return &dltQnxSlogger2Context;
     } else {
         return search->second;
     }
 }
 
+template <class time, class period>
+static void wait_for_buffer_space(const double max_usage_threshold,
+                                  const std::chrono::duration<time, period> max_wait_time) {
+    int total_size = 0;
+    int used_size = 0;
+    double used_percent = 100.0;
+    bool timeout = false;
+    const auto end_time = std::chrono::steady_clock::now() + max_wait_time;
+
+    do {
+        dlt_user_check_buffer(&total_size, &used_size);
+        used_percent = static_cast<double>(used_size) / total_size;
+        if (used_percent < max_usage_threshold) {
+            break;
+        }
+
+        dlt_user_log_resend_buffer();
+
+        std::this_thread::sleep_for(10ms);
+        timeout = std::chrono::steady_clock::now() < end_time;
+    } while (!timeout);
+
+    if (timeout) {
+        DLT_LOG(dltQnxSystem, DLT_LOG_ERROR,
+                DLT_STRING("failed to get enough buffer space"));
+
+    }
+}
+
 /**
  *  Function which is invoked by slog2_parse_all()
  *  See slog2_parse_all api docs on qnx.com for details
@@ -177,6 +217,8 @@ static int sloggerinfo_callback(slog2_packet_info_t *info, void *payload, void *
     DltContextData log_local; /* Used in DLT_* macros, do not rename */
     DltContext *ctxt = dlt_context_from_slog2file(info->file_name);
 
+    wait_for_buffer_space(0.8, std::chrono::milliseconds(DLT_QNX_SLOG_ADAPTER_WAIT_BUFFER_TIMEOUT_MS));
+
     int ret;
     ret = dlt_user_log_write_start(ctxt, &log_local, loglevel);