Skip to content

Commit efb3b60

Browse files
odygrdodygrd
committed
add_metadata_to_multi_line_logs will also now work for the Sink override pattern formatter options
1 parent f249eb0 commit efb3b60

File tree

8 files changed

+328
-199
lines changed

8 files changed

+328
-199
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,10 @@
179179
- `QUILL_LOG_RUNTIME_METADATA_SHALLOW` - Will take everything as reference. This is used when logging with
180180
compile-time metadata and using, for example, a dynamic log-level such as `LOG_DYNAMIC`.
181181

182+
- When using a sink with overridden `PatternFormatterOptions`, the option `add_metadata_to_multi_line_logs` will now be
183+
correctly applied at the Sink level. Previously, this option was only available and effective at the Logger level
184+
`PatternFormatter`.
185+
182186
- When a Sink with override `PatternFormatterOptions` is used and if no other sink exists using the `Logger`
183187
`PatternFormatterOptions`, then the backend thread will no longer perform a redundant format log statement.
184188

‎include/quill/backend/BackendWorker.h

Lines changed: 3 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -917,65 +917,9 @@ class BackendWorker
917917
log_level_to_string(transit_event.log_level(), _options.log_level_short_codes.data(),
918918
_options.log_level_short_codes.size());
919919

920-
if (transit_event.logger_base->_pattern_formatter->get_options().add_metadata_to_multi_line_logs &&
921-
(!transit_event.get_named_args() || transit_event.get_named_args()->empty()))
922-
{
923-
// This is only supported when named_args are not used
924-
_process_multi_line_message(transit_event, thread_id, thread_name, log_level_description, log_level_short_code);
925-
}
926-
else
927-
{
928-
// if the log_message ends with \n we should exclude it
929-
size_t const log_message_size =
930-
((transit_event.formatted_msg->size() > 0) &&
931-
(transit_event.formatted_msg->data()[transit_event.formatted_msg->size() - 1] == '\n'))
932-
? transit_event.formatted_msg->size() - 1
933-
: transit_event.formatted_msg->size();
934-
935-
// process the whole message without adding metadata to each line
936-
_write_log_statement(transit_event, thread_id, thread_name, log_level_description, log_level_short_code,
937-
std::string_view{transit_event.formatted_msg->data(), log_message_size});
938-
}
939-
}
940-
941-
/**
942-
* Splits and writes a transit event that has multiple lines
943-
*/
944-
QUILL_ATTRIBUTE_HOT void _process_multi_line_message(TransitEvent const& transit_event,
945-
std::string_view const& thread_id,
946-
std::string_view const& thread_name,
947-
std::string_view const& log_level_description,
948-
std::string_view const& log_level_short_code) const
949-
{
950-
auto const msg =
951-
std::string_view{transit_event.formatted_msg->data(), transit_event.formatted_msg->size()};
952-
953-
if (QUILL_UNLIKELY(msg.empty()))
954-
{
955-
// Process an empty message
956-
_write_log_statement(transit_event, thread_id, thread_name, log_level_description,
957-
log_level_short_code, msg);
958-
return;
959-
}
960-
961-
size_t start = 0;
962-
while (start < msg.size())
963-
{
964-
size_t const end = msg.find_first_of('\n', start);
965-
966-
if (end == std::string_view::npos)
967-
{
968-
// Handle the last line or a single line without a newline
969-
_write_log_statement(transit_event, thread_id, thread_name, log_level_description, log_level_short_code,
970-
std::string_view(msg.data() + start, msg.size() - start));
971-
break;
972-
}
973-
974-
// Write the current line
975-
_write_log_statement(transit_event, thread_id, thread_name, log_level_description,
976-
log_level_short_code, std::string_view(msg.data() + start, end - start));
977-
start = end + 1;
978-
}
920+
_write_log_statement(
921+
transit_event, thread_id, thread_name, log_level_description, log_level_short_code,
922+
std::string_view{transit_event.formatted_msg->data(), transit_event.formatted_msg->size()});
979923
}
980924

981925
/**

‎include/quill/backend/PatternFormatter.h

Lines changed: 157 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@ class PatternFormatter
9494
/***/
9595
~PatternFormatter() = default;
9696

97-
/***/
9897
QUILL_NODISCARD QUILL_ATTRIBUTE_HOT std::string_view format(
9998
uint64_t timestamp, std::string_view thread_id, std::string_view thread_name,
10099
std::string_view process_id, std::string_view logger, std::string_view log_level_description,
@@ -109,122 +108,59 @@ class PatternFormatter
109108
return std::string_view{};
110109
}
111110

112-
// clear out existing buffer
111+
// clear out the existing buffer
113112
_formatted_log_message_buffer.clear();
114113

115-
if (_is_set_in_pattern[Attribute::Time])
116-
{
117-
_set_arg_val<Attribute::Time>(_timestamp_formatter.format_timestamp(std::chrono::nanoseconds{timestamp}));
118-
}
119-
120-
if (_is_set_in_pattern[Attribute::FileName])
121-
{
122-
_set_arg_val<Attribute::FileName>(log_statement_metadata.file_name());
123-
}
124-
125-
if (_is_set_in_pattern[Attribute::CallerFunction])
126-
{
127-
std::string_view const function_name = _options.process_function_name
128-
? _options.process_function_name(log_statement_metadata.caller_function())
129-
: std::string_view{log_statement_metadata.caller_function()};
130-
131-
_set_arg_val<Attribute::CallerFunction>(function_name);
132-
}
133-
134-
if (_is_set_in_pattern[Attribute::LogLevel])
135-
{
136-
_set_arg_val<Attribute::LogLevel>(log_level_description);
137-
}
138-
139-
if (_is_set_in_pattern[Attribute::LogLevelShortCode])
140-
{
141-
_set_arg_val<Attribute::LogLevelShortCode>(log_level_short_code);
142-
}
143-
144-
if (_is_set_in_pattern[Attribute::LineNumber])
145-
{
146-
_set_arg_val<Attribute::LineNumber>(log_statement_metadata.line());
147-
}
148-
149-
if (_is_set_in_pattern[Attribute::Logger])
150-
{
151-
_set_arg_val<Attribute::Logger>(logger);
152-
}
153-
154-
if (_is_set_in_pattern[Attribute::FullPath])
155-
{
156-
_set_arg_val<Attribute::FullPath>(log_statement_metadata.full_path());
157-
}
158-
159-
if (_is_set_in_pattern[Attribute::ThreadId])
114+
if (QUILL_UNLIKELY(log_msg.empty()))
160115
{
161-
_set_arg_val<Attribute::ThreadId>(thread_id);
116+
// Process an empty message
117+
return _format(timestamp, thread_id, thread_name, process_id, logger, log_level_description,
118+
log_level_short_code, log_statement_metadata, named_args, log_msg);
162119
}
163120

164-
if (_is_set_in_pattern[Attribute::ThreadName])
165-
{
166-
_set_arg_val<Attribute::ThreadName>(thread_name);
167-
}
121+
std::string_view formatted_log_msg;
168122

169-
if (_is_set_in_pattern[Attribute::ProcessId])
123+
// Check if we need to handle multi-line formatting
124+
if (_options.add_metadata_to_multi_line_logs && (!named_args || named_args->empty()))
170125
{
171-
_set_arg_val<Attribute::ProcessId>(process_id);
172-
}
126+
// multi line metadata only supported when named_args are not used
127+
size_t start = 0;
173128

174-
if (_is_set_in_pattern[Attribute::SourceLocation])
175-
{
176-
_set_arg_val<Attribute::SourceLocation>(_process_source_location_path(
177-
log_statement_metadata.source_location(), _options.source_location_path_strip_prefix,
178-
_options.source_location_remove_relative_paths));
179-
;
180-
}
181-
182-
if (_is_set_in_pattern[Attribute::ShortSourceLocation])
183-
{
184-
_set_arg_val<Attribute::ShortSourceLocation>(log_statement_metadata.short_source_location());
185-
}
186-
187-
if (_is_set_in_pattern[Attribute::NamedArgs])
188-
{
189-
_formatted_named_args_buffer.clear();
190-
191-
if (named_args)
129+
while (start < log_msg.size())
192130
{
193-
for (size_t i = 0; i < named_args->size(); ++i)
194-
{
195-
_formatted_named_args_buffer.append((*named_args)[i].first);
196-
_formatted_named_args_buffer.append(std::string_view{": "});
197-
_formatted_named_args_buffer.append((*named_args)[i].second);
131+
size_t const end = log_msg.find_first_of('\n', start);
198132

199-
if (i != named_args->size() - 1)
200-
{
201-
_formatted_named_args_buffer.append(std::string_view{", "});
202-
}
133+
if (end == std::string_view::npos)
134+
{
135+
// Handle the last line or a single line without a newline
136+
formatted_log_msg =
137+
_format(timestamp, thread_id, thread_name, process_id, logger, log_level_description,
138+
log_level_short_code, log_statement_metadata, named_args,
139+
std::string_view(log_msg.data() + start, log_msg.size() - start));
140+
break;
203141
}
204-
}
205142

206-
_set_arg_val<Attribute::NamedArgs>(
207-
std::string_view{_formatted_named_args_buffer.data(), _formatted_named_args_buffer.size()});
208-
}
143+
// Write the current line
144+
formatted_log_msg = _format(timestamp, thread_id, thread_name, process_id, logger, log_level_description,
145+
log_level_short_code, log_statement_metadata, named_args, std::string_view(log_msg.data() + start, end - start));
209146

210-
if (_is_set_in_pattern[Attribute::Tags])
211-
{
212-
if (log_statement_metadata.tags())
213-
{
214-
_set_arg_val<Attribute::Tags>(std::string_view{log_statement_metadata.tags()});
215-
}
216-
else
217-
{
218-
_set_arg_val<Attribute::Tags>(std::string_view{});
147+
start = end + 1;
219148
}
220149
}
150+
else
151+
{
152+
// Use the regular format method for single-line messages
221153

222-
_set_arg_val<Attribute::Message>(log_msg);
154+
// if the log_message ends with \n we should exclude it
155+
assert(!log_msg.empty() && "Already checked non empty message earlier");
156+
size_t const log_message_size =
157+
(log_msg[log_msg.size() - 1] == '\n') ? log_msg.size() - 1 : log_msg.size();
223158

224-
fmtquill::vformat_to(std::back_inserter(_formatted_log_message_buffer), _fmt_format,
225-
fmtquill::basic_format_args(_args.data(), static_cast<int>(_args.size())));
159+
formatted_log_msg = _format(timestamp, thread_id, thread_name, process_id, logger, log_level_description,
160+
log_level_short_code, log_statement_metadata, named_args, std::string_view{log_msg.data(), log_message_size});
161+
}
226162

227-
return std::string_view{_formatted_log_message_buffer.data(), _formatted_log_message_buffer.size()};
163+
return formatted_log_msg;
228164
}
229165

230166
/***/
@@ -513,6 +449,128 @@ class PatternFormatter
513449
return std::make_pair(pattern, order_index);
514450
}
515451

452+
/***/
453+
QUILL_ATTRIBUTE_HOT std::string_view _format(
454+
uint64_t timestamp, std::string_view thread_id, std::string_view thread_name,
455+
std::string_view process_id, std::string_view logger, std::string_view log_level_description,
456+
std::string_view log_level_short_code, MacroMetadata const& log_statement_metadata,
457+
std::vector<std::pair<std::string, std::string>> const* named_args, std::string_view log_msg)
458+
{
459+
if (_is_set_in_pattern[Attribute::Time])
460+
{
461+
_set_arg_val<Attribute::Time>(_timestamp_formatter.format_timestamp(std::chrono::nanoseconds{timestamp}));
462+
}
463+
464+
if (_is_set_in_pattern[Attribute::FileName])
465+
{
466+
_set_arg_val<Attribute::FileName>(log_statement_metadata.file_name());
467+
}
468+
469+
if (_is_set_in_pattern[Attribute::CallerFunction])
470+
{
471+
std::string_view const function_name = _options.process_function_name
472+
? _options.process_function_name(log_statement_metadata.caller_function())
473+
: std::string_view{log_statement_metadata.caller_function()};
474+
475+
_set_arg_val<Attribute::CallerFunction>(function_name);
476+
}
477+
478+
if (_is_set_in_pattern[Attribute::LogLevel])
479+
{
480+
_set_arg_val<Attribute::LogLevel>(log_level_description);
481+
}
482+
483+
if (_is_set_in_pattern[Attribute::LogLevelShortCode])
484+
{
485+
_set_arg_val<Attribute::LogLevelShortCode>(log_level_short_code);
486+
}
487+
488+
if (_is_set_in_pattern[Attribute::LineNumber])
489+
{
490+
_set_arg_val<Attribute::LineNumber>(log_statement_metadata.line());
491+
}
492+
493+
if (_is_set_in_pattern[Attribute::Logger])
494+
{
495+
_set_arg_val<Attribute::Logger>(logger);
496+
}
497+
498+
if (_is_set_in_pattern[Attribute::FullPath])
499+
{
500+
_set_arg_val<Attribute::FullPath>(log_statement_metadata.full_path());
501+
}
502+
503+
if (_is_set_in_pattern[Attribute::ThreadId])
504+
{
505+
_set_arg_val<Attribute::ThreadId>(thread_id);
506+
}
507+
508+
if (_is_set_in_pattern[Attribute::ThreadName])
509+
{
510+
_set_arg_val<Attribute::ThreadName>(thread_name);
511+
}
512+
513+
if (_is_set_in_pattern[Attribute::ProcessId])
514+
{
515+
_set_arg_val<Attribute::ProcessId>(process_id);
516+
}
517+
518+
if (_is_set_in_pattern[Attribute::SourceLocation])
519+
{
520+
_set_arg_val<Attribute::SourceLocation>(_process_source_location_path(
521+
log_statement_metadata.source_location(), _options.source_location_path_strip_prefix,
522+
_options.source_location_remove_relative_paths));
523+
;
524+
}
525+
526+
if (_is_set_in_pattern[Attribute::ShortSourceLocation])
527+
{
528+
_set_arg_val<Attribute::ShortSourceLocation>(log_statement_metadata.short_source_location());
529+
}
530+
531+
if (_is_set_in_pattern[Attribute::NamedArgs])
532+
{
533+
_formatted_named_args_buffer.clear();
534+
535+
if (named_args)
536+
{
537+
for (size_t i = 0; i < named_args->size(); ++i)
538+
{
539+
_formatted_named_args_buffer.append((*named_args)[i].first);
540+
_formatted_named_args_buffer.append(std::string_view{": "});
541+
_formatted_named_args_buffer.append((*named_args)[i].second);
542+
543+
if (i != named_args->size() - 1)
544+
{
545+
_formatted_named_args_buffer.append(std::string_view{", "});
546+
}
547+
}
548+
}
549+
550+
_set_arg_val<Attribute::NamedArgs>(
551+
std::string_view{_formatted_named_args_buffer.data(), _formatted_named_args_buffer.size()});
552+
}
553+
554+
if (_is_set_in_pattern[Attribute::Tags])
555+
{
556+
if (log_statement_metadata.tags())
557+
{
558+
_set_arg_val<Attribute::Tags>(std::string_view{log_statement_metadata.tags()});
559+
}
560+
else
561+
{
562+
_set_arg_val<Attribute::Tags>(std::string_view{});
563+
}
564+
}
565+
566+
_set_arg_val<Attribute::Message>(log_msg);
567+
568+
fmtquill::vformat_to(std::back_inserter(_formatted_log_message_buffer), _fmt_format,
569+
fmtquill::basic_format_args(_args.data(), static_cast<int>(_args.size())));
570+
571+
return std::string_view{_formatted_log_message_buffer.data(), _formatted_log_message_buffer.size()};
572+
}
573+
516574
private:
517575
PatternFormatterOptions _options;
518576
std::string _fmt_format;

‎include/quill/sinks/Sink.h

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -47,20 +47,6 @@ class Sink
4747
explicit Sink(std::optional<PatternFormatterOptions> override_pattern_formatter_options = std::nullopt)
4848
: _override_pattern_formatter_options(std::move(override_pattern_formatter_options))
4949
{
50-
if (_override_pattern_formatter_options)
51-
{
52-
PatternFormatterOptions default_options{};
53-
if (_override_pattern_formatter_options->add_metadata_to_multi_line_logs != default_options.add_metadata_to_multi_line_logs)
54-
{
55-
QUILL_THROW(QuillError{
56-
"The 'add_metadata_to_multi_line_logs' option cannot be configured at the Sink level. "
57-
"This setting affects message processing and must be configured at the Logger level by "
58-
"passing PatternFormatterOptions to quill::Frontend::create_or_get_logger() instead. If "
59-
"your application requires both multiline messages with metadata and without metadata, "
60-
"create two separate Logger instances with unique names and different "
61-
"PatternFormatterOptions configurations."});
62-
}
63-
}
6450
}
6551

6652
/**

0 commit comments

Comments
 (0)