Add option to split stdout and stderr

When enabled ninja prints rule output into
appropriate stream: either stdout or stderr. Previously both outputs
were always concatenated into single buffer to be printed to stdout
when rule is finished.

When enabled behavior is next:

Subprocess sequentially appends rule output into internal buffer
looking at the source channel of incoming output. When channel
changes already collected buffer is printed and cleared.

Author: dendy

src/build.cc

@@ -643,7 +643,7 @@ size_t RealCommandRunner::CanRunMore() const {
 
 bool RealCommandRunner::StartCommand(Edge* edge) {
   string command = edge->EvaluateCommand();
-  Subprocess* subproc = subprocs_.Add(command, edge->use_console());
+  Subprocess* subproc = subprocs_.Add(command, edge->use_console(), edge->use_stderr());
   if (!subproc)
     return false;
   subproc_to_edge_.insert(make_pair(subproc, edge));
@@ -659,8 +659,10 @@ bool RealCommandRunner::WaitForCommand(Result* result) {
       return false;
   }
 
+  result->use_stderr = subproc->UseStdErr();
   result->status = subproc->Finish();
   result->output = subproc->GetOutput();
+  result->error = subproc->GetError();
 
   map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc);
   result->edge = e->second;
@@ -944,7 +946,9 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
         result->success()) {
       if (!result->output.empty())
         result->output.append("\n");
-      result->output.append(extract_err);
+      if (!result->error.empty())
+        result->error.append("\n");
+      (result->use_stderr ? result->error : result->output).append(extract_err);
       result->status = ExitFailure;
     }
   }
@@ -956,7 +960,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
   running_edges_.erase(it);
 
   status_->BuildEdgeFinished(edge, start_time_millis, end_time_millis,
-                             result->success(), result->output);
+                             result->success(), result->output, result->error);
 
   // The rest of this function only applies to successful commands.
   if (!result->success()) {

src/build.h

@@ -153,8 +153,10 @@ struct CommandRunner {
   struct Result {
     Result() : edge(NULL) {}
     Edge* edge;
+    bool use_stderr;
     ExitStatus status;
     std::string output;
+    std::string error;
     bool success() const { return status == ExitSuccess; }
   };
   /// Wait for a command to complete, or return false if interrupted.

src/eval_env.cc

@@ -75,7 +75,8 @@ bool Rule::IsReservedBinding(const string& var) {
       var == "restat" ||
       var == "rspfile" ||
       var == "rspfile_content" ||
-      var == "msvc_deps_prefix";
+      var == "msvc_deps_prefix" ||
+      var == "stderr";
 }
 
 const map<string, const Rule*>& BindingEnv::GetRules() const {

src/graph.cc

@@ -588,6 +588,17 @@ bool Edge::use_console() const {
   return pool() == &State::kConsolePool;
 }
 
+static bool s_use_stderr()
+{
+    const char* const use_string = getenv("NINJA_USE_STDERR");
+    return !use_string || strcmp(use_string, "0") != 0;
+}
+
+bool Edge::use_stderr() const {
+    static const bool use = s_use_stderr();
+    return use;
+}
+
 bool Edge::maybe_phonycycle_diagnostic() const {
   // CMake 2.8.12.x and 3.0.x produced self-referencing phony rules
   // of the form "build a: phony ... a ...".   Restrict our

src/graph.h

@@ -264,6 +264,7 @@ struct Edge {
 
   bool is_phony() const;
   bool use_console() const;
+  bool use_stderr() const;
   bool maybe_phonycycle_diagnostic() const;
 
   // Historical info: how long did this edge take last time,

src/line_printer.cc

@@ -32,7 +32,7 @@
 
 using namespace std;
 
-LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) {
+LinePrinter::LinePrinter(bool out) : out_(out), have_blank_line_(true), console_locked_(false) {
   const char* term = getenv("TERM");
 #ifndef _WIN32
   smart_terminal_ = isatty(1) && term && string(term) != "dumb";
@@ -128,7 +128,14 @@ void LinePrinter::PrintOrBuffer(const char* data, size_t size) {
   } else {
     // Avoid printf and C strings, since the actual output might contain null
     // bytes like UTF-16 does (yuck).
-    fwrite(data, 1, size, stdout);
+    fwrite(data, 1, size, file());
+  }
+}
+
+void LinePrinter::CompleteLine() {
+  if (!have_blank_line_) {
+    PrintOrBuffer("\n", 1);
+    have_blank_line_ = false;
   }
 }
 
@@ -138,9 +145,7 @@ void LinePrinter::PrintOnNewLine(const string& to_print) {
     output_buffer_.append(1, '\n');
     line_buffer_.clear();
   }
-  if (!have_blank_line_) {
-    PrintOrBuffer("\n", 1);
-  }
+  CompleteLine();
   if (!to_print.empty()) {
     PrintOrBuffer(&to_print[0], to_print.size());
   }

src/line_printer.h

@@ -16,12 +16,13 @@
 #define NINJA_LINE_PRINTER_H_
 
 #include <stddef.h>
+#include <stdio.h>
 #include <string>
 
 /// Prints lines of text, possibly overprinting previously printed lines
 /// if the terminal supports it.
 struct LinePrinter {
-  LinePrinter();
+  LinePrinter(bool out);
 
   bool is_smart_terminal() const { return smart_terminal_; }
   void set_smart_terminal(bool smart) { smart_terminal_ = smart; }
@@ -36,6 +37,9 @@ struct LinePrinter {
   /// one line.
   void Print(std::string to_print, LineType type);
 
+  /// Prints line break if needed.
+  void CompleteLine();
+
   /// Prints a string on a new line, not overprinting previous output.
   void PrintOnNewLine(const std::string& to_print);
 
@@ -44,6 +48,10 @@ struct LinePrinter {
   void SetConsoleLocked(bool locked);
 
  private:
+  FILE* file() const { return out_ ? stdout : stderr; }
+
+  const bool out_;
+
   /// Whether we can do fancy terminal control codes.
   bool smart_terminal_;
 

src/status.h

@@ -30,7 +30,7 @@ struct Status {
                                 int64_t start_time_millis) = 0;
   virtual void BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
                                  int64_t end_time_millis, bool success,
-                                 const std::string& output) = 0;
+                                 const std::string& output, const std::string& error) = 0;
   virtual void BuildStarted() = 0;
   virtual void BuildFinished() = 0;
 

src/status_printer.cc

@@ -42,7 +42,7 @@ Status* Status::factory(const BuildConfig& config) {
 
 StatusPrinter::StatusPrinter(const BuildConfig& config)
     : config_(config), started_edges_(0), finished_edges_(0), total_edges_(0),
-      running_edges_(0), progress_status_format_(NULL),
+      running_edges_(0), printer_(true), err_printer_(false), progress_status_format_(NULL),
       current_rate_(config.parallelism) {
   // Don't do anything fancy in verbose mode.
   if (config_.verbosity != BuildConfig::NORMAL)
@@ -90,8 +90,10 @@ void StatusPrinter::BuildEdgeStarted(const Edge* edge,
   if (edge->use_console() || printer_.is_smart_terminal())
     PrintStatus(edge, start_time_millis);
 
-  if (edge->use_console())
+  if (edge->use_console()) {
     printer_.SetConsoleLocked(true);
+    err_printer_.SetConsoleLocked(true);
+  }
 }
 
 void StatusPrinter::RecalculateProgressPrediction() {
@@ -175,7 +177,7 @@ void StatusPrinter::RecalculateProgressPrediction() {
 
 void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
                                       int64_t end_time_millis, bool success,
-                                      const string& output) {
+                                      const string& output, const std::string& error) {
   time_millis_ = end_time_millis;
   ++finished_edges_;
 
@@ -190,8 +192,10 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
   } else
     --eta_unpredictable_edges_remaining_;
 
-  if (edge->use_console())
+  if (edge->use_console()) {
     printer_.SetConsoleLocked(false);
+    err_printer_.SetConsoleLocked(false);
+  }
 
   if (config_.verbosity == BuildConfig::QUIET)
     return;
@@ -208,12 +212,14 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
          o != edge->outputs_.end(); ++o)
       outputs += (*o)->path() + " ";
 
-    if (printer_.supports_color()) {
-        printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n");
+    printer_.CompleteLine();
+
+    if (err_printer_.supports_color()) {
+        err_printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n");
     } else {
-        printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
+        err_printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
     }
-    printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
+    err_printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
   }
 
   if (!output.empty()) {
@@ -247,6 +253,22 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
     _setmode(_fileno(stdout), _O_TEXT);  // End Windows extra CR fix
 #endif
   }
+
+  if (!error.empty()) {
+    string::size_type begin = 0;
+    while (true) {
+      const std::string::size_type crpos = error.find('\xd', begin);
+      if (crpos == string::npos) {
+        fwrite(error.c_str() + begin, error.size() - begin, 1, stderr);
+        break;
+      }
+      const std::string::size_type size = crpos - begin;
+      if (size != 0)
+        fwrite(error.c_str() + begin, crpos - begin, 1, stderr);
+      begin = crpos + 1;
+    }
+    fflush(stderr);
+  }
 }
 
 void StatusPrinter::BuildStarted() {

src/status_printer.h

@@ -32,7 +32,7 @@ struct StatusPrinter : Status {
   virtual void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis);
   virtual void BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
                                  int64_t end_time_millis, bool success,
-                                 const std::string& output);
+                                 const std::string& output, const std::string& error);
   virtual void BuildStarted();
   virtual void BuildFinished();
 
@@ -88,6 +88,7 @@ struct StatusPrinter : Status {
 
   /// Prints progress output.
   LinePrinter printer_;
+  LinePrinter err_printer_;
 
   /// An optional Explanations pointer, used to implement `-d explain`.
   Explanations* explanations_ = nullptr;