make tests quieter (#2468)

Don't mix every http request in with the tests output.
Don't print that the file servers are starting unless
-vv flag is passed.

Capture the output of run with run_output which returns
stdout, stderr and exit_code. Test against this rather
than relying on sys.exit.

Author: hayd

.appveyor.yml

@@ -196,7 +196,7 @@ build_script:
 test_script:
   - python tools\lint.py
   - python tools\test_format.py
-  - ps: Exec { & python tools\test.py -v --build-dir $env:DENO_BUILD_PATH }
+  - ps: Exec { & python tools\test.py --build-dir $env:DENO_BUILD_PATH }
 
 after_test:
   # Delete the the rollup cache, which is unreliable, so that it doesn't get

.travis.yml

@@ -73,7 +73,7 @@ script:
 - ./tools/lint.py
 - ./tools/test_format.py
 - ./tools/build.py -C target/release
-- DENO_BUILD_MODE=release ./tools/test.py -v
+- DENO_BUILD_MODE=release ./tools/test.py
 
 jobs:
   fast_finish: true

tools/benchmark.py

@@ -11,7 +11,7 @@
 import json
 import time
 import shutil
-from util import run, run_output, root_path, build_path, executable_suffix
+from util import root_path, run, run_output, build_path, executable_suffix
 import tempfile
 import http_server
 import throughput_benchmark
@@ -212,7 +212,8 @@ def main(argv):
         print "Usage: tools/benchmark.py [build_dir]"
         sys.exit(1)
 
-    sha1 = run_output(["git", "rev-parse", "HEAD"]).strip()
+    sha1 = run_output(["git", "rev-parse", "HEAD"],
+                      exit_on_fail=True).out.strip()
     http_server.spawn()
 
     deno_exe = os.path.join(build_dir, "deno")

tools/deno_dir_test.py

@@ -5,7 +5,7 @@
 import os
 
 from test_util import DenoTestCase, run_tests
-from util import mkdtemp, rmtree, run
+from util import mkdtemp, rmtree, run_output
 
 
 class TestDenoDir(DenoTestCase):
@@ -38,7 +38,8 @@ def test_deno_dir(self):
     def run_deno(self, deno_dir=None):
         cmd = [self.deno_exe, "run", "tests/002_hello.ts"]
         deno_dir_env = {"DENO_DIR": deno_dir} if deno_dir is not None else None
-        run(cmd, quiet=True, env=deno_dir_env)
+        res = run_output(cmd, quiet=True, env=deno_dir_env)
+        self.assertEqual(res.code, 0)
 
 
 if __name__ == '__main__':

tools/fetch_test.py

@@ -2,20 +2,23 @@
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
 import shutil
+import sys
 
 import http_server
 from test_util import DenoTestCase, run_tests
 from util import mkdtemp, tests_path, run_output
 
 
-class FetchTest(DenoTestCase):
+class TestFetch(DenoTestCase):
     def test_fetch(self):
         deno_dir = mkdtemp()
         try:
             t = os.path.join(tests_path, "006_url_imports.ts")
-            output = run_output([self.deno_exe, "fetch", t],
+            result = run_output([self.deno_exe, "fetch", t],
+                                quiet=True,
                                 merge_env={"DENO_DIR": deno_dir})
-            assert output == ""
+            self.assertEqual(result.out, "")
+            self.assertEqual(result.code, 0)
             # Check that we actually did the prefetch.
             os.path.exists(
                 os.path.join(

tools/fmt_test.py

@@ -4,10 +4,10 @@
 import shutil
 
 from test_util import DenoTestCase, run_tests
-from util import mkdtemp, root_path, tests_path, run
+from util import mkdtemp, root_path, tests_path, run_output
 
 
-class FmtTest(DenoTestCase):
+class TestFmt(DenoTestCase):
     def test_fmt(self):
         d = mkdtemp()
         try:
@@ -26,12 +26,15 @@ def test_fmt(self):
             # TODO(kt3k) Below can be run([deno_exe, "fmt", dst], ...)
             # once the following issue is addressed:
             # https://github.com/denoland/deno_std/issues/330
-            run([
+            result = run_output([
                 os.path.join(root_path, self.deno_exe), "fmt",
                 "badly_formatted.js"
             ],
-                cwd=d,
-                merge_env={"DENO_DIR": deno_dir})
+                                cwd=d,
+                                merge_env={"DENO_DIR": deno_dir},
+                                exit_on_fail=True,
+                                quiet=True)
+            self.assertEqual(result.code, 0)
             with open(fixed_filename) as f:
                 expected = f.read()
             with open(dst) as f:

tools/http_server.py

@@ -17,8 +17,17 @@
 ANOTHER_REDIRECT_PORT = 4547
 DOUBLE_REDIRECTS_PORT = 4548
 
+QUIET = '-v' not in sys.argv and '--verbose' not in sys.argv
 
-class ContentTypeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+
+class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    def log_request(self, code='-', size='-'):
+        if not QUIET:
+            SimpleHTTPServer.SimpleHTTPRequestHandler.log_request(
+                self, code, size)
+
+
+class ContentTypeHandler(QuietSimpleHTTPRequestHandler):
     def do_GET(self):
         if "multipart_form_data.txt" in self.path:
             self.protocol_version = 'HTTP/1.1'
@@ -102,15 +111,16 @@ def server():
     })
     SocketServer.TCPServer.allow_reuse_address = True
     s = SocketServer.TCPServer(("", PORT), Handler)
-    print "Deno test server http://localhost:%d/" % PORT
+    if not QUIET:
+        print "Deno test server http://localhost:%d/" % PORT
     return RunningServer(s, start(s))
 
 
 def base_redirect_server(host_port, target_port, extra_path_segment=""):
     os.chdir(root_path)
     target_host = "http://localhost:%d" % target_port
 
-    class RedirectHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    class RedirectHandler(QuietSimpleHTTPRequestHandler):
         def do_GET(self):
             self.send_response(301)
             self.send_header('Location',
@@ -120,8 +130,9 @@ def do_GET(self):
     Handler = RedirectHandler
     SocketServer.TCPServer.allow_reuse_address = True
     s = SocketServer.TCPServer(("", host_port), Handler)
-    print "redirect server http://localhost:%d/ -> http://localhost:%d/" % (
-        host_port, target_port)
+    if not QUIET:
+        print "redirect server http://localhost:%d/ -> http://localhost:%d/" % (
+            host_port, target_port)
     return RunningServer(s, start(s))
 
 
@@ -153,7 +164,8 @@ def start(s):
 def spawn():
     servers = (server(), redirect_server(), another_redirect_server(),
                double_redirects_server())
-    sleep(1)  # TODO I'm too lazy to figure out how to do this properly.
+    while any(not s.thread.is_alive() for s in servers):
+        sleep(0.01)
     try:
         yield
     finally:

tools/setup.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import third_party
-from util import build_mode, build_path, enable_ansi_colors, root_path, run
-from util import shell_quote, run_output
+from util import (build_mode, build_path, enable_ansi_colors, root_path, run,
+                  shell_quote)
 import os
 import re
 import sys

tools/target_test.py

@@ -2,7 +2,7 @@
 import sys
 
 from test_util import DenoTestCase, run_tests
-from util import executable_suffix, run, tests_path, run_output
+from util import executable_suffix, tests_path, run, run_output
 
 
 class TestTarget(DenoTestCase):
@@ -20,7 +20,7 @@ def _test(self, executable):
         "Test executable runs and exits with code 0."
         bin_file = os.path.join(self.build_dir, executable + executable_suffix)
         self.check_exists(bin_file)
-        run([bin_file])
+        run([bin_file], quiet=True)
 
     def test_libdeno(self):
         self._test("libdeno_test")
@@ -35,26 +35,31 @@ def test_core_http_benchmark(self):
         self._test("deno_core_http_bench_test")
 
     def test_ts_library_builder(self):
-        run([
+        result = run_output([
             "node", "./node_modules/.bin/ts-node", "--project",
             "tools/ts_library_builder/tsconfig.json",
             "tools/ts_library_builder/test.ts"
-        ])
+        ],
+                            quiet=True)
+        self.assertEqual(result.code, 0)
+        assert "ts_library_builder ok" in result.out
 
     def test_no_color(self):
         t = os.path.join(tests_path, "no_color.js")
-        output = run_output([self.deno_exe, "run", t],
-                            merge_env={"NO_COLOR": "1"})
-        assert output.strip() == "noColor true"
+        result = run_output([self.deno_exe, "run", t],
+                            merge_env={"NO_COLOR": "1"},
+                            quiet=True)
+        assert result.out.strip() == "noColor true"
         t = os.path.join(tests_path, "no_color.js")
-        output = run_output([self.deno_exe, "run", t])
-        assert output.strip() == "noColor false"
+        result = run_output([self.deno_exe, "run", t], quiet=True)
+        assert result.out.strip() == "noColor false"
 
     def test_exec_path(self):
         cmd = [self.deno_exe, "run", "tests/exec_path.ts"]
-        output = run_output(cmd)
-        assert self.deno_exe in output.strip()
+        result = run_output(cmd, quiet=True)
+        assert self.deno_exe in result.out.strip()
+        self.assertEqual(result.code, 0)
 
 
-if __name__ == "main":
+if __name__ == "__main__":
     run_tests()

tools/test.py

@@ -6,8 +6,8 @@
 
 from benchmark_test import TestBenchmark
 from deno_dir_test import TestDenoDir
-from fetch_test import FetchTest
-from fmt_test import FmtTest
+from fetch_test import TestFetch
+from fmt_test import TestFmt
 from integration_tests import TestIntegrations
 from repl_test import TestRepl
 from setup_test import TestSetup
@@ -21,7 +21,7 @@
 
 import http_server
 from util import (enable_ansi_colors, build_path, RESET, FG_RED, FG_GREEN,
-                  executable_suffix, run, run_output, rmtree, tests_path)
+                  executable_suffix, rmtree, tests_path)
 from test_util import parse_test_args, run_tests
 
 
@@ -40,8 +40,8 @@ def main():
         TestUtil,
         TestTarget,
         JsUnitTests,
-        FetchTest,
-        FmtTest,
+        TestFetch,
+        TestFmt,
         TestIntegrations,
         TestRepl,
         TestDenoDir,

tools/test_format.py

@@ -9,11 +9,12 @@
 
 def main():
     util.run([sys.executable, "tools/format.py"])
-    output = util.run_output(
-        ["git", "status", "-uno", "--porcelain", "--ignore-submodules"])
-    if len(output) > 0:
+    result = util.run_output(
+        ["git", "status", "-uno", "--porcelain", "--ignore-submodules"],
+        exit_on_fail=True)
+    if result.out:
         print "Run tools/format.py "
-        print output
+        print result.out
         sys.exit(1)
 
 

tools/test_util.py

@@ -3,12 +3,13 @@
 # Runs the full test suite.
 # Usage: ./tools/test.py out/Debug
 import argparse
+import contextlib
 import os
 import sys
 import unittest
 
 from util import (enable_ansi_colors, build_path, RESET, FG_RED, FG_GREEN,
-                  executable_suffix, run, run_output, rmtree, tests_path)
+                  executable_suffix, rmtree, tests_path)
 
 
 class DenoTestCase(unittest.TestCase):
@@ -22,32 +23,31 @@ def setUpClass(cls):
 
 # overload the test result class
 class ColorTextTestResult(unittest.TextTestResult):
+    @contextlib.contextmanager
+    def color(self, code):
+        self.stream.write(code)
+        try:
+            yield
+        finally:
+            self.stream.write(RESET)
+
     def getDescription(self, test):
         name = str(test)
         if name.startswith("test_"):
             name = name[5:]
         return name
 
     def addSuccess(self, test):
-        if self.showAll:
-            self.stream.write(FG_GREEN)
-        super(ColorTextTestResult, self).addSuccess(test)
-        if self.showAll:
-            self.stream.write(RESET)
+        with self.color(FG_GREEN):
+            super(ColorTextTestResult, self).addSuccess(test)
 
     def addError(self, test, err):
-        if self.showAll:
-            self.stream.write(FG_RED)
-        super(ColorTextTestResult, self).addError(test, err)
-        if self.showAll:
-            self.stream.write(RESET)
+        with self.color(FG_RED):
+            super(ColorTextTestResult, self).addError(test, err)
 
     def addFailure(self, test, err):
-        if self.showAll:
-            self.stream.write(FG_RED)
-        super(ColorTextTestResult, self).addFailure(test, err)
-        if self.showAll:
-            self.stream.write(RESET)
+        with self.color(FG_RED):
+            super(ColorTextTestResult, self).addFailure(test, err)
 
 
 class ColorTextTestRunner(unittest.TextTestRunner):
@@ -133,7 +133,7 @@ def run_tests(test_cases=None):
         suite = unittest.TestSuite(filtered_tests)
 
     runner = ColorTextTestRunner(
-        verbosity=args.verbose + 1, failfast=args.failfast)
+        verbosity=args.verbose + 2, failfast=args.failfast)
 
     result = runner.run(suite)
     if not result.wasSuccessful():