chore: Increase test parallelism, enable work stealing

Author: TrevorBergeron

noxfile.py

@@ -322,7 +322,7 @@ def run_system(
     print_duration=False,
     extra_pytest_options=(),
     timeout_seconds=900,
-    num_workers=20,
+    num_workers=100,
 ):
     """Run the system test suite."""
     constraints_path = str(
@@ -343,6 +343,7 @@ def run_system(
         "py.test",
         "--quiet",
         f"-n={num_workers}",
+        "--dist worksteal",
         # Any individual test taking longer than 15 mins will be terminated.
         f"--timeout={timeout_seconds}",
         # Log 20 slowest tests
@@ -413,7 +414,7 @@ def doctest(session: nox.sessions.Session):
         ),
         test_folder="bigframes",
         check_cov=True,
-        num_workers=5,
+        num_workers=20,
     )