feat: reconfigure tqdm progress bar in %%bigquery magic (#1355)

* feat: add bigquery job id to tqdm progress bar description

Change-Id: I2add62e3cdd5f25f88ace2d08f212796918158b6

* write to sys.stdout instead of sys.stderr

Change-Id: I6c4001608af1bd8c305c53c6089d64f99605bd8c

* configure progress bar

Change-Id: I5788448d580b53898e75fba68ff5d5a9d12e33d6

* tqdm.notebook

Change-Id: I87e45085b7535083327a5fe2e51dba4b6411db00

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* reinclude ipywidgets

Change-Id: Ibe0fc01db05fcfaacdbe0c074b841ead3a39afc9

* reinclude ipywidgets

Change-Id: I56f8f98853e83ead0e0ca743c03407a521370233

* change test assertions to tqdm_notebook

Change-Id: I2d55e529142ad0024ef4a98c2f15d10a73535380

* change test assertions in test_magics

Change-Id: I7961ff1c5e9c54930d077e67ef9e01d79e351c5f

* remove ipywidgets

Change-Id: I183e277fc7be8797c85d6802f4f8c3947871d4cc

* update assertions in test

Change-Id: I3b4a1b9460227ca49bf344362efbcc2c895d804d

* update method args in query.py and table.py

Change-Id: I9a2bf2b54579668ff36ed992e599f4c7fabe918c

* string formatting

* fix typo

* fix incorrect import structure for tqdm notebook

* change default decorator back to tqdm

* modify system test

* add ipywidgets package for tqdm.notebook feature, set tqdm.notebook as default decorator for bq magic

* change test assertion in test_query_pandas

* revert test changes

* reformat import statement

* reformat import statement

* remove timeouterror side effect

* add tqdm mock patch

* Revert "reformat import statement"

This reverts commit 4114221.

* Revert "add tqdm mock patch"

This reverts commit ef809a0.

* add timeout side effect

* fix assertion

* fix import

* change mock patch to tqdm

* move assertion

* move assertions

* add timeout side effect

* adjust import statement, mock.patch tqdm

* create fixture

* revert import change

* add import from helper

* fix linting

* remove unused imort

* set ipywidgets version to 7.7.1

* set ipywidgets version to 7.7.1

* set ipywidgets version to 7.7.1

* bump sphinx version

* bump sphinx version

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: aribray

google/cloud/bigquery/_tqdm_helpers.py

@@ -15,13 +15,15 @@
 """Shared helper functions for tqdm progress bar."""
 
 import concurrent.futures
+import sys
 import time
 import typing
 from typing import Optional
 import warnings
 
 try:
     import tqdm  # type: ignore
+    import tqdm.notebook as notebook  # type: ignore
 
 except ImportError:  # pragma: NO COVER
     tqdm = None
@@ -47,9 +49,22 @@ def get_progress_bar(progress_bar_type, description, total, unit):
 
     try:
         if progress_bar_type == "tqdm":
-            return tqdm.tqdm(desc=description, total=total, unit=unit)
+            return tqdm.tqdm(
+                bar_format="{l_bar}{bar}|",
+                colour="green",
+                desc=description,
+                file=sys.stdout,
+                total=total,
+                unit=unit,
+            )
         elif progress_bar_type == "tqdm_notebook":
-            return tqdm.notebook.tqdm(desc=description, total=total, unit=unit)
+            return notebook.tqdm(
+                bar_format="{l_bar}{bar}|",
+                desc=description,
+                file=sys.stdout,
+                total=total,
+                unit=unit,
+            )
         elif progress_bar_type == "tqdm_gui":
             return tqdm.tqdm_gui(desc=description, total=total, unit=unit)
     except (KeyError, TypeError):
@@ -80,7 +95,7 @@ def wait_for_query(
     """
     default_total = 1
     current_stage = None
-    start_time = time.time()
+    start_time = time.perf_counter()
 
     progress_bar = get_progress_bar(
         progress_bar_type, "Query is running", default_total, "query"
@@ -95,19 +110,15 @@ def wait_for_query(
             current_stage = query_job.query_plan[i]
             progress_bar.total = len(query_job.query_plan)
             progress_bar.set_description(
-                "Query executing stage {} and status {} : {:0.2f}s".format(
-                    current_stage.name,
-                    current_stage.status,
-                    time.time() - start_time,
-                ),
+                f"Query executing stage {current_stage.name} and status {current_stage.status} : {time.perf_counter() - start_time:.2f}s"
             )
         try:
             query_result = query_job.result(
                 timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=max_results
             )
             progress_bar.update(default_total)
             progress_bar.set_description(
-                "Query complete after {:0.2f}s".format(time.time() - start_time),
+                f"Job ID {query_job.job_id} successfully executed",
             )
             break
         except concurrent.futures.TimeoutError:

google/cloud/bigquery/job/query.py

@@ -1556,9 +1556,9 @@ def to_arrow(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a

google/cloud/bigquery/magics/magics.py

@@ -125,7 +125,7 @@ def __init__(self):
         self._default_query_job_config = bigquery.QueryJobConfig()
         self._bigquery_client_options = client_options.ClientOptions()
         self._bqstorage_client_options = client_options.ClientOptions()
-        self._progress_bar_type = "tqdm"
+        self._progress_bar_type = "tqdm_notebook"
 
     @property
     def credentials(self):
@@ -269,7 +269,7 @@ def progress_bar_type(self):
             Manually setting the progress_bar_type:
 
             >>> from google.cloud.bigquery import magics
-            >>> magics.context.progress_bar_type = "tqdm"
+            >>> magics.context.progress_bar_type = "tqdm_notebook"
         """
         return self._progress_bar_type
 
@@ -286,7 +286,7 @@ def _handle_error(error, destination_var=None):
 
     Args:
         error (Exception):
-            An exception that ocurred during the query exectution.
+            An exception that ocurred during the query execution.
         destination_var (Optional[str]):
             The name of the IPython session variable to store the query job.
     """
@@ -329,22 +329,25 @@ def _run_query(client, query, job_config=None):
         Query complete after 2.07s
         'bf633912-af2c-4780-b568-5d868058632b'
     """
-    start_time = time.time()
+    start_time = time.perf_counter()
     query_job = client.query(query, job_config=job_config)
 
     if job_config and job_config.dry_run:
         return query_job
 
-    print("Executing query with job ID: {}".format(query_job.job_id))
+    print(f"Executing query with job ID: {query_job.job_id}")
 
     while True:
-        print("\rQuery executing: {:0.2f}s".format(time.time() - start_time), end="")
+        print(
+            f"\rQuery executing: {time.perf_counter() - start_time:.2f}s".format(),
+            end="",
+        )
         try:
             query_job.result(timeout=0.5)
             break
         except futures.TimeoutError:
             continue
-    print("\nQuery complete after {:0.2f}s".format(time.time() - start_time))
+    print(f"\nJob ID {query_job.job_id} successfully executed")
     return query_job
 
 
@@ -365,7 +368,7 @@ def _create_dataset_if_necessary(client, dataset_id):
         pass
     dataset = bigquery.Dataset(dataset_reference)
     dataset.location = client.location
-    print("Creating dataset: {}".format(dataset_id))
+    print(f"Creating dataset: {dataset_id}")
     dataset = client.create_dataset(dataset)
 
 
@@ -500,7 +503,7 @@ def _create_dataset_if_necessary(client, dataset_id):
     default=None,
     help=(
         "Sets progress bar type to display a progress bar while executing the query."
-        "Defaults to use tqdm. Install the ``tqdm`` package to use this feature."
+        "Defaults to use tqdm_notebook. Install the ``tqdm`` package to use this feature."
     ),
 )
 def _cell_magic(line, query):

google/cloud/bigquery/table.py

@@ -1728,9 +1728,9 @@ def to_arrow(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a
@@ -1921,9 +1921,9 @@ def to_dataframe(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a
@@ -2075,9 +2075,9 @@ def to_geodataframe(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a

noxfile.py

@@ -81,7 +81,7 @@ def default(session, install_extras=True):
     )
 
     if install_extras and session.python == "3.10":
-        install_target = ".[bqstorage,pandas,tqdm,opentelemetry]"
+        install_target = ".[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
     elif install_extras:
         install_target = ".[all]"
     else:
@@ -186,7 +186,7 @@ def system(session):
     session.install("google-cloud-datacatalog", "-c", constraints_path)
 
     if session.python == "3.10":
-        extras = "[bqstorage,pandas,tqdm,opentelemetry]"
+        extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
     else:
         extras = "[all]"
     session.install("-e", f".{extras}", "-c", constraints_path)
@@ -235,7 +235,7 @@ def snippets(session):
     session.install("grpcio", "-c", constraints_path)
 
     if session.python == "3.10":
-        extras = "[bqstorage,pandas,tqdm,opentelemetry]"
+        extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
     else:
         extras = "[all]"
     session.install("-e", f".{extras}", "-c", constraints_path)
@@ -387,7 +387,7 @@ def blacken(session):
 def docs(session):
     """Build the docs."""
 
-    session.install("recommonmark", "sphinx==4.0.1", "sphinx_rtd_theme")
+    session.install("recommonmark", "sphinx==4.0.2", "sphinx_rtd_theme")
     session.install("google-cloud-storage")
     session.install("-e", ".[all]")
 
@@ -412,7 +412,7 @@ def docfx(session):
 
     session.install("-e", ".")
     session.install(
-        "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml"
+        "sphinx==4.0.2", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml"
     )
 
     shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)

samples/magics/requirements.txt

@@ -2,6 +2,7 @@ db-dtypes==1.0.4
 google-cloud-bigquery-storage==2.16.1
 google-auth-oauthlib==0.5.3
 grpcio==1.49.1
+ipywidgets==7.7.1
 ipython===7.31.1; python_version == '3.7'
 ipython===8.0.1; python_version == '3.8'
 ipython==8.5.0; python_version >= '3.9'

samples/snippets/requirements.txt

@@ -3,6 +3,7 @@ google-cloud-bigquery==3.3.3
 google-cloud-bigquery-storage==2.16.1
 google-auth-oauthlib==0.5.3
 grpcio==1.49.1
+ipywidgets==7.7.1
 ipython===7.31.1; python_version == '3.7'
 ipython===8.0.1; python_version == '3.8'
 ipython==8.5.0; python_version >= '3.9'

setup.py

@@ -52,6 +52,7 @@
     # See: https://github.com/googleapis/python-bigquery/issues/757
     "bqstorage": [],
     "pandas": ["pandas>=1.0.0", "db-dtypes>=0.3.0,<2.0.0dev"],
+    "ipywidgets": ["ipywidgets==7.7.1"],
     "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.6.0, <2.0dev"],
     "ipython": ["ipython>=7.0.1,!=8.1.0"],
     "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"],

testing/constraints-3.7.txt

@@ -12,6 +12,7 @@ google-cloud-bigquery-storage==2.0.0
 google-cloud-core==1.4.1
 google-resumable-media==0.6.0
 grpcio==1.47.0
+ipywidgets==7.7.1
 ipython==7.0.1
 opentelemetry-api==1.1.0
 opentelemetry-instrumentation==0.20b0

tests/system/test_magics.py

@@ -71,8 +71,7 @@ def test_bigquery_magic(ipython_interactive):
     # Removes blanks & terminal code (result of display clearing)
     updates = list(filter(lambda x: bool(x) and x != "\x1b[2K", lines))
     assert re.match("Executing query with job ID: .*", updates[0])
-    assert all(re.match("Query executing: .*s", line) for line in updates[1:-1])
-    assert re.match("Query complete after .*s", updates[-1])
+    assert (re.match("Query executing: .*s", line) for line in updates[1:-1])
     assert isinstance(result, pandas.DataFrame)
     assert len(result) == 10  # verify row count
     assert list(result) == ["url", "view_count"]  # verify column names