From 2c64f6ea3afba3d0a49bbc92f0a432fd33083311 Mon Sep 17 00:00:00 2001 From: DouEnergy Date: Mon, 24 Feb 2025 17:30:42 +0800 Subject: [PATCH 1/5] scientific notation formatting --- ibis-server/app/util.py | 10 +++++++--- .../tests/routers/v2/connector/test_bigquery.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ibis-server/app/util.py b/ibis-server/app/util.py index f05f61e2e..25884c7f8 100644 --- a/ibis-server/app/util.py +++ b/ibis-server/app/util.py @@ -29,9 +29,13 @@ def _to_datetime_and_format(series: pd.Series) -> pd.Series: def _to_json_obj(df: pd.DataFrame) -> dict: - data = df.map(lambda x: f"{x:.9g}" if isinstance(x, float) else x).to_dict( - orient="split", index=False - ) + data = df.map( + lambda x: f"{x:.9g}" + if isinstance(x, float) + else f"{x:.3f}" + if isinstance(x, decimal.Decimal) + else x + ).to_dict(orient="split") def default(obj): if pd.isna(obj): diff --git a/ibis-server/tests/routers/v2/connector/test_bigquery.py b/ibis-server/tests/routers/v2/connector/test_bigquery.py index a5be020c7..50cfc43ac 100644 --- a/ibis-server/tests/routers/v2/connector/test_bigquery.py +++ b/ibis-server/tests/routers/v2/connector/test_bigquery.py @@ -198,6 +198,20 @@ async def test_query_values(client, manifest_str): assert response.status_code == 204 +async def test_scientific_notation(client, manifest_str): + response = await client.post( + url=f"{base_url}/query", + json={ + "connectionInfo": connection_info, + "manifestStr": manifest_str, + "sql": "SELECT cast(0 as numeric) as col", + }, + ) + assert response.status_code == 200 + result = response.json() + assert result["data"][0] == ["0.000"] + + async def test_query_empty_json(client, manifest_str): """Test the empty result with json column.""" response = await client.post( From 4bca4387e29ea3eca10e282cdffa62d63343284a Mon Sep 17 00:00:00 2001 From: DouEnergy Date: Tue, 25 Feb 2025 16:50:42 +0800 Subject: [PATCH 2/5] only handel special case 0 --- ibis-server/app/util.py | 17 ++++++++++------- .../tests/routers/v2/connector/test_bigquery.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ibis-server/app/util.py b/ibis-server/app/util.py index 25884c7f8..8576f1c93 100644 --- a/ibis-server/app/util.py +++ b/ibis-server/app/util.py @@ -29,13 +29,16 @@ def _to_datetime_and_format(series: pd.Series) -> pd.Series: def _to_json_obj(df: pd.DataFrame) -> dict: - data = df.map( - lambda x: f"{x:.9g}" - if isinstance(x, float) - else f"{x:.3f}" - if isinstance(x, decimal.Decimal) - else x - ).to_dict(orient="split") + def format_value(x): + if isinstance(x, float): + # Special case for zero to avoid scientific notation + if x == 0: + return "0" + return f"{x:.9g}" + else: + return x + + data = df.map(format_value).to_dict(orient="split", index=False) def default(obj): if pd.isna(obj): diff --git a/ibis-server/tests/routers/v2/connector/test_bigquery.py b/ibis-server/tests/routers/v2/connector/test_bigquery.py index 50cfc43ac..764fbd5d8 100644 --- a/ibis-server/tests/routers/v2/connector/test_bigquery.py +++ b/ibis-server/tests/routers/v2/connector/test_bigquery.py @@ -209,7 +209,7 @@ async def test_scientific_notation(client, manifest_str): ) assert response.status_code == 200 result = response.json() - assert result["data"][0] == ["0.000"] + assert result["data"][0] == ["0"] async def test_query_empty_json(client, manifest_str): From 5d622040315e231d4ec29fa2c2088f4b189d0911 Mon Sep 17 00:00:00 2001 From: DouEnergy Date: Tue, 25 Feb 2025 17:03:36 +0800 Subject: [PATCH 3/5] fix local file flaky test --- ibis-server/tests/routers/v2/connector/test_local_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/tests/routers/v2/connector/test_local_file.py b/ibis-server/tests/routers/v2/connector/test_local_file.py index e882307e2..0f0b6c99c 100644 --- a/ibis-server/tests/routers/v2/connector/test_local_file.py +++ b/ibis-server/tests/routers/v2/connector/test_local_file.py @@ -331,7 +331,7 @@ async def test_list_csv_files(client): assert "type-test" in table_names # `invalid` will be considered as a one column csv file assert "invalid" in table_names - columns = result[0]["columns"] + columns = next(filter(lambda x: x["name"] == "type-test-csv", result))["columns"] assert columns[0]["name"] == "c_bigint" assert columns[0]["type"] == "INT64" assert columns[1]["name"] == "c_bit" From 6af964ebbcea397ebb3c98313154dd9f66185dd8 Mon Sep 17 00:00:00 2001 From: DouEnergy Date: Tue, 25 Feb 2025 17:25:34 +0800 Subject: [PATCH 4/5] decimal 0 handle --- ibis-server/app/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ibis-server/app/util.py b/ibis-server/app/util.py index 8576f1c93..0dbbe3a44 100644 --- a/ibis-server/app/util.py +++ b/ibis-server/app/util.py @@ -31,10 +31,12 @@ def _to_datetime_and_format(series: pd.Series) -> pd.Series: def _to_json_obj(df: pd.DataFrame) -> dict: def format_value(x): if isinstance(x, float): - # Special case for zero to avoid scientific notation - if x == 0: - return "0" return f"{x:.9g}" + elif isinstance(x, decimal.Decimal): + if x.to_integral_value() == 0: + return "0" + else: + return x else: return x From 9342cf5c914539151a7045c293257e9f9e8e7b67 Mon Sep 17 00:00:00 2001 From: DouEnergy Date: Tue, 25 Feb 2025 17:46:30 +0800 Subject: [PATCH 5/5] remove to_integral_value --- ibis-server/app/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/app/util.py b/ibis-server/app/util.py index 0dbbe3a44..9cbed3b77 100644 --- a/ibis-server/app/util.py +++ b/ibis-server/app/util.py @@ -33,7 +33,7 @@ def format_value(x): if isinstance(x, float): return f"{x:.9g}" elif isinstance(x, decimal.Decimal): - if x.to_integral_value() == 0: + if x == 0: return "0" else: return x