feat(ibis): bump ibis to 10.0.0 (#1060)

Author: goldmedal

ibis-server/Dockerfile

@@ -49,6 +49,9 @@ RUN apt-get update \
 # Install msodbcsql 18 driver for mssql
 RUN ACCEPT_EULA=Y apt-get -y install unixodbc-dev msodbcsql18
 
+# Install libmysqlclient-dev for mysql
+RUN apt-get install -y default-libmysqlclient-dev
+
 # libpq-dev is required for psycopg2
 RUN apt-get -y install libpq-dev \
     && rm -rf /var/lib/apt/lists/*

ibis-server/app/model/__init__.py

@@ -112,7 +112,11 @@ class MySqlConnectionInfo(BaseModel):
     database: SecretStr
     user: SecretStr
     password: SecretStr | None = None
-    ssl_mode: SecretStr | None = Field(alias="sslMode", default=None)
+    ssl_mode: SecretStr | None = Field(
+        alias="sslMode",
+        default="ENABLED",
+        description="Use ssl connection or not. The default value is `ENABLED` because MySQL uses `caching_sha2_password` by default and the driver MySQLdb support caching_sha2_password with ssl only.",
+    )
     ssl_ca: SecretStr | None = Field(alias="sslCA", default=None)
     kwargs: dict[str, str] | None = Field(
         description="Additional keyword arguments to pass to PyMySQL", default=None

ibis-server/app/model/data_source.py

@@ -140,12 +140,7 @@ def get_mssql_connection(cls, info: MSSqlConnectionInfo) -> BaseBackend:
             port=info.port.get_secret_value(),
             database=info.database.get_secret_value(),
             user=info.user.get_secret_value(),
-            password=(
-                info.password
-                and cls._escape_special_characters_for_odbc(
-                    info.password.get_secret_value()
-                )
-            ),
+            password=info.password.get_secret_value(),
             driver=info.driver,
             TDS_Version=info.tds_version,
             **info.kwargs if info.kwargs else dict(),
@@ -197,10 +192,6 @@ def get_trino_connection(info: TrinoConnectionInfo) -> BaseBackend:
             password=(info.password and info.password.get_secret_value()),
         )
 
-    @staticmethod
-    def _escape_special_characters_for_odbc(value: str) -> str:
-        return "{" + value.replace("}", "}}") + "}"
-
     @staticmethod
     def _create_ssl_context(info: ConnectionInfo) -> Optional[ssl.SSLContext]:
         ssl_mode = (

ibis-server/docs/development.md

@@ -92,3 +92,23 @@ If you encounter this error, you can add the `TrustServerCertificate` parameter
   }
 }
 ```
+
+### No driver for MySQL Server
+
+If you want run tests related to MySQL Server or connect to MySQL through Wren Engine, you need to install the MySQL client libraries (e.g. `libmysqlclient`) by yourself.
+- For linux, you can install `libmysqlclient-dev`. By the way, there are some different names for different linux versions. You should take care about it.
+- For Mac, you can install `mysql-connector-c`
+- For Windows, you can dowanload [the libraries](https://dev.mysql.com/downloads/c-api)
+
+
+### Connect MySQL without SSL
+
+By default, SSL mode is enabled and uses `caching_sha2_password` authentication, which only supports SSL connections. If you need to disable SSL, you must set SSLMode to DISABLED in your connection configuration to use `mysql_native_password` instead.
+
+```json
+{
+  "connectionInfo": {
+    "ssl_mode": "DISABLED"
+  }
+}
+```

ibis-server/poetry.lock

@@ -10,7 +10,7 @@ packages = [{ include = "app" }]
 python = ">=3.11,<3.12"
 fastapi = { version = "0.115.8", extras = ["standard"] }
 pydantic = "2.10.6"
-ibis-framework = { version = "9.5.0", extras = [
+ibis-framework = { version = "10.0.0", extras = [
   "bigquery",
   "clickhouse",
   "mssql",
@@ -26,13 +26,14 @@ orjson = "3.10.15"
 pandas = "2.2.3"
 sqlglot = { extras = [
   "rs",
-], version = ">=23.4,<25.21" } # the version should follow the ibis-framework
+], version = ">=23.4,<26.5" } # the version should follow the ibis-framework
 loguru = "0.7.3"
 asgi-correlation-id = "4.3.4"
 gql = { extras = ["aiohttp"], version = "3.5.0" }
 anyio = "4.8.0"
 duckdb = "1.1.3"
 opendal = ">=0.45"
+mysqlclient = { version = ">=2.2.4,<3", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = "8.3.4"

ibis-server/pyproject.toml

@@ -91,7 +91,7 @@ async def test_query(client, manifest_str):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000 UTC",

ibis-server/tests/routers/v2/connector/test_bigquery.py

@@ -95,7 +95,7 @@ async def test_query(client, manifest_str):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000 UTC",

ibis-server/tests/routers/v2/connector/test_canner.py

@@ -180,7 +180,7 @@ async def test_query(client, manifest_str, clickhouse: ClickHouseContainer):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000 UTC",
@@ -519,18 +519,20 @@ async def test_metadata_list_tables(client, clickhouse: ClickHouseContainer):
     )
     assert response.status_code == 200
 
-    result = response.json()[1]
-    assert result["name"] == "test.orders"
-    assert result["primaryKey"] is not None
-    assert result["description"] == "This is a table comment"
-    assert result["properties"] == {
+    results = response.json()
+    orders = next((x for x in results if x["name"] == "test.orders"), None)
+    assert orders is not None
+    assert orders["name"] == "test.orders"
+    assert orders["primaryKey"] is not None
+    assert orders["description"] == "This is a table comment"
+    assert orders["properties"] == {
         "catalog": None,
         "schema": "test",
         "table": "orders",
         "path": None,
     }
-    assert len(result["columns"]) == 9
-    assert result["columns"][8] == {
+    assert len(orders["columns"]) == 9
+    assert orders["columns"][8] == {
         "name": "o_comment",
         "nestedColumns": None,
         "type": "VARCHAR",

ibis-server/tests/routers/v2/connector/test_clickhouse.py

@@ -1,4 +1,5 @@
 import base64
+import urllib
 
 import orjson
 import pandas as pd
@@ -125,7 +126,7 @@ async def test_query(client, manifest_str, mssql: SqlServerContainer):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000 UTC",
@@ -427,6 +428,15 @@ async def test_password_with_special_characters(client):
         assert response.status_code == 200
         assert "Microsoft SQL Server 2019" in response.text
 
+        connection_url = _to_connection_url(mssql)
+        response = await client.post(
+            url=f"{base_url}/metadata/version",
+            json={"connectionInfo": {"connectionUrl": connection_url}},
+        )
+
+        assert response.status_code == 200
+        assert "Microsoft SQL Server 2019" in response.text
+
 
 def _to_connection_info(mssql: SqlServerContainer):
     return {
@@ -441,4 +451,4 @@ def _to_connection_info(mssql: SqlServerContainer):
 
 def _to_connection_url(mssql: SqlServerContainer):
     info = _to_connection_info(mssql)
-    return f"mssql://{info['user']}:{info['password']}@{info['host']}:{info['port']}/{info['database']}?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=YES"
+    return f"mssql://{info['user']}:{urllib.parse.quote_plus(info['password'])}@{info['host']}:{info['port']}/{info['database']}?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=YES"

ibis-server/tests/routers/v2/connector/test_mssql.py

@@ -2,9 +2,10 @@
 
 import orjson
 import pandas as pd
+import pymysql
 import pytest
 import sqlalchemy
-from pymysql import OperationalError
+from MySQLdb import OperationalError
 from sqlalchemy import text
 from testcontainers.mysql import MySqlContainer
 
@@ -116,6 +117,26 @@ def mysql(request) -> MySqlContainer:
 @pytest.fixture(scope="module")
 def mysql_ssl_off(request) -> MySqlContainer:
     mysql = MySqlContainer(image="mysql:8.0.40").with_command("--ssl=0").start()
+    # We disable SSL for this container to test SSLMode.ENABLED.
+    # However, Mysql use caching_sha2_password as default authentication plugin which requires the connection to use SSL.
+    # MysqlDB used by ibis supports caching_sha2_password only with SSL. So, we need to change the authentication plugin to mysql_native_password.
+    # Before changing the authentication plugin, we need to connect to the database using caching_sha2_password. That's why we use pymysql to connect to the database.
+    # pymsql supports caching_sha2_password without SSL.
+    conn = pymysql.connect(
+        host="127.0.0.1",
+        user="root",
+        passwd="test",
+        port=int(mysql.get_exposed_port(mysql.port)),
+    )
+
+    cur = conn.cursor()
+    cur.execute(
+        "ALTER USER 'test'@'%' IDENTIFIED WITH mysql_native_password BY 'test';"
+    )
+    cur.execute("FLUSH PRIVILEGES;")
+    conn.commit()
+    conn.close()
+
     request.addfinalizer(mysql.stop)
     return mysql
 
@@ -139,7 +160,7 @@ async def test_query(client, manifest_str, mysql: MySqlContainer):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000",
@@ -417,7 +438,11 @@ async def test_metadata_db_version(client, mysql: MySqlContainer):
 @pytest.mark.parametrize(
     "ssl_mode, expected_exception, expected_error",
     [
-        (SSLMode.ENABLED, OperationalError, "Bad handshake"),
+        (
+            SSLMode.ENABLED,
+            OperationalError,
+            "SSL connection error: SSL is required but the server doesn't support it",
+        ),
         (
             SSLMode.VERIFY_CA,
             ValueError,

ibis-server/tests/routers/v2/connector/test_mysql.py

@@ -3,7 +3,7 @@
 
 import orjson
 import pandas as pd
-import psycopg2
+import psycopg
 import pytest
 import sqlalchemy
 from sqlalchemy import text
@@ -143,7 +143,7 @@ async def test_query(client, manifest_str, postgres: PostgresContainer):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000 UTC",
@@ -296,8 +296,8 @@ async def test_dry_run_with_connection_url_and_password_with_bracket_should_not_
     ).geturl()
 
     with pytest.raises(
-        psycopg2.OperationalError,
-        match='FATAL:  password authentication failed for user "test"',
+        psycopg.OperationalError,
+        match=r'.*FATAL:  password authentication failed for user "test".*',
     ):
         await client.post(
             url=f"{base_url}/query",

ibis-server/tests/routers/v2/connector/test_postgres.py

@@ -90,7 +90,7 @@ async def test_query(client, manifest_str):
         36901,
         "O",
         "173665.47",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_36901",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000 UTC",

ibis-server/tests/routers/v2/connector/test_snowflake.py

@@ -112,7 +112,7 @@ async def test_query(client, manifest_str, trino: TrinoContainer):
         370,
         "O",
         "172799.49",
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         "1_370",
         "2024-01-01 23:59:59.000000",
         "2024-01-01 23:59:59.000000",

ibis-server/tests/routers/v2/connector/test_trino.py

@@ -94,7 +94,7 @@ async def test_scalar_function(client, manifest_str: str, connection_info):
     result = response.json()
     assert result == {
         "columns": ["col"],
-        "data": [["2024-01-02"]],
+        "data": [["2024-01-02 00:00:00.000000"]],
         "dtypes": {"col": "object"},
     }
 

ibis-server/tests/routers/v3/connector/bigquery/test_functions.py

@@ -84,7 +84,7 @@ async def test_query(client, manifest_str, connection_info):
         "2024-07-16 03:00:00.000000 UTC",  # utc-4
         "36485_1202",
         1202,
-        "1992-06-06",
+        "1992-06-06 00:00:00.000000",
         36485,
         "F",
         "356711.63",

ibis-server/tests/routers/v3/connector/bigquery/test_query.py

@@ -118,7 +118,7 @@ async def test_query(client, manifest_str, connection_info):
         "172799.49",
         "1_370",
         370,
-        "1996-01-02",
+        "1996-01-02 00:00:00.000000",
         1,
         "O",
     ]