fix(pyspark): avoid potentially different field names produced by SQL by using python-native APIs

Author: cpcloud

ibis/backends/pyspark/__init__.py

@@ -229,13 +229,11 @@ def version(self):
 
     @property
     def current_database(self) -> str:
-        [(db,)] = self._session.sql("SELECT CURRENT_DATABASE()").collect()
-        return db
+        return self._session.catalog.currentDatabase()
 
     @property
     def current_catalog(self) -> str:
-        [(catalog,)] = self._session.sql("SELECT CURRENT_CATALOG()").collect()
-        return catalog
+        return self._session.catalog.currentCatalog()
 
     @contextlib.contextmanager
     def _active_catalog_database(self, catalog: str | None, db: str | None):
@@ -349,7 +347,7 @@ def _active_catalog(self, name: str | None):
                 catalog_api.setCurrentDatabase(prev_database)
 
     def list_catalogs(self, *, like: str | None = None) -> list[str]:
-        catalogs = [res.catalog for res in self._session.sql("SHOW CATALOGS").collect()]
+        catalogs = [cat.name for cat in self._session.catalog.listCatalogs()]
         return self._filter_with_like(catalogs, like)
 
     def list_databases(
@@ -378,12 +376,11 @@ def list_tables(
         """
         table_loc = self._to_sqlglot_table(database)
         catalog, db = self._to_catalog_db_tuple(table_loc)
+        if db is None:
+            db = self.current_database
         with self._active_catalog(catalog):
             tables = [
-                row.tableName
-                for row in self._session.sql(
-                    f"SHOW TABLES IN {db or self.current_database}"
-                ).collect()
+                table.name for table in self._session.catalog.listTables(dbName=db)
             ]
         return self._filter_with_like(tables, like)