fix(bigquery)!: Canonicalize struct & array inline constructor (#3751)

* fix(bigquery): Fix parsing of struct & array inline constructor

* Add more comments on _parse_type

Author: VaggelisD

sqlglot/dialects/duckdb.py

@@ -116,17 +116,24 @@ def _build_make_timestamp(args: t.List) -> exp.Expression:
 
 def _struct_sql(self: DuckDB.Generator, expression: exp.Struct) -> str:
     args: t.List[str] = []
+
+    # BigQuery allows inline construction such as "STRUCT<a STRING, b INTEGER>('str', 1)" which is
+    # canonicalized to "ROW('str', 1) AS STRUCT(a TEXT, b INT)" in DuckDB
+    is_struct_cast = expression.find_ancestor(exp.Cast)
+
     for i, expr in enumerate(expression.expressions):
-        if isinstance(expr, exp.PropertyEQ):
-            key = expr.name
-            value = expr.expression
+        is_property_eq = isinstance(expr, exp.PropertyEQ)
+        value = expr.expression if is_property_eq else expr
+
+        if is_struct_cast:
+            args.append(self.sql(value))
         else:
-            key = f"_{i}"
-            value = expr
+            key = expr.name if is_property_eq else f"_{i}"
+            args.append(f"{self.sql(exp.Literal.string(key))}: {self.sql(value)}")
 
-        args.append(f"{self.sql(exp.Literal.string(key))}: {self.sql(value)}")
+    csv_args = ", ".join(args)
 
-    return f"{{{', '.join(args)}}}"
+    return f"ROW({csv_args})" if is_struct_cast else f"{{{csv_args}}}"
 
 
 def _datatype_sql(self: DuckDB.Generator, expression: exp.DataType) -> str:

sqlglot/parser.py

@@ -4264,6 +4264,13 @@ def _parse_type(
         index = self._index
         data_type = self._parse_types(check_func=True, allow_identifiers=False)
 
+        # parse_types() returns a Cast if we parsed BQ's inline constructor <type>(<values>) e.g.
+        # STRUCT<a INT, b STRING>(1, 'foo'), which is canonicalized to CAST(<values> AS <type>)
+        if isinstance(data_type, exp.Cast):
+            # This constructor can contain ops directly after it, for instance struct unnesting:
+            # STRUCT<a INT, b STRING>(1, 'foo').* --> CAST(STRUCT(1, 'foo') AS STRUCT<a iNT, b STRING).*
+            return self._parse_column_ops(data_type)
+
         if data_type:
             index2 = self._index
             this = self._parse_primary()
@@ -4483,9 +4490,14 @@ def _parse_types(
                 this=exp.DataType.Type[type_token.value],
                 expressions=expressions,
                 nested=nested,
-                values=values,
                 prefix=prefix,
             )
+
+            # Empty arrays/structs are allowed
+            if values is not None:
+                cls = exp.Struct if is_struct else exp.Array
+                this = exp.cast(cls(expressions=values), this, copy=False)
+
         elif expressions:
             this.set("expressions", expressions)
 

tests/dialects/test_bigquery.py

@@ -138,7 +138,6 @@ def test_bigquery(self):
         self.validate_identity("SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day")
         self.validate_identity("SAFE_CAST(encrypted_value AS STRING FORMAT 'BASE64')")
         self.validate_identity("CAST(encrypted_value AS STRING FORMAT 'BASE64')")
-        self.validate_identity("CAST(STRUCT<a INT64>(1) AS STRUCT<a INT64>)")
         self.validate_identity("STRING_AGG(a)")
         self.validate_identity("STRING_AGG(a, ' & ')")
         self.validate_identity("STRING_AGG(DISTINCT a, ' & ')")
@@ -162,12 +161,9 @@ def test_bigquery(self):
         self.validate_identity("x <> ''")
         self.validate_identity("DATE_TRUNC(col, WEEK(MONDAY))")
         self.validate_identity("SELECT b'abc'")
-        self.validate_identity("""SELECT * FROM UNNEST(ARRAY<STRUCT<x INT64>>[])""")
         self.validate_identity("SELECT AS STRUCT 1 AS a, 2 AS b")
         self.validate_identity("SELECT DISTINCT AS STRUCT 1 AS a, 2 AS b")
         self.validate_identity("SELECT AS VALUE STRUCT(1 AS a, 2 AS b)")
-        self.validate_identity("SELECT STRUCT<ARRAY<STRING>>(['2023-01-17'])")
-        self.validate_identity("SELECT STRUCT<STRING>((SELECT a FROM b.c LIMIT 1)).*")
         self.validate_identity("SELECT * FROM q UNPIVOT(values FOR quarter IN (b, c))")
         self.validate_identity("""CREATE TABLE x (a STRUCT<values ARRAY<INT64>>)""")
         self.validate_identity("""CREATE TABLE x (a STRUCT<b STRING OPTIONS (description='b')>)""")
@@ -1613,7 +1609,7 @@ def test_gap_fill(self):
             "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'null')], origin => CAST('2023-11-01 09:30:01' AS DATETIME)) ORDER BY time"
         )
         self.validate_identity(
-            "SELECT * FROM GAP_FILL(TABLE (SELECT * FROM UNNEST(ARRAY<STRUCT<device_id INT64, time DATETIME, signal INT64, state STRING>>[STRUCT(1, CAST('2023-11-01 09:34:01' AS DATETIME), 74, 'INACTIVE'), STRUCT(2, CAST('2023-11-01 09:36:00' AS DATETIME), 77, 'ACTIVE'), STRUCT(3, CAST('2023-11-01 09:37:00' AS DATETIME), 78, 'ACTIVE'), STRUCT(4, CAST('2023-11-01 09:38:01' AS DATETIME), 80, 'ACTIVE')])), ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'linear')]) ORDER BY time"
+            "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time"
         )
 
     def test_models(self):
@@ -1763,3 +1759,49 @@ def test_mod(self):
             "MOD((a + 1), b)",
             "MOD(a + 1, b)",
         )
+
+    def test_inline_constructor(self):
+        self.validate_identity(
+            """SELECT STRUCT<ARRAY<STRING>>(["2023-01-17"])""",
+            """SELECT CAST(STRUCT(['2023-01-17']) AS STRUCT<ARRAY<STRING>>)""",
+        )
+        self.validate_identity(
+            """SELECT STRUCT<STRING>((SELECT 'foo')).*""",
+            """SELECT CAST(STRUCT((SELECT 'foo')) AS STRUCT<STRING>).*""",
+        )
+
+        self.validate_all(
+            "SELECT ARRAY<INT>[1, 2, 3]",
+            write={
+                "bigquery": "SELECT CAST([1, 2, 3] AS ARRAY<INT64>)",
+                "duckdb": "SELECT CAST([1, 2, 3] AS INT[])",
+            },
+        )
+        self.validate_all(
+            "CAST(STRUCT<a INT64>(1) AS STRUCT<a INT64>)",
+            write={
+                "bigquery": "CAST(CAST(STRUCT(1) AS STRUCT<a INT64>) AS STRUCT<a INT64>)",
+                "duckdb": "CAST(CAST(ROW(1) AS STRUCT(a BIGINT)) AS STRUCT(a BIGINT))",
+            },
+        )
+        self.validate_all(
+            "SELECT * FROM UNNEST(ARRAY<STRUCT<x INT64>>[])",
+            write={
+                "bigquery": "SELECT * FROM UNNEST(CAST([] AS ARRAY<STRUCT<x INT64>>))",
+                "duckdb": "SELECT * FROM UNNEST(CAST([] AS STRUCT(x BIGINT)[]))",
+            },
+        )
+        self.validate_all(
+            "SELECT * FROM UNNEST(ARRAY<STRUCT<device_id INT64, time DATETIME, signal INT64, state STRING>>[STRUCT(1, DATETIME '2023-11-01 09:34:01', 74, 'INACTIVE'),STRUCT(4, DATETIME '2023-11-01 09:38:01', 80, 'ACTIVE')])",
+            write={
+                "bigquery": "SELECT * FROM UNNEST(CAST([STRUCT(1, CAST('2023-11-01 09:34:01' AS DATETIME), 74, 'INACTIVE'), STRUCT(4, CAST('2023-11-01 09:38:01' AS DATETIME), 80, 'ACTIVE')] AS ARRAY<STRUCT<device_id INT64, time DATETIME, signal INT64, state STRING>>))",
+                "duckdb": "SELECT * FROM UNNEST(CAST([ROW(1, CAST('2023-11-01 09:34:01' AS TIMESTAMP), 74, 'INACTIVE'), ROW(4, CAST('2023-11-01 09:38:01' AS TIMESTAMP), 80, 'ACTIVE')] AS STRUCT(device_id BIGINT, time TIMESTAMP, signal BIGINT, state TEXT)[]))",
+            },
+        )
+        self.validate_all(
+            "SELECT STRUCT<a INT64, b STRUCT<c STRING>>(1, STRUCT('c_str'))",
+            write={
+                "bigquery": "SELECT CAST(STRUCT(1, STRUCT('c_str')) AS STRUCT<a INT64, b STRUCT<c STRING>>)",
+                "duckdb": "SELECT CAST(ROW(1, ROW('c_str')) AS STRUCT(a BIGINT, b STRUCT(c TEXT)))",
+            },
+        )

tests/dialects/test_duckdb.py

@@ -1042,11 +1042,11 @@ def test_cast(self):
         )
         self.validate_identity(
             "CAST([[STRUCT_PACK(a := 1)]] AS STRUCT(a BIGINT)[][])",
-            "CAST([[{'a': 1}]] AS STRUCT(a BIGINT)[][])",
+            "CAST([[ROW(1)]] AS STRUCT(a BIGINT)[][])",
         )
         self.validate_identity(
             "CAST([STRUCT_PACK(a := 1)] AS STRUCT(a BIGINT)[])",
-            "CAST([{'a': 1}] AS STRUCT(a BIGINT)[])",
+            "CAST([ROW(1)] AS STRUCT(a BIGINT)[])",
         )
 
         self.validate_all(

tests/fixtures/identity.sql

@@ -777,14 +777,6 @@ ALTER TABLE table1 RENAME COLUMN IF EXISTS c1 TO c2
 SELECT div.a FROM test_table AS div
 WITH view AS (SELECT 1 AS x) SELECT * FROM view
 ARRAY<STRUCT<INT, DOUBLE, ARRAY<INT>>>
-ARRAY<INT>[1, 2, 3]
-ARRAY<INT>[]
-STRUCT<x VARCHAR(10)>
-STRUCT<x VARCHAR(10)>("bla")
-STRUCT<VARCHAR(10)>("bla")
-STRUCT<INT>(5)
-STRUCT<DATE>("2011-05-05")
-STRUCT<x INT, y TEXT>(1, t.str_col)
 STRUCT<int INT>
 SELECT CAST(NULL AS ARRAY<INT>) IS NULL AS array_is_null
 ALTER TABLE "schema"."tablename" ADD CONSTRAINT "CHK_Name" CHECK (NOT "IdDwh" IS NULL AND "IdDwh" <> (0))