Skip to content

Commit b5e5502

Browse files
goldmedalgoldmedal
committed
fix upgrade conflict
1 parent 2ea6996 commit b5e5502

File tree

9 files changed

+46
-37
lines changed

9 files changed

+46
-37
lines changed

‎wren-core/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ version = "0.1.0"
1515
[workspace.dependencies]
1616
async-trait = "0.1.80"
1717
# We require the latest sqlparser-rs to support the latest SQL syntax
18-
datafusion = { git = "https://github.com/goldmedal/datafusion.git", branch = "wren/foward-to-compund-access" }
18+
datafusion = { version = "46.0.0" }
1919
env_logger = "0.11.3"
2020
hashbrown = "0.15.2"
2121
log = { version = "0.4.14" }

‎wren-core/core/src/logical_plan/analyze/model_anlayze.rs

+5-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@ use crate::logical_plan::analyze::plan::ModelPlanNode;
22
use crate::logical_plan::utils::{belong_to_mdl, expr_to_columns};
33
use crate::mdl::utils::quoted;
44
use crate::mdl::{AnalyzedWrenMDL, Dataset, SessionStateRef};
5-
use datafusion::catalog_common::TableReference;
65
use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
7-
use datafusion::common::{internal_err, plan_err, Column, DFSchemaRef, Result};
6+
use datafusion::common::{internal_err, plan_err, Column, DFSchemaRef, Result, Spans};
87
use datafusion::config::ConfigOptions;
98
use datafusion::error::DataFusionError;
109
use datafusion::logical_expr::expr::Alias;
@@ -14,6 +13,7 @@ use datafusion::logical_expr::{
1413
Window,
1514
};
1615
use datafusion::optimizer::AnalyzerRule;
16+
use datafusion::sql::TableReference;
1717
use std::cell::{RefCell, RefMut};
1818
use std::collections::{HashMap, HashSet, VecDeque};
1919
use std::fmt::Debug;
@@ -255,6 +255,7 @@ impl ModelAnalyzeRule {
255255
Expr::Column(Column {
256256
relation: Some(relation),
257257
name,
258+
..
258259
}) => {
259260
// only collect the required column if the relation belongs to the mdl
260261
if belong_to_mdl(
@@ -661,7 +662,7 @@ impl ModelAnalyzeRule {
661662
schema: DFSchemaRef,
662663
) -> Result<Transformed<Expr>> {
663664
match expr {
664-
Expr::Column(Column { relation, name }) => {
665+
Expr::Column(Column { relation, name, .. }) => {
665666
if let Some(relation) = relation {
666667
Ok(self.rewrite_column_qualifier(relation, name, alias_model))
667668
} else {
@@ -725,6 +726,7 @@ impl ModelAnalyzeRule {
725726
Transformed::no(Expr::Column(Column {
726727
relation: Some(relation),
727728
name,
729+
spans: Spans::new(),
728730
}))
729731
}
730732
}

‎wren-core/core/src/logical_plan/analyze/plan.rs

+4
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ impl ModelPlanNodeBuilder {
291291
if required_fields.is_empty() {
292292
source_required_fields.insert(
293293
0,
294+
// TODO: remove deprecated wildcard
295+
#[allow(deprecated)]
294296
Expr::Wildcard {
295297
qualifier: None,
296298
options: Box::new(WildcardOptions::default()),
@@ -740,6 +742,8 @@ impl ModelSourceNode {
740742
let mut required_exprs_buffer = BTreeSet::new();
741743
let mut fields_buffer = BTreeSet::new();
742744
for expr in required_exprs.iter() {
745+
// TODO: remove deprecated wildcard
746+
#[allow(deprecated)]
743747
if let Expr::Wildcard { qualifier, .. } = expr {
744748
let model = if let Some(model) = qualifier {
745749
let Some(model) =

‎wren-core/core/src/logical_plan/utils.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ use crate::mdl::{Dataset, SessionStateRef};
66
use datafusion::arrow::datatypes::{
77
DataType, Field, IntervalUnit, Schema, SchemaBuilder, SchemaRef, TimeUnit,
88
};
9-
use datafusion::catalog_common::TableReference;
109
use datafusion::common::plan_err;
1110
use datafusion::common::tree_node::{
1211
Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
@@ -18,6 +17,7 @@ use datafusion::logical_expr::sqlparser::dialect::GenericDialect;
1817
use datafusion::logical_expr::{builder::LogicalTableSource, Expr, TableSource};
1918
use datafusion::sql::sqlparser::ast;
2019
use datafusion::sql::sqlparser::parser::Parser;
20+
use datafusion::sql::TableReference;
2121
use log::debug;
2222
use petgraph::dot::{Config, Dot};
2323
use petgraph::Graph;
@@ -256,6 +256,8 @@ pub fn expr_to_columns(
256256
accum: &mut HashSet<datafusion::common::Column>,
257257
) -> Result<()> {
258258
expr.apply(|expr| {
259+
// TODO: remove deprecated wildcard
260+
#[allow(deprecated)]
259261
match expr {
260262
Expr::Column(qc) => {
261263
accum.insert(qc.clone());

‎wren-core/core/src/mdl/context.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,13 @@ use crate::mdl::manifest::Model;
1111
use crate::mdl::{AnalyzedWrenMDL, SessionStateRef, WrenMDL};
1212
use async_trait::async_trait;
1313
use datafusion::arrow::datatypes::SchemaRef;
14-
use datafusion::catalog::Session;
15-
use datafusion::catalog_common::memory::{MemoryCatalogProvider, MemorySchemaProvider};
14+
use datafusion::catalog::memory::MemoryCatalogProvider;
15+
use datafusion::catalog::{MemorySchemaProvider, Session};
1616
use datafusion::catalog_common::CatalogProvider;
1717
use datafusion::common::Result;
1818
use datafusion::datasource::{TableProvider, TableType, ViewTable};
1919
use datafusion::execution::session_state::SessionStateBuilder;
2020
use datafusion::logical_expr::Expr;
21-
use datafusion::optimizer::analyzer::count_wildcard_rule::CountWildcardRule;
2221
use datafusion::optimizer::analyzer::expand_wildcard_rule::ExpandWildcardRule;
2322
use datafusion::optimizer::analyzer::inline_table_scan::InlineTableScan;
2423
use datafusion::optimizer::analyzer::type_coercion::TypeCoercion;
@@ -111,7 +110,6 @@ fn analyze_rule_for_local_runtime(
111110
Arc::new(ExpandWildcardRule::new()),
112111
// [Expr::Wildcard] should be expanded before [TypeCoercion]
113112
Arc::new(TypeCoercion::new()),
114-
Arc::new(CountWildcardRule::new()),
115113
]
116114
}
117115

‎wren-core/core/src/mdl/dialect/utils.rs

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ pub(crate) fn function_args_to_sql(
2929
) -> Result<Vec<ast::FunctionArg>> {
3030
args.iter()
3131
.map(|e| {
32+
// TODO: remove deprecated wildcard
33+
#[allow(deprecated)]
3234
if matches!(
3335
e,
3436
Expr::Wildcard {

‎wren-core/core/src/mdl/lineage.rs

+5-3
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ fn get_dataset_link_revers_if_need(
321321

322322
#[cfg(test)]
323323
mod test {
324-
use datafusion::common::Column;
324+
use datafusion::common::{Column, Spans};
325325
use datafusion::error::Result;
326326
use datafusion::sql::TableReference;
327327
use std::collections::HashSet;
@@ -409,7 +409,8 @@ mod test {
409409
assert_eq!(a1_concat_b1.len(), 2);
410410
assert!(a1_concat_b1.contains(&Column {
411411
relation: Some(TableReference::full("wrenai", "public", "a")),
412-
name: "b.b1".to_string()
412+
name: "b.b1".to_string(),
413+
spans: Spans::new(),
413414
}));
414415

415416
let a1_concat_c1 = lineage
@@ -419,7 +420,8 @@ mod test {
419420
assert_eq!(a1_concat_c1.len(), 2);
420421
assert!(a1_concat_c1.contains(&Column {
421422
relation: Some(TableReference::full("wrenai", "public", "a")),
422-
name: "b.c.c1".to_string()
423+
name: "b.c.c1".to_string(),
424+
spans: Spans::new(),
423425
}));
424426
Ok(())
425427
}

‎wren-core/core/src/mdl/mod.rs

+23-19
Original file line numberDiff line numberDiff line change
@@ -560,13 +560,13 @@ mod test {
560560
sql,
561561
)
562562
.await?;
563-
let expected = "SELECT profile.totalcost FROM (SELECT totalcost.totalcost FROM \
563+
let expected = "SELECT \"profile\".totalcost FROM (SELECT totalcost.totalcost FROM \
564564
(SELECT __relation__2.p_custkey AS p_custkey, sum(CAST(__relation__2.o_totalprice AS BIGINT)) AS totalcost FROM \
565565
(SELECT __relation__1.c_custkey, orders.o_custkey, orders.o_totalprice, __relation__1.p_custkey FROM \
566566
(SELECT __source.o_custkey AS o_custkey, __source.o_totalprice AS o_totalprice FROM orders AS __source) AS orders RIGHT JOIN \
567-
(SELECT customer.c_custkey, profile.p_custkey FROM (SELECT __source.c_custkey AS c_custkey FROM customer AS __source) AS customer RIGHT JOIN \
568-
(SELECT __source.p_custkey AS p_custkey FROM profile AS __source) AS profile ON customer.c_custkey = profile.p_custkey) AS __relation__1 \
569-
ON orders.o_custkey = __relation__1.c_custkey) AS __relation__2 GROUP BY __relation__2.p_custkey) AS totalcost) AS profile";
567+
(SELECT customer.c_custkey, \"profile\".p_custkey FROM (SELECT __source.c_custkey AS c_custkey FROM customer AS __source) AS customer RIGHT JOIN \
568+
(SELECT __source.p_custkey AS p_custkey FROM \"profile\" AS __source) AS \"profile\" ON customer.c_custkey = \"profile\".p_custkey) AS __relation__1 \
569+
ON orders.o_custkey = __relation__1.c_custkey) AS __relation__2 GROUP BY __relation__2.p_custkey) AS totalcost) AS \"profile\"";
570570
assert_eq!(result, expected);
571571

572572
let sql = "select totalcost from profile where p_sex = 'M'";
@@ -578,16 +578,16 @@ mod test {
578578
)
579579
.await?;
580580
assert_eq!(result,
581-
"SELECT profile.totalcost FROM (SELECT __relation__1.p_sex, __relation__1.totalcost FROM \
582-
(SELECT totalcost.p_custkey, profile.p_sex, totalcost.totalcost FROM (SELECT __relation__2.p_custkey AS p_custkey, \
581+
"SELECT \"profile\".totalcost FROM (SELECT __relation__1.p_sex, __relation__1.totalcost FROM \
582+
(SELECT totalcost.p_custkey, \"profile\".p_sex, totalcost.totalcost FROM (SELECT __relation__2.p_custkey AS p_custkey, \
583583
sum(CAST(__relation__2.o_totalprice AS BIGINT)) AS totalcost FROM (SELECT __relation__1.c_custkey, orders.o_custkey, \
584584
orders.o_totalprice, __relation__1.p_custkey FROM (SELECT __source.o_custkey AS o_custkey, __source.o_totalprice AS o_totalprice \
585-
FROM orders AS __source) AS orders RIGHT JOIN (SELECT customer.c_custkey, profile.p_custkey FROM \
585+
FROM orders AS __source) AS orders RIGHT JOIN (SELECT customer.c_custkey, \"profile\".p_custkey FROM \
586586
(SELECT __source.c_custkey AS c_custkey FROM customer AS __source) AS customer RIGHT JOIN \
587-
(SELECT __source.p_custkey AS p_custkey FROM profile AS __source) AS profile ON customer.c_custkey = profile.p_custkey) AS __relation__1 \
587+
(SELECT __source.p_custkey AS p_custkey FROM \"profile\" AS __source) AS \"profile\" ON customer.c_custkey = \"profile\".p_custkey) AS __relation__1 \
588588
ON orders.o_custkey = __relation__1.c_custkey) AS __relation__2 GROUP BY __relation__2.p_custkey) AS totalcost RIGHT JOIN \
589-
(SELECT __source.p_custkey AS p_custkey, __source.p_sex AS p_sex FROM profile AS __source) AS profile \
590-
ON totalcost.p_custkey = profile.p_custkey) AS __relation__1) AS profile WHERE profile.p_sex = 'M'");
589+
(SELECT __source.p_custkey AS p_custkey, __source.p_sex AS p_sex FROM \"profile\" AS __source) AS \"profile\" \
590+
ON totalcost.p_custkey = \"profile\".p_custkey) AS __relation__1) AS \"profile\" WHERE \"profile\".p_sex = 'M'");
591591
Ok(())
592592
}
593593

@@ -618,7 +618,7 @@ mod test {
618618
assert_eq!(actual,
619619
"SELECT \"Customer\".\"Custkey\", \"Customer\".\"Name\" FROM \
620620
(SELECT __source.\"Custkey\" AS \"Custkey\", __source.\"Name\" AS \"Name\" FROM \
621-
datafusion.public.customer AS __source) AS \"Customer\"");
621+
datafusion.\"public\".customer AS __source) AS \"Customer\"");
622622
Ok(())
623623
}
624624

@@ -655,7 +655,7 @@ mod test {
655655
)
656656
.await?;
657657
assert_eq!(actual, "SELECT add_two(\"Customer\".\"Custkey\") FROM (SELECT \"Customer\".\"Custkey\" \
658-
FROM (SELECT __source.\"Custkey\" AS \"Custkey\" FROM datafusion.public.customer AS __source) AS \"Customer\") AS \"Customer\"");
658+
FROM (SELECT __source.\"Custkey\" AS \"Custkey\" FROM datafusion.\"public\".customer AS __source) AS \"Customer\") AS \"Customer\"");
659659

660660
let actual = transform_sql_with_ctx(
661661
&ctx,
@@ -665,7 +665,7 @@ mod test {
665665
)
666666
.await?;
667667
assert_eq!(actual, "SELECT median(\"Customer\".\"Custkey\") FROM (SELECT \"Customer\".\"Custkey\", \"Customer\".\"Name\" \
668-
FROM (SELECT __source.\"Custkey\" AS \"Custkey\", __source.\"Name\" AS \"Name\" FROM datafusion.public.customer AS __source) AS \"Customer\") AS \"Customer\" \
668+
FROM (SELECT __source.\"Custkey\" AS \"Custkey\", __source.\"Name\" AS \"Name\" FROM datafusion.\"public\".customer AS __source) AS \"Customer\") AS \"Customer\" \
669669
GROUP BY \"Customer\".\"Name\"");
670670

671671
// TODO: support window functions analysis
@@ -925,7 +925,9 @@ mod test {
925925
let sql = "select count(*) from (select 1)";
926926
let actual =
927927
transform_sql_with_ctx(&ctx, Arc::clone(&analyzed_mdl), &[], sql).await?;
928-
assert_eq!(actual, "SELECT count(*) FROM (SELECT 1)");
928+
// TODO: BigQuery doesn't support the alias include invalid characters (e.g. `*`, `()`).
929+
// We should remove the invalid characters for the alias.
930+
assert_eq!(actual, "SELECT count(1) AS \"count(*)\" FROM (SELECT 1)");
929931
Ok(())
930932
}
931933

@@ -1062,7 +1064,9 @@ mod test {
10621064
)
10631065
.await?;
10641066
assert_eq!(actual,
1065-
"SELECT count(*) FROM (SELECT artist.cast_timestamptz FROM \
1067+
// TODO: BigQuery doesn't support the alias include invalid characters (e.g. `*`, `()`).
1068+
// We should remove the invalid characters for the alias.
1069+
"SELECT count(1) AS \"count(*)\" FROM (SELECT artist.cast_timestamptz FROM \
10661070
(SELECT CAST(__source.\"出道時間\" AS TIMESTAMP WITH TIME ZONE) AS cast_timestamptz \
10671071
FROM artist AS __source) AS artist) AS artist WHERE CAST(artist.cast_timestamptz AS TIMESTAMP) > CAST('2011-01-01 21:00:00' AS TIMESTAMP)");
10681072
Ok(())
@@ -1148,7 +1152,7 @@ mod test {
11481152
"SELECT CAST(timestamp_table.timestamp_col AS TIMESTAMP WITH TIME ZONE) = timestamp_table.timestamptz_col \
11491153
FROM (SELECT timestamp_table.timestamp_col, timestamp_table.timestamptz_col FROM \
11501154
(SELECT __source.timestamp_col AS timestamp_col, __source.timestamptz_col AS timestamptz_col \
1151-
FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table");
1155+
FROM datafusion.\"public\".timestamp_table AS __source) AS timestamp_table) AS timestamp_table");
11521156

11531157
let sql = r#"select timestamptz_col > cast('2011-01-01 18:00:00' as TIMESTAMP WITH TIME ZONE) from wren.test.timestamp_table"#;
11541158
let actual = transform_sql_with_ctx(
@@ -1160,7 +1164,7 @@ mod test {
11601164
.await?;
11611165
// assert the simplified literal will be casted to the timestamp tz
11621166
assert_eq!(actual,
1163-
"SELECT timestamp_table.timestamptz_col > CAST(CAST('2011-01-01 18:00:00' AS TIMESTAMP) AS TIMESTAMP WITH TIME ZONE) FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT __source.timestamptz_col AS timestamptz_col FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table"
1167+
"SELECT timestamp_table.timestamptz_col > CAST(CAST('2011-01-01 18:00:00' AS TIMESTAMP) AS TIMESTAMP WITH TIME ZONE) FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT __source.timestamptz_col AS timestamptz_col FROM datafusion.\"public\".timestamp_table AS __source) AS timestamp_table) AS timestamp_table"
11641168
);
11651169

11661170
let sql = r#"select timestamptz_col > '2011-01-01 18:00:00' from wren.test.timestamp_table"#;
@@ -1175,7 +1179,7 @@ mod test {
11751179
assert_eq!(actual,
11761180
"SELECT timestamp_table.timestamptz_col > CAST('2011-01-01 18:00:00' AS TIMESTAMP WITH TIME ZONE) \
11771181
FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT __source.timestamptz_col AS timestamptz_col \
1178-
FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table");
1182+
FROM datafusion.\"public\".timestamp_table AS __source) AS timestamp_table) AS timestamp_table");
11791183

11801184
let sql = r#"select timestamp_col > cast('2011-01-01 18:00:00' as TIMESTAMP WITH TIME ZONE) from wren.test.timestamp_table"#;
11811185
let actual = transform_sql_with_ctx(
@@ -1189,7 +1193,7 @@ mod test {
11891193
assert_eq!(actual,
11901194
"SELECT timestamp_table.timestamp_col > CAST('2011-01-01 18:00:00' AS TIMESTAMP) \
11911195
FROM (SELECT timestamp_table.timestamp_col FROM (SELECT __source.timestamp_col AS timestamp_col \
1192-
FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table");
1196+
FROM datafusion.\"public\".timestamp_table AS __source) AS timestamp_table) AS timestamp_table");
11931197
}
11941198
Ok(())
11951199
}

‎wren-core/sqllogictest/test_files/tpch/q22.slt.part

+1-6
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,7 @@
1616
# specific language governing permissions and limitations
1717
# under the License.
1818

19-
# TODO:
20-
#External error: query failed: DataFusion error: ModelAnalyzeRule
21-
#caused by
22-
#Schema error: No field named customer.c_phone.
23-
24-
query TIR
19+
query ?IR
2520
select
2621
cntrycode,
2722
count(*) as numcust,

0 commit comments

Comments
 (0)