Skip to content

feat(core): enhance scope analysis and implement ByPass Function #802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Sep 25, 2024
Merged

feat(core): enhance scope analysis and implement ByPass Function #802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
42aa7da
remove unused return result
goldmedal Sep 19, 2024
1b5fba3
implement ByPass function for udf, udaf and udwf
goldmedal Sep 23, 2024
9b3169f
format and enhance the doc
goldmedal Sep 23, 2024
300e632
create table schema refer to the column type
goldmedal Sep 23, 2024
3480c16
fix the scope analyze for the CTE and subquery
goldmedal Sep 23, 2024
89359db
fix the interval unparsing and enable the tests
goldmedal Sep 23, 2024
39c993b
enable the model test
goldmedal Sep 23, 2024
bba4f62
fix compile fails
goldmedal Sep 24, 2024
1112d8a
fix the python test
goldmedal Sep 24, 2024
b2edc1a
fix ibis test
goldmedal Sep 24, 2024
c38e406
fix ibis test
goldmedal Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 20 additions & 22 deletions ibis-server/tests/routers/v3/connector/test_postgres.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,17 @@
"table": "orders",
},
"columns": [
{"name": "orderkey", "expression": "o_orderkey", "type": "integer"},
{"name": "custkey", "expression": "o_custkey", "type": "integer"},
{"name": "o_orderkey", "type": "integer"},
{"name": "o_custkey", "type": "integer"},
Comment on lines -30 to +31
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't purely rename the column name after this PR.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. In this case, we won't provide any remote schema for them. Wren core will use the column without expression as the remote column.

{
"name": "orderstatus",
"expression": "o_orderstatus",
"name": "o_orderstatus",
"type": "varchar",
},
{
"name": "totalprice",
"expression": "o_totalprice",
"name": "o_totalprice",
"type": "double",
},
{"name": "orderdate", "expression": "o_orderdate", "type": "date"},
{"name": "o_orderdate", "type": "date"},
{
"name": "order_cust_key",
"expression": "concat(o_orderkey, '_', o_custkey)",
Expand All @@ -56,7 +54,7 @@
"type": "timestamp",
},
],
"primaryKey": "orderkey",
"primaryKey": "o_orderkey",
},
{
"name": "customer",
Expand All @@ -65,10 +63,10 @@
"table": "customer",
},
"columns": [
{"name": "custkey", "expression": "c_custkey", "type": "integer"},
{"name": "name", "expression": "c_name", "type": "varchar"},
{"name": "c_custkey", "type": "integer"},
{"name": "c_name", "type": "varchar"},
],
"primaryKey": "custkey",
"primaryKey": "c_custkey",
},
],
}
Expand Down Expand Up @@ -109,17 +107,17 @@ def test_query(postgres: PostgresContainer):
"2024-01-01 23:59:59.000000 UTC",
"1_370",
370,
1,
"1996-01-02",
1,
"O",
"172799.49",
]
assert result["dtypes"] == {
"orderkey": "int32",
"custkey": "int32",
"orderstatus": "object",
"totalprice": "object",
"orderdate": "object",
"o_orderkey": "int32",
"o_custkey": "int32",
"o_orderstatus": "object",
"o_totalprice": "object",
"o_orderdate": "object",
"order_cust_key": "object",
"timestamp": "object",
"timestamptz": "object",
Expand Down Expand Up @@ -270,7 +268,7 @@ def test_validate_with_unknown_rule(postgres: PostgresContainer):
json={
"connectionInfo": connection_info,
"manifestStr": manifest_str,
"parameters": {"modelName": "orders", "columnName": "orderkey"},
"parameters": {"modelName": "orders", "columnName": "o_orderkey"},
},
)
assert response.status_code == 422
Expand All @@ -287,7 +285,7 @@ def test_validate_rule_column_is_valid(postgres: PostgresContainer):
json={
"connectionInfo": connection_info,
"manifestStr": manifest_str,
"parameters": {"modelName": "orders", "columnName": "orderkey"},
"parameters": {"modelName": "orders", "columnName": "o_orderkey"},
},
)
assert response.status_code == 204
Expand All @@ -302,7 +300,7 @@ def test_validate_rule_column_is_valid_with_invalid_parameters(
json={
"connectionInfo": connection_info,
"manifestStr": manifest_str,
"parameters": {"modelName": "X", "columnName": "orderkey"},
"parameters": {"modelName": "X", "columnName": "o_orderkey"},
},
)
assert response.status_code == 422
Expand Down Expand Up @@ -352,7 +350,7 @@ def test_validate_rule_column_is_valid_without_one_parameter(
json={
"connectionInfo": connection_info,
"manifestStr": manifest_str,
"parameters": {"columnName": "orderkey"},
"parameters": {"columnName": "o_orderkey"},
},
)
assert response.status_code == 422
Expand All @@ -364,7 +362,7 @@ def test_dry_plan():
url=f"{base_url}/dry-plan",
json={
"manifestStr": manifest_str,
"sql": "SELECT orderkey, order_cust_key FROM wren.public.orders LIMIT 1",
"sql": "SELECT o_orderkey, order_cust_key FROM wren.public.orders LIMIT 1",
},
)
assert response.status_code == 200
Expand Down
8 changes: 4 additions & 4 deletions wren-modeling-py/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@ mod tests {
"table": "customer"
},
"columns": [
{"name": "custkey", "expression": "c_custkey", "type": "integer"},
{"name": "name", "expression": "c_name", "type": "varchar"}
{"name": "c_custkey", "type": "integer"},
{"name": "c_name", "type": "varchar"}
],
"primaryKey": "custkey"
"primaryKey": "c_custkey"
}
]
}"#;
Expand All @@ -71,7 +71,7 @@ mod tests {
.unwrap();
assert_eq!(
transformed_sql,
r#"SELECT * FROM (SELECT main.customer.c_custkey AS custkey, main.customer.c_name AS "name" FROM main.customer) AS customer"#
r#"SELECT * FROM (SELECT main.customer.c_custkey AS c_custkey, main.customer.c_name AS c_name FROM main.customer) AS customer"#
);
}
}
8 changes: 4 additions & 4 deletions wren-modeling-py/tests/test_modeling_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
"table": "customer",
},
"columns": [
{"name": "custkey", "expression": "c_custkey", "type": "integer"},
{"name": "name", "expression": "c_name", "type": "varchar"},
{"name": "c_custkey", "type": "integer"},
{"name": "c_name", "type": "varchar"},
],
"primaryKey": "custkey",
"primaryKey": "c_custkey",
},
],
}
Expand All @@ -30,5 +30,5 @@ def test_transform_sql():
rewritten_sql = wren_core.transform_sql(manifest_str, sql)
assert (
rewritten_sql
== 'SELECT * FROM (SELECT main.customer.c_custkey AS custkey, main.customer.c_name AS "name" FROM main.customer) AS customer'
== 'SELECT * FROM (SELECT main.customer.c_custkey AS c_custkey, main.customer.c_name AS c_name FROM main.customer) AS customer'
)
Loading
Loading