Skip to content

Commit 44191a7

Browse files
wangrunji0408wangrunji0408
authored
refactor(expr): generate build-from-prost with procedural macros (risingwavelabs#8499)
Signed-off-by: Runji Wang <[email protected]>
1 parent 58f29a6 commit 44191a7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+3269
-5364
lines changed

‎Cargo.lock

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ members = [
1111
"src/connector",
1212
"src/ctl",
1313
"src/expr",
14+
"src/expr/macro",
1415
"src/frontend",
1516
"src/frontend/planner_test",
1617
"src/java_binding",

‎Makefile.toml

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -622,12 +622,13 @@ tar xf ${TARGET_PATH} -C "${PREFIX_BIN}/connector-node"
622622
category = "RiseDev - Build in simulation mode"
623623
description = "Build in simulation mode"
624624
dependencies = ["warn-on-missing-tools"]
625-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
625+
env = { CARGO_TARGET_DIR = "target/sim" }
626626
script = """
627627
#!/usr/bin/env bash
628628
set -e
629629
630630
cargo build \
631+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
631632
-p risingwave_batch \
632633
-p risingwave_common \
633634
-p risingwave_compute \
@@ -646,12 +647,13 @@ cargo build \
646647
category = "RiseDev - Deterministic Simulation Test"
647648
description = "Run unit tests in deterministic simulation mode"
648649
dependencies = ["warn-on-missing-tools"]
649-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
650+
env = { CARGO_TARGET_DIR = "target/sim" }
650651
script = """
651652
#!/usr/bin/env bash
652653
set -e
653654
654655
cargo nextest run \
656+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
655657
-p risingwave_batch \
656658
-p risingwave_common \
657659
-p risingwave_compute \
@@ -670,12 +672,13 @@ cargo nextest run \
670672
category = "RiseDev - Simulation scaling tests"
671673
description = "Run integration scaling tests in deterministic simulation mode"
672674
dependencies = ["warn-on-missing-tools"]
673-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
675+
env = { CARGO_TARGET_DIR = "target/sim" }
674676
script = """
675677
#!/usr/bin/env bash
676678
set -e
677679
678680
cargo nextest run \
681+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
679682
-p risingwave_simulation \
680683
"$@"
681684
"""
@@ -684,12 +687,13 @@ cargo nextest run \
684687
category = "RiseDev - Simulation scaling tests"
685688
description = "Archive integration scaling tests in deterministic simulation mode"
686689
dependencies = ["warn-on-missing-tools"]
687-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
690+
env = { CARGO_TARGET_DIR = "target/sim" }
688691
script = """
689692
#!/usr/bin/env bash
690693
set -e
691694
692695
cargo nextest archive \
696+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
693697
-p risingwave_simulation \
694698
--archive-file scale-test.tar.zst \
695699
"$@"
@@ -699,48 +703,58 @@ cargo nextest archive \
699703
category = "RiseDev - Deterministic Simulation End-to-end Test"
700704
description = "Run cargo check in deterministic simulation mode"
701705
dependencies = ["warn-on-missing-tools"]
702-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
706+
env = { CARGO_TARGET_DIR = "target/sim" }
703707
script = """
704708
#!/usr/bin/env bash
705709
set -e
706710
707-
cargo check -p risingwave_simulation --all-targets "$@"
711+
cargo check \
712+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
713+
-p risingwave_simulation --all-targets "$@"
708714
"""
709715

710716
[tasks.sslt]
711717
category = "RiseDev - Deterministic Simulation End-to-end Test"
712718
description = "Run e2e tests in deterministic simulation mode"
713719
dependencies = ["warn-on-missing-tools"]
714-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
720+
env = { CARGO_TARGET_DIR = "target/sim" }
715721
script = """
716722
#!/usr/bin/env bash
717723
set -e
718724
719-
cargo run -p risingwave_simulation "$@"
725+
cargo run \
726+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
727+
-p risingwave_simulation "$@"
720728
"""
721729

722730
[tasks.sslt-build-all]
723731
category = "RiseDev - Deterministic Simulation End-to-end Test"
724732
description = "Build deterministic simulation runner and tests"
725733
dependencies = ["warn-on-missing-tools"]
726-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
734+
env = { CARGO_TARGET_DIR = "target/sim" }
727735
script = """
728736
#!/usr/bin/env bash
729737
set -e
730738
731-
cargo build -p risingwave_simulation --tests "$@"
739+
cargo build \
740+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
741+
-p risingwave_simulation \
742+
--tests "$@"
732743
"""
733744

734745
[tasks.sslt-cov]
735746
category = "RiseDev - Deterministic Simulation End-to-end Test"
736747
description = "Run e2e tests in deterministic simulation mode and report code coverage"
737748
dependencies = ["warn-on-missing-tools"]
738-
env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim-cov" }
749+
env = { CARGO_TARGET_DIR = "target/sim-cov" }
739750
script = """
740751
#!/usr/bin/env bash
741752
set -e
742753
743-
cargo llvm-cov run -p risingwave_simulation --html "$@"
754+
cargo llvm-cov run \
755+
--config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
756+
-p risingwave_simulation \
757+
--html "$@"
744758
"""
745759

746760
[tasks.check-java]

‎src/batch/src/executor/join/hash_join.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1799,8 +1799,8 @@ mod tests {
17991799
use risingwave_common::test_prelude::DataChunkTestExt;
18001800
use risingwave_common::types::DataType;
18011801
use risingwave_common::util::iter_util::ZipEqDebug;
1802-
use risingwave_expr::expr::{new_binary_expr, BoxedExpression, InputRefExpression};
1803-
use risingwave_pb::expr::expr_node::Type;
1802+
use risingwave_expr::expr::{build, BoxedExpression, Expression, InputRefExpression};
1803+
use risingwave_pb::expr::expr_node::PbType;
18041804

18051805
use super::{
18061806
ChunkedData, HashJoinExecutor, JoinType, LeftNonEquiJoinState, RightNonEquiJoinState, RowId,
@@ -1985,13 +1985,13 @@ mod tests {
19851985
}
19861986

19871987
fn create_cond() -> BoxedExpression {
1988-
let left_expr = InputRefExpression::new(DataType::Float32, 1);
1989-
let right_expr = InputRefExpression::new(DataType::Float64, 3);
1990-
new_binary_expr(
1991-
Type::LessThan,
1988+
build(
1989+
PbType::LessThan,
19921990
DataType::Boolean,
1993-
Box::new(left_expr),
1994-
Box::new(right_expr),
1991+
vec![
1992+
InputRefExpression::new(DataType::Float32, 1).boxed(),
1993+
InputRefExpression::new(DataType::Float64, 3).boxed(),
1994+
],
19951995
)
19961996
.unwrap()
19971997
}

‎src/batch/src/executor/join/local_lookup_join.rs

Lines changed: 26 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -462,9 +462,9 @@ mod tests {
462462
use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
463463
use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
464464
use risingwave_expr::expr::{
465-
new_binary_expr, BoxedExpression, InputRefExpression, LiteralExpression,
465+
build, BoxedExpression, Expression, InputRefExpression, LiteralExpression,
466466
};
467-
use risingwave_pb::expr::expr_node::Type;
467+
use risingwave_pb::expr::expr_node::PbType;
468468

469469
use super::LocalLookupJoinExecutorArgs;
470470
use crate::executor::join::JoinType;
@@ -676,14 +676,13 @@ mod tests {
676676
);
677677

678678
let condition = Some(
679-
new_binary_expr(
680-
Type::LessThan,
679+
build(
680+
PbType::LessThan,
681681
DataType::Boolean,
682-
Box::new(LiteralExpression::new(
683-
DataType::Int32,
684-
Some(ScalarImpl::Int32(5)),
685-
)),
686-
Box::new(InputRefExpression::new(DataType::Float32, 3)),
682+
vec![
683+
LiteralExpression::new(DataType::Int32, Some(ScalarImpl::Int32(5))).boxed(),
684+
InputRefExpression::new(DataType::Float32, 3).boxed(),
685+
],
687686
)
688687
.unwrap(),
689688
);
@@ -705,14 +704,13 @@ mod tests {
705704
);
706705

707706
let condition = Some(
708-
new_binary_expr(
709-
Type::LessThan,
707+
build(
708+
PbType::LessThan,
710709
DataType::Boolean,
711-
Box::new(LiteralExpression::new(
712-
DataType::Int32,
713-
Some(ScalarImpl::Int32(5)),
714-
)),
715-
Box::new(InputRefExpression::new(DataType::Float32, 3)),
710+
vec![
711+
LiteralExpression::new(DataType::Int32, Some(ScalarImpl::Int32(5))).boxed(),
712+
InputRefExpression::new(DataType::Float32, 3).boxed(),
713+
],
716714
)
717715
.unwrap(),
718716
);
@@ -730,14 +728,13 @@ mod tests {
730728
);
731729

732730
let condition = Some(
733-
new_binary_expr(
734-
Type::LessThan,
731+
build(
732+
PbType::LessThan,
735733
DataType::Boolean,
736-
Box::new(LiteralExpression::new(
737-
DataType::Int32,
738-
Some(ScalarImpl::Int32(5)),
739-
)),
740-
Box::new(InputRefExpression::new(DataType::Float32, 3)),
734+
vec![
735+
LiteralExpression::new(DataType::Int32, Some(ScalarImpl::Int32(5))).boxed(),
736+
InputRefExpression::new(DataType::Float32, 3).boxed(),
737+
],
741738
)
742739
.unwrap(),
743740
);
@@ -756,14 +753,13 @@ mod tests {
756753
);
757754

758755
let condition = Some(
759-
new_binary_expr(
760-
Type::LessThan,
756+
build(
757+
PbType::LessThan,
761758
DataType::Boolean,
762-
Box::new(LiteralExpression::new(
763-
DataType::Int32,
764-
Some(ScalarImpl::Int32(5)),
765-
)),
766-
Box::new(InputRefExpression::new(DataType::Float32, 3)),
759+
vec![
760+
LiteralExpression::new(DataType::Int32, Some(ScalarImpl::Int32(5))).boxed(),
761+
InputRefExpression::new(DataType::Float32, 3).boxed(),
762+
],
767763
)
768764
.unwrap(),
769765
);

‎src/batch/src/executor/join/nested_loop_join.rs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -470,8 +470,8 @@ mod tests {
470470
use risingwave_common::array::*;
471471
use risingwave_common::catalog::{Field, Schema};
472472
use risingwave_common::types::DataType;
473-
use risingwave_expr::expr::{new_binary_expr, InputRefExpression};
474-
use risingwave_pb::expr::expr_node::Type;
473+
use risingwave_expr::expr::{build, InputRefExpression};
474+
use risingwave_pb::expr::expr_node::PbType;
475475

476476
use crate::executor::join::nested_loop_join::NestedLoopJoinExecutor;
477477
use crate::executor::join::JoinType;
@@ -587,11 +587,13 @@ mod tests {
587587
};
588588

589589
Box::new(NestedLoopJoinExecutor::new(
590-
new_binary_expr(
591-
Type::Equal,
590+
build(
591+
PbType::Equal,
592592
DataType::Boolean,
593-
Box::new(InputRefExpression::new(DataType::Int32, 0)),
594-
Box::new(InputRefExpression::new(DataType::Int32, 2)),
593+
vec![
594+
Box::new(InputRefExpression::new(DataType::Int32, 0)),
595+
Box::new(InputRefExpression::new(DataType::Int32, 2)),
596+
],
595597
)
596598
.unwrap(),
597599
join_type,

‎src/common/src/array/jsonb_array.rs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,26 @@ impl JsonbRef<'_> {
274274
}
275275
}
276276

277+
impl FromIterator<Option<JsonbVal>> for JsonbArray {
278+
fn from_iter<I: IntoIterator<Item = Option<JsonbVal>>>(iter: I) -> Self {
279+
let iter = iter.into_iter();
280+
let mut builder = <Self as Array>::Builder::new(iter.size_hint().0);
281+
for i in iter {
282+
match i {
283+
Some(x) => builder.append(Some(x.as_scalar_ref())),
284+
None => builder.append(None),
285+
}
286+
}
287+
builder.finish()
288+
}
289+
}
290+
291+
impl FromIterator<JsonbVal> for JsonbArray {
292+
fn from_iter<I: IntoIterator<Item = JsonbVal>>(iter: I) -> Self {
293+
iter.into_iter().map(Some).collect()
294+
}
295+
}
296+
277297
#[derive(Debug)]
278298
pub struct JsonbArrayBuilder {
279299
bitmap: BitmapBuilder,

‎src/common/src/array/list_array.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ pub struct ListArray {
156156
bitmap: Bitmap,
157157
pub(super) offsets: Vec<u32>,
158158
pub(super) value: Box<ArrayImpl>,
159-
value_type: DataType,
159+
pub(super) value_type: DataType,
160160
}
161161

162162
impl Array for ListArray {

‎src/common/src/array/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,13 @@ pub use list_array::{ListArray, ListArrayBuilder, ListRef, ListValue};
5959
use paste::paste;
6060
pub use primitive_array::{PrimitiveArray, PrimitiveArrayBuilder, PrimitiveArrayItemType};
6161
use risingwave_pb::data::{PbArray, PbArrayType};
62+
pub use serial_array::{Serial, SerialArray, SerialArrayBuilder};
6263
pub use stream_chunk::{Op, StreamChunk, StreamChunkTestExt};
6364
pub use struct_array::{StructArray, StructArrayBuilder, StructRef, StructValue};
6465
pub use utf8_array::*;
6566
pub use vis::{Vis, VisRef};
6667

6768
pub use self::error::ArrayError;
68-
use crate::array::serial_array::{Serial, SerialArray, SerialArrayBuilder};
6969
use crate::buffer::Bitmap;
7070
use crate::types::*;
7171
use crate::util::iter_util::ZipEqFast;

0 commit comments

Comments
 (0)