Merge branch 'main' into nulls-last

Author: max-sixty

.github/workflows/nightly.yaml

@@ -47,7 +47,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:
-          config-file: .config/.markdown-link-check-global.json
+          config-file: .config/.markdown-link-check-all.json
 
   time-compilation:
     runs-on: ubuntu-latest
@@ -64,17 +64,17 @@ jobs:
   update-rust-toolchain:
     runs-on: ubuntu-latest
     # Don't run on forks; will attempt to create a PR into the fork...
-    # if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork }}
 
-    # Disabling until we resolve what version Debian supports, ref https://github.com/PRQL/prql/pull/1561
-    if: false
+    # Note that this doesn't change the minimum supported version, only the
+    # default to run on. The minimum is defined by Cargo.toml's metadata.msrv.
     steps:
       - name: 📂 Checkout code
         uses: actions/checkout@v3
       - uses: a-kenji/update-rust-toolchain@main
         with:
           # Discussion in #1561
-          minor-version-delta: 3
+          minor-version-delta: 1
           toolchain-path: "./rust-toolchain.toml"
           pr-title: "build: Update rust toolchain version"
 

.github/workflows/test-all.yaml

@@ -144,9 +144,12 @@ jobs:
         with:
           crate: cargo-msrv
         # Note this currently uses a manually maintained key in
-        # `prql-compiler/Cargo.toml`, because of
+        # `prql-compiler/Cargo.toml` (and `prql-compiler/prqlc/Cargo.toml` below), because of
         # https://github.com/foresterre/cargo-msrv/issues/590
-      - name: Verify minimum rust version
+      - name: Verify minimum rust version — prql-compiler
         # Ideally we'd check all crates, ref https://github.com/foresterre/cargo-msrv/issues/295
         working-directory: prql-compiler
         run: cargo msrv verify
+      - name: Verify minimum rust version — prqlc
+        working-directory: prql-compiler/prqlc
+        run: cargo msrv verify

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## 0.8.2 — [unreleased]
 
+_The following unreleased features are only available in the `main` branch. They
+will become the public version at the next release._
+
 **Features**:
 
 - Add a `~=` regex search operator (@max-sixty, #2458). An example:
@@ -26,6 +29,15 @@
   [Regex docs](https://prql-lang.org/book/language-features/regex.html) for more
   details.
 
+- Change to function syntax: `let f = x -> ...` See
+  https://github.com/PRQL/prql/blob/main/web/book/src/queries/functions.md
+
+- Modules allow importing declarations from other files: See
+  https://github.com/PRQL/prql/blob/main/web/book/src/internals/modules.md
+
+- Relation literals create in-line tables or array literals: See
+  https://github.com/PRQL/prql/pull/2605
+
 **Fixes**:
 
 **Documentation**:

README.md

@@ -127,7 +127,8 @@ To stay in touch with PRQL:
 This repo is composed of:
 
 - **[prql-compiler](./prql-compiler/)** — the compiler, written in rust, whose
-  main role is to compile PRQL into SQL. It also includes `prqlc`, the CLI.
+  main role is to compile PRQL into SQL. It also includes
+  [prqlc](./prql-compiler/prqlc/), the CLI.
 - **[web](./web/)** — our web content: the [Book][prql book],
   [Website][prql website], and [Playground][prql playground].
 - **[bindings](./bindings/)** — bindings from various languages to

prql-compiler/prqlc/Cargo.toml

@@ -8,6 +8,8 @@ repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
 
+metadata.msrv = "1.65.0"
+
 [target.'cfg(not(target_family="wasm"))'.dependencies]
 anyhow = {version = "1.0.57"}
 ariadne = "0.2.0"

prql-compiler/src/ast/rq/fold.rs

@@ -171,7 +171,6 @@ pub fn fold_transform<T: ?Sized + RqFold>(
             partition: fold.fold_cids(partition)?,
             compute: fold.fold_cids(compute)?,
         },
-
         Select(ids) => Select(fold.fold_cids(ids)?),
         Filter(i) => Filter(fold.fold_expr(i)?),
         Sort(sorts) => Sort(fold_column_sorts(fold, sorts)?),

prql-compiler/src/sql/dialect.rs

@@ -158,6 +158,10 @@ pub(super) trait DialectHandler: Any + Debug {
         true
     }
 
+    fn supports_distinct_on(&self) -> bool {
+        false
+    }
+
     fn translate_regex(
         &self,
         search: sql_ast::Expr,
@@ -370,6 +374,10 @@ impl DialectHandler for DuckDbDialect {
         false
     }
 
+    fn supports_distinct_on(&self) -> bool {
+        true
+    }
+
     fn translate_regex(
         &self,
         search: sql_ast::Expr,

prql-compiler/src/sql/gen_query.rs

@@ -107,8 +107,29 @@ fn translate_select_pipeline(
     let order_by = pipeline.pluck(|t| t.into_sort());
     let takes = pipeline.pluck(|t| t.into_take());
     let is_distinct = pipeline.iter().any(|t| matches!(t, SqlTransform::Distinct));
+    let distinct_ons = pipeline.pluck(|t| t.into_distinct_on());
     let distinct = if is_distinct {
         Some(sql_ast::Distinct::Distinct)
+    } else if !distinct_ons.is_empty() {
+        // FIXME: this "works" but is very hacky — it's using the
+        // `translate_select_item` to translate each id into an Expr — is that
+        // correct?
+        //
+        // Do we know we won't have more than one DistinctOn? We have that for
+        // `Distinct`. But this will panic if we have more than one.
+        // And if there the `SelectItem` is not an `UnnamedExpr`, it'll panic.
+        Some(sql_ast::Distinct::On(
+            distinct_ons
+                .into_iter()
+                .exactly_one()?
+                .into_iter()
+                .map(|id| translate_select_item(id, ctx).unwrap())
+                .map(|item| match item {
+                    SelectItem::UnnamedExpr(expr) => expr,
+                    _ => unreachable!(),
+                })
+                .collect::<Vec<sql_ast::Expr>>(),
+        ))
     } else {
         None
     };

prql-compiler/src/sql/srq/anchor.rs

@@ -475,7 +475,6 @@ pub(super) fn get_requirements(
         Super(Sort(_)) => (Complexity::Aggregation, true),
         Super(Take(_)) => (Complexity::Plain, false),
         Super(Transform::Join { .. }) => (Complexity::Plain, false),
-
         _ => unreachable!(),
     };
 

prql-compiler/src/sql/srq/ast.rs

@@ -98,6 +98,7 @@ pub enum SqlTransform<Rel = RelationExpr, Super = rq::Transform> {
     },
 
     Distinct,
+    DistinctOn(Vec<rq::CId>),
     Except {
         bottom: Rel,
         distinct: bool,
@@ -173,6 +174,7 @@ pub fn fold_sql_transform<
         },
 
         SqlTransform::Distinct => SqlTransform::Distinct,
+        SqlTransform::DistinctOn(ids) => SqlTransform::DistinctOn(fold.fold_cids(ids)?),
         SqlTransform::Union { bottom, distinct } => SqlTransform::Union {
             bottom: fold.fold_rel(bottom)?,
             distinct,

prql-compiler/src/sql/srq/preprocess.rs

@@ -130,13 +130,14 @@ pub(in crate::sql) fn distinct(
 
                 if take_only_first && sort.is_empty() && matching_columns {
                     res.push(SqlTransform::Distinct);
-                    continue;
+                } else if ctx.dialect.supports_distinct_on() {
+                    res.push(SqlTransform::DistinctOn(partition));
+                } else {
+                    // convert `take range` into:
+                    //   derive _rn = s"ROW NUMBER"
+                    //   filter (_rn | in range)
+                    res.extend(create_filter_by_row_number(range, sort, partition, ctx));
                 }
-
-                // convert `take range` into:
-                //   derive _rn = s"ROW NUMBER"
-                //   filter (_rn | in range)
-                res.extend(create_filter_by_row_number(range, sort, partition, ctx));
             }
             _ => {
                 res.push(transform);

prql-compiler/src/tests/test.rs

@@ -1485,6 +1485,23 @@ fn test_distinct() {
     "###);
 }
 
+#[test]
+fn test_distinct_on() {
+    assert_display_snapshot!((compile(r###"
+    prql target:sql.duckdb
+
+    from x
+    select [class, begins]
+    group [begins] (take 1)
+    "###).unwrap()), @r###"
+    SELECT
+      DISTINCT ON (begins) class,
+      begins
+    FROM
+      x
+    "###);
+}
+
 #[test]
 fn test_join() {
     assert_display_snapshot!((compile(r###"

rust-toolchain.toml

@@ -1,5 +1,5 @@
 [toolchain]
-channel = "1.65.0"
+channel = "1.66.0"
 components = ["rustfmt", "clippy"]
 # We want two targets: wasm32, and the default target for the platform, which we
 # don't list here. (i.e. we use each platform to test each platform)

web/book/src/contributing/README.md

@@ -91,7 +91,7 @@ general channels such as GitHub or Discord, feel free to reach out to:
 
 - [**@aljazerzen**](https://github.com/aljazerzen) — Aljaž Mur Eržen
 - [**@max-sixty**](https://github.com/max-sixty) — Maximilian Roos
-- [**@eitsupi**](https://twitter.com/eitsupi) — SHIMA Tatsuya
+- [**@eitsupi**](https://github.com/eitsupi) — SHIMA Tatsuya
 - [**@snth**](https://github.com/snth) — Tobias Brandt
 
 ### Core team Emeritus

web/book/src/integrations/README.md

@@ -2,7 +2,6 @@
 
 PRQL is building integrations with lots of external tools, including:
 
-- [dbt](./dbt.md)
 - [Jupyter](./jupyter.md)
 - [DuckDB](./duckdb.md)
 - [Prefect](./prefect.md)

web/playground/package-lock.json

@@ -12,7 +12,7 @@
     ]
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.24.0",
+    "@duckdb/duckdb-wasm": "^1.26.0",
     "@monaco-editor/react": "^4.5.0",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^14.0.0",