Merge pull request #179 from drizzle-team/orm-99-with-statement

Implement WITH clause, fix subquery bugs

Author: dankochetov

README.md

@@ -108,7 +108,7 @@ const allUsers /* : User[] */ = await db.select().from(users);
 const upperCaseNames /* : { id: number; name: string }[] */ = await db
   .select({
     id: users.id,
-    name: sql`upper(${users.fullName})`.as<string>(),
+    name: sql<string>`upper(${users.fullName})`,
   })
   .from(users);
 

changelogs/drizzle-orm/0.20.0.md

@@ -0,0 +1,23 @@
+
+- 🎉 **Implemented support for WITH clause ([docs](/drizzle-orm/src/pg-core/README.md#with-clause)). Example usage:**
+
+  ```ts
+  const sq = db
+    .select()
+    .from(users)
+    .prepareWithSubquery('sq');
+
+  const result = await db
+    .with(sq)
+    .select({
+      id: sq.id,
+      name: sq.name,
+      total: sql<number>`count(${sq.id})::int`(),
+    })
+    .from(sq)
+    .groupBy(sq.id, sq.name);
+  ```
+
+- 🐛 Fixed various bugs with selecting/joining of subqueries.
+- ❗ Renamed `.subquery('alias')` to `.as('alias')`.
+- ❗ ``sql`query`.as<type>()`` is now ``sql<type>`query`()``. Old syntax is still supported, but is deprecated and will be removed in one of the next releases.

dprint.json

@@ -11,8 +11,8 @@
 	"includes": ["**/*.{ts,tsx,js,jsx,cjs,mjs,json,md}"],
 	"excludes": ["**/node_modules", "**/*-lock.json"],
 	"plugins": [
-		"https://plugins.dprint.dev/typescript-0.70.0.wasm",
-		"https://plugins.dprint.dev/json-0.15.3.wasm",
-		"https://plugins.dprint.dev/markdown-0.13.3.wasm"
+		"https://plugins.dprint.dev/typescript-0.83.0.wasm",
+		"https://plugins.dprint.dev/json-0.17.0.wasm",
+		"https://plugins.dprint.dev/markdown-0.15.2.wasm"
 	]
 }

drizzle-orm/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "drizzle-orm",
-	"version": "0.19.1",
+	"version": "0.20.0",
 	"description": "Drizzle ORM package for SQL databases",
 	"scripts": {
 		"build": "tsc && resolve-tspaths && cp ../README.md package.json dist/",

drizzle-orm/src/aws-data-api/pg/migrator.ts

@@ -1,13 +1,12 @@
 import { MigrationConfig, readMigrationFiles } from '~/migrator';
 import { sql } from '~/sql';
 import { AwsDataApiPgDatabase } from './driver';
-import { AwsDataApiSession } from './session';
 
 export async function migrate(db: AwsDataApiPgDatabase, config: string | MigrationConfig) {
 	const migrations = readMigrationFiles(config);
 
-	// Write own aws datapi migrator
-	const session = db.session as AwsDataApiSession;
+	// TODO: Write own aws datapi migrator
+	const { session } = db;
 
 	const migrationTableCreate = sql`CREATE TABLE IF NOT EXISTS "drizzle"."__drizzle_migrations" (
 		id SERIAL PRIMARY KEY,

drizzle-orm/src/mysql-core/README.md

@@ -375,6 +375,47 @@ await db.select().from(users).orderBy(desc(users.name));
 await db.select().from(users).orderBy(asc(users.name), desc(users.name));
 ```
 
+#### Conditionally select fields
+
+```typescript
+async function selectUsers(withName: boolean) {
+  return db
+    .select({
+      id: users.id,
+      ...(withName ? { name: users.name } : {}),
+    })
+    .from(users);
+}
+
+const users = await selectUsers(true);
+```
+
+#### WITH clause
+
+```typescript
+const sq = db.select().from(users).where(eq(users.id, 42)).prepareWithSubquery('sq');
+const result = await db.with(sq).select().from(sq);
+```
+
+> **Note**: Keep in mind, that if you need to select raw `sql` in a WITH subquery and reference that field in other queries, you must add an alias to it:
+
+```typescript
+const sq = db
+  .select({
+    name: sql<string>`upper(${users.name})`.as('name'),
+  })
+  .from(users)
+  .prepareWithSubquery('sq');
+
+const result = await db
+  .select({
+    name: sq.name,
+  })
+  .from(sq);
+```
+
+Otherwise, the field type will become `DrizzleTypeError` and you won't be able to reference it in other queries. If you ignore the type error and still try to reference the field, you will get a runtime error, because we cannot reference that field without an alias.
+
 #### Select from subquery
 
 ```typescript
@@ -430,8 +471,8 @@ notIlike(column, value)
 
 not(sqlExpression)
 
-and(expressions: SQL[])
-or(expressions: SQL[])
+and(...expressions: SQL[])
+or(...expressions: SQL[])
 
 ```
 

drizzle-orm/src/mysql-core/db.ts

@@ -1,5 +1,6 @@
 import { ResultSetHeader } from 'mysql2/promise';
 import { SQLWrapper } from '~/sql';
+import { WithSubquery } from '~/subquery';
 import { MySqlDialect } from './dialect';
 import { MySqlDelete, MySqlInsertBuilder, MySqlSelectBuilder, MySqlUpdateBuilder } from './query-builders';
 import { SelectFields } from './query-builders/select.types';
@@ -14,6 +15,18 @@ export class MySqlDatabase<TQueryResult extends QueryResultHKT, TSession extends
 		readonly session: TSession,
 	) {}
 
+	with(...queries: WithSubquery[]) {
+		const self = this;
+
+		function select(): MySqlSelectBuilder<undefined>;
+		function select<TSelection extends SelectFields>(fields: TSelection): MySqlSelectBuilder<TSelection>;
+		function select(fields?: SelectFields): MySqlSelectBuilder<SelectFields | undefined> {
+			return new MySqlSelectBuilder(fields ?? undefined, self.session, self.dialect, queries);
+		}
+
+		return { select };
+	}
+
 	select(): MySqlSelectBuilder<undefined>;
 	select<TSelection extends SelectFields>(fields: TSelection): MySqlSelectBuilder<TSelection>;
 	select(fields?: SelectFields): MySqlSelectBuilder<SelectFields | undefined> {

drizzle-orm/src/mysql-core/dialect.ts

@@ -1,6 +1,6 @@
 import { AnyColumn, Column } from '~/column';
 import { MigrationMeta } from '~/migrator';
-import { Name, Query, SQL, sql, SQLResponse, SQLSourceParam } from '~/sql';
+import { Name, Query, SQL, sql, SQLSourceParam } from '~/sql';
 import { Subquery, SubqueryConfig } from '~/subquery';
 import { getTableName, Table } from '~/table';
 import { UpdateSet } from '~/utils';
@@ -126,8 +126,10 @@ export class MySqlDialect {
 			.map(({ field }, i) => {
 				const chunk: SQLSourceParam[] = [];
 
-				if (field instanceof SQLResponse || field instanceof SQL) {
-					const query = field instanceof SQLResponse ? field.sql : field;
+				if (field instanceof SQL.Aliased && field.isSubquerySelectionField) {
+					chunk.push(new Name(field.fieldAlias));
+				} else if (field instanceof SQL.Aliased || field instanceof SQL) {
+					const query = field instanceof SQL.Aliased ? field.sql : field;
 
 					if (isSingleTable) {
 						chunk.push(
@@ -143,6 +145,10 @@ export class MySqlDialect {
 					} else {
 						chunk.push(query);
 					}
+
+					if (field instanceof SQL.Aliased) {
+						chunk.push(sql` as ${new Name(field.fieldAlias)}`);
+					}
 				} else if (field instanceof Column) {
 					if (isSingleTable) {
 						chunk.push(new Name(field.name));
@@ -163,25 +169,39 @@ export class MySqlDialect {
 	}
 
 	buildSelectQuery(
-		{ fieldsList: fields, where, table, joins, orderBy, groupBy, limit, offset }: MySqlSelectConfig,
+		{ withList, fieldsList: fields, where, table, joins, orderBy, groupBy, limit, offset }: MySqlSelectConfig,
 	): SQL {
 		fields.forEach((f) => {
 			let tableName: string;
 			if (
-				f.field instanceof Column && f.field.table !== table && !((tableName = getTableName(f.field.table)) in joins)
+				f.field instanceof Column
+				&& getTableName(f.field.table)
+					!== (table instanceof Subquery ? table[SubqueryConfig].alias : getTableName(table))
+				&& !((tableName = getTableName(f.field.table)) in joins)
 			) {
 				throw new Error(
-					`Column "${f.field.name}" was selected, but its table "${tableName}" was not joined`,
+					`Your "${
+						f.path.join('->')
+					}" field references a column "${tableName}"."${f.field.name}", but the table "${tableName}" is not part of the query! Did you forget to join it?`,
 				);
 			}
 		});
 
 		const joinKeys = Object.keys(joins);
 		const isSingleTable = joinKeys.length === 0;
 
-		const tableSql = table instanceof Subquery
-			? sql`(${table})${isSingleTable ? undefined : new Name(table[SubqueryConfig].alias)}`
-			: table;
+		let withSql: SQL | undefined;
+		if (withList.length) {
+			const withSqlChunks = [sql`with `];
+			withList.forEach((w, i) => {
+				withSqlChunks.push(sql`${new Name(w[SubqueryConfig].alias)} as (${w[SubqueryConfig].sql})`);
+				if (i < withList.length - 1) {
+					withSqlChunks.push(sql`, `);
+				}
+			});
+			withSqlChunks.push(sql` `);
+			withSql = sql.fromList(withSqlChunks);
+		}
 
 		const selection = this.buildSelection(fields, { isSingleTable });
 
@@ -206,9 +226,7 @@ export class MySqlDialect {
 				);
 			} else {
 				joinsArray.push(
-					sql`${sql.raw(joinMeta.joinType)} join (${table[SubqueryConfig].sql}) ${new Name(
-						tableAlias,
-					)} on ${joinMeta.on}`,
+					sql`${sql.raw(joinMeta.joinType)} join ${table} on ${joinMeta.on}`,
 				);
 			}
 			if (index < joinKeys.length - 1) {
@@ -246,7 +264,7 @@ export class MySqlDialect {
 
 		const offsetSql = offset ? sql` offset ${offset}` : undefined;
 
-		return sql`select ${selection} from ${tableSql}${joinsSql}${whereSql}${groupBySql}${orderBySql}${limitSql}${offsetSql}`;
+		return sql`${withSql}select ${selection} from ${table}${joinsSql}${whereSql}${groupBySql}${orderBySql}${limitSql}${offsetSql}`;
 	}
 
 	buildInsertQuery({ table, values, onConflict, returning }: MySqlInsertConfig): SQL {

drizzle-orm/src/mysql-core/expressions.ts

@@ -1,19 +1,23 @@
-import { param, SQL, sql, SQLSourceParam } from '~/sql';
+import { bindIfParam } from '~/expressions';
+import { Placeholder, SQL, sql, SQLSourceParam, SQLWrapper } from '~/sql';
 import { AnyMySqlColumn } from './columns/common';
 
 export * from '~/expressions';
 
-export function concat(column: AnyMySqlColumn, value: string): SQL {
-	return sql`${column} || ${param(value, column)}`;
+export function concat(column: AnyMySqlColumn | SQL.Aliased, value: string | Placeholder | SQLWrapper): SQL {
+	return sql`${column} || ${bindIfParam(value, column)}`;
 }
 
-export function substring(column: AnyMySqlColumn, { from, for: _for }: { from?: number; for?: number }): SQL {
+export function substring(
+	column: AnyMySqlColumn | SQL.Aliased,
+	{ from, for: _for }: { from?: number | Placeholder | SQLWrapper; for?: number | Placeholder | SQLWrapper },
+): SQL {
 	const chunks: SQLSourceParam[] = [sql`substring(`, column];
 	if (from !== undefined) {
-		chunks.push(sql` from `, param(from, column));
+		chunks.push(sql` from `, bindIfParam(from, column));
 	}
 	if (_for !== undefined) {
-		chunks.push(sql` for `, param(_for, column));
+		chunks.push(sql` for `, bindIfParam(_for, column));
 	}
 	chunks.push(sql`)`);
 	return sql.fromList(chunks);

drizzle-orm/src/mysql-core/query-builders/delete.ts

@@ -20,13 +20,11 @@ export interface MySqlDeleteConfig {
 export interface MySqlDelete<
 	TTable extends AnyMySqlTable,
 	TQueryResult extends QueryResultHKT,
-	TReturning = undefined,
 > extends QueryPromise<QueryResultKind<TQueryResult, never>> {}
 
 export class MySqlDelete<
 	TTable extends AnyMySqlTable,
 	TQueryResult extends QueryResultHKT,
-	TReturning = undefined,
 > extends QueryPromise<QueryResultKind<TQueryResult, never>> implements SQLWrapper {
 	private config: MySqlDeleteConfig;
 
@@ -46,17 +44,6 @@ export class MySqlDelete<
 		return this;
 	}
 
-	// returning(): Omit<MySqlDelete<TTable, InferModel<TTable>>, 'where' | 'returning'>;
-	// returning<TSelectedFields extends SelectFields>(
-	// 	fields: TSelectedFields,
-	// ): Omit<MySqlDelete<TTable, SelectResultFields<TSelectedFields>>, 'where' | 'returning'>;
-	// returning(
-	// 	fields: SelectFields = this.config.table[MySqlTable.Symbol.Columns],
-	// ): Omit<MySqlDelete<TTable, any>, 'where' | 'returning'> {
-	// 	this.config.returning = orderSelectedFields(fields);
-	// 	return this;
-	// }
-
 	/** @internal */
 	getSQL(): SQL {
 		return this.dialect.buildDeleteQuery(this.config);

drizzle-orm/src/mysql-core/query-builders/insert.ts

@@ -74,23 +74,6 @@ export class MySqlInsert<TTable extends AnyMySqlTable, TQueryResult extends Quer
 		this.config = { table, values };
 	}
 
-	// onDuplicateDoUpdate(
-	// 	target:
-	// 		| SQL<GetTableConfig<TTable, 'name'>>
-	// 		| ((constraints: GetTableConflictConstraints<TTable>) => Check<GetTableConfig<TTable, 'name'>>),
-	// 	set: MySqlUpdateSet<TTable>,
-	// ): Pick<this, 'returning' | 'getQuery' | 'execute'> {
-	// 	const setSql = this.dialect.buildUpdateSet<GetTableConfig<TTable, 'name'>>(this.config.table, set);
-
-	// 	if (target instanceof SQL) {
-	// 		this.config.onConflict = sql<GetTableConfig<TTable, 'name'>>`${target} do update set ${setSql}`;
-	// 	} else {
-	// 		const targetSql = new Name(target(this.config.table[tableConflictConstraints]).name);
-	// 		this.config.onConflict = sql`on constraint ${targetSql} do update set ${setSql}`;
-	// 	}
-	// 	return this;
-	// }
-
 	onDuplicateKeyUpdate(config: {
 		// target?: IndexColumn | IndexColumn[];
 		set: MySqlUpdateSetSource<TTable>;