feat: Upgrade to Mosaic v0.11 (#56)

Author: manzt

deno.json

@@ -24,10 +24,10 @@
 		"@js-temporal/polyfill": "npm:@js-temporal/polyfill@~0.4.4",
 		"@lukeed/uuid": "npm:@lukeed/uuid@^2.0.1",
 		"@preact/signals-core": "npm:@preact/signals-core@^1.8.0",
-		"@uwdata/mosaic-core": "npm:@uwdata/mosaic-core@~0.10.0",
-		"@uwdata/mosaic-plot": "npm:@uwdata/mosaic-plot@~0.10.0",
-		"@uwdata/mosaic-sql": "npm:@uwdata/mosaic-sql@~0.10.0",
-		"apache-arrow": "npm:apache-arrow@^17.0.0",
+		"@uwdata/flechette": "npm:@uwdata/flechette@^1.1.0",
+		"@uwdata/mosaic-core": "npm:@uwdata/mosaic-core@~0.11.0",
+		"@uwdata/mosaic-plot": "npm:@uwdata/mosaic-plot@~0.11.0",
+		"@uwdata/mosaic-sql": "npm:@uwdata/mosaic-sql@~0.11.0",
 		"d3": "npm:d3@^7.9.0",
 		"htl": "npm:htl@~0.3.1"
 	},

deno.lock

@@ -1,4 +1,4 @@
-import * as arrow from "apache-arrow";
+import * as flech from "@uwdata/flechette";
 // @ts-types="../deps/mosaic-core.d.ts"
 import {
 	type Coordinator,
@@ -24,12 +24,13 @@ import { StatusBar } from "./StatusBar.ts";
 
 interface DataTableOptions {
 	table: string;
-	schema: arrow.Schema;
+	schema: flech.Schema;
 	height?: number;
 }
 
 // TODO: more
 type ColumnSummaryClient = Histogram | ValueCounts;
+type TableRow = Record<string, unknown>;
 
 /**
  * Create a DataTable client.
@@ -67,7 +68,7 @@ export async function datatable(
 
 export class DataTable extends MosaicClient {
 	/** source of the data */
-	#meta: { table: string; schema: arrow.Schema };
+	#meta: { table: string; schema: flech.Schema };
 	/** for the component */
 	#root: HTMLElement = document.createElement("div");
 	/** shadow root for the component */
@@ -97,8 +98,8 @@ export class DataTable extends MosaicClient {
 	/** the formatter for the data table entries */
 	#format: Record<string, (value: unknown) => string>;
 
-	/** @type {AsyncBatchReader<arrow.StructRowProxy> | null} */
-	#reader: AsyncBatchReader<arrow.StructRowProxy> | null = null;
+	/** @type {AsyncBatchReader<flech.StructRowProxy> | null} */
+	#reader: AsyncBatchReader<TableRow> | null = null;
 
 	#sql = signal(undefined as string | undefined);
 
@@ -199,7 +200,7 @@ export class DataTable extends MosaicClient {
 	 * A Mosiac lifecycle function called with arrow results from `query`.
 	 * Must be synchronous, and return `this`.
 	 */
-	queryResult(table: arrow.Table): this {
+	queryResult(table: flech.Table): this {
 		if (!this.#pendingInternalRequest) {
 			// data is not from an internal request, so reset table
 			this.#reader = new AsyncBatchReader(() => {
@@ -362,7 +363,7 @@ export class DataTable extends MosaicClient {
 		}
 	}
 
-	#appendRow(d: arrow.StructRowProxy, i: number) {
+	#appendRow(d: TableRow, i: number) {
 		let itr = this.#templateRow?.cloneNode(true);
 		assert(itr, "Must have a data row");
 		let td = itr.childNodes[0] as HTMLTableCellElement;
@@ -389,7 +390,7 @@ const TRUNCATE = /** @type {const} */ ({
 });
 
 function thcol(
-	field: arrow.Field,
+	field: flech.Field,
 	minWidth: number,
 	vis?: ColumnSummaryClient,
 ) {
@@ -502,7 +503,7 @@ function thcol(
 /**
  * Return a formatter for each field in the schema
  */
-function formatof(schema: arrow.Schema) {
+function formatof(schema: flech.Schema) {
 	const format: Record<string, (value: unknown) => string> = Object.create(
 		null,
 	);
@@ -515,20 +516,20 @@ function formatof(schema: arrow.Schema) {
 /**
  * Return a class type of each field in the schema.
  */
-function classof(schema: arrow.Schema): Record<string, "number" | "date"> {
+function classof(schema: flech.Schema): Record<string, "number" | "date"> {
 	const classes: Record<string, "number" | "date"> = Object.create(null);
 	for (const field of schema.fields) {
-		if (
-			arrow.DataType.isInt(field.type) ||
-			arrow.DataType.isFloat(field.type)
-		) {
-			classes[field.name] = "number";
-		}
-		if (
-			arrow.DataType.isDate(field.type) ||
-			arrow.DataType.isTimestamp(field.type)
-		) {
-			classes[field.name] = "date";
+		switch (field.type.typeId) {
+			case flech.Type.Int:
+			case flech.Type.Float:
+				classes[field.name] = "number";
+				break;
+			case flech.Type.Date:
+			case flech.Type.Timestamp:
+				classes[field.name] = "date";
+				break;
+			default:
+				break;
 		}
 	}
 	return classes;

lib/clients/DataTable.ts

@@ -9,7 +9,7 @@ import {
 // @ts-types="../deps/mosaic-sql.d.ts";
 import { count, Query, type SQLExpression } from "@uwdata/mosaic-sql";
 import * as mplot from "@uwdata/mosaic-plot";
-import type * as arrow from "apache-arrow";
+import type * as flech from "@uwdata/flechette";
 
 import { CrossfilterHistogramPlot } from "../utils/CrossfilterHistogramPlot.ts";
 
@@ -21,7 +21,7 @@ interface HistogramOptions {
 	/** The table to query. */
 	table: string;
 	/** An arrow Field containing the column info to use for the histogram. */
-	field: arrow.Field;
+	field: flech.Field;
 	/** The column to use for the histogram. */
 	column: string;
 	/** The type of the column. Must be "number" or "date". */
@@ -30,14 +30,14 @@ interface HistogramOptions {
 	filterBy: Selection;
 }
 
-type BinTable = arrow.Table<{ x1: arrow.Int; x2: arrow.Int; y: arrow.Int }>;
+type BinTable = flech.Table;
 
 /** Represents a Cross-filtered Histogram */
 export class Histogram extends MosaicClient implements Mark {
 	#source: {
 		table: string;
 		column: string;
-		field: arrow.Field;
+		field: flech.Field;
 		type: "number" | "date";
 	};
 	#el: HTMLElement = document.createElement("div");
@@ -102,11 +102,9 @@ export class Histogram extends MosaicClient implements Mark {
 	 * Provide query result data to the mark.
 	 */
 	queryResult(data: BinTable) {
-		let bins = Array.from(data, (d) => ({
-			x0: d.x1,
-			x1: d.x2,
-			length: d.y,
-		}));
+		let bins: Array<{ x0: number; x1: number; length: number }> = data
+			.toArray()
+			.map((d) => ({ x0: d.x1, x1: d.x2, length: d.y }));
 		let nullCount = 0;
 		let nullBinIndex = bins.findIndex((b) => b.x0 == null);
 		if (nullBinIndex >= 0) {

lib/clients/Histogram.ts

@@ -1,4 +1,4 @@
-import type * as arrow from "apache-arrow";
+import * as flech from "@uwdata/flechette";
 // @ts-types="../deps/mosaic-core.d.ts"
 import {
 	type Interactor,
@@ -7,6 +7,7 @@ import {
 } from "@uwdata/mosaic-core";
 // @ts-types="../deps/mosaic-sql.d.ts"
 import { count, Query } from "@uwdata/mosaic-sql";
+import { assert } from "../utils/assert.ts";
 
 interface StatusBarOptions {
 	table: string;
@@ -66,7 +67,13 @@ export class StatusBar extends MosaicClient {
 		return query;
 	}
 
-	queryResult(table: arrow.Table<{ count: arrow.Int }>) {
+	queryResult(table: flech.Table) {
+		assert(
+			table.schema.fields.find((f) => f.name === "count")?.type.typeId ===
+				flech.Type.Int,
+			"Expected count field to be an integer",
+		);
+
 		let count = Number(table.get(0)?.count ?? 0);
 		if (!this.#totalRows) {
 			// we need to know the total number of rows to display

lib/clients/StatusBar.ts

@@ -9,7 +9,7 @@ import {
 	type SQLExpression,
 	sum,
 } from "@uwdata/mosaic-sql";
-import type * as arrow from "apache-arrow";
+import type * as flech from "@uwdata/flechette";
 import { effect } from "@preact/signals-core";
 
 import { ValueCountsPlot } from "../utils/ValueCountsPlot.ts";
@@ -19,17 +19,17 @@ interface UniqueValuesOptions {
 	/** The table to query. */
 	table: string;
 	/** An arrow Field containing the column info to use for the histogram. */
-	field: arrow.Field;
+	field: flech.Field;
 	/** A mosaic selection to filter the data. */
 	filterBy: Selection;
 }
 
-type CountTable = arrow.Table<{ key: arrow.Utf8; total: arrow.Int }>;
+type CountTable = flech.Table;
 
 export class ValueCounts extends MosaicClient {
 	#table: string;
 	#column: string;
-	#field: arrow.Field;
+	#field: flech.Field;
 	#el: HTMLElement = document.createElement("div");
 	#plot: ReturnType<typeof ValueCountsPlot> | undefined;
 

lib/clients/ValueCounts.ts

@@ -1,4 +1,5 @@
 /// <reference types="npm:vite/client" />
+// @ts-types="./deps/mosaic-core.d.ts";
 import * as mc from "@uwdata/mosaic-core";
 import * as msql from "@uwdata/mosaic-sql";
 

lib/demo.ts

@@ -1,6 +1,6 @@
 // @ts-types="./mosaic-sql.d.ts";
 import type { Query, SQLExpression } from "@uwdata/mosaic-sql";
-import type * as arrow from "apache-arrow";
+import type * as flechette from "@uwdata/flechette";
 
 export interface Interactor<T = unknown> {
 	reset(): void;
@@ -113,7 +113,7 @@ export class MosaicClient {
 	query(filter?: Array<unknown>): Query;
 	/** Called before the coordinator submitting a query to inform the client */
 	queryPending(): this;
-	queryResult(data: arrow.Table): this;
+	queryResult(data: flechette.Table): this;
 	queryError(error: unknown): this;
 	requestQuery(query: Query): void;
 	requestUpdate(query: Query): void;
@@ -124,7 +124,7 @@ export type ConnectorQuery = { type: "arrow" | "json"; sql: string };
 export interface Connector {
 	query(
 		query: ConnectorQuery,
-	): Promise<arrow.Table | Record<string, unknown>>;
+	): Promise<flechette.Table | Record<string, unknown>>;
 }
 
 export class Coordinator {
@@ -134,8 +134,11 @@ export class Coordinator {
 	prefetch(query: Query): void;
 	logger(impl?: unknown): Logger;
 	databaseConnector(connector: Connector): void;
-	query(query: Query): Promise<arrow.Table>;
+	query(query: Query): Promise<flechette.Table>;
 	clear(): void;
+	dataCubeIndexer: {
+		schema: string | undefined;
+	};
 }
 
 type Logger = typeof console & {

lib/deps/mosaic-core.d.ts

@@ -22,13 +22,13 @@ import { assert } from "./assert.ts";
  */
 export class AsyncBatchReader<T> {
 	/** the iterable batches to read */
-	#batches: Array<{ data: Iterator<T>; last: boolean }> = [];
+	#batches: Array<{ data: Generator<T>; last: boolean }> = [];
 	/** the index of the current row */
 	#index: number = 0;
 	/** resolves a promise for when the next batch is available */
 	#resolve: (() => void) | null = null;
 	/** the current batch */
-	#current: { data: Iterator<T>; last: boolean } | null = null;
+	#current: { data: Generator<T>; last: boolean } | null = null;
 	/** A function to request more data. */
 	#requestNextBatch: () => void;
 	/**
@@ -50,7 +50,7 @@ export class AsyncBatchReader<T> {
 	 * @param options
 	 * @param options.last - whether this is the last batch
 	 */
-	enqueueBatch(batch: Iterator<T>, { last }: { last: boolean }) {
+	enqueueBatch(batch: Generator<T>, { last }: { last: boolean }) {
 		this.#batches.push({ data: batch, last });
 		if (this.#resolve) {
 			this.#resolve();

lib/utils/AsyncBatchReader.ts

@@ -4,7 +4,7 @@ import * as d3 from "d3";
 import { assert } from "../utils/assert.ts";
 import { tickFormatterForBins } from "./tick-formatter-for-bins.ts";
 import type { Bin, Scale } from "../types.ts";
-import type * as arrow from "apache-arrow";
+import type * as flech from "@uwdata/flechette";
 import { formatDataType, percentFormatter } from "./formatting.ts";
 
 interface HistogramOptions {
@@ -29,7 +29,7 @@ interface HistogramOptions {
  */
 export function CrossfilterHistogramPlot(
 	bins: Array<Bin>,
-	field: arrow.Field,
+	field: flech.Field,
 	{
 		type = "number",
 		width = 125,

lib/utils/CrossfilterHistogramPlot.ts

@@ -1,14 +1,11 @@
 import { effect, signal } from "@preact/signals-core";
-import type * as arrow from "apache-arrow";
+import type * as flech from "@uwdata/flechette";
 // @ts-types="npm:@types/d3"
 import * as d3 from "d3";
 import { assert } from "./assert.ts";
 import { formatDataType, percentFormatter } from "./formatting.ts";
 
-type CountTableData = arrow.Table<{
-	key: arrow.Utf8;
-	total: arrow.Int;
-}>;
+type CountTableData = flech.Table;
 
 interface ValueCountsPlot {
 	width?: number;
@@ -24,7 +21,7 @@ interface ValueCountsPlot {
 
 export function ValueCountsPlot(
 	data: CountTableData,
-	field: arrow.Field,
+	field: flech.Field,
 	{
 		width = 125,
 		height = 30,
@@ -178,9 +175,11 @@ function createBar(opts: {
 }
 
 function prepareData(data: CountTableData) {
-	let arr: Array<{ key: string; total: number }> = data
+	let arr = data
 		.toArray()
-		.toSorted((a, b) => b.total - a.total);
+		.toSorted((a, b) => b.total - a.total) as Array<
+			{ key: string; total: number }
+		>;
 	let total = arr.reduce((acc, d) => acc + d.total, 0);
 	return {
 		bins: arr.filter((d) =>