Skip to content

Commit 14f5754

Browse files
arpowersarpowers
committed
wip: charts 2
1 parent 64658d2 commit 14f5754

File tree

11 files changed

+934
-515
lines changed

11 files changed

+934
-515
lines changed

‎@fiction/analytics/plugin-clickhouse/test/fillData.test.ts ‎@fiction/analytics/.ref/__fillData.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { dayjs } from '@fiction/core'
22
import { describe, expect, it } from 'vitest'
3-
import { fillData } from '../utils.js'
3+
import { fillData } from '../plugin-clickhouse/utils.js'
44

55
const data = [
66
{

‎@fiction/analytics/endpointMetrics.ts

+41-35
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import type {
1010
QueryParamsRefined,
1111
} from './types'
1212
import { AnalyticsEndpoint } from './endpoints'
13-
import { refineParams } from './utils/refine'
13+
import { refineComparedData, refineParams } from './utils/refine'
1414

1515
type MetricDataPoint = DataPointChart<string>
1616
type ReturnData = DataCompared<DataPointChart<string>>
@@ -68,32 +68,17 @@ export class QueryCompiledMetrics extends AnalyticsEndpoint {
6868
metrics.flatMap(m => m.events || []),
6969
)]
7070

71-
const subQueryBase = ch.clickhouseDateQuery({ params: refinedParams, table: 'event', isCompare })
71+
// Return individual event values for each date
72+
return ch.clickhouseDateQuery({ params: refinedParams, table: 'event', isCompare })
7273
.whereIn('event', eventNames)
7374
.select([
7475
client.raw(`${dateSelect} as date`),
75-
'event',
76-
client.raw('argMax(value, timeAt) as event_value'),
76+
...eventNames.map(event =>
77+
ch.client().raw(`argMax(if(event = '${event}', value, null), timeAt) as ${event}`),
78+
),
7779
])
78-
.groupBy(['date', 'event'])
79-
80-
// Aggregate snapshots by metric keys
81-
const mainQuery = client
82-
.from(subQueryBase.as('snapshots'))
83-
.select(['date'])
84-
.select(metrics.map((metric) => {
85-
const events = metric.events || []
86-
return client.raw(
87-
`sum(CASE WHEN event IN (${
88-
events.map(() => '?').join(',')
89-
}) THEN event_value ELSE 0 END) as ??`,
90-
[...events, metric.key],
91-
)
92-
}))
9380
.groupByRaw('date WITH ROLLUP')
94-
.orderBy('date', 'asc')
95-
96-
return mainQuery
81+
.orderBy('date')
9782
}
9883

9984
private async getSnapshotData(args: {
@@ -109,6 +94,7 @@ export class QueryCompiledMetrics extends AnalyticsEndpoint {
10994

11095
// Aggregate snapshots by metric keys
11196
const mainQuery = this.getSnapshotQuery({ refinedParams, metrics })
97+
11298
const compareQuery = this.getSnapshotQuery({ refinedParams, metrics, isCompare: true })
11399

114100
const [mainResult, compareResult] = await Promise.all([
@@ -126,8 +112,9 @@ export class QueryCompiledMetrics extends AnalyticsEndpoint {
126112
compareTotals: compareTotals || {},
127113
params: refinedParams,
128114
}
115+
const finalData = this.getMetricResults({ metrics, data })
129116

130-
return { status: 'success', data: this.getMetricResults({ metrics, data }) }
117+
return { status: 'success', data: finalData }
131118
}
132119

133120
private getEventQuery(args: {
@@ -257,18 +244,37 @@ export class QueryCompiledMetrics extends AnalyticsEndpoint {
257244

258245
getMetricResults(args: { metrics: MetricSelector[], data: ReturnData }): MetricSelectorResult[] {
259246
const { metrics, data } = args
247+
const snapshotKeys = metrics
248+
.filter(m => m.type === 'snapshot')
249+
.flatMap(m => m.events || [])
260250

261-
return metrics.map((metric) => {
262-
const key = metric.key
263-
const main = data.main?.map(d => ({ ...d, value: d[key] }))
264-
const compare = data.compare?.map(d => ({ ...d, value: d[key] }))
265-
const mainTotals = { ...data.mainTotals, value: data.mainTotals?.[key] || 0 }
266-
const compareTotals = { ...data.compareTotals, value: data.compareTotals?.[key] || 0 }
267-
268-
return {
269-
...metric,
270-
data: { main, compare, mainTotals, compareTotals, params: data.params },
271-
}
272-
})
251+
const refinedData = refineComparedData({ data, snapshotKeys })
252+
253+
function transformPoints(points: DataPointChart[] = [], metric: MetricSelector) {
254+
return points.map(point => transformPoint({ point, metric })) as DataPointChart[]
255+
}
256+
257+
function transformPoint(args: { point?: DataPointChart, metric: MetricSelector }) {
258+
const { point, metric } = args
259+
if (!point)
260+
return undefined
261+
262+
const value = metric?.type === 'snapshot' && metric.events?.length
263+
? metric.events.reduce((sum, key) => sum + (Number(point[key]) || 0), 0)
264+
: point[metric?.key] || 0
265+
266+
return { ...point, value }
267+
}
268+
269+
return metrics.map(metric => ({
270+
...metric,
271+
data: {
272+
...refinedData,
273+
main: transformPoints(refinedData.main, metric),
274+
compare: transformPoints(refinedData.compare, metric),
275+
mainTotals: transformPoint({ point: refinedData.mainTotals, metric }),
276+
compareTotals: transformPoint({ point: refinedData.compareTotals, metric }),
277+
},
278+
}))
273279
}
274280
}

‎@fiction/analytics/plugin-clickhouse/index.ts

+11-6
Original file line numberDiff line numberDiff line change
@@ -291,14 +291,19 @@ export class FictionClickHouse extends FictionPlugin<FictionClickHouseSettings>
291291
timeZone: string
292292
timeField?: 'timestamp' | 'session__timestamp'
293293
}): string {
294-
let startOf = `toStartOf${capitalize(interval)}(${timeField}, '${timeZone}')`
294+
// Handle minute-based intervals
295+
const minutes = interval.match(/(\d+)min/)?.[1]
296+
if (minutes) {
297+
return `formatDateTime(toStartOfInterval(${timeField}, INTERVAL ${minutes} MINUTE, '${timeZone}'), '%FT%T.000Z', 'UTC')`
298+
}
295299

296-
// clickhouse doesn't seem to support timezone in week/month/year intervals
297-
// week requires mode = 1 to set monday to be the start of the week
298-
if (interval === 'week')
299-
startOf = `toMonday(${timeField}, '${timeZone}')`
300+
// Special case for week
301+
if (interval === 'week') {
302+
return `formatDateTime(toMonday(${timeField}, '${timeZone}'), '%FT%T.000Z', 'UTC')`
303+
}
300304

301-
return `formatDateTime(${startOf}, '%FT%T.000Z', 'UTC')`
305+
// Default case for other intervals
306+
return `formatDateTime(toStartOf${capitalize(interval)}(${timeField}, '${timeZone}'), '%FT%T.000Z', 'UTC')`
302307
}
303308

304309
naiveDateTime = (time: number): string => {

‎@fiction/analytics/plugin-clickhouse/utils.ts

+71-72
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
import type { ColDefaultValue, ColSettings, FictionDbTableSettings } from '@fiction/core'
2-
import type { TimeLineInterval } from '../types.js'
32
import type { FictionEvent } from '../typesTracking.js'
4-
import type { BaseChartData, ClickHouseDatatype, FictionClickHouse } from './index.js'
5-
import { Col, dayjs, FictionDbTable } from '@fiction/core'
3+
import type { ClickHouseDatatype, FictionClickHouse } from './index.js'
4+
import { Col, FictionDbTable } from '@fiction/core'
65
import { z } from 'zod'
76

87
type ValueCallback = (params: {
@@ -95,72 +94,72 @@ export class FictionAnalyticsTable extends FictionDbTable {
9594
}
9695
}
9796

98-
export function fillData<T extends BaseChartData>(args: {
99-
timeZone: string
100-
timeStartAt: dayjs.Dayjs
101-
timeEndAt: dayjs.Dayjs
102-
interval: TimeLineInterval
103-
withRollup?: boolean
104-
data: T[]
105-
}): T[] {
106-
const { timeStartAt, timeEndAt, timeZone, interval, data = [], withRollup } = args
107-
108-
const newData: { date: string, [key: string]: any }[]
109-
= withRollup && data[0] ? [data[0]] : [{} as T]
110-
111-
// clickhouse returns different timezone handling for weeks/months/years vs days/hours
112-
// appropriate timezone is returned for < weeks but always utc otherwise
113-
let loopTime: dayjs.Dayjs
114-
let finishTime: dayjs.Dayjs
115-
if (interval === 'week' || interval === 'month') {
116-
loopTime = timeStartAt.utc().startOf(interval)
117-
finishTime = timeEndAt.utc().endOf(interval)
118-
}
119-
else {
120-
loopTime = timeStartAt.clone().tz(timeZone)
121-
finishTime = timeEndAt.clone().tz(timeZone)
122-
}
123-
124-
const duration = Math.abs(finishTime.diff(loopTime, 'day'))
125-
126-
const sample = data[0] ?? {}
127-
// create default object from sample set to zeros
128-
const defaultObjectIfMissing = Object.fromEntries(
129-
Object.entries(sample)
130-
.map(([k, v]) => {
131-
return ((typeof v === 'string' && /^-?\d+$/.test(v)) || typeof v === 'number') ? [k, 0] : undefined
132-
})
133-
.filter(Boolean) as [string, number][],
134-
)
135-
136-
while (
137-
loopTime.isBefore(finishTime, interval)
138-
|| loopTime.isSame(finishTime, interval)
139-
) {
140-
const date = loopTime.toISOString()
141-
const displayDate = loopTime.tz(timeZone)
142-
143-
const now = dayjs()
144-
const found = data.find(_ => _.date === date) || defaultObjectIfMissing
145-
146-
const dateFormat
147-
= duration < 3 ? 'ha' : duration > 180 ? 'MMM D, YYYY' : 'MMM D'
148-
149-
const d: BaseChartData = {
150-
...found,
151-
date,
152-
label: displayDate.format(dateFormat),
153-
tense: displayDate.isSame(now, interval)
154-
? 'present'
155-
: displayDate.isAfter(now, interval)
156-
? 'future'
157-
: 'past',
158-
}
159-
160-
newData.push(d)
161-
162-
loopTime = loopTime.add(1, interval)
163-
}
164-
165-
return newData as T[]
166-
}
97+
// export function fillData<T extends BaseChartData>(args: {
98+
// timeZone: string
99+
// timeStartAt: dayjs.Dayjs
100+
// timeEndAt: dayjs.Dayjs
101+
// interval: TimeLineInterval
102+
// withRollup?: boolean
103+
// data: T[]
104+
// }): T[] {
105+
// const { timeStartAt, timeEndAt, timeZone, interval, data = [], withRollup } = args
106+
107+
// const newData: { date: string, [key: string]: any }[]
108+
// = withRollup && data[0] ? [data[0]] : [{} as T]
109+
110+
// // clickhouse returns different timezone handling for weeks/months/years vs days/hours
111+
// // appropriate timezone is returned for < weeks but always utc otherwise
112+
// let loopTime: dayjs.Dayjs
113+
// let finishTime: dayjs.Dayjs
114+
// if (interval === 'week' || interval === 'month') {
115+
// loopTime = timeStartAt.utc().startOf(interval)
116+
// finishTime = timeEndAt.utc().endOf(interval)
117+
// }
118+
// else {
119+
// loopTime = timeStartAt.clone().tz(timeZone)
120+
// finishTime = timeEndAt.clone().tz(timeZone)
121+
// }
122+
123+
// const duration = Math.abs(finishTime.diff(loopTime, 'day'))
124+
125+
// const sample = data[0] ?? {}
126+
// // create default object from sample set to zeros
127+
// const defaultObjectIfMissing = Object.fromEntries(
128+
// Object.entries(sample)
129+
// .map(([k, v]) => {
130+
// return ((typeof v === 'string' && /^-?\d+$/.test(v)) || typeof v === 'number') ? [k, 0] : undefined
131+
// })
132+
// .filter(Boolean) as [string, number][],
133+
// )
134+
135+
// while (
136+
// loopTime.isBefore(finishTime, interval)
137+
// || loopTime.isSame(finishTime, interval)
138+
// ) {
139+
// const date = loopTime.toISOString()
140+
// const displayDate = loopTime.tz(timeZone)
141+
142+
// const now = dayjs()
143+
// const found = data.find(_ => _.date === date) || defaultObjectIfMissing
144+
145+
// const dateFormat
146+
// = duration < 3 ? 'ha' : duration > 180 ? 'MMM D, YYYY' : 'MMM D'
147+
148+
// const d: BaseChartData = {
149+
// ...found,
150+
// date,
151+
// label: displayDate.format(dateFormat),
152+
// tense: displayDate.isSame(now, interval)
153+
// ? 'present'
154+
// : displayDate.isAfter(now, interval)
155+
// ? 'future'
156+
// : 'past',
157+
// }
158+
159+
// newData.push(d)
160+
161+
// loopTime = loopTime.add(1, interval)
162+
// }
163+
164+
// return newData as T[]
165+
// }

0 commit comments

Comments
 (0)