diff --git a/src/app/app.component.ts b/src/app/app.component.ts index 15180f3..6e6a976 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -7,8 +7,6 @@ import { IconField } from 'primeng/iconfield'; import { InputIcon } from 'primeng/inputicon'; import { SplitButtonModule } from 'primeng/splitbutton'; import { InputTextModule } from 'primeng/inputtext'; -import { TableLazyLoadEvent, TableModule } from 'primeng/table'; -import { Skeleton } from 'primeng/skeleton'; import { FileTreeComponent } from './file-tree/file-tree.component'; import { FileViewerComponent } from './file-viewer/file-viewer.component'; diff --git a/src/app/duckdb.service.ts b/src/app/duckdb.service.ts index 5b0b9d8..1406724 100644 --- a/src/app/duckdb.service.ts +++ b/src/app/duckdb.service.ts @@ -7,37 +7,82 @@ export const Column = z.object({ type: z.string(), }); -const SortColumn = z.object({ +export const SortColumn = z.object({ name: z.string(), sortType: z.enum(['asc', 'desc']), }); -const Filter = z.object({ - value: z.string().array(), - operator: z.enum(['and', 'or']), - matchType: z.enum([ - 'startsWith', - 'contains', - 'notContains', - 'endsWith', - 'equals', - 'notEquals', - ]), +export const FilterOperator = z.enum([ + 'startsWith', + 'contains', + 'notContains', + 'endsWith', + 'equals', + 'notEquals', +]); + +export const FilterValue = z.object({ + value: z.string().trim(), + matchType: FilterOperator, }); -const RowsResponse = z.object({ +export const Filter = z.object({ + column: z.string(), + value: FilterValue.optional().array(), + operator: z.enum(['and', 'or']), +}); + +export const RowsResponse = z.object({ + // TODO: Can't this just be a string[]? rows: z.any(), - totalRows: z.bigint(), + totalRows: z.bigint().nonnegative(), }); export type Column = z.infer; export type SortColumn = z.infer; +export type FilterValue = z.infer; +export type FilterOperator = z.infer; export type Filter = z.infer; export type RowsResponse = z.infer; const sanitisedFileName = (file: File) => file.name.toLowerCase().replaceAll("'", '').replaceAll(/\s*/g, ''); +const sqlOperator = (operator: FilterOperator) => { + switch (operator) { + case 'startsWith': + case 'endsWith': + case 'contains': + case 'equals': + return 'ILIKE'; + case 'notContains': + case 'notEquals': + return 'NOT ILIKE'; + } +}; + +const prefix = (operator: FilterOperator) => { + switch (operator) { + case 'endsWith': + case 'contains': + case 'notContains': + return '%'; + default: + return ''; + } +}; + +const suffix = (operator: FilterOperator) => { + switch (operator) { + case 'startsWith': + case 'contains': + case 'notContains': + return '%'; + default: + return ''; + } +}; + //https://www.npmjs.com/package/@duckdb/duckdb-wasm @Injectable({ providedIn: 'root', @@ -127,19 +172,30 @@ export class DuckdbService { ): Promise { const conn = await this.db.connect(); try { - const totalRowResponse = await conn.query( - `SELECT COUNT(1) totalRows FROM ${sanitisedFileName(file)}`, + const whereClause = this.getWhereClause(filters); + const mappedFilterValues = filters.flatMap((filter) => + filter.value + .filter((value) => value?.value) + .map( + (value) => + `${prefix(value?.matchType!)}${value?.value}${suffix(value?.matchType!)}`, + ), ); + const rowCountQuery = `SELECT COUNT(1) totalRows FROM ${sanitisedFileName(file)} ${whereClause}`; + const totalRowStmt = await conn.prepare(rowCountQuery); + const totalRowResponse = await totalRowStmt.query(...mappedFilterValues); const { totalRows } = totalRowResponse.get(0)?.toJSON()!; - let query = `SELECT ${columns.map((column) => `"${column.name}"`).join(', ')} FROM ${sanitisedFileName(file)} `; + let query = `SELECT ${columns.map((column) => `"${column.name}"`).join(', ')} FROM ${sanitisedFileName(file)} ${whereClause}`; if (sorts.length > 0) { query += ` ORDER BY ${sorts.map((sort) => `"${sort.name}" ${sort.sortType}`).join(', ')}`; } query += ` LIMIT ${numRows} OFFSET ${start}`; - const response = await conn.query(query); + const stmt = await conn.prepare(query); const rows = []; - for (let i = 0; i < response.numRows; i++) { - rows.push(response.get(i)?.toJSON()!); + for await (const batch of await stmt.send(...mappedFilterValues)) { + for (const row of batch) { + rows.push(row.toJSON()!); + } } return { rows, totalRows }; } catch (err) { @@ -149,4 +205,27 @@ export class DuckdbService { conn.close(); } } + + // Where clause that gets attached to the main and rowCount selects. + private getWhereClause(filters: Filter[]) { + let query = ''; + if (filters.length > 0) { + let and = 'WHERE'; + for (const filter of filters) { + if (filter.value.find((value) => value?.value)) { + query += ` ${and} (`; + let or = ''; + for (const value of filter.value) { + if (value?.value) { + query += ` ${or} ${filter.column} ${sqlOperator(value?.matchType!)} ? `; + or = filter.operator; + } + } + query += ') '; + and = 'and'; + } + } + } + return query; + } } diff --git a/src/app/file-viewer/file-viewer.component.ts b/src/app/file-viewer/file-viewer.component.ts index 4e1fb80..c9474bb 100644 --- a/src/app/file-viewer/file-viewer.component.ts +++ b/src/app/file-viewer/file-viewer.component.ts @@ -7,16 +7,20 @@ import { signal, } from '@angular/core'; import { Skeleton } from 'primeng/skeleton'; -import { TableLazyLoadEvent, TableModule } from 'primeng/table'; -import { Column, DuckdbService } from '../duckdb.service'; +import { Table, TableLazyLoadEvent, TableModule } from 'primeng/table'; +import { Column, DuckdbService, Filter, FilterValue } from '../duckdb.service'; +import { ButtonModule } from 'primeng/button'; +import { IconField } from 'primeng/iconfield'; +import { InputIconModule } from 'primeng/inputicon'; @Component({ selector: 'app-file-viewer', standalone: true, - imports: [TableModule, Skeleton], + imports: [TableModule, Skeleton, ButtonModule, IconField, InputIconModule], template: ` @if (file() && columns().length > 0) { + +
+ + + + + + + +
+
@for (col of columns; track $index) { @@ -86,25 +106,16 @@ export class FileViewerComponent { constructor() { effect(async () => { - const file = this.file(); - if (file) { - await this.duckdbService.addFile(file); - this.columns.set(await this.duckdbService.getColumns(file)); - const rows = await this.duckdbService.getRows( - file, - 0, - this.numRows, - this.columns(), - [], - [], - [], - ); - const newValue = Array.from({ length: Number(rows.totalRows) }); - this.currentValue.set(newValue); - } + this.loadEmpty(); }); } + protected clear(table: Table) { + table.clear(); + table.multiSortMeta = [{ field: this.columns()[0].name, order: 1 }]; + table.sortMultiple(); + } + protected async onLazyLoad(event: TableLazyLoadEvent) { const file = this.file(); if (file) { @@ -117,17 +128,43 @@ export class FileViewerComponent { name: meta.field, sortType: meta.order < 0 ? 'desc' : 'asc', })) ?? [], - Array.isArray(event.filters) - ? Object.values(event.filters).map((filter) => ({ - value: Array.isArray(filter.value) - ? filter.value - : [filter.value], - matchType: filter.matchMode, - operator: filter.operator, - })) + event.filters + ? Object.entries(event.filters).flatMap(([column, filter]) => { + if (Array.isArray(filter)) { + const firstFilter = filter[0]; + return Filter.parse({ + column, + value: filter + .filter((f) => f.value != null) + .map((f) => + FilterValue.parse({ + value: f.value, + matchType: f.matchMode, + }), + ), + operator: firstFilter.operator, + }); + } else if (filter) { + return Filter.parse({ + column, + value: [ + FilterValue.parse({ + value: filter.value, + matchType: filter.matchMode, + }), + ], + operator: filter.operator, + }); + } + return []; + }) : [], [], ); + if (this.currentValue().length !== Number(rows.totalRows)) { + this.currentValue.set(Array.from({ length: Number(rows.totalRows) })); + return; + } // First clear out existing data, don't want to risk loading entire file into memory // Keep data in previous/next page so when user scrolls between 2 pages they don't see missing data for (let i = 0; i < this.currentValue().length; i++) { @@ -138,9 +175,29 @@ export class FileViewerComponent { this.currentValue()[i] = undefined; } } + // Can't update the current value, otherwise we get an infinite loop this.currentValue().splice(event.first!, event.rows!, ...rows.rows); event.forceUpdate!(); } } + + private async loadEmpty() { + const file = this.file(); + if (file) { + await this.duckdbService.addFile(file); + this.columns.set(await this.duckdbService.getColumns(file)); + const rows = await this.duckdbService.getRows( + file, + 0, + this.numRows, + this.columns(), + [], + [], + [], + ); + const newValue = Array.from({ length: Number(rows.totalRows) }); + this.currentValue.set(newValue); + } + } }