Add basic filtering

src/app/app.component.ts

@@ -7,8 +7,6 @@ import { IconField } from 'primeng/iconfield';
 import { InputIcon } from 'primeng/inputicon';
 import { SplitButtonModule } from 'primeng/splitbutton';
 import { InputTextModule } from 'primeng/inputtext';
-import { TableLazyLoadEvent, TableModule } from 'primeng/table';
-import { Skeleton } from 'primeng/skeleton';
 import { FileTreeComponent } from './file-tree/file-tree.component';
 import { FileViewerComponent } from './file-viewer/file-viewer.component';
 

src/app/duckdb.service.ts

@@ -7,37 +7,82 @@ export const Column = z.object({
   type: z.string(),
 });
 
-const SortColumn = z.object({
+export const SortColumn = z.object({
   name: z.string(),
   sortType: z.enum(['asc', 'desc']),
 });
 
-const Filter = z.object({
-  value: z.string().array(),
-  operator: z.enum(['and', 'or']),
-  matchType: z.enum([
+export const FilterOperator = z.enum([
   'startsWith',
   'contains',
   'notContains',
   'endsWith',
   'equals',
   'notEquals',
-  ]),
+]);
+
+export const FilterValue = z.object({
+  value: z.string().trim(),
+  matchType: FilterOperator,
 });
 
-const RowsResponse = z.object({
+export const Filter = z.object({
+  column: z.string(),
+  value: FilterValue.optional().array(),
+  operator: z.enum(['and', 'or']),
+});
+
+export const RowsResponse = z.object({
+  // TODO: Can't this just be a string[]?
   rows: z.any(),
-  totalRows: z.bigint(),
+  totalRows: z.bigint().nonnegative(),
 });
 
 export type Column = z.infer<typeof Column>;
 export type SortColumn = z.infer<typeof SortColumn>;
+export type FilterValue = z.infer<typeof FilterValue>;
+export type FilterOperator = z.infer<typeof FilterOperator>;
 export type Filter = z.infer<typeof Filter>;
 export type RowsResponse = z.infer<typeof RowsResponse>;
 
 const sanitisedFileName = (file: File) =>
   file.name.toLowerCase().replaceAll("'", '').replaceAll(/\s*/g, '');
 
+const sqlOperator = (operator: FilterOperator) => {
+  switch (operator) {
+    case 'startsWith':
+    case 'endsWith':
+    case 'contains':
+    case 'equals':
+      return 'ILIKE';
+    case 'notContains':
+    case 'notEquals':
+      return 'NOT ILIKE';
+  }
+};
+
+const prefix = (operator: FilterOperator) => {
+  switch (operator) {
+    case 'endsWith':
+    case 'contains':
+    case 'notContains':
+      return '%';
+    default:
+      return '';
+  }
+};
+
+const suffix = (operator: FilterOperator) => {
+  switch (operator) {
+    case 'startsWith':
+    case 'contains':
+    case 'notContains':
+      return '%';
+    default:
+      return '';
+  }
+};
+
 //https://www.npmjs.com/package/@duckdb/duckdb-wasm
 @Injectable({
   providedIn: 'root',
@@ -127,19 +172,30 @@ export class DuckdbService {
   ): Promise<RowsResponse> {
     const conn = await this.db.connect();
     try {
-      const totalRowResponse = await conn.query(
-        `SELECT COUNT(1) totalRows FROM ${sanitisedFileName(file)}`,
+      const whereClause = this.getWhereClause(filters);
+      const mappedFilterValues = filters.flatMap((filter) =>
+        filter.value
+          .filter((value) => value?.value)
+          .map(
+            (value) =>
+              `${prefix(value?.matchType!)}${value?.value}${suffix(value?.matchType!)}`,
+          ),
       );
+      const rowCountQuery = `SELECT COUNT(1) totalRows FROM ${sanitisedFileName(file)} ${whereClause}`;
+      const totalRowStmt = await conn.prepare(rowCountQuery);
+      const totalRowResponse = await totalRowStmt.query(...mappedFilterValues);
       const { totalRows } = totalRowResponse.get(0)?.toJSON()!;
-      let query = `SELECT ${columns.map((column) => `"${column.name}"`).join(', ')} FROM ${sanitisedFileName(file)} `;
+      let query = `SELECT ${columns.map((column) => `"${column.name}"`).join(', ')} FROM ${sanitisedFileName(file)} ${whereClause}`;
       if (sorts.length > 0) {
         query += ` ORDER BY ${sorts.map((sort) => `"${sort.name}" ${sort.sortType}`).join(', ')}`;
       }
       query += ` LIMIT ${numRows} OFFSET ${start}`;
-      const response = await conn.query(query);
+      const stmt = await conn.prepare(query);
       const rows = [];
-      for (let i = 0; i < response.numRows; i++) {
-        rows.push(response.get(i)?.toJSON()!);
+      for await (const batch of await stmt.send(...mappedFilterValues)) {
+        for (const row of batch) {
+          rows.push(row.toJSON()!);
+        }
       }
       return { rows, totalRows };
     } catch (err) {
@@ -149,4 +205,27 @@ export class DuckdbService {
       conn.close();
     }
   }
+
+  // Where clause that gets attached to the main and rowCount selects.
+  private getWhereClause(filters: Filter[]) {
+    let query = '';
+    if (filters.length > 0) {
+      let and = 'WHERE';
+      for (const filter of filters) {
+        if (filter.value.find((value) => value?.value)) {
+          query += ` ${and} (`;
+          let or = '';
+          for (const value of filter.value) {
+            if (value?.value) {
+              query += ` ${or} ${filter.column} ${sqlOperator(value?.matchType!)} ? `;
+              or = filter.operator;
+            }
+          }
+          query += ') ';
+          and = 'and';
+        }
+      }
+    }
+    return query;
+  }
 }

src/app/file-viewer/file-viewer.component.ts

@@ -7,16 +7,20 @@ import {
   signal,
 } from '@angular/core';
 import { Skeleton } from 'primeng/skeleton';
-import { TableLazyLoadEvent, TableModule } from 'primeng/table';
-import { Column, DuckdbService } from '../duckdb.service';
+import { Table, TableLazyLoadEvent, TableModule } from 'primeng/table';
+import { Column, DuckdbService, Filter, FilterValue } from '../duckdb.service';
+import { ButtonModule } from 'primeng/button';
+import { IconField } from 'primeng/iconfield';
+import { InputIconModule } from 'primeng/inputicon';
 
 @Component({
   selector: 'app-file-viewer',
   standalone: true,
-  imports: [TableModule, Skeleton],
+  imports: [TableModule, Skeleton, ButtonModule, IconField, InputIconModule],
   template: `
     @if (file() && columns().length > 0) {
       <p-table
+        #table
         sortMode="multiple"
         [reorderableColumns]="true"
         [resizableColumns]="true"
@@ -34,6 +38,22 @@ import { Column, DuckdbService } from '../duckdb.service';
         [multiSortMeta]="[{ field: columns()[0].name, order: 1 }]"
         stripedRows
       >
+        <ng-template #caption>
+          <div class="flex">
+            <p-button
+              label="Reset"
+              [outlined]="true"
+              icon="pi pi-filter-slash"
+              (click)="clear(table)"
+            />
+            <p-iconfield iconPosition="left" class="ml-auto">
+              <p-inputicon>
+                <i class="pi pi-search"></i>
+              </p-inputicon>
+              <input pInputText type="text" placeholder="Search keyword" />
+            </p-iconfield>
+          </div>
+        </ng-template>
         <ng-template #header let-columns>
           <tr>
             @for (col of columns; track $index) {
@@ -86,6 +106,83 @@ export class FileViewerComponent {
 
   constructor() {
     effect(async () => {
+      this.loadEmpty();
+    });
+  }
+
+  protected clear(table: Table) {
+    table.clear();
+    table.multiSortMeta = [{ field: this.columns()[0].name, order: 1 }];
+    table.sortMultiple();
+  }
+
+  protected async onLazyLoad(event: TableLazyLoadEvent) {
+    const file = this.file();
+    if (file) {
+      const rows = await this.duckdbService.getRows(
+        file,
+        event.first ?? 0,
+        event.rows ?? 0,
+        this.columns(),
+        event.multiSortMeta?.map((meta) => ({
+          name: meta.field,
+          sortType: meta.order < 0 ? 'desc' : 'asc',
+        })) ?? [],
+        event.filters
+          ? Object.entries(event.filters).flatMap(([column, filter]) => {
+              if (Array.isArray(filter)) {
+                const firstFilter = filter[0];
+                return Filter.parse({
+                  column,
+                  value: filter
+                    .filter((f) => f.value != null)
+                    .map((f) =>
+                      FilterValue.parse({
+                        value: f.value,
+                        matchType: f.matchMode,
+                      }),
+                    ),
+                  operator: firstFilter.operator,
+                });
+              } else if (filter) {
+                return Filter.parse({
+                  column,
+                  value: [
+                    FilterValue.parse({
+                      value: filter.value,
+                      matchType: filter.matchMode,
+                    }),
+                  ],
+                  operator: filter.operator,
+                });
+              }
+              return [];
+            })
+          : [],
+        [],
+      );
+      if (this.currentValue().length !== Number(rows.totalRows)) {
+        this.currentValue.set(Array.from({ length: Number(rows.totalRows) }));
+        return;
+      }
+      // First clear out existing data, don't want to risk loading entire file into memory
+      // Keep data in previous/next page so when user scrolls between 2 pages they don't see missing data
+      for (let i = 0; i < this.currentValue().length; i++) {
+        if (
+          i < event.first! - this.numRows ||
+          i > event.first! + event.rows! + this.numRows
+        ) {
+          this.currentValue()[i] = undefined;
+        }
+      }
+
+      // Can't update the current value, otherwise we get an infinite loop
+      this.currentValue().splice(event.first!, event.rows!, ...rows.rows);
+      event.forceUpdate!();
+    }
+  }
+
+  private async loadEmpty() {
     const file = this.file();
     if (file) {
       await this.duckdbService.addFile(file);
@@ -102,45 +199,5 @@ export class FileViewerComponent {
       const newValue = Array.from({ length: Number(rows.totalRows) });
       this.currentValue.set(newValue);
     }
-    });
-  }
-
-  protected async onLazyLoad(event: TableLazyLoadEvent) {
-    const file = this.file();
-    if (file) {
-      const rows = await this.duckdbService.getRows(
-        file,
-        event.first ?? 0,
-        event.rows ?? 0,
-        this.columns(),
-        event.multiSortMeta?.map((meta) => ({
-          name: meta.field,
-          sortType: meta.order < 0 ? 'desc' : 'asc',
-        })) ?? [],
-        Array.isArray(event.filters)
-          ? Object.values(event.filters).map((filter) => ({
-              value: Array.isArray(filter.value)
-                ? filter.value
-                : [filter.value],
-              matchType: filter.matchMode,
-              operator: filter.operator,
-            }))
-          : [],
-        [],
-      );
-      // First clear out existing data, don't want to risk loading entire file into memory
-      // Keep data in previous/next page so when user scrolls between 2 pages they don't see missing data
-      for (let i = 0; i < this.currentValue().length; i++) {
-        if (
-          i < event.first! - this.numRows ||
-          i > event.first! + event.rows! + this.numRows
-        ) {
-          this.currentValue()[i] = undefined;
-        }
-      }
-      // Can't update the current value, otherwise we get an infinite loop
-      this.currentValue().splice(event.first!, event.rows!, ...rows.rows);
-      event.forceUpdate!();
-    }
   }
 }