Add basic filtering

src/app/duckdb.service.ts

@@ -7,37 +7,82 @@ export const Column = z.object({
   type: z.string(),
 });
 
-const SortColumn = z.object({
+export const SortColumn = z.object({
   name: z.string(),
   sortType: z.enum(['asc', 'desc']),
 });
 
-const Filter = z.object({
-  value: z.string().array(),
-  operator: z.enum(['and', 'or']),
-  matchType: z.enum([
-    'startsWith',
-    'contains',
-    'notContains',
-    'endsWith',
-    'equals',
-    'notEquals',
-  ]),
+export const FilterOperator = z.enum([
+  'startsWith',
+  'contains',
+  'notContains',
+  'endsWith',
+  'equals',
+  'notEquals',
+]);
+
+export const FilterValue = z.object({
+  value: z.string().trim(),
+  matchType: FilterOperator,
 });
 
-const RowsResponse = z.object({
+export const Filter = z.object({
+  column: z.string(),
+  value: FilterValue.optional().array(),
+  operator: z.enum(['and', 'or']),
+});
+
+export const RowsResponse = z.object({
+  // TODO: Can't this just be a string[]?
   rows: z.any(),
-  totalRows: z.bigint(),
+  totalRows: z.bigint().nonnegative(),
 });
 
 export type Column = z.infer<typeof Column>;
 export type SortColumn = z.infer<typeof SortColumn>;
+export type FilterValue = z.infer<typeof FilterValue>;
+export type FilterOperator = z.infer<typeof FilterOperator>;
 export type Filter = z.infer<typeof Filter>;
 export type RowsResponse = z.infer<typeof RowsResponse>;
 
 const sanitisedFileName = (file: File) =>
   file.name.toLowerCase().replaceAll("'", '').replaceAll(/\s*/g, '');
 
+const sqlOperator = (operator: FilterOperator) => {
+  switch (operator) {
+    case 'startsWith':
+    case 'endsWith':
+    case 'contains':
+    case 'equals':
+      return 'ILIKE';
+    case 'notContains':
+    case 'notEquals':
+      return 'NOT ILIKE';
+  }
+};
+
+const prefix = (operator: FilterOperator) => {
+  switch (operator) {
+    case 'endsWith':
+    case 'contains':
+    case 'notContains':
+      return '%';
+    default:
+      return '';
+  }
+};
+
+const suffix = (operator: FilterOperator) => {
+  switch (operator) {
+    case 'startsWith':
+    case 'contains':
+    case 'notContains':
+      return '%';
+    default:
+      return '';
+  }
+};
+
 //https://www.npmjs.com/package/@duckdb/duckdb-wasm
 @Injectable({
   providedIn: 'root',
@@ -127,19 +172,30 @@ export class DuckdbService {
   ): Promise<RowsResponse> {
     const conn = await this.db.connect();
     try {
-      const totalRowResponse = await conn.query(
-        `SELECT COUNT(1) totalRows FROM ${sanitisedFileName(file)}`,
+      const whereClause = this.getWhereClause(filters);
+      const mappedFilterValues = filters.flatMap((filter) =>
+        filter.value
+          .filter((value) => value?.value)
+          .map(
+            (value) =>
+              `${prefix(value?.matchType!)}${value?.value}${suffix(value?.matchType!)}`,
+          ),
       );
+      const rowCountQuery = `SELECT COUNT(1) totalRows FROM ${sanitisedFileName(file)} ${whereClause}`;
+      const totalRowStmt = await conn.prepare(rowCountQuery);
+      const totalRowResponse = await totalRowStmt.query(...mappedFilterValues);
       const { totalRows } = totalRowResponse.get(0)?.toJSON()!;
-      let query = `SELECT ${columns.map((column) => `"${column.name}"`).join(', ')} FROM ${sanitisedFileName(file)} `;
+      let query = `SELECT ${columns.map((column) => `"${column.name}"`).join(', ')} FROM ${sanitisedFileName(file)} ${whereClause}`;
       if (sorts.length > 0) {
         query += ` ORDER BY ${sorts.map((sort) => `"${sort.name}" ${sort.sortType}`).join(', ')}`;
       }
       query += ` LIMIT ${numRows} OFFSET ${start}`;
-      const response = await conn.query(query);
+      const stmt = await conn.prepare(query);
       const rows = [];
-      for (let i = 0; i < response.numRows; i++) {
-        rows.push(response.get(i)?.toJSON()!);
+      for await (const batch of await stmt.send(...mappedFilterValues)) {
+        for (const row of batch) {
+          rows.push(row.toJSON()!);
+        }
       }
       return { rows, totalRows };
     } catch (err) {
@@ -149,4 +205,27 @@ export class DuckdbService {
       conn.close();
     }
   }
+
+  // Where clause that gets attached to the main and rowCount selects.
+  private getWhereClause(filters: Filter[]) {
+    let query = '';
+    if (filters.length > 0) {
+      let and = 'WHERE';
+      for (const filter of filters) {
+        if (filter.value.find((value) => value?.value)) {
+          query += ` ${and} (`;
+          let or = '';
+          for (const value of filter.value) {
+            if (value?.value) {
+              query += ` ${or} ${filter.column} ${sqlOperator(value?.matchType!)} ? `;
+              or = filter.operator;
+            }
+          }
+          query += ') ';
+          and = 'and';
+        }
+      }
+    }
+    return query;
+  }
 }