Switch to zoneless change detection, add paging for large datasets

src/app/app.config.ts

@@ -1,7 +1,6 @@
 import {
   ApplicationConfig,
-  provideAppInitializer,
-  provideZoneChangeDetection,
+  provideZonelessChangeDetection,
 } from '@angular/core';
 import { provideRouter } from '@angular/router';
 import { provideAnimationsAsync } from '@angular/platform-browser/animations/async';
@@ -12,7 +11,7 @@ import { routes } from './app.routes';
 
 export const appConfig: ApplicationConfig = {
   providers: [
-    provideZoneChangeDetection({ eventCoalescing: true }),
+    provideZonelessChangeDetection(),
     provideRouter(routes),
     provideAnimationsAsync(),
     providePrimeNG({

src/app/file-viewer/file-viewer.component.css

@@ -1,4 +1,5 @@
 :host {
-  display: block;
+  display: flex;
+  flex-direction: column;
   height: 100%;
 }

src/app/file-viewer/file-viewer.component.ts

@@ -1,5 +1,6 @@
 import {
   ChangeDetectionStrategy,
+  ChangeDetectorRef,
   Component,
   effect,
   inject,
@@ -30,6 +31,7 @@ import { InputTextModule } from 'primeng/inputtext';
 import { Menu } from 'primeng/menu';
 import { MenuItem } from 'primeng/api';
 import { DecimalPipe } from '@angular/common';
+import { PaginatorModule, PaginatorState } from 'primeng/paginator';
 
 @Component({
   selector: 'app-file-viewer',
@@ -43,107 +45,123 @@ import { DecimalPipe } from '@angular/common';
     InputTextModule,
     Menu,
     DecimalPipe,
+    PaginatorModule,
   ],
   template: `
     @if (file() && columns().length > 0) {
-      <p-table
-        #table
-        sortMode="multiple"
-        [resizableColumns]="true"
-        columnResizeMode="expand"
-        [columns]="columns()"
-        [value]="currentValue()"
-        showGridlines
-        [scrollable]="true"
-        scrollHeight="flex"
-        [rows]="numRows"
-        [virtualScroll]="true"
-        [virtualScrollItemSize]="46"
-        [lazy]="true"
-        (onLazyLoad)="onLazyLoad($event)"
-        [multiSortMeta]="[{ field: columns()[0].name, order: 1 }]"
-        stripedRows
-      >
-        <ng-template #caption>
-          <div class="flex">
-            <p-button
-              label="Reset"
-              [outlined]="true"
-              icon="pi pi-filter-slash"
-              (click)="clear(table)"
-            />
-            <p-iconfield iconPosition="left" class="ml-auto">
-              <p-inputicon>
-                <i class="pi pi-search"></i>
-              </p-inputicon>
-              <input pInputText type="text" placeholder="Search keyword" />
-            </p-iconfield>
-          </div>
-        </ng-template>
-        <ng-template #header let-columns>
-          <tr>
-            @for (col of columns; track $index) {
-              <th
-                pReorderableColumn
-                pResizableColumn
-                [pSortableColumn]="col.name"
-                class="w-3xs min-w-3xs max-w-3xs"
-              >
-                {{ col.name }}
-                <p-columnFilter type="text" [field]="col.name" display="menu" />
-                <p-sortIcon [field]="col.name" />
-                <!-- More options can go here... integrate it with the column filter? -->
-              </th>
-            }
-          </tr>
-        </ng-template>
-        <ng-template #body let-rowData let-columns="columns">
-          <tr style="height: 46px">
-            @for (col of columns; track $index) {
-              <td pReorderableRowHandle class="w-3xs min-w-3xs max-w-3xs">
-                {{ rowData[col.name] }}
-              </td>
-            }
-          </tr>
-        </ng-template>
-        <ng-template #loadingbody let-columns="columns">
-          <tr style="height: 46px">
-            @for (col of columns; track $index) {
-              <td class="w-3xs min-w-3xs max-w-3xs">
-                <p-skeleton />
-              </td>
-            }
-          </tr>
-        </ng-template>
-        <ng-template #footer>
-          <tr>
-            @for (col of columns(); track $index) {
-              <td #attachment>
-                @if (col.type === 'DOUBLE') {
-                  <div class="flex items-baseline">
-                    <p-button
-                      (click)="
-                        currentAggregateColumn.set($index);
-                        aggregateMenu.toggle($event)
-                      "
-                      [label]="aggregates()[$index]?.type?.toUpperCase()"
-                    >
-                      <ng-template #icon>
-                        <span class="material-symbols-outlined">
-                          mediation
-                        </span>
-                      </ng-template>
-                    </p-button>
-                    <span class="flex-1 text-end">
-                      {{ aggregateValues()[$index] | number }}
-                    </span>
-                  </div>
-                }
-              </td>
-            }
-          </tr>
-        </ng-template>
-      </p-table>
+      <div class="h-full">
+        <p-table
+          #table
+          sortMode="multiple"
+          [resizableColumns]="true"
+          columnResizeMode="expand"
+          [columns]="columns()"
+          [value]="currentValue()"
+          showGridlines
+          [scrollable]="true"
+          scrollHeight="flex"
+          [rows]="NUM_ROWS"
+          [virtualScroll]="true"
+          [virtualScrollItemSize]="46"
+          [lazy]="true"
+          (onLazyLoad)="onLazyLoad($event)"
+          [multiSortMeta]="[{ field: columns()[0].name, order: 1 }]"
+          stripedRows
+        >
+          <ng-template #caption>
+            <div class="flex">
+              <p-button
+                label="Reset"
+                [outlined]="true"
+                icon="pi pi-filter-slash"
+                (click)="clear(table)"
+              />
+              <p-iconfield iconPosition="left" class="ml-auto">
+                <p-inputicon>
+                  <i class="pi pi-search"></i>
+                </p-inputicon>
+                <input pInputText type="text" placeholder="Search keyword" />
+              </p-iconfield>
+            </div>
+          </ng-template>
+          <ng-template #header let-columns>
+            <tr>
+              @for (col of columns; track $index) {
+                <th
+                  pReorderableColumn
+                  pResizableColumn
+                  [pSortableColumn]="col.name"
+                  class="w-3xs min-w-3xs max-w-3xs"
+                >
+                  {{ col.name }}
+                  <p-columnFilter
+                    type="text"
+                    [field]="col.name"
+                    display="menu"
+                  />
+                  <p-sortIcon [field]="col.name" />
+                  <!-- More options can go here... integrate it with the column filter? -->
+                </th>
+              }
+            </tr>
+          </ng-template>
+          <ng-template #body let-rowData let-columns="columns">
+            <tr style="height: 46px">
+              @for (col of columns; track $index) {
+                <td pReorderableRowHandle class="w-3xs min-w-3xs max-w-3xs">
+                  {{ rowData[col.name] }}
+                </td>
+              }
+            </tr>
+          </ng-template>
+          <ng-template #loadingbody let-columns="columns">
+            <tr style="height: 46px">
+              @for (col of columns; track $index) {
+                <td class="w-3xs min-w-3xs max-w-3xs">
+                  <p-skeleton />
+                </td>
+              }
+            </tr>
+          </ng-template>
+          <ng-template #footer>
+            <tr>
+              @for (col of columns(); track $index) {
+                <td #attachment>
+                  @if (col.type === 'DOUBLE' || col.type === 'BIGINT') {
+                    <div class="flex items-baseline">
+                      <p-button
+                        (click)="
+                          currentAggregateColumn.set($index);
+                          aggregateMenu.toggle($event)
+                        "
+                        [label]="aggregates()[$index]?.type?.toUpperCase()"
+                      >
+                        <ng-template #icon>
+                          <span class="material-symbols-outlined">
+                            mediation
+                          </span>
+                        </ng-template>
+                      </p-button>
+                      <span class="flex-1 text-end">
+                        {{ aggregateValues()[$index] | number }}
+                      </span>
+                    </div>
+                  }
+                </td>
+              }
+            </tr>
+          </ng-template>
+        </p-table>
+      </div>
+      @if (currentRowCount() > PAGE_SIZE) {
+        <!-- Can't use default table paginator as it doesn't work with lazy, it ends up loading all values rather than just the standard 200 -->
+        <p-paginator
+          (onPageChange)="onPageChange($event)"
+          [first]="currentPage() * PAGE_SIZE"
+          [rows]="PAGE_SIZE"
+          [totalRecords]="currentRowCount()"
+        />
+      }
       <p-menu #aggregateMenu [model]="aggregateItems" [popup]="true" />
     }
   `,
@@ -160,14 +178,16 @@ export class FileViewerComponent {
   protected aggregates = signal<(Aggregate | undefined)[]>([]);
   protected currentAggregateColumn = signal<number | undefined>(undefined);
   protected aggregateValues = signal<(number | undefined)[]>([]);
-  protected numRows = 200;
+  protected readonly NUM_ROWS = 200;
+  protected currentPage = signal(0);
+
+  // This is required since once the number of rows exceeds ~300000, the footer gets blocked
+  // and the user can't even scroll to those bottom rows.
+  protected readonly PAGE_SIZE = 300_000;
 
-  // TODO: This needs to be limited to 500000 or something, since exceeding the max safe integer
-  //       will cause weird layout and prevent the bottom rows/footer from being viewed. Alternately
-  //       we can use paging instead, but allow paging to be virtual with a max page size
-  //       of 500000
   protected currentValue = signal<any[]>([]);
   protected appliedFilters = signal<Filter[]>([]);
+  protected currentRowCount = signal(0n);
 
   private table = viewChild(Table);
 
@@ -226,7 +246,7 @@ export class FileViewerComponent {
       );
       const rows = await this.duckdbService.getRows(
         file,
-        event.first ?? 0,
+        (event.first ?? 0) + this.currentPage() * this.PAGE_SIZE,
         event.rows ?? 0,
         this.columns(),
         event.multiSortMeta?.map((meta) => ({
@@ -236,23 +256,21 @@ export class FileViewerComponent {
         this.appliedFilters(),
         this.aggregates().filter((agg) => !!agg) as Aggregate[],
       );
-      if (this.currentValue().length !== Number(rows.totalRows)) {
-        this.currentValue.set(Array.from({ length: Number(rows.totalRows) }));
-        return;
-      }
       // First clear out existing data, don't want to risk loading entire file into memory
       // Keep data in previous/next page so when user scrolls between 2 pages they don't see missing data
       for (let i = 0; i < this.currentValue().length; i++) {
         if (
-          i < event.first! - this.numRows ||
-          i > event.first! + event.rows! + this.numRows
+          i < event.first! - this.NUM_ROWS ||
+          i > event.first! + event.rows! + this.NUM_ROWS
         ) {
           this.currentValue()[i] = undefined;
         }
       }
 
-      // Can't update the current value, otherwise we get an infinite loop
-      this.currentValue().splice(event.first!, event.rows!, ...rows.rows);
+      // Can't update the current value, otherwise we get an infinite loop due to primeng change detection rerunning (plus it's faster to mutate)
+      for (let i = 0; i < event.rows!; i++) {
+        this.currentValue()[event.first! + i] = rows.rows![i];
+      }
       event.forceUpdate!();
     }
   }
@@ -266,13 +284,19 @@ export class FileViewerComponent {
       const rows = await this.duckdbService.getRows(
         file,
         0,
-        this.numRows,
+        this.NUM_ROWS,
         this.columns(),
         [],
         [],
         [],
       );
-      const newValue = Array.from({ length: Number(rows.totalRows) });
+      this.currentRowCount.set(rows.totalRows);
+      this.currentPage.set(0);
+      const newValue = Array.from({
+        length: Number(
+          rows.totalRows > this.PAGE_SIZE ? this.PAGE_SIZE : rows.totalRows,
+        ),
+      });
       this.currentValue.set(newValue);
     }
     // TODO: Replace this with previous state once persistence is implemented
@@ -304,4 +328,16 @@ export class FileViewerComponent {
       return copy;
     });
   }
+
+  protected onPageChange(event: PaginatorState) {
+    this.currentPage.set(event.page!);
+    this.currentValue.set(
+      Array.from({
+        length:
+          event.page! === event.pageCount! - 1
+            ? Number(this.currentRowCount()) % this.PAGE_SIZE
+            : this.PAGE_SIZE,
+      }),
+    );
+  }
 }