Switch to zoneless change detection, add paging for large datasets

src/app/app.config.ts

@@ -1,7 +1,6 @@
 import {
   ApplicationConfig,
-  provideAppInitializer,
+  provideZonelessChangeDetection,
-  provideZoneChangeDetection,
 } from '@angular/core';
 import { provideRouter } from '@angular/router';
 import { provideAnimationsAsync } from '@angular/platform-browser/animations/async';
@@ -12,7 +11,7 @@ import { routes } from './app.routes';
 
 export const appConfig: ApplicationConfig = {
   providers: [
-    provideZoneChangeDetection({ eventCoalescing: true }),
+    provideZonelessChangeDetection(),
     provideRouter(routes),
     provideAnimationsAsync(),
     providePrimeNG({

src/app/file-viewer/file-viewer.component.css

@@ -1,4 +1,5 @@
 :host {
-  display: block;
+  display: flex;
+  flex-direction: column;
   height: 100%;
 }

src/app/file-viewer/file-viewer.component.ts

@@ -1,5 +1,6 @@
 import {
   ChangeDetectionStrategy,
+  ChangeDetectorRef,
   Component,
   effect,
   inject,
@@ -30,6 +31,7 @@ import { InputTextModule } from 'primeng/inputtext';
 import { Menu } from 'primeng/menu';
 import { MenuItem } from 'primeng/api';
 import { DecimalPipe } from '@angular/common';
+import { PaginatorModule, PaginatorState } from 'primeng/paginator';
 
 @Component({
   selector: 'app-file-viewer',
@@ -43,9 +45,11 @@ import { DecimalPipe } from '@angular/common';
     InputTextModule,
     Menu,
     DecimalPipe,
+    PaginatorModule,
   ],
   template: `
     @if (file() && columns().length > 0) {
+      <div class="h-full">
         <p-table
           #table
           sortMode="multiple"
@@ -56,7 +60,7 @@ import { DecimalPipe } from '@angular/common';
           showGridlines
           [scrollable]="true"
           scrollHeight="flex"
-        [rows]="numRows"
+          [rows]="NUM_ROWS"
           [virtualScroll]="true"
           [virtualScrollItemSize]="46"
           [lazy]="true"
@@ -90,7 +94,11 @@ import { DecimalPipe } from '@angular/common';
                   class="w-3xs min-w-3xs max-w-3xs"
                 >
                   {{ col.name }}
-                <p-columnFilter type="text" [field]="col.name" display="menu" />
+                  <p-columnFilter
+                    type="text"
+                    [field]="col.name"
+                    display="menu"
+                  />
                   <p-sortIcon [field]="col.name" />
                   <!-- More options can go here... integrate it with the column filter? -->
                 </th>
@@ -119,7 +127,7 @@ import { DecimalPipe } from '@angular/common';
             <tr>
               @for (col of columns(); track $index) {
                 <td #attachment>
-                @if (col.type === 'DOUBLE') {
+                  @if (col.type === 'DOUBLE' || col.type === 'BIGINT') {
                     <div class="flex items-baseline">
                       <p-button
                         (click)="
@@ -144,6 +152,16 @@ import { DecimalPipe } from '@angular/common';
             </tr>
           </ng-template>
         </p-table>
+      </div>
+      @if (currentRowCount() > PAGE_SIZE) {
+        <!-- Can't use default table paginator as it doesn't work with lazy, it ends up loading all values rather than just the standard 200 -->
+        <p-paginator
+          (onPageChange)="onPageChange($event)"
+          [first]="currentPage() * PAGE_SIZE"
+          [rows]="PAGE_SIZE"
+          [totalRecords]="currentRowCount()"
+        />
+      }
       <p-menu #aggregateMenu [model]="aggregateItems" [popup]="true" />
     }
   `,
@@ -160,14 +178,16 @@ export class FileViewerComponent {
   protected aggregates = signal<(Aggregate | undefined)[]>([]);
   protected currentAggregateColumn = signal<number | undefined>(undefined);
   protected aggregateValues = signal<(number | undefined)[]>([]);
-  protected numRows = 200;
+  protected readonly NUM_ROWS = 200;
+  protected currentPage = signal(0);
+
+  // This is required since once the number of rows exceeds ~300000, the footer gets blocked
+  // and the user can't even scroll to those bottom rows.
+  protected readonly PAGE_SIZE = 300_000;
 
-  // TODO: This needs to be limited to 500000 or something, since exceeding the max safe integer
-  //       will cause weird layout and prevent the bottom rows/footer from being viewed. Alternately
-  //       we can use paging instead, but allow paging to be virtual with a max page size
-  //       of 500000
   protected currentValue = signal<any[]>([]);
   protected appliedFilters = signal<Filter[]>([]);
+  protected currentRowCount = signal(0n);
 
   private table = viewChild(Table);
 
@@ -226,7 +246,7 @@ export class FileViewerComponent {
       );
       const rows = await this.duckdbService.getRows(
         file,
-        event.first ?? 0,
+        (event.first ?? 0) + this.currentPage() * this.PAGE_SIZE,
         event.rows ?? 0,
         this.columns(),
         event.multiSortMeta?.map((meta) => ({
@@ -236,23 +256,21 @@ export class FileViewerComponent {
         this.appliedFilters(),
         this.aggregates().filter((agg) => !!agg) as Aggregate[],
       );
-      if (this.currentValue().length !== Number(rows.totalRows)) {
-        this.currentValue.set(Array.from({ length: Number(rows.totalRows) }));
-        return;
-      }
       // First clear out existing data, don't want to risk loading entire file into memory
       // Keep data in previous/next page so when user scrolls between 2 pages they don't see missing data
       for (let i = 0; i < this.currentValue().length; i++) {
         if (
-          i < event.first! - this.numRows ||
+          i < event.first! - this.NUM_ROWS ||
-          i > event.first! + event.rows! + this.numRows
+          i > event.first! + event.rows! + this.NUM_ROWS
         ) {
           this.currentValue()[i] = undefined;
         }
       }
 
-      // Can't update the current value, otherwise we get an infinite loop
+      // Can't update the current value, otherwise we get an infinite loop due to primeng change detection rerunning (plus it's faster to mutate)
-      this.currentValue().splice(event.first!, event.rows!, ...rows.rows);
+      for (let i = 0; i < event.rows!; i++) {
+        this.currentValue()[event.first! + i] = rows.rows![i];
+      }
       event.forceUpdate!();
     }
   }
@@ -266,13 +284,19 @@ export class FileViewerComponent {
       const rows = await this.duckdbService.getRows(
         file,
         0,
-        this.numRows,
+        this.NUM_ROWS,
         this.columns(),
         [],
         [],
         [],
       );
-      const newValue = Array.from({ length: Number(rows.totalRows) });
+      this.currentRowCount.set(rows.totalRows);
+      this.currentPage.set(0);
+      const newValue = Array.from({
+        length: Number(
+          rows.totalRows > this.PAGE_SIZE ? this.PAGE_SIZE : rows.totalRows,
+        ),
+      });
       this.currentValue.set(newValue);
     }
     // TODO: Replace this with previous state once persistence is implemented
@@ -304,4 +328,16 @@ export class FileViewerComponent {
       return copy;
     });
   }
+
+  protected onPageChange(event: PaginatorState) {
+    this.currentPage.set(event.page!);
+    this.currentValue.set(
+      Array.from({
+        length:
+          event.page! === event.pageCount! - 1
+            ? Number(this.currentRowCount()) % this.PAGE_SIZE
+            : this.PAGE_SIZE,
+      }),
+    );
+  }
 }