Rework definition import to support multiple files and imports, nested messages, add configuration export/import

src/app/editor/editor.component.html

@@ -10,8 +10,10 @@
   } }
 </div>
 
+@if(showRaw()) {
 <h2>Preview</h2>
 <div class="actions">
   <button mat-flat-button (click)="copyToClipboard()">Copy</button>
 </div>
-<pre *ngIf="showRaw()"><code #code></code></pre>
+<pre><code #code></code></pre>
+}

src/app/editor/editor.component.scss

@@ -11,7 +11,7 @@ div {
 }
 
 div {
-  padding: 10px;
+  padding: 16px;
 }
 
 .actions {

src/app/model/proto-message.model.ts

@@ -93,9 +93,12 @@ export interface ProtoMessageField<T extends MessageConfiguration> {
 
 export interface ProtoBase {
   name: string;
+  fullName?: string;
+  packageName?: string;
 }
 
 export interface ProtoMessage extends ProtoBase {
+  fileName?: string;
   values: ProtoMessageField<any>[];
 }
 
@@ -103,7 +106,11 @@ export interface ProtoEnum extends ProtoBase {
   options: string[];
 }
 
-export const UnknownProto = (name: string): ProtoMessage => ({
+export const UnknownProto = (
+  name: string,
+  fullName?: string
+): ProtoMessage => ({
   name,
+  fullName,
   values: [{ name: 'Raw JSON', configuration: RawMessage() }],
 });

src/app/proto-definition-selector/proto-definition-selector.component.scss

@@ -14,3 +14,6 @@
 input {
   display: none;
 }
+a {
+  display: none;
+}

src/app/proto-definition-selector/proto-definition-selector.component.ts

@@ -5,6 +5,7 @@ import {
   ElementRef,
   HostBinding,
   HostListener,
+  computed,
   output,
   signal,
   viewChild,
@@ -13,25 +14,37 @@ import { MatButtonModule } from '@angular/material/button';
 import { MatListModule } from '@angular/material/list';
 import { ProtoMessage } from '../model/proto-message.model';
 import { ProtoDefinitionService } from './proto-definition.service';
+import { MatTreeModule } from '@angular/material/tree';
+import { MatSnackBar } from '@angular/material/snack-bar';
 
 @Component({
   selector: 'app-proto-definition-selector',
   standalone: true,
-  imports: [CommonModule, MatButtonModule, MatListModule],
+  imports: [CommonModule, MatButtonModule, MatListModule, MatTreeModule],
   template: `
     <h2>Protobuf Definitions</h2>
     <button mat-button (click)="protoSelector.click()">
       Select definitions
     </button>
+    @if(currentFiles().length > 0) {
+    <button mat-button (click)="exporter.click()">Export Configuration</button>
+    }
+    <button mat-button (click)="configurationSelector.click()">
+      Import Configuration
+    </button>
+    <!-- TODO: Make this a selection list to indicate which file is selected, and allow deselecting a file -->
     <mat-action-list>
-      @for (item of definitionFiles(); track $index) {
+      @for (item of currentFiles(); track $index) {
       <button mat-list-item (click)="selectProtoDefinition(item)">
-        {{ item.name }}
+        {{ item }}
       </button>
       }
     </mat-action-list>
-    @if(selectedProtoFile()) {
+    @if(selectedDefinition().length > 0) {
-    <h3>Messages in {{ selectedProtoFile()?.name }}</h3>
+    <h3>
+      @if(selectedProtoFile()) { Messages in {{ selectedProtoFile() }} } @else
+      {Showing all messages}
+    </h3>
     <mat-action-list>
       @for (item of selectedDefinition(); track $index) {
       <button mat-list-item (click)="messageSelected.emit(item)">
@@ -47,49 +60,77 @@ import { ProtoDefinitionService } from './proto-definition.service';
       (change)="addDefinitionFiles()"
       accept=".proto"
     />
+    <input
+      #configurationSelector
+      type="file"
+      (change)="importConfiguration()"
+      accept=".json"
+    />
+    <a #exporter [href]="exportHref()" download="bufpiv.json"></a>
   `,
   styleUrl: './proto-definition-selector.component.scss',
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class ProtoDefinitionSelectorComponent {
-  protoSelector = viewChild<ElementRef<HTMLInputElement>>('protoSelector');
   messageSelected = output<ProtoMessage>();
 
-  protected definitionFiles = signal<File[]>([]);
+  protected protoSelector =
-  protected selectedDefinition = signal<ProtoMessage[]>([]);
+    viewChild<ElementRef<HTMLInputElement>>('protoSelector');
-  protected selectedProtoFile = signal<File | null>(null);
+  protected configurationSelector = viewChild<ElementRef<HTMLInputElement>>(
-  protected isDragging = signal(false);
+    'configurationSelector'
+  );
 
-  private currentFiles: string[] = [];
+  protected selectedDefinition = signal<ProtoMessage[]>([]);
+  protected selectedProtoFile = signal<string | null>(null);
+  protected isDragging = signal(false);
+  protected currentFiles = signal<string[]>([]);
+
+  private allDefinitionFiles: File[] = [];
+  private allProtoFiles = signal<ProtoMessage[]>([]);
+
+  protected exportHref = computed(() => {
+    const blob = new Blob([JSON.stringify(this.allProtoFiles())], {
+      type: 'application/json',
+    });
+    return URL.createObjectURL(blob);
+  });
 
   @HostBinding('class.droppable')
   get droppable() {
     return this.isDragging();
   }
 
-  constructor(private protoDefinitionService: ProtoDefinitionService) {}
+  constructor(
+    private protoDefinitionService: ProtoDefinitionService,
+    private snackBar: MatSnackBar
+  ) {}
 
   protected async addDefinitionFiles() {
     const files = this.protoSelector()?.nativeElement.files;
     if (files) {
-      const definitionFiles = this.definitionFiles();
+      const definitionFiles = this.allDefinitionFiles;
       for (let i = 0; i < files.length; i++) {
-        if (!this.currentFiles.includes(files[i].name)) {
+        if (
+          !this.allDefinitionFiles.find((file) => file.name === files[i].name)
+        ) {
           definitionFiles.push(files[i]);
-          this.currentFiles.push(files[i].name);
+          if (!this.currentFiles().includes(files[i].name)) {
+            this.currentFiles.update((currentFiles) => [
+              ...currentFiles,
+              files[i].name,
+            ]);
           }
         }
-      this.definitionFiles.set(definitionFiles);
       }
-  }
+      this.allDefinitionFiles = definitionFiles;
-
-  protected async selectProtoDefinition(file: File) {
       try {
-      const protoContents = await file.text();
         const messageObjects =
-        await this.protoDefinitionService.parseProtoDefinition(protoContents);
+          await this.protoDefinitionService.parseAllDefinitions(
+            this.allDefinitionFiles
+          );
+        this.allProtoFiles.set(messageObjects);
         this.selectedDefinition.set(messageObjects);
-      this.selectedProtoFile.set(file);
+        this.selectedProtoFile.set(null);
       } catch (err) {
         console.error(err);
         alert(
@@ -97,6 +138,51 @@ export class ProtoDefinitionSelectorComponent {
         );
       }
     }
+  }
+
+  protected async selectProtoDefinition(file: string) {
+    this.selectedProtoFile.set(file);
+    this.selectedDefinition.set(
+      this.allProtoFiles().filter(
+        (protoMessage) => protoMessage.fileName === file
+      )
+    );
+  }
+
+  protected async importConfiguration() {
+    const configurationFile = this.configurationSelector()?.nativeElement.files;
+    try {
+      const fileContents = await configurationFile?.item(0)?.text();
+      if (!fileContents) {
+        this.snackBar.open('Failed to read file, please try again', 'Dismiss', {
+          duration: 5000,
+        });
+        return;
+      }
+      const parsed = JSON.parse(fileContents) as ProtoMessage[];
+      this.allProtoFiles.set(parsed);
+      this.selectedDefinition.set(parsed);
+      this.currentFiles.set([]);
+      for (const message of parsed) {
+        if (
+          message.fileName &&
+          !this.currentFiles().includes(message.fileName)
+        ) {
+          this.currentFiles.update((currentFiles) => [
+            ...currentFiles,
+            message.fileName!,
+          ]);
+        }
+      }
+    } catch (err) {
+      console.error(err);
+      this.snackBar.open(
+        "Could not parse configuration file, please ensure it's valid, otherwise consider recreating the configuration",
+        'Dismiss',
+        { duration: 5000 }
+      );
+    }
+  }
 
   @HostListener('dragover', ['$event'])
   onDrag(event: DragEvent) {

src/app/proto-definition-selector/proto-definition.service.spec.ts

@@ -14,6 +14,10 @@ import { ProtoDefinitionService } from './proto-definition.service';
 let testProto = `
 syntax="proto3";
 
+package mytestpackage;
+
+import "./myimport.proto";
+
 message Test {
   message NestedMessage {
     string nested = 1;
@@ -29,10 +33,12 @@ message Test {
   ReferenceMessage hello9 = 9;
   NestedMessage hello10 = 10;
   EnumTest enum_test = 11;
+  myimportedpackage.ImportedMessage imported_message = 12;
 }
 
 message ReferenceMessage {
   string test = 1; 
+  Test.NestedMessage nested_message = 2;
 }
 
 enum EnumTest {
@@ -51,13 +57,18 @@ describe('TestService', () => {
     service = TestBed.inject(ProtoDefinitionService);
   });
 
-  it('should convert parse protobuf correctly', async () => {
+  it('should convert parsed protobuf correctly', async () => {
-    const testMessages = await service.parseProtoDefinition(testProto);
+    const testMessages = await service.parseAllDefinitions([
+      new File(
+        [new Blob([testProto], { type: 'text/plain' })],
+        'TestFile.proto'
+      ),
+    ]);
     const converted = testMessages[1];
 
     expect(converted.name).toBe('Test');
 
-    expect(converted.values.length).toBe(11);
+    expect(converted.values.length).toBe(12);
     checkNameAndType(converted, 'hello', MessageTypeEnum.String);
     checkNameAndType(converted, 'hello2', MessageTypeEnum.Numeric);
     checkNameAndType(converted, 'hello3', MessageTypeEnum.Numeric);
@@ -69,6 +80,7 @@ describe('TestService', () => {
     checkNameAndType(converted, 'hello9', MessageTypeEnum.Object);
     checkNameAndType(converted, 'hello10', MessageTypeEnum.Object);
     checkNameAndType(converted, 'enumTest', MessageTypeEnum.Enum);
+    checkNameAndType(converted, 'importedMessage', MessageTypeEnum.Object);
 
     const listMessage = converted.values[6].configuration as ListMessage;
     expect(listMessage.subConfiguration.type).toBe(MessageTypeEnum.String);
@@ -78,13 +90,31 @@ describe('TestService', () => {
     expect(mapMessage.valueConfiguration.type).toBe(MessageTypeEnum.String);
 
     const referenceMessage = converted.values[8].configuration as ObjectMessage;
-    expect(referenceMessage.messageDefinition.values.length).toBe(1);
+    expect(referenceMessage.messageDefinition.values.length).toBe(2);
     expect(referenceMessage.messageDefinition.name).toBe('ReferenceMessage');
     const nestedReferenceMessage = referenceMessage.messageDefinition.values[0];
     expect(nestedReferenceMessage.configuration.type).toBe(
       MessageTypeEnum.String
     );
     expect(nestedReferenceMessage.name).toBe('test');
+    const parentNestedMessage = referenceMessage.messageDefinition.values[1];
+    expect(parentNestedMessage.configuration.type).toBe(MessageTypeEnum.Object);
+    expect(parentNestedMessage.name).toBe('nestedMessage');
+    const parentNestedMessageConfiguration =
+      parentNestedMessage.configuration as ObjectMessage;
+    expect(parentNestedMessageConfiguration.messageDefinition.name).toBe(
+      'Test.NestedMessage'
+    );
+    expect(
+      parentNestedMessageConfiguration.messageDefinition.values.length
+    ).toBe(1);
+    expect(
+      parentNestedMessageConfiguration.messageDefinition.values[0].name
+    ).toBe('nested');
+    expect(
+      parentNestedMessageConfiguration.messageDefinition.values[0].configuration
+        .type
+    ).toBe(MessageTypeEnum.String);
 
     const nestedMessage = converted.values[9].configuration as ObjectMessage;
     expect(nestedMessage.messageDefinition.values.length).toBe(1);

src/app/proto-definition-selector/proto-definition.service.ts

@@ -15,19 +15,69 @@ import {
   StringMessage,
   UnknownProto,
 } from '../model/proto-message.model';
+import { readTextFile } from '@tauri-apps/api/fs';
 
 @Injectable({
   providedIn: 'root',
 })
 export class ProtoDefinitionService {
-  constructor() {}
+  async parseAllDefinitions(allProtoFiles: File[]): Promise<ProtoMessage[]> {
+    const messages: (ProtoBase | undefined)[] = [];
+    for (const protoFile of allProtoFiles) {
+      const definition = await parse(await protoFile.text());
+      // Try resolve imports, if they fail (due to invalid path or running app in browser)
+      // they'll be skipped
+      if (definition.imports) {
+        for (const importPath of definition.imports) {
+          try {
+            const fileName = importPath.substring(
+              importPath.lastIndexOf('/') + 1
+            );
+            allProtoFiles.push(
+              new File(
+                [
+                  new Blob([await readTextFile(importPath)], {
+                    type: 'text/plain',
+                  }),
+                ],
+                fileName
+              )
+            );
+          } catch (err) {
+            console.error(
+              `Failed to import file ${importPath}. Skipping...`,
+              err
+            );
+          }
+        }
+      }
+      const messageDefinitions = definition.root;
+      const parsedMessages = this.parseAllNestedMessages(
+        messageDefinitions,
+        protoFile.name,
+        definition.package
+      );
+      messages.push(...parsedMessages);
+    }
 
-  async parseProtoDefinition(protoContents: string): Promise<ProtoMessage[]> {
+    if (messages.length === 0) {
-    const definition = await parse(protoContents);
+      return [];
-    const messages = definition.root;
+    }
-    if (messages) {
+
-      const messageObjects: ProtoBase[] = this.parseAllNestedMessages(messages);
+    // Now check for duplicates in case we auto-imported files
-      const standardMessages = messageObjects
+    for (let i = 0; i < messages.length - 1; i++) {
+      for (let j = i + 1; j < messages.length; j++) {
+        if (messages[i]?.fullName === messages[j]?.fullName) {
+          messages[j] = undefined;
+        }
+      }
+    }
+    const distinctMessages = messages.filter(
+      (message) => message
+    ) as ProtoBase[];
+
+    // Then populate all object/enum messages for the final passes
+    const standardMessages = distinctMessages
       .filter((messageObject) => 'values' in messageObject)
       .map((messageObject) => messageObject as ProtoMessage);
     for (const messageObject of standardMessages) {
@@ -35,12 +85,13 @@ export class ProtoDefinitionService {
         if (value.configuration.type === MessageTypeEnum.Object) {
           this.populateNestedObject(
             value.configuration as ObjectMessage,
+            messageObject.packageName!,
             standardMessages
           );
         }
       }
     }
-      const enumMessages = messageObjects
+    const enumMessages = distinctMessages
       .filter((messageObject) => 'options' in messageObject)
       .map((messageObject) => messageObject as ProtoEnum);
     for (const messageObject of standardMessages) {
@@ -56,15 +107,36 @@ export class ProtoDefinitionService {
 
     return standardMessages;
   }
-    return [];
-  }
 
   private populateEnumMessages(
     objectMessage: ObjectMessage,
     enumMessages: ProtoEnum[]
   ): MessageConfiguration {
+    let useFullName = false;
+    let objectMessageName = objectMessage.messageDefinition.name;
+    if (objectMessageName.includes('.')) {
+      // First do a check for current package.
       for (const enumMessage of enumMessages) {
-      if (enumMessage.name === objectMessage.messageDefinition.name) {
+        if (
+          enumMessage.packageName ===
+            objectMessage.messageDefinition.packageName &&
+          enumMessage.fullName ===
+            `${objectMessage.messageDefinition.packageName}.${objectMessageName}`
+        ) {
+          return EnumMessage(enumMessage.options);
+        }
+      }
+
+      // If there's no match, then we'll be matching on full name, assuming a separate package
+      useFullName = true;
+    }
+
+    // None matching messages in this package found, check external package
+    for (const enumMessage of enumMessages) {
+      if (
+        (useFullName && enumMessage.fullName === objectMessageName) ||
+        (!useFullName && enumMessage.name === objectMessageName)
+      ) {
         return EnumMessage(enumMessage.options);
       }
     }
@@ -83,23 +155,34 @@ export class ProtoDefinitionService {
     return objectMessage;
   }
 
-  private parseAllNestedMessages(obj: ReflectionObject) {
+  private parseAllNestedMessages(
+    obj: ReflectionObject,
+    fileName: string,
+    packageName?: string
+  ) {
     const messageObjects: ProtoBase[] = [];
     // Nested messages/enums
     if ('nestedArray' in obj && obj.nestedArray) {
       const nestedArray = obj.nestedArray as ReflectionObject[];
       for (const nestedObj of nestedArray) {
-        messageObjects.push(...this.parseAllNestedMessages(nestedObj));
+        messageObjects.push(
+          ...this.parseAllNestedMessages(nestedObj, fileName, packageName)
+        );
       }
     }
     // Message
     if ('fieldsArray' in obj) {
-      messageObjects.push(this.convertProtoDefinitionToModel(obj));
+      messageObjects.push(
+        this.convertProtoDefinitionToModel(obj, fileName, packageName)
+      );
     }
     // Enum
     if ('values' in obj) {
       messageObjects.push({
         name: obj.name,
+        fullName: obj.fullName,
+        packageName,
+        fileName,
         options: Object.entries(obj.values as Record<string, string>)
           .sort((a, b) => Number(a[1]) - Number(b[1]))
           .map(([key, _]) => key),
@@ -110,36 +193,73 @@ export class ProtoDefinitionService {
 
   private populateNestedObject(
     objectMessage: ObjectMessage,
+    packageName: string,
     availableMessages: ProtoMessage[]
   ) {
+    let useFullName = false;
+    let objectMessageName = objectMessage.messageDefinition.name;
+    if (objectMessageName.includes('.')) {
+      // First do a check for current package.
       for (const message of availableMessages) {
-      if (message.name === objectMessage.messageDefinition.name) {
+        if (
+          message.packageName === packageName &&
+          message.fullName === `${packageName}.${objectMessageName}`
+        ) {
+          this.updateObjectMessage(objectMessage, message, availableMessages);
+          return;
+        }
+      }
+
+      // If there's no match, then we'll be matching on full name, assuming a separate package
+      useFullName = true;
+    }
+
+    for (const message of availableMessages) {
+      if (
+        (useFullName && message.fullName === objectMessageName) ||
+        (!useFullName && message.name === objectMessageName)
+      ) {
+        this.updateObjectMessage(objectMessage, message, availableMessages);
+        return;
+      }
+    }
+    objectMessage.messageDefinition = UnknownProto(
+      objectMessage.messageDefinition.name,
+      objectMessage.messageDefinition.fullName
+    );
+  }
+
+  private updateObjectMessage(
+    objectMessage: ObjectMessage,
+    message: ProtoMessage,
+    availableMessages: ProtoMessage[]
+  ) {
     objectMessage.messageDefinition = {
-          name: message.name,
+      name: objectMessage.messageDefinition.name,
       values: message.values.map((value) => {
         if (value.configuration.type === MessageTypeEnum.Object) {
           this.populateNestedObject(
             value.configuration as ObjectMessage,
+            message.packageName!,
             availableMessages
           );
         }
         return structuredClone(value);
       }),
     };
-        return;
-      }
-    }
-    objectMessage.messageDefinition = UnknownProto(
-      objectMessage.messageDefinition.name
-    );
   }
 
   private convertProtoDefinitionToModel(
-    messageDefinition?: ReflectionObject
+    messageDefinition: ReflectionObject,
+    fileName: string,
+    packageName?: string
   ): ProtoBase {
     if (messageDefinition && 'fieldsArray' in messageDefinition) {
       const convertedFields: ProtoMessage = {
         name: messageDefinition.name,
+        fullName: messageDefinition.fullName.substring(1), // trim leading .
+        packageName,
+        fileName,
         values: [],
       };
       const fields = messageDefinition.fieldsArray as FieldBase[];