vato007/bufpiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework definition import to support multiple files and imports, nested messages, add configuration export/import

Browse Source
This commit is contained in:
vato007
2024-07-13 22:01:41 +09:30
parent 72c24a70ae
commit cf0a96591c
7 changed files with 330 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/app/editor/editor.component.html
View File

@@ -10,8 +10,10 @@
} }
</div>
@if(showRaw()) {
<h2>Preview</h2>
<div class="actions">
<button mat-flat-button (click)="copyToClipboard()">Copy</button>
</div>
<pre *ngIf="showRaw()"><code #code></code></pre>
<pre><code #code></code></pre>
}

2
src/app/editor/editor.component.scss
View File

@@ -11,7 +11,7 @@ div {
}
div {
padding: 10px;
padding: 16px;
}
.actions {

9
src/app/model/proto-message.model.ts
View File

@@ -93,9 +93,12 @@ export interface ProtoMessageField<T extends MessageConfiguration> {
export interface ProtoBase {
name: string;
fullName?: string;
packageName?: string;
}
export interface ProtoMessage extends ProtoBase {
fileName?: string;
values: ProtoMessageField<any>[];
}
@@ -103,7 +106,11 @@ export interface ProtoEnum extends ProtoBase {
options: string[];
}
export const UnknownProto = (name: string): ProtoMessage => ({
export const UnknownProto = (
name: string,
fullName?: string
): ProtoMessage => ({
name,
fullName,
values: [{ name: 'Raw JSON', configuration: RawMessage() }],
});

3
src/app/proto-definition-selector/proto-definition-selector.component.scss
View File

@@ -14,3 +14,6 @@
input {
display: none;
}
a {
display: none;
}

130
src/app/proto-definition-selector/proto-definition-selector.component.ts
View File

@@ -5,6 +5,7 @@ import {
ElementRef,
HostBinding,
HostListener,
computed,
output,
signal,
viewChild,
@@ -13,25 +14,37 @@ import { MatButtonModule } from '@angular/material/button';
import { MatListModule } from '@angular/material/list';
import { ProtoMessage } from '../model/proto-message.model';
import { ProtoDefinitionService } from './proto-definition.service';
import { MatTreeModule } from '@angular/material/tree';
import { MatSnackBar } from '@angular/material/snack-bar';
@Component({
selector: 'app-proto-definition-selector',
standalone: true,
imports: [CommonModule, MatButtonModule, MatListModule],
imports: [CommonModule, MatButtonModule, MatListModule, MatTreeModule],
template: `
<h2>Protobuf Definitions</h2>
<button mat-button (click)="protoSelector.click()">
Select definitions
</button>
@if(currentFiles().length > 0) {
<button mat-button (click)="exporter.click()">Export Configuration</button>
}
<button mat-button (click)="configurationSelector.click()">
Import Configuration
</button>
<!-- TODO: Make this a selection list to indicate which file is selected, and allow deselecting a file -->
<mat-action-list>
@for (item of definitionFiles(); track $index) {
@for (item of currentFiles(); track $index) {
<button mat-list-item (click)="selectProtoDefinition(item)">
{{ item.name }}
{{ item }}
</button>
}
</mat-action-list>
@if(selectedProtoFile()) {
<h3>Messages in {{ selectedProtoFile()?.name }}</h3>
@if(selectedDefinition().length > 0) {
<h3>
@if(selectedProtoFile()) { Messages in {{ selectedProtoFile() }} } @else
{Showing all messages}
</h3>
<mat-action-list>
@for (item of selectedDefinition(); track $index) {
<button mat-list-item (click)="messageSelected.emit(item)">
@@ -47,49 +60,77 @@ import { ProtoDefinitionService } from './proto-definition.service';
(change)="addDefinitionFiles()"
accept=".proto"
/>
<input
#configurationSelector
type="file"
(change)="importConfiguration()"
accept=".json"
/>
<a #exporter [href]="exportHref()" download="bufpiv.json"></a>
`,
styleUrl: './proto-definition-selector.component.scss',
changeDetection: ChangeDetectionStrategy.OnPush,
})
export class ProtoDefinitionSelectorComponent {
protoSelector = viewChild<ElementRef<HTMLInputElement>>('protoSelector');
messageSelected = output<ProtoMessage>();
protected definitionFiles = signal<File[]>([]);
protected selectedDefinition = signal<ProtoMessage[]>([]);
protected selectedProtoFile = signal<File | null>(null);
protected isDragging = signal(false);
protected protoSelector =
viewChild<ElementRef<HTMLInputElement>>('protoSelector');
protected configurationSelector = viewChild<ElementRef<HTMLInputElement>>(
'configurationSelector'
);
private currentFiles: string[] = [];
protected selectedDefinition = signal<ProtoMessage[]>([]);
protected selectedProtoFile = signal<string | null>(null);
protected isDragging = signal(false);
protected currentFiles = signal<string[]>([]);
private allDefinitionFiles: File[] = [];
private allProtoFiles = signal<ProtoMessage[]>([]);
protected exportHref = computed(() => {
const blob = new Blob([JSON.stringify(this.allProtoFiles())], {
type: 'application/json',
});
return URL.createObjectURL(blob);
});
@HostBinding('class.droppable')
get droppable() {
return this.isDragging();
}
constructor(private protoDefinitionService: ProtoDefinitionService) {}
constructor(
private protoDefinitionService: ProtoDefinitionService,
private snackBar: MatSnackBar
) {}
protected async addDefinitionFiles() {
const files = this.protoSelector()?.nativeElement.files;
if (files) {
const definitionFiles = this.definitionFiles();
const definitionFiles = this.allDefinitionFiles;
for (let i = 0; i < files.length; i++) {
if (!this.currentFiles.includes(files[i].name)) {
if (
!this.allDefinitionFiles.find((file) => file.name === files[i].name)
) {
definitionFiles.push(files[i]);
this.currentFiles.push(files[i].name);
if (!this.currentFiles().includes(files[i].name)) {
this.currentFiles.update((currentFiles) => [
...currentFiles,
files[i].name,
]);
}
}
this.definitionFiles.set(definitionFiles);
}
}
protected async selectProtoDefinition(file: File) {
this.allDefinitionFiles = definitionFiles;
try {
const protoContents = await file.text();
const messageObjects =
await this.protoDefinitionService.parseProtoDefinition(protoContents);
await this.protoDefinitionService.parseAllDefinitions(
this.allDefinitionFiles
);
this.allProtoFiles.set(messageObjects);
this.selectedDefinition.set(messageObjects);
this.selectedProtoFile.set(file);
this.selectedProtoFile.set(null);
} catch (err) {
console.error(err);
alert(
@@ -97,6 +138,51 @@ export class ProtoDefinitionSelectorComponent {
);
}
}
}
protected async selectProtoDefinition(file: string) {
this.selectedProtoFile.set(file);
this.selectedDefinition.set(
this.allProtoFiles().filter(
(protoMessage) => protoMessage.fileName === file
)
);
}
protected async importConfiguration() {
const configurationFile = this.configurationSelector()?.nativeElement.files;
try {
const fileContents = await configurationFile?.item(0)?.text();
if (!fileContents) {
this.snackBar.open('Failed to read file, please try again', 'Dismiss', {
duration: 5000,
});
return;
}
const parsed = JSON.parse(fileContents) as ProtoMessage[];
this.allProtoFiles.set(parsed);
this.selectedDefinition.set(parsed);
this.currentFiles.set([]);
for (const message of parsed) {
if (
message.fileName &&
!this.currentFiles().includes(message.fileName)
) {
this.currentFiles.update((currentFiles) => [
...currentFiles,
message.fileName!,
]);
}
}
} catch (err) {
console.error(err);
this.snackBar.open(
"Could not parse configuration file, please ensure it's valid, otherwise consider recreating the configuration",
'Dismiss',
{ duration: 5000 }
);
}
}
@HostListener('dragover', ['$event'])
onDrag(event: DragEvent) {

38
src/app/proto-definition-selector/proto-definition.service.spec.ts
View File

@@ -14,6 +14,10 @@ import { ProtoDefinitionService } from './proto-definition.service';
let testProto = `
syntax="proto3";
package mytestpackage;
import "./myimport.proto";
message Test {
message NestedMessage {
string nested = 1;
@@ -29,10 +33,12 @@ message Test {
ReferenceMessage hello9 = 9;
NestedMessage hello10 = 10;
EnumTest enum_test = 11;
myimportedpackage.ImportedMessage imported_message = 12;
}
message ReferenceMessage {
string test = 1;
Test.NestedMessage nested_message = 2;
}
enum EnumTest {
@@ -51,13 +57,18 @@ describe('TestService', () => {
service = TestBed.inject(ProtoDefinitionService);
});
it('should convert parse protobuf correctly', async () => {
const testMessages = await service.parseProtoDefinition(testProto);
it('should convert parsed protobuf correctly', async () => {
const testMessages = await service.parseAllDefinitions([
new File(
[new Blob([testProto], { type: 'text/plain' })],
'TestFile.proto'
),
]);
const converted = testMessages[1];
expect(converted.name).toBe('Test');
expect(converted.values.length).toBe(11);
expect(converted.values.length).toBe(12);
checkNameAndType(converted, 'hello', MessageTypeEnum.String);
checkNameAndType(converted, 'hello2', MessageTypeEnum.Numeric);
checkNameAndType(converted, 'hello3', MessageTypeEnum.Numeric);
@@ -69,6 +80,7 @@ describe('TestService', () => {
checkNameAndType(converted, 'hello9', MessageTypeEnum.Object);
checkNameAndType(converted, 'hello10', MessageTypeEnum.Object);
checkNameAndType(converted, 'enumTest', MessageTypeEnum.Enum);
checkNameAndType(converted, 'importedMessage', MessageTypeEnum.Object);
const listMessage = converted.values[6].configuration as ListMessage;
expect(listMessage.subConfiguration.type).toBe(MessageTypeEnum.String);
@@ -78,13 +90,31 @@ describe('TestService', () => {
expect(mapMessage.valueConfiguration.type).toBe(MessageTypeEnum.String);
const referenceMessage = converted.values[8].configuration as ObjectMessage;
expect(referenceMessage.messageDefinition.values.length).toBe(1);
expect(referenceMessage.messageDefinition.values.length).toBe(2);
expect(referenceMessage.messageDefinition.name).toBe('ReferenceMessage');
const nestedReferenceMessage = referenceMessage.messageDefinition.values[0];
expect(nestedReferenceMessage.configuration.type).toBe(
MessageTypeEnum.String
);
expect(nestedReferenceMessage.name).toBe('test');
const parentNestedMessage = referenceMessage.messageDefinition.values[1];
expect(parentNestedMessage.configuration.type).toBe(MessageTypeEnum.Object);
expect(parentNestedMessage.name).toBe('nestedMessage');
const parentNestedMessageConfiguration =
parentNestedMessage.configuration as ObjectMessage;
expect(parentNestedMessageConfiguration.messageDefinition.name).toBe(
'Test.NestedMessage'
);
expect(
parentNestedMessageConfiguration.messageDefinition.values.length
).toBe(1);
expect(
parentNestedMessageConfiguration.messageDefinition.values[0].name
).toBe('nested');
expect(
parentNestedMessageConfiguration.messageDefinition.values[0].configuration
.type
).toBe(MessageTypeEnum.String);
const nestedMessage = converted.values[9].configuration as ObjectMessage;
expect(nestedMessage.messageDefinition.values.length).toBe(1);

166
src/app/proto-definition-selector/proto-definition.service.ts
View File

@@ -15,19 +15,69 @@ import {
StringMessage,
UnknownProto,
} from '../model/proto-message.model';
import { readTextFile } from '@tauri-apps/api/fs';
@Injectable({
providedIn: 'root',
})
export class ProtoDefinitionService {
constructor() {}
async parseAllDefinitions(allProtoFiles: File[]): Promise<ProtoMessage[]> {
const messages: (ProtoBase | undefined)[] = [];
for (const protoFile of allProtoFiles) {
const definition = await parse(await protoFile.text());
// Try resolve imports, if they fail (due to invalid path or running app in browser)
// they'll be skipped
if (definition.imports) {
for (const importPath of definition.imports) {
try {
const fileName = importPath.substring(
importPath.lastIndexOf('/') + 1
);
allProtoFiles.push(
new File(
[
new Blob([await readTextFile(importPath)], {
type: 'text/plain',
}),
],
fileName
)
);
} catch (err) {
console.error(
`Failed to import file ${importPath}. Skipping...`,
err
);
}
}
}
const messageDefinitions = definition.root;
const parsedMessages = this.parseAllNestedMessages(
messageDefinitions,
protoFile.name,
definition.package
);
messages.push(...parsedMessages);
}
async parseProtoDefinition(protoContents: string): Promise<ProtoMessage[]> {
const definition = await parse(protoContents);
const messages = definition.root;
if (messages) {
const messageObjects: ProtoBase[] = this.parseAllNestedMessages(messages);
const standardMessages = messageObjects
if (messages.length === 0) {
return [];
}
// Now check for duplicates in case we auto-imported files
for (let i = 0; i < messages.length - 1; i++) {
for (let j = i + 1; j < messages.length; j++) {
if (messages[i]?.fullName === messages[j]?.fullName) {
messages[j] = undefined;
}
}
}
const distinctMessages = messages.filter(
(message) => message
) as ProtoBase[];
// Then populate all object/enum messages for the final passes
const standardMessages = distinctMessages
.filter((messageObject) => 'values' in messageObject)
.map((messageObject) => messageObject as ProtoMessage);
for (const messageObject of standardMessages) {
@@ -35,12 +85,13 @@ export class ProtoDefinitionService {
if (value.configuration.type === MessageTypeEnum.Object) {
this.populateNestedObject(
value.configuration as ObjectMessage,
messageObject.packageName!,
standardMessages
);
}
}
}
const enumMessages = messageObjects
const enumMessages = distinctMessages
.filter((messageObject) => 'options' in messageObject)
.map((messageObject) => messageObject as ProtoEnum);
for (const messageObject of standardMessages) {
@@ -56,15 +107,36 @@ export class ProtoDefinitionService {
return standardMessages;
}
return [];
}
private populateEnumMessages(
objectMessage: ObjectMessage,
enumMessages: ProtoEnum[]
): MessageConfiguration {
let useFullName = false;
let objectMessageName = objectMessage.messageDefinition.name;
if (objectMessageName.includes('.')) {
// First do a check for current package.
for (const enumMessage of enumMessages) {
if (enumMessage.name === objectMessage.messageDefinition.name) {
if (
enumMessage.packageName ===
objectMessage.messageDefinition.packageName &&
enumMessage.fullName ===
`${objectMessage.messageDefinition.packageName}.${objectMessageName}`
) {
return EnumMessage(enumMessage.options);
}
}
// If there's no match, then we'll be matching on full name, assuming a separate package
useFullName = true;
}
// None matching messages in this package found, check external package
for (const enumMessage of enumMessages) {
if (
(useFullName && enumMessage.fullName === objectMessageName) ||
(!useFullName && enumMessage.name === objectMessageName)
) {
return EnumMessage(enumMessage.options);
}
}
@@ -83,23 +155,34 @@ export class ProtoDefinitionService {
return objectMessage;
}
private parseAllNestedMessages(obj: ReflectionObject) {
private parseAllNestedMessages(
obj: ReflectionObject,
fileName: string,
packageName?: string
) {
const messageObjects: ProtoBase[] = [];
// Nested messages/enums
if ('nestedArray' in obj && obj.nestedArray) {
const nestedArray = obj.nestedArray as ReflectionObject[];
for (const nestedObj of nestedArray) {
messageObjects.push(...this.parseAllNestedMessages(nestedObj));
messageObjects.push(
...this.parseAllNestedMessages(nestedObj, fileName, packageName)
);
}
}
// Message
if ('fieldsArray' in obj) {
messageObjects.push(this.convertProtoDefinitionToModel(obj));
messageObjects.push(
this.convertProtoDefinitionToModel(obj, fileName, packageName)
);
}
// Enum
if ('values' in obj) {
messageObjects.push({
name: obj.name,
fullName: obj.fullName,
packageName,
fileName,
options: Object.entries(obj.values as Record<string, string>)
.sort((a, b) => Number(a[1]) - Number(b[1]))
.map(([key, _]) => key),
@@ -110,36 +193,73 @@ export class ProtoDefinitionService {
private populateNestedObject(
objectMessage: ObjectMessage,
packageName: string,
availableMessages: ProtoMessage[]
) {
let useFullName = false;
let objectMessageName = objectMessage.messageDefinition.name;
if (objectMessageName.includes('.')) {
// First do a check for current package.
for (const message of availableMessages) {
if (message.name === objectMessage.messageDefinition.name) {
if (
message.packageName === packageName &&
message.fullName === `${packageName}.${objectMessageName}`
) {
this.updateObjectMessage(objectMessage, message, availableMessages);
return;
}
}
// If there's no match, then we'll be matching on full name, assuming a separate package
useFullName = true;
}
for (const message of availableMessages) {
if (
(useFullName && message.fullName === objectMessageName) ||
(!useFullName && message.name === objectMessageName)
) {
this.updateObjectMessage(objectMessage, message, availableMessages);
return;
}
}
objectMessage.messageDefinition = UnknownProto(
objectMessage.messageDefinition.name,
objectMessage.messageDefinition.fullName
);
}
private updateObjectMessage(
objectMessage: ObjectMessage,
message: ProtoMessage,
availableMessages: ProtoMessage[]
) {
objectMessage.messageDefinition = {
name: message.name,
name: objectMessage.messageDefinition.name,
values: message.values.map((value) => {
if (value.configuration.type === MessageTypeEnum.Object) {
this.populateNestedObject(
value.configuration as ObjectMessage,
message.packageName!,
availableMessages
);
}
return structuredClone(value);
}),
};
return;
}
}
objectMessage.messageDefinition = UnknownProto(
objectMessage.messageDefinition.name
);
}
private convertProtoDefinitionToModel(
messageDefinition?: ReflectionObject
messageDefinition: ReflectionObject,
fileName: string,
packageName?: string
): ProtoBase {
if (messageDefinition && 'fieldsArray' in messageDefinition) {
const convertedFields: ProtoMessage = {
name: messageDefinition.name,
fullName: messageDefinition.fullName.substring(1), // trim leading .
packageName,
fileName,
values: [],
};
const fields = messageDefinition.fieldsArray as FieldBase[];