vato007/bufpiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework proto parsing to use javascript objects, add nested and object message support

Browse Source 
Objects are preferred to classes as they're easier to clone and serialise/deserialise
This commit is contained in:
vato007
2024-06-26 21:51:43 +09:30
parent d484f75540
commit da8861cde2
3 changed files with 191 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
src/app/model/proto-message.model.ts
View File

@@ -1,27 +1,80 @@
export class StringMessage implements MessageConfiguration {
constructor(public maxLength?: number) {}
export enum MessageTypeEnum {
String = 'string',
Boolean = 'boolean',
Numeric = 'numeric',
List = 'list',
Map = 'map',
Object = 'object',
Raw = 'raw',
}
export class BooleanMessage implements MessageConfiguration {}
export class NumericMessage implements MessageConfiguration {
constructor(public min?: number, public max?: number) {}
export interface StringMessage extends MessageConfiguration {
maxLength?: number;
}
export class ListMessage implements MessageConfiguration {
constructor(readonly subConfiguration: MessageConfiguration) {}
export interface BooleanMessage extends MessageConfiguration {}
export interface NumericMessage extends MessageConfiguration {
min?: number;
max?: number;
}
export class MapMessage implements MessageConfiguration {
constructor(
readonly keyConfiguration: MessageConfiguration,
readonly valueConfiguration: MessageConfiguration
) {}
export interface ListMessage extends MessageConfiguration {
subConfiguration: MessageConfiguration;
}
export class ObjectMessage implements MessageConfiguration {}
export interface MapMessage extends MessageConfiguration {
keyConfiguration: MessageConfiguration;
valueConfiguration: MessageConfiguration;
}
export class MessageConfiguration {}
export interface ObjectMessage extends MessageConfiguration {
messageDefinition: ProtoMessage;
}
export interface RawMessage extends MessageConfiguration {}
export interface MessageConfiguration {
type: MessageTypeEnum;
}
export const StringMessage = (maxLength?: number): StringMessage => ({
type: MessageTypeEnum.String,
maxLength,
});
export const BooleanMessage = (): BooleanMessage => ({
type: MessageTypeEnum.Boolean,
});
export const NumericMessage = (min?: number, max?: number): NumericMessage => ({
type: MessageTypeEnum.Numeric,
min,
max,
});
export const ListMessage = (
subConfiguration: MessageConfiguration
): ListMessage => ({
type: MessageTypeEnum.List,
subConfiguration,
});
export const MapMessage = (
keyConfiguration: MessageConfiguration,
valueConfiguration: MessageConfiguration
): MapMessage => ({
type: MessageTypeEnum.Map,
keyConfiguration,
valueConfiguration,
});
export const ObjectMessage = (messageDefinition: ProtoMessage) => ({
type: MessageTypeEnum.Object,
messageDefinition,
});
export const RawMessage = (): RawMessage => ({ type: MessageTypeEnum.Raw });
export interface ProtoMessageField {
name: string;
@@ -33,3 +86,8 @@ export interface ProtoMessage {
name: string;
values: ProtoMessageField[];
}
export const UnknownProto = (name: string): ProtoMessage => ({
name,
values: [{ name: 'Raw JSON', configuration: RawMessage(), value: '' }],
});

87
src/app/proto-definition.service.spec.ts
View File

@@ -1,17 +1,12 @@
import { TestBed } from '@angular/core/testing';
import { provideExperimentalZonelessChangeDetection } from '@angular/core';
import {
Type,
provideExperimentalZonelessChangeDetection,
} from '@angular/core';
import {
BooleanMessage,
ListMessage,
MapMessage,
MessageConfiguration,
NumericMessage,
MessageTypeEnum,
ObjectMessage,
ProtoMessage,
StringMessage,
} from './model/proto-message.model';
import { ProtoDefinitionService } from './proto-definition.service';
@@ -19,15 +14,25 @@ let testProto = `
syntax="proto3";
message Test {
string hello = 1;
int32 hello2 = 2;
int64 hello3 = 3;
float hello4 = 4;
double hello5 = 5;
bool hello6 = 6;
repeated string hello7 = 7;
map<string, string> hello8 = 8;
message NestedMessage {
string nested = 1;
}
string hello = 1;
int32 hello2 = 2;
int64 hello3 = 3;
float hello4 = 4;
double hello5 = 5;
bool hello6 = 6;
repeated string hello7 = 7;
map<string, string> hello8 = 8;
ReferenceMessage hello9 = 9;
NestedMessage hello10 = 10;
}
message ReferenceMessage {
string test = 1;
}
`;
describe('TestService', () => {
@@ -42,34 +47,54 @@ describe('TestService', () => {
it('should convert parse protobuf correctly', async () => {
const testMessages = await service.parseProtoDefinition(testProto);
const converted = testMessages[0];
const converted = testMessages[1];
expect(converted.name).toBe('Test');
expect(converted.values.length).toBe(8);
checkNameAndType(converted, 'hello', StringMessage);
checkNameAndType(converted, 'hello2', NumericMessage);
checkNameAndType(converted, 'hello3', NumericMessage);
checkNameAndType(converted, 'hello4', NumericMessage);
checkNameAndType(converted, 'hello5', NumericMessage);
checkNameAndType(converted, 'hello6', BooleanMessage);
checkNameAndType(converted, 'hello7', ListMessage);
checkNameAndType(converted, 'hello8', MapMessage);
expect(converted.values.length).toBe(10);
checkNameAndType(converted, 'hello', MessageTypeEnum.String);
checkNameAndType(converted, 'hello2', MessageTypeEnum.Numeric);
checkNameAndType(converted, 'hello3', MessageTypeEnum.Numeric);
checkNameAndType(converted, 'hello4', MessageTypeEnum.Numeric);
checkNameAndType(converted, 'hello5', MessageTypeEnum.Numeric);
checkNameAndType(converted, 'hello6', MessageTypeEnum.Boolean);
checkNameAndType(converted, 'hello7', MessageTypeEnum.List);
checkNameAndType(converted, 'hello8', MessageTypeEnum.Map);
checkNameAndType(converted, 'hello9', MessageTypeEnum.Object);
checkNameAndType(converted, 'hello10', MessageTypeEnum.Object);
const listMessage = converted.values[6].configuration as ListMessage;
expect(listMessage.subConfiguration).toBeInstanceOf(StringMessage);
expect(listMessage.subConfiguration.type).toBe(MessageTypeEnum.String);
const mapMessage = converted.values[7].configuration as MapMessage;
expect(mapMessage.keyConfiguration).toBeInstanceOf(StringMessage);
expect(mapMessage.valueConfiguration).toBeInstanceOf(StringMessage);
expect(mapMessage.keyConfiguration.type).toBe(MessageTypeEnum.String);
expect(mapMessage.valueConfiguration.type).toBe(MessageTypeEnum.String);
const referenceMessage = converted.values[8].configuration as ObjectMessage;
expect(referenceMessage.messageDefinition.values.length).toBe(1);
expect(referenceMessage.messageDefinition.name).toBe('ReferenceMessage');
const nestedReferenceMessage = referenceMessage.messageDefinition.values[0];
expect(nestedReferenceMessage.configuration.type).toBe(
MessageTypeEnum.String
);
expect(nestedReferenceMessage.name).toBe('test');
const nestedMessage = converted.values[9].configuration as ObjectMessage;
expect(nestedMessage.messageDefinition.values.length).toBe(1);
expect(nestedMessage.messageDefinition.name).toBe('NestedMessage');
const nestedNestedMessage = nestedMessage.messageDefinition.values[0];
expect(nestedNestedMessage.configuration.type).toBe(MessageTypeEnum.String);
expect(nestedNestedMessage.name).toBe('nested');
// TODO: Enum type
});
});
const checkNameAndType = (
converted: ProtoMessage,
name: string,
type: Type<MessageConfiguration>
type: MessageTypeEnum
) => {
const field = converted.values.find((value) => value.name === name);
expect(field?.configuration).toBeInstanceOf(type);
expect(field?.configuration.type).toBe(type);
};

78
src/app/proto-definition.service.ts
View File

@@ -5,9 +5,13 @@ import {
ListMessage,
MapMessage,
MessageConfiguration,
MessageTypeEnum,
NumericMessage,
ObjectMessage,
ProtoMessage,
RawMessage,
StringMessage,
UnknownProto,
} from './model/proto-message.model';
@Injectable({
@@ -18,16 +22,19 @@ export class ProtoDefinitionService {
async parseProtoDefinition(protoContents: string): Promise<ProtoMessage[]> {
const definition = await parse(protoContents);
const messages = definition.root.nested;
const messages = definition.root;
if (messages) {
const objectDefinitions = Object.values(messages);
const messageObjects = [];
for (const objectDefinition of objectDefinitions) {
// TODO: Better way to check for message type (i.e. not enum)?
if ('fieldsArray' in objectDefinition) {
messageObjects.push(
this.convertProtoDefinitionToModel(objectDefinition)
);
const messageObjects: ProtoMessage[] =
this.parseAllNestedMessages(messages);
// Now do another pass, where we populate each of the object fields based on their definitions... need a way to detect recursion? Maybe only go up to 5 layers deep?
for (const messageObject of messageObjects) {
for (const value of messageObject.values) {
if (value.configuration.type === MessageTypeEnum.Object) {
this.populateNestedObject(
value.configuration as ObjectMessage,
messageObjects
);
}
}
}
return messageObjects;
@@ -35,6 +42,46 @@ export class ProtoDefinitionService {
return [];
}
private parseAllNestedMessages(obj: ReflectionObject) {
const messageObjects: ProtoMessage[] = [];
if ('nestedArray' in obj && obj.nestedArray) {
const nestedArray = obj.nestedArray as ReflectionObject[];
for (const nestedObj of nestedArray) {
messageObjects.push(...this.parseAllNestedMessages(nestedObj));
}
}
if ('fieldsArray' in obj) {
messageObjects.push(this.convertProtoDefinitionToModel(obj));
}
return messageObjects;
}
private populateNestedObject(
objectMessage: ObjectMessage,
availableMessages: ProtoMessage[]
) {
for (const message of availableMessages) {
if (message.name === objectMessage.messageDefinition.name) {
objectMessage.messageDefinition = {
name: message.name,
values: message.values.map((value) => {
if (value.configuration.type === MessageTypeEnum.Object) {
this.populateNestedObject(
value.configuration as ObjectMessage,
availableMessages
);
}
return structuredClone(value);
}),
};
return;
}
}
objectMessage.messageDefinition = UnknownProto(
objectMessage.messageDefinition.name
);
}
private convertProtoDefinitionToModel(
messageDefinition?: ReflectionObject
): ProtoMessage {
@@ -44,20 +91,19 @@ export class ProtoDefinitionService {
values: [],
};
const fields = messageDefinition.fieldsArray as FieldBase[];
// TODO: Needs to be a visitor/tree search/recursive since messages/enums may be nested
for (const field of fields) {
let type: MessageConfiguration | undefined;
if ('rule' in field && field.rule === 'repeated') {
const subType = this.getTypeFromField(field.type);
if (subType) {
type = new ListMessage(subType);
type = ListMessage(subType);
}
} else if (field instanceof MapField) {
// Map
const keyConfiguration = this.getTypeFromField(field.keyType);
const valueConfiguration = this.getTypeFromField(field.type);
if (keyConfiguration && valueConfiguration) {
type = new MapMessage(keyConfiguration, valueConfiguration);
type = MapMessage(keyConfiguration, valueConfiguration);
}
} else {
type = this.getTypeFromField(field.type);
@@ -82,19 +128,19 @@ export class ProtoDefinitionService {
): MessageConfiguration | undefined {
switch (fieldType) {
case 'string':
return new StringMessage();
return StringMessage();
break;
case 'int32':
case 'int64':
case 'float':
case 'double':
return new NumericMessage();
return NumericMessage();
break;
case 'bool':
return new BooleanMessage();
return BooleanMessage();
break;
// TODO: bytes as well, though that's pretty useless (can't really represent/edit it?)
}
return undefined;
return ObjectMessage({ name: fieldType, values: [] });
}
}