From d54de3b54bdc07396bcc7c16ca571f8e7628b160 Mon Sep 17 00:00:00 2001 From: Daniel Orner Date: Thu, 23 Nov 2023 14:59:21 -0500 Subject: [PATCH] Add namespace_map param, fix incorrect case statements, fix nesting records unnecessarily --- README.md | 16 ++++++++++++++++ avro/field.go | 8 ++++++++ avro/record.go | 3 ++- avro/type.go | 3 +++ avro/typeRepo.go | 25 ++++++++++++++++++++----- main.go | 28 +++++++++++++++++++++++++--- 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9708c91..2882e38 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,22 @@ protoc --avro_out=. --avro_opt=emit_only=Foo,Bar:. *.proto This will generate only `Foo.avsc` and `Bar.avsc` files. +You can also change the namespaces being mapped: + +```bash +protoc --avro_out=. --avro_opt=namespace_map=foo:bar,baz:spam *.proto +``` + +...will change the output namespace for `foo` to `bar` and `baz` to `spam`. + +--- + +To Do List: + +* Add tests +* Map is currently outputting as Array due to how Protobuf handles [maps](https://protobuf.com/docs/descriptors#map-fields) (as repeated entries). Need to fix. +* Need to decide on how to truly differentiate between optional and required fields (technically all fields are optional on Protobuf, but maybe we should use the actual `optional` keyword and only have those be optional in Avro?) + --- This project supported by [Flipp](https://corp.flipp.com/). diff --git a/avro/field.go b/avro/field.go index ba91d7a..27dde12 100644 --- a/avro/field.go +++ b/avro/field.go @@ -54,15 +54,23 @@ func BasicFieldTypeFromProto(proto *descriptorpb.FieldDescriptorProto) Type { case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE: return Bare("double") case descriptorpb.FieldDescriptorProto_TYPE_INT64: + return Bare("long") case descriptorpb.FieldDescriptorProto_TYPE_UINT64: + return Bare("long") case descriptorpb.FieldDescriptorProto_TYPE_FIXED64: + return Bare("long") case descriptorpb.FieldDescriptorProto_TYPE_SINT64: + return Bare("long") case descriptorpb.FieldDescriptorProto_TYPE_SFIXED64: return Bare("long") case descriptorpb.FieldDescriptorProto_TYPE_INT32: + return Bare("int") case descriptorpb.FieldDescriptorProto_TYPE_UINT32: + return Bare("int") case descriptorpb.FieldDescriptorProto_TYPE_FIXED32: + return Bare("int") case descriptorpb.FieldDescriptorProto_TYPE_SINT32: + return Bare("int") case descriptorpb.FieldDescriptorProto_TYPE_SFIXED32: return Bare("int") case descriptorpb.FieldDescriptorProto_TYPE_BOOL: diff --git a/avro/record.go b/avro/record.go index 1a4ba2b..addae82 100644 --- a/avro/record.go +++ b/avro/record.go @@ -21,10 +21,11 @@ func (t Record) GetNamespace() string { } func (t Record) ToJSON(types *TypeRepo) (any, error) { + types.SeenType(t) jsonMap := orderedmap.New() jsonMap.Set("type", "record") jsonMap.Set("name", t.Name) - jsonMap.Set("namespace", t.Namespace) + jsonMap.Set("namespace", types.MappedNamespace(t.Namespace)) fields := make([]any, len(t.Fields)) for i, field := range t.Fields { fieldJson, err := field.ToJSON(types) diff --git a/avro/type.go b/avro/type.go index 80269f4..b74c743 100644 --- a/avro/type.go +++ b/avro/type.go @@ -36,15 +36,18 @@ func DefaultValue(t Type) any { case Bare("boolean"): return false case Bare("int"): + return 0 case Bare("long"): return 0 case Bare("float"): + return 0.0 case Bare("double"): return 0.0 } switch typedT := t.(type) { case Record: + return map[string]any{} case Map: return map[string]any{} case Array: diff --git a/avro/typeRepo.go b/avro/typeRepo.go index 93893d8..30a6b6e 100644 --- a/avro/typeRepo.go +++ b/avro/typeRepo.go @@ -2,15 +2,20 @@ package avro import ( "fmt" + "strings" ) type TypeRepo struct { Types map[string]NamedType seenTypes map[string]bool // go "set" + NamespaceMap map[string]string } -func NewTypeRepo() *TypeRepo { - return &TypeRepo{Types: make(map[string]NamedType)} +func NewTypeRepo(namespaceMap map[string]string) *TypeRepo { + return &TypeRepo{ + Types: make(map[string]NamedType), + NamespaceMap: namespaceMap, + } } func (r *TypeRepo) AddType(t NamedType) { @@ -27,16 +32,19 @@ func (r *TypeRepo) GetTypeByBareName(name string) Type { return nil } +func (r *TypeRepo) SeenType(t NamedType) { + r.seenTypes[FullName(t)] = true +} + func (r *TypeRepo) GetType(name string) (Type, error) { if r.seenTypes[name] { - return Bare(name[1:]), nil + return Bare(r.MappedNamespace(name[1:])), nil } t, ok := r.Types[name] if !ok { -// r.LogTypes() return nil, fmt.Errorf("type %s not found", name) } - r.seenTypes[FullName(t)] = true + r.SeenType(t) return t, nil } @@ -52,3 +60,10 @@ func (r *TypeRepo) LogTypes() { LogObj(keys) } +func (r *TypeRepo) MappedNamespace(namespace string) string { + out := namespace + for k, v := range r.NamespaceMap { + out = strings.Replace(out, k, v, -1) + } + return out +} diff --git a/main.go b/main.go index 3d60c42..0d04447 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,12 @@ import ( "strings" ) +type Params struct { + EmitOnly []string + NamespaceMap map[string]string +} + +var params Params var typeRepo *avro.TypeRepo func readRequest() (*pluginpb.CodeGeneratorRequest, error) { @@ -104,8 +110,9 @@ func processAll(fileProto *descriptorpb.FileDescriptorProto) { } } -func writeResponse(req *pluginpb.CodeGeneratorRequest) { +func parseParams(req *pluginpb.CodeGeneratorRequest) Params { var recordsToEmit []string + namespaceMap := map[string]string{} param := req.GetParameter() if len(param) > 0 { paramTokens := strings.Split(param, " ") @@ -113,10 +120,24 @@ func writeResponse(req *pluginpb.CodeGeneratorRequest) { paramStrings := strings.Split(token, "=") if len(paramStrings) == 2 && paramStrings[0] == "emit_only" { recordsToEmit = strings.Split(paramStrings[1], ",") + } else if len(paramStrings) == 2 && paramStrings[0] == "namespace_map" { + namespaces := strings.Split(paramStrings[1], ",") + for _, namespaceMapToken := range namespaces { + namespaceTokens := strings.Split(namespaceMapToken, ":") + namespaceMap[namespaceTokens[0]] = namespaceTokens[1] + } } } } - response := generateResponse(recordsToEmit) + return Params{ + EmitOnly: recordsToEmit, + NamespaceMap: namespaceMap, + } + +} + +func writeResponse(req *pluginpb.CodeGeneratorRequest) { + response := generateResponse(params.EmitOnly) out, err := proto.Marshal(response) if err != nil { log.Fatalf("%s", fmt.Errorf("error marshalling response: %w", err)) @@ -128,11 +149,12 @@ func writeResponse(req *pluginpb.CodeGeneratorRequest) { } func main() { - typeRepo = avro.NewTypeRepo() req, err := readRequest() if err != nil { log.Fatalf("%s", fmt.Errorf("error reading request: %w", err)) } + params = parseParams(req) + typeRepo = avro.NewTypeRepo(params.NamespaceMap) for _, file := range req.ProtoFile { if !slices.Contains(req.FileToGenerate, *file.Name) {