mapstructuredecoder that produces structpb safe output (#522)

changelog/v0.24.7/mapstructure-utils.yaml

@@ -0,0 +1,11 @@
+changelog:
+  - type: NON_USER_FACING
+    description: >
+      Normalizes and decodes map interfaces that can have (deeply nested) numbers of type json.Number
+      by setting those to either to int64 or float64.
+
+      Usually you don't need to normalize a map[string]interface{}, that you are decoding into a struct with mapstructure
+      Unless, you will use it with `structpb.NewValue()` which doesn't handle that type and throws an invalid type error.
+
+    issueLink: https://github.com/solo-io/gloo-mesh-enterprise/issues/12608
+    resolvesIssue: false

go.mod

@@ -95,6 +95,7 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/prometheus/client_golang v1.15.1 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect

go.sum

@@ -296,6 +296,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=

mapstructureutils/mapdecoder.go

@@ -0,0 +1,74 @@
+package mapdecoder
+
+import (
+	"encoding/json"
+	"reflect"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+// NormalizeMapDecode decodes a map[string]interface{} into the given result object
+// and handles converting json.Number to int64 or float64 which would cause errors in structpb.NewValue
+func NormalizeMapDecode(input interface{}, result interface{}) error {
+	config := &mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.ComposeDecodeHookFunc(jsonNumberToNumberHook()),
+		Result:     result,
+	}
+
+	decoder, err := mapstructure.NewDecoder(config)
+	if err != nil {
+		return err
+	}
+
+	return decoder.Decode(input)
+}
+
+// jsonNumberToNumberHook creates a DecodeHookFuncType that converts json.Number to int64 or float64
+func jsonNumberToNumberHook() mapstructure.DecodeHookFuncType {
+	return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+		if numberStr, ok := data.(json.Number); ok {
+			if t.Kind() == reflect.Int64 {
+				return numberStr.Int64()
+			}
+
+			if t.Kind() == reflect.Float64 {
+				return numberStr.Float64()
+			}
+		}
+
+		if f.Kind() == reflect.Map {
+			// Recursively process the map
+			return convertNumbersInMap(data)
+		}
+
+		return data, nil
+	}
+}
+
+// convertNumbersInMap takes a map and converts all json.Number values to the appropriate numeric type
+func convertNumbersInMap(original interface{}) (interface{}, error) {
+	resultMap := make(map[string]interface{})
+	for key, val := range original.(map[string]interface{}) {
+		switch v := val.(type) {
+		case json.Number:
+			if intVal, err := v.Int64(); err == nil {
+				resultMap[key] = intVal
+			} else if floatVal, err := v.Float64(); err == nil {
+				resultMap[key] = floatVal
+			} else {
+				// If it's not a number, just keep the original string
+				resultMap[key] = val
+			}
+		case map[string]interface{}:
+			// Recursively convert nested maps
+			convertedMap, err := convertNumbersInMap(v)
+			if err != nil {
+				return nil, err
+			}
+			resultMap[key] = convertedMap
+		default:
+			resultMap[key] = val
+		}
+	}
+	return resultMap, nil
+}

mapstructureutils/mapdecoder_suite_test.go

@@ -0,0 +1,25 @@
+package mapdecoder_test
+
+import (
+	"testing"
+
+	"go.uber.org/zap"
+
+	"github.com/fgrosse/zaptest"
+	"github.com/solo-io/go-utils/contextutils"
+
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2/reporters"
+	. "github.com/onsi/gomega"
+)
+
+func TestMapDecoderServer(t *testing.T) {
+	zaptest.Level = zap.InfoLevel
+	logger := zaptest.LoggerWriter(GinkgoWriter)
+
+	contextutils.SetFallbackLogger(logger.Sugar())
+
+	RegisterFailHandler(Fail)
+	junitReporter := reporters.NewJUnitReporter("junit.xml")
+	RunSpecsWithDefaultAndCustomReporters(t, "Map Decoder Suite", []Reporter{junitReporter})
+}

mapstructureutils/mapdecoder_test.go

@@ -0,0 +1,51 @@
+package mapdecoder_test
+
+import (
+	"encoding/json"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/solo-io/go-utils/mapstructureutils"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+var _ = Describe("Normalize Map Decode", func() {
+	DescribeTable("should correctly decode JSON numbers into int64 or float64",
+		func(input map[string]interface{}, expectedResult map[string]interface{}) {
+			result := make(map[string]interface{})
+			err := NormalizeMapDecode(input, &result)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(result).To(Equal(expectedResult))
+			value, err := structpb.NewValue(result)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(value).ToNot(BeNil())
+		},
+		Entry("deeply nested JSON number to int",
+			map[string]interface{}{
+				"number": json.Number("10"),
+				"float":  float64(10.5),
+				"nested": map[string]interface{}{
+					"number": json.Number("100"),
+					"float":  float64(100.5),
+					"deeplyNested": map[string]interface{}{
+						"number": json.Number("1000"),
+						"float":  float64(100.5),
+					},
+				},
+			},
+			map[string]interface{}{
+				"number": int64(10),
+				"float":  float64(10.5),
+				"nested": map[string]interface{}{
+					"number": int64(100),
+					"float":  float64(100.5),
+					"deeplyNested": map[string]interface{}{
+						"number": int64(1000),
+						"float":  float64(100.5),
+					},
+				},
+			},
+		),
+	)
+})