use many config files

cfg/config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"log"
 	"os"
 	"reflect"
 	"regexp"
@@ -13,7 +14,8 @@ import (
 
 	"github.com/bitly/go-simplejson"
 	"github.com/ozontech/file.d/logger"
-	"sigs.k8s.io/yaml"
+	"gopkg.in/yaml.v2"
+	k8s_yaml "sigs.k8s.io/yaml"
 )
 
 const trueValue = "true"
@@ -77,22 +79,37 @@ func NewConfig() *Config {
 	}
 }
 
-func NewConfigFromFile(path string) *Config {
-	logger.Infof("reading config %q", path)
-	yamlContents, err := os.ReadFile(path)
+func NewConfigFromFile(paths []string) *Config {
+	mergedConfig := make(map[interface{}]interface{})
+
+	for _, path := range paths {
+		logger.Infof("reading config %q", path)
+		yamlContents, err := os.ReadFile(path)
+		if err != nil {
+			logger.Fatalf("can't read config file %q: %s", path, err)
+		}
+		var currentConfig map[interface{}]interface{}
+		if err := yaml.Unmarshal(yamlContents, &currentConfig); err != nil {
+			logger.Fatalf("can't parse config file yaml %q: %s", path, err)
+		}
+
+		mergedConfig = mergeYAMLs(mergedConfig, currentConfig)
+	}
+
+	mergedYAML, err := yaml.Marshal(mergedConfig)
 	if err != nil {
-		logger.Fatalf("can't read config file %q: %s", path, err)
+		log.Fatalf("can't marshal merged config to YAML: %s", err)
 	}
 
-	jsonContents, err := yaml.YAMLToJSON(yamlContents)
+	jsonContents, err := k8s_yaml.YAMLToJSON(mergedYAML)
 	if err != nil {
-		logger.Infof("config content:\n%s", logger.Numerate(string(yamlContents)))
-		logger.Fatalf("can't parse config file yaml %q: %s", path, err.Error())
+		logger.Infof("config content:\n%s", logger.Numerate(string(mergedYAML)))
+		logger.Fatalf("can't parse config file yaml %q: %s", paths, err.Error())
 	}
 
 	object, err := simplejson.NewJson(jsonContents)
 	if err != nil {
-		logger.Fatalf("can't convert config to json %q: %s", path, err.Error())
+		logger.Fatalf("can't convert config to json %q: %s", paths, err.Error())
 	}
 
 	err = applyEnvs(object)
@@ -637,3 +654,22 @@ func CompileRegex(s string) (*regexp.Regexp, error) {
 
 	return regexp.Compile(s[1 : len(s)-1])
 }
+
+func mergeYAMLs(a, b map[interface{}]interface{}) map[interface{}]interface{} {
+	merged := make(map[interface{}]interface{})
+	for k, v := range a {
+		merged[k] = v
+	}
+	for k, v := range b {
+		if existingValue, exists := merged[k]; exists {
+			if existingMap, ok := existingValue.(map[interface{}]interface{}); ok {
+				if newMap, ok := v.(map[interface{}]interface{}); ok {
+					merged[k] = mergeYAMLs(existingMap, newMap)
+					continue
+				}
+			}
+		}
+		merged[k] = v
+	}
+	return merged
+}

cfg/config_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func NewTestConfig(name string) *Config {
-	return NewConfigFromFile("../testdata/config/" + name)
+	return NewConfigFromFile([]string{"../testdata/config/" + name})
 }
 
 func TestSimple(t *testing.T) {

cmd/file.d/file.d.go

@@ -65,7 +65,7 @@ var (
 	fileD *fd.FileD
 	exit  = make(chan bool)
 
-	config        = kingpin.Flag("config", `Config file name`).Required().ExistingFile()
+	config        = kingpin.Flag("config", `Config file name (to add a config file you can repeat the argument)`).Required().ExistingFiles()
 	http          = kingpin.Flag("http", `HTTP listen addr eg. ":9000", "off" to disable`).Default(":9000").String()
 	memLimitRatio = kingpin.Flag(
 		"mem-limit-ratio",

cmd/file.d/file.d_test.go

@@ -85,7 +85,7 @@ func TestEndToEnd(t *testing.T) {
 	filesDir := t.TempDir()
 	offsetsDir := t.TempDir()
 
-	config := cfg.NewConfigFromFile(configFilename)
+	config := cfg.NewConfigFromFile([]string{configFilename})
 	input := config.Pipelines["test"].Raw.Get("input")
 	input.Set("watching_dir", filesDir)
 	input.Set("offsets_file", filepath.Join(offsetsDir, "offsets.yaml"))