Support for Split output Logging (#463)

* Add support for split output logging
* Only use utils to set fallbackLogger level
* Add changelog entry
* Don't use boolean environment variable values
* Remove debug logger configuration
* Code review

changelog/v0.21.17/add-split-output-logging.yaml

@@ -0,0 +1,9 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/gloo/issues/5048
+    resolvesIssue: false
+    description: >
+      When the SPLIT_LOG_OUTPUT environment variable is specified, a global logger is created with the following characteristics:
+      debug/info/warning are written to stdout, error/fatal/panic are written to stderr
+      Configuration is otherwise identical to the default logger
+      When SPLIT_LOG_OUTPUT is unspecified, logging configuration is unchanged

contextutils/context_and_logging.go

@@ -19,6 +19,7 @@ package contextutils
 
 import (
 	"context"
+	"os"
 
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
@@ -36,15 +37,61 @@ var (
 	level zap.AtomicLevel
 )
 
-func buildLogger() (*zap.Logger, error) {
+func buildProductionLogger() (*zap.Logger, error) {
 	config := zap.NewProductionConfig()
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	level = zap.NewAtomicLevel()
 	config.Level = level
 	return config.Build()
 }
 
+func buildSplitOutputProductionLogger() (*zap.Logger, error) {
+	logger, err := buildProductionLogger()
+	if err != nil {
+		return nil, err
+	}
+
+	// Define level-handling logic.
+	highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
+		return lvl >= zapcore.ErrorLevel
+	})
+	lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
+		return lvl < zapcore.ErrorLevel
+	})
+
+	// High-priority output should go to standard error, and low-priority
+	// output should go to standard out.
+	consoleDebugging := zapcore.Lock(os.Stdout)
+	consoleErrors := zapcore.Lock(os.Stderr)
+
+	buildEncoder := func() zapcore.Encoder {
+		encoderConfig := zap.NewProductionEncoderConfig()
+		encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+		return zapcore.NewJSONEncoder(encoderConfig)
+	}
+
+	splitOutput := zap.WrapCore(func(c zapcore.Core) zapcore.Core {
+		// Join the outputs, encoders, and level-handling functions into
+		// zapcore.Cores, then tee the cores together.
+		return zapcore.NewTee(
+			zapcore.NewCore(
+				buildEncoder(), consoleErrors, highPriority,
+			),
+			zapcore.NewCore(
+				buildEncoder(), consoleDebugging, lowPriority,
+			),
+		)
+	})
+
+	return logger.WithOptions(splitOutput, zap.AddCaller(), zap.AddStacktrace(highPriority)), nil
+}
+
 func init() {
+	buildLogger := buildProductionLogger
+	// Specify gloo.splitLogOutput=true when installing Gloo via Helm to set the SPLIT_LOG_OUTPUT env var
+	if os.Getenv("SPLIT_LOG_OUTPUT") == "true" {
+		buildLogger = buildSplitOutputProductionLogger
+	}
 	if logger, err := buildLogger(); err != nil {
 
 		// We failed to create a fallback logger. Our fallback

stats/stats.go

@@ -77,23 +77,11 @@ func StartStatsServerWithPort(startupOpts StartupOptions, addhandlers ...func(mu
 		default:
 			setLevel = zapcore.InfoLevel
 		}
-		logLevel = zap.NewAtomicLevelAt(setLevel)
 
-		logConfig := zap.NewProductionConfig()
-		logConfig.Level = logLevel
-		logger, logErr := logConfig.Build()
-		if logErr == nil {
-			contextutils.SetFallbackLogger(logger.Sugar())
-		}
+		contextutils.SetLogLevel(setLevel)
+
 	} else if startupOpts.LogLevel != nil {
 		logLevel = *startupOpts.LogLevel
-	} else {
-		logConfig := zap.NewProductionConfig()
-		logger, logErr := logConfig.Build()
-		if logErr == nil {
-			logLevel = logConfig.Level
-			contextutils.SetFallbackLogger(logger.Sugar())
-		}
 	}
 
 	go RunGoroutineStat()