Add context to throughout the project and update logging (#2209)

propagate context from all the API calls and log request ID, account ID and peer ID --------- Co-authored-by: Zoltan Papp <[email protected]>
netbirdio · Jul 3, 2024 · 765aba2 · 765aba2
1 parent 7cb81f1
commit 765aba2
Show file tree

Hide file tree

Showing 127 changed files with 2,936 additions and 2,642 deletions.
diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go
@@ -76,7 +76,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, cleanUp, err := mgmt.NewTestStoreFromJson(config.Datadir)
+	store, cleanUp, err := mgmt.NewTestStoreFromJson(context.Background(), config.Datadir)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -87,13 +87,13 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 	if err != nil {
 		return nil, nil
 	}
-	iv, _ := integrations.NewIntegratedValidator(eventStore)
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv)
+	iv, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
+	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv)
 	if err != nil {
 		t.Fatal(err)
 	}
 	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go
@@ -174,7 +174,7 @@ func TestEngine_SSH(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	//time.Sleep(250 * time.Millisecond)
+	// time.Sleep(250 * time.Millisecond)
 	assert.NotNil(t, engine.sshServer)
 	assert.Contains(t, sshPeersRemoved, "MNHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=")
 
@@ -1057,7 +1057,7 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
 
-	store, cleanUp, err := server.NewTestStoreFromJson(config.Datadir)
+	store, cleanUp, err := server.NewTestStoreFromJson(context.Background(), config.Datadir)
 	if err != nil {
 		return nil, "", err
 	}
@@ -1068,13 +1068,13 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 	if err != nil {
 		return nil, "", err
 	}
-	ia, _ := integrations.NewIntegratedValidator(eventStore)
-	accountManager, err := server.BuildManager(store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
+	ia, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
+	accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
 	if err != nil {
 		return nil, "", err
 	}
 	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		return nil, "", err
 	}

diff --git a/client/server/server_test.go b/client/server/server_test.go
@@ -108,7 +108,7 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, cleanUp, err := server.NewTestStoreFromJson(config.Datadir)
+	store, cleanUp, err := server.NewTestStoreFromJson(context.Background(), config.Datadir)
 	if err != nil {
 		return nil, "", err
 	}
@@ -119,13 +119,13 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve
 	if err != nil {
 		return nil, "", err
 	}
-	ia, _ := integrations.NewIntegratedValidator(eventStore)
-	accountManager, err := server.BuildManager(store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
+	ia, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
+	accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
 	if err != nil {
 		return nil, "", err
 	}
 	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		return nil, "", err
 	}

diff --git a/formatter/hook.go b/formatter/hook.go
@@ -7,6 +7,18 @@ import (
 	"strings"
 
 	"github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/management/server/context"
+)
+
+type ExecutionContext string
+
+const (
+	ExecutionContextKey = "executionContext"
+
+	HTTPSource   ExecutionContext = "HTTP"
+	GRPCSource   ExecutionContext = "GRPC"
+	SystemSource ExecutionContext = "SYSTEM"
 )
 
 // ContextHook is a custom hook for add the source information for the entry
@@ -30,6 +42,27 @@ func (hook ContextHook) Levels() []logrus.Level {
 func (hook ContextHook) Fire(entry *logrus.Entry) error {
 	src := hook.parseSrc(entry.Caller.File)
 	entry.Data["source"] = fmt.Sprintf("%s:%v", src, entry.Caller.Line)
+
+	if entry.Context == nil {
+		return nil
+	}
+
+	source, ok := entry.Context.Value(ExecutionContextKey).(ExecutionContext)
+	if !ok {
+		return nil
+	}
+
+	entry.Data["context"] = source
+
+	switch source {
+	case HTTPSource:
+		addHTTPFields(entry)
+	case GRPCSource:
+		addGRPCFields(entry)
+	case SystemSource:
+		addSystemFields(entry)
+	}
+
 	return nil
 }
 
@@ -59,3 +92,42 @@ func (hook ContextHook) parseSrc(filePath string) string {
 	file := path.Base(filePath)
 	return fmt.Sprintf("%s/%s", pkg, file)
 }
+
+func addHTTPFields(entry *logrus.Entry) {
+	if ctxReqID, ok := entry.Context.Value(context.RequestIDKey).(string); ok {
+		entry.Data[context.RequestIDKey] = ctxReqID
+	}
+	if ctxAccountID, ok := entry.Context.Value(context.AccountIDKey).(string); ok {
+		entry.Data[context.AccountIDKey] = ctxAccountID
+	}
+	if ctxInitiatorID, ok := entry.Context.Value(context.UserIDKey).(string); ok {
+		entry.Data[context.UserIDKey] = ctxInitiatorID
+	}
+}
+
+func addGRPCFields(entry *logrus.Entry) {
+	if ctxReqID, ok := entry.Context.Value(context.RequestIDKey).(string); ok {
+		entry.Data[context.RequestIDKey] = ctxReqID
+	}
+	if ctxAccountID, ok := entry.Context.Value(context.AccountIDKey).(string); ok {
+		entry.Data[context.AccountIDKey] = ctxAccountID
+	}
+	if ctxDeviceID, ok := entry.Context.Value(context.PeerIDKey).(string); ok {
+		entry.Data[context.PeerIDKey] = ctxDeviceID
+	}
+}
+
+func addSystemFields(entry *logrus.Entry) {
+	if ctxReqID, ok := entry.Context.Value(context.RequestIDKey).(string); ok {
+		entry.Data[context.RequestIDKey] = ctxReqID
+	}
+	if ctxInitiatorID, ok := entry.Context.Value(context.UserIDKey).(string); ok {
+		entry.Data[context.UserIDKey] = ctxInitiatorID
+	}
+	if ctxAccountID, ok := entry.Context.Value(context.AccountIDKey).(string); ok {
+		entry.Data[context.AccountIDKey] = ctxAccountID
+	}
+	if ctxDeviceID, ok := entry.Context.Value(context.PeerIDKey).(string); ok {
+		entry.Data[context.PeerIDKey] = ctxDeviceID
+	}
+}
diff --git a/formatter/set.go b/formatter/set.go
@@ -1,6 +1,8 @@
 package formatter
 
-import "github.com/sirupsen/logrus"
+import (
+	"github.com/sirupsen/logrus"
+)
 
 // SetTextFormatter set the text formatter for given logger.
 func SetTextFormatter(logger *logrus.Logger) {
@@ -9,6 +11,13 @@ func SetTextFormatter(logger *logrus.Logger) {
 	logger.AddHook(NewContextHook())
 }
 
+// SetJSONFormatter set the JSON formatter for given logger.
+func SetJSONFormatter(logger *logrus.Logger) {
+	logger.Formatter = &logrus.JSONFormatter{}
+	logger.ReportCaller = true
+	logger.AddHook(NewContextHook())
+}
+
 // SetLogcatFormatter set the logcat formatter for given logger.
 func SetLogcatFormatter(logger *logrus.Logger) {
 	logger.Formatter = NewLogcatFormatter()

diff --git a/go.mod b/go.mod
@@ -44,7 +44,6 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/gopacket v1.1.19
-	github.com/google/martian/v3 v3.0.0
 	github.com/google/nftables v0.0.0-20220808154552-2eca00135732
 	github.com/gopacket/gopacket v1.1.1
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357
@@ -58,7 +57,7 @@ require (
 	github.com/miekg/dns v1.1.43
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/nadoo/ipset v0.5.0
-	github.com/netbirdio/management-integrations/integrations v0.0.0-20240524104853-69c6d89826cd
+	github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e
 	github.com/okta/okta-sdk-golang/v2 v2.18.0
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible

diff --git a/go.sum b/go.sum
@@ -209,8 +209,6 @@ github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/nftables v0.0.0-20220808154552-2eca00135732 h1:csc7dT82JiSLvq4aMyQMIQDL7986NH6Wxf/QrvOj55A=
 github.com/google/nftables v0.0.0-20220808154552-2eca00135732/go.mod h1:b97ulCCFipUC+kSin+zygkvUVpx0vyIAwxXFdY3PlNc=
 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
@@ -335,8 +333,8 @@ github.com/netbirdio/go-netroute v0.0.0-20240611143515-f59b0e1d3944 h1:TDtJKmM6S
 github.com/netbirdio/go-netroute v0.0.0-20240611143515-f59b0e1d3944/go.mod h1:sHA6TRxjQ6RLbnI+3R4DZo2Eseg/iKiPRfNmcuNySVQ=
 github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e h1:PURA50S8u4mF6RrkYYCAvvPCixhqqEiEy3Ej6avh04c=
 github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e/go.mod h1:YMLU7qbKfVjmEv7EoZPIVEI+kNYxWCdPK3VS0BU+U4Q=
-github.com/netbirdio/management-integrations/integrations v0.0.0-20240524104853-69c6d89826cd h1:IzGGIJMpz07aPs3R6/4sxZv63JoCMddftLpVodUK+Ec=
-github.com/netbirdio/management-integrations/integrations v0.0.0-20240524104853-69c6d89826cd/go.mod h1:kxks50DrZnhW+oRTdHOkVOJbcTcyo766am8RBugo+Yc=
+github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e h1:LYxhAmiEzSldLELHSMVoUnRPq3ztTNQImrD27frrGsI=
+github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e/go.mod h1:nykwWZnxb+sJz2Z//CEq45CMRWSHllH8pODKRB8eY7Y=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0 h1:hirFRfx3grVA/9eEyjME5/z3nxdJlN9kfQpvWWPk32g=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/netbirdio/systray v0.0.0-20231030152038-ef1ed2a27949 h1:xbWM9BU6mwZZLHxEjxIX/V8Hv3HurQt4mReIE4mY4DM=
@@ -565,7 +563,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

diff --git a/management/client/client_test.go b/management/client/client_test.go
@@ -62,21 +62,21 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, cleanUp, err := mgmt.NewTestStoreFromJson(config.Datadir)
+	store, cleanUp, err := mgmt.NewTestStoreFromJson(context.Background(), config.Datadir)
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Cleanup(cleanUp)
 
 	peersUpdateManager := mgmt.NewPeersUpdateManager(nil)
 	eventStore := &activity.InMemoryEventStore{}
-	ia, _ := integrations.NewIntegratedValidator(eventStore)
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
+	ia, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
+	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
 	if err != nil {
 		t.Fatal(err)
 	}
 	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}