Merge pull request #850 from forta-network/kisel/forta-1563-chain-id-…

…from-bot-config-for-logs

Сhain id from bot config for logs

clients/docker/client.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -798,13 +799,18 @@ func (d *dockerClient) ListDigestReferences(ctx context.Context) (imgs []string,
 	return
 }
 
+const (
+	defaultAgentLogAvgMaxCharsPerLine = 200
+)
+
 // GetContainerLogs gets the container logs.
-func (d *dockerClient) GetContainerLogs(ctx context.Context, containerID, tail string, truncate int) (string, error) {
+func (d *dockerClient) GetContainerLogs(ctx context.Context, containerID, since string, tail int) (string, error) {
 	r, err := d.cli.ContainerLogs(ctx, containerID, types.ContainerLogsOptions{
 		ShowStdout: true,
 		ShowStderr: true,
 		Timestamps: true,
-		Tail:       tail,
+		Since:      since,
+		Tail:       strconv.Itoa(tail),
 	})
 	if err != nil {
 		return "", err
@@ -813,22 +819,21 @@ func (d *dockerClient) GetContainerLogs(ctx context.Context, containerID, tail s
 	if err != nil {
 		return "", err
 	}
-	if truncate >= 0 && len(b) > truncate {
-		b = b[:truncate]
+
+	// limit the log size
+	if tail >= 0 && len(b) > defaultAgentLogAvgMaxCharsPerLine*tail {
+		b = b[:defaultAgentLogAvgMaxCharsPerLine*tail]
 	}
-	// remove strange 8-byte prefix in each line
-	lines := strings.Split(string(b), "\n")
-	for i, line := range lines {
-		if len(line) == 0 {
-			continue
-		}
-		prefixEnd := strings.Index(line, "2") // timestamp beginning
-		if prefixEnd < 0 || prefixEnd > len(line) {
-			continue
+
+	// remove 8-byte prefix in each line
+	// https://github.com/moby/moby/issues/8223
+	bs := bytes.Split(b, []byte("\n"))
+	for i, v := range bs {
+		if len(v) > 8 {
+			bs[i] = v[8:]
 		}
-		lines[i] = line[prefixEnd:]
 	}
-	return strings.Join(lines, "\n"), nil
+	return string(bytes.Join(bs, []byte("\n"))), nil
 }
 
 func (d *dockerClient) labelFilter() filters.Args {

clients/interfaces.go

@@ -43,7 +43,7 @@ type DockerClient interface {
 	EnsureLocalImage(ctx context.Context, name, ref string) error
 	EnsureLocalImages(ctx context.Context, timeoutPerPull time.Duration, imagePulls []docker.ImagePull) []error
 	ListDigestReferences(ctx context.Context) ([]string, error)
-	GetContainerLogs(ctx context.Context, containerID, tail string, truncate int) (string, error)
+	GetContainerLogs(ctx context.Context, containerID, since string, truncate int) (string, error)
 	GetContainerFromRemoteAddr(ctx context.Context, hostPort string) (*types.Container, error)
 	SetImagePullCooldown(threshold int, cooldownDuration time.Duration)
 	Events(ctx context.Context, since time.Time) (<-chan events.Message, <-chan error)

go.mod

@@ -41,7 +41,7 @@ replace github.com/docker/docker => github.com/moby/moby v20.10.25+incompatible
 require (
 	github.com/docker/docker v1.6.2
 	github.com/docker/go-connections v0.4.0
-	github.com/forta-network/forta-core-go v0.0.0-20240129180226-af53540338f3
+	github.com/forta-network/forta-core-go v0.0.0-20240207125602-ede00282c520
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
 	github.com/prometheus/common v0.39.0

go.sum

@@ -329,10 +329,8 @@ github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe
 github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
 github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
-github.com/forta-network/forta-core-go v0.0.0-20240129095537-dad5459b7283 h1:MmvZ3so59eNLtsJgEnRS1cwy/uqI/PazAS0x9Xkl3+E=
-github.com/forta-network/forta-core-go v0.0.0-20240129095537-dad5459b7283/go.mod h1:iNehCWOypwVeO8b1GKmsrEWReHTvO5qw8SsGvZsBINo=
-github.com/forta-network/forta-core-go v0.0.0-20240129180226-af53540338f3 h1:tfuCghhFdyolM3CiapTxtdLVHcy7ssRUjo5JxwwJnGc=
-github.com/forta-network/forta-core-go v0.0.0-20240129180226-af53540338f3/go.mod h1:iNehCWOypwVeO8b1GKmsrEWReHTvO5qw8SsGvZsBINo=
+github.com/forta-network/forta-core-go v0.0.0-20240207125602-ede00282c520 h1:4RGJtf8/9K8nPCpIxcBWrXt7wE+pcJzwDl/tELuvb3c=
+github.com/forta-network/forta-core-go v0.0.0-20240207125602-ede00282c520/go.mod h1:iNehCWOypwVeO8b1GKmsrEWReHTvO5qw8SsGvZsBINo=
 github.com/forta-network/go-multicall v0.0.0-20230609185354-1436386c6707 h1:f6I7K43i2m6AwHSsDxh0Mf3qFzYt8BKnabSl/zGFmh0=
 github.com/forta-network/go-multicall v0.0.0-20230609185354-1436386c6707/go.mod h1:nqTUF1REklpWLZ/M5HfzqhSHNz4dPVKzJvbLziqTZpw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=

services/components/components.go

@@ -130,6 +130,7 @@ func GetBotLifecycleComponents(
 	botLogger := lifecycle.NewBotLogger(
 		botClient,
 		dockerClient,
+		botLifeConfig.BotRegistry,
 		botLifeConfig.Key,
 		agentlogs.NewClient(botLifeConfig.Config.AgentLogsConfig.URL).SendLogs,
 	)

services/components/lifecycle/bot_logger.go

@@ -3,7 +3,6 @@ package lifecycle
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -12,17 +11,19 @@ import (
 	"github.com/forta-network/forta-node/clients"
 	"github.com/forta-network/forta-node/clients/docker"
 	"github.com/forta-network/forta-node/services/components/containers"
+	"github.com/forta-network/forta-node/services/components/registry"
 	log "github.com/sirupsen/logrus"
 )
 
 // BotLogger manages bots logging.
 type BotLogger interface {
-	SendBotLogs(ctx context.Context) error
+	SendBotLogs(ctx context.Context, snapshotInterval time.Duration) error
 }
 
 type botLogger struct {
 	botClient     containers.BotClient
 	dockerClient  clients.DockerClient
+	agentRegistry registry.BotRegistry
 	key           *keystore.Key
 	prevAgentLogs agentlogs.Agents
 
@@ -34,25 +35,25 @@ var _ BotLogger = &botLogger{}
 func NewBotLogger(
 	botClient containers.BotClient,
 	dockerClient clients.DockerClient,
+	agentRegistry registry.BotRegistry,
 	key *keystore.Key,
 	sendAgentLogs func(agents agentlogs.Agents, authToken string) error,
 ) *botLogger {
 	return &botLogger{
 		botClient:     botClient,
 		dockerClient:  dockerClient,
+		agentRegistry: agentRegistry,
 		key:           key,
 		sendAgentLogs: sendAgentLogs,
 	}
 }
 
 // adjust these better with auto-upgrade later
 const (
-	defaultAgentLogSendInterval       = time.Minute
-	defaultAgentLogTailLines          = 50
-	defaultAgentLogAvgMaxCharsPerLine = 200
+	defaultAgentLogTailLines = 120
 )
 
-func (bl *botLogger) SendBotLogs(ctx context.Context) error {
+func (bl *botLogger) SendBotLogs(ctx context.Context, snapshotInterval time.Duration) error {
 	var (
 		sendLogs agentlogs.Agents
 		keepLogs agentlogs.Agents
@@ -69,18 +70,30 @@ func (bl *botLogger) SendBotLogs(ctx context.Context) error {
 		}
 		logs, err := bl.dockerClient.GetContainerLogs(
 			ctx, container.ID,
-			strconv.Itoa(defaultAgentLogTailLines),
-			defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+			fmt.Sprintf("%ds", int64(snapshotInterval.Seconds())),
+			defaultAgentLogTailLines,
 		)
 		if err != nil {
 			log.WithError(err).Warn("failed to get agent container logs")
 			continue
 		}
 
+		if len(logs) == 0 {
+			log.WithField("agent", container.Labels[docker.LabelFortaBotID]).Debug("no logs found for agent")
+			continue
+		}
+
+		agentID := container.Labels[docker.LabelFortaBotID]
 		agent := &agentlogs.Agent{
-			ID:   container.Labels[docker.LabelFortaBotID],
+			ID:   agentID,
 			Logs: logs,
 		}
+
+		agentConfig, err := bl.agentRegistry.GetConfigByID(agentID)
+		if err == nil {
+			agent.ChainID = int64(agentConfig.ChainID)
+		}
+
 		// don't send if it's the same with previous logs but keep it for next time
 		// so we can check
 		keepLogs = append(keepLogs, agent)
@@ -94,7 +107,7 @@ func (bl *botLogger) SendBotLogs(ctx context.Context) error {
 
 	if len(sendLogs) > 0 {
 		scannerJwt, err := security.CreateScannerJWT(bl.key, map[string]interface{}{
-			"access": "bot_logger",
+			"access": "agent_logs",
 		})
 		if err != nil {
 			return fmt.Errorf("failed to create scanner token: %v", err)

services/components/lifecycle/bot_logger_test.go

@@ -3,16 +3,18 @@ package lifecycle
 import (
 	"context"
 	"errors"
-	"strconv"
 	"testing"
+	"time"
 
 	"github.com/docker/docker/api/types"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/forta-network/forta-core-go/clients/agentlogs"
 	"github.com/forta-network/forta-core-go/security"
 	"github.com/forta-network/forta-node/clients/docker"
 	mock_clients "github.com/forta-network/forta-node/clients/mocks"
+	"github.com/forta-network/forta-node/config"
 	mock_containers "github.com/forta-network/forta-node/services/components/containers/mocks"
+	mock_registry "github.com/forta-network/forta-node/services/components/registry/mocks"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -25,10 +27,10 @@ func TestSendBotLogsSuite(t *testing.T) {
 type BotLoggerSuite struct {
 	r *require.Assertions
 
-	botLogger    *botLogger
-	botClient    *mock_containers.MockBotClient
-	dockerClient *mock_clients.MockDockerClient
-	key          *keystore.Key
+	botClient     *mock_containers.MockBotClient
+	dockerClient  *mock_clients.MockDockerClient
+	agentRegistry *mock_registry.MockBotRegistry
+	key           *keystore.Key
 	suite.Suite
 }
 
@@ -39,6 +41,7 @@ func (s *BotLoggerSuite) SetupTest() {
 
 	botClient := mock_containers.NewMockBotClient(ctrl)
 	dockerClient := mock_clients.NewMockDockerClient(ctrl)
+	agentRegistry := mock_registry.NewMockBotRegistry(ctrl)
 
 	dir := t.TempDir()
 	ks := keystore.NewKeyStore(dir, keystore.StandardScryptN, keystore.StandardScryptP)
@@ -51,17 +54,22 @@ func (s *BotLoggerSuite) SetupTest() {
 
 	s.botClient = botClient
 	s.dockerClient = dockerClient
+	s.agentRegistry = agentRegistry
 	s.key = key
 	s.r = r
 }
 
 func (s *BotLoggerSuite) TestSendBotLogs() {
 	botLogger := NewBotLogger(
-		s.botClient, s.dockerClient, s.key,
+		s.botClient, s.dockerClient, s.agentRegistry, s.key,
 		func(agents agentlogs.Agents, authToken string) error {
 			s.r.Equal(2, len(agents))
+
 			s.r.Equal("bot1ID", agents[0].ID)
+			s.r.EqualValues(1, agents[0].ChainID)
+
 			s.r.Equal("bot2ID", agents[1].ID)
+			s.r.EqualValues(2, agents[1].ChainID)
 			return nil
 		},
 	)
@@ -87,25 +95,28 @@ func (s *BotLoggerSuite) TestSendBotLogs() {
 	}
 	s.dockerClient.EXPECT().GetContainerLogs(
 		ctx, "bot1",
-		strconv.Itoa(defaultAgentLogTailLines),
-		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+		"60s",
+		defaultAgentLogTailLines,
 	).Return("some log", nil).Times(1)
 
 	s.dockerClient.EXPECT().GetContainerLogs(
 		ctx, "bot2",
-		strconv.Itoa(defaultAgentLogTailLines),
-		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+		"60s",
+		defaultAgentLogTailLines,
 	).Return("some log", nil).Times(1)
 
+	s.agentRegistry.EXPECT().GetConfigByID("bot1ID").Return(&config.AgentConfig{ChainID: 1}, nil).Times(1)
+	s.agentRegistry.EXPECT().GetConfigByID("bot2ID").Return(&config.AgentConfig{ChainID: 2}, nil).Times(1)
+
 	s.botClient.EXPECT().LoadBotContainers(ctx).Return(mockContainers, nil)
-	s.r.NoError(botLogger.SendBotLogs(ctx))
+	s.r.NoError(botLogger.SendBotLogs(ctx, time.Minute))
 }
 
 // should fail if there is an error loading
 // bot containers
 func (s *BotLoggerSuite) TestLoadBotContainersError() {
 	botLogger := NewBotLogger(
-		s.botClient, s.dockerClient, s.key,
+		s.botClient, s.dockerClient, s.agentRegistry, s.key,
 		func(agents agentlogs.Agents, authToken string) error {
 			return nil
 		},
@@ -115,14 +126,14 @@ func (s *BotLoggerSuite) TestLoadBotContainersError() {
 	mockContainers := []types.Container{}
 
 	s.botClient.EXPECT().LoadBotContainers(ctx).Return(mockContainers, errors.New("test"))
-	s.r.EqualError(botLogger.SendBotLogs(ctx), "failed to load the bot containers: test")
+	s.r.EqualError(botLogger.SendBotLogs(ctx, time.Minute), "failed to load the bot containers: test")
 }
 
 // Should not send agent logs if fails
 // to get container logs but continue processing
 func (s *BotLoggerSuite) TestGetContainerLogsError() {
 	botLogger := NewBotLogger(
-		s.botClient, s.dockerClient, s.key,
+		s.botClient, s.dockerClient, s.agentRegistry, s.key,
 		func(agents agentlogs.Agents, authToken string) error {
 			s.r.Equal(1, len(agents))
 			s.r.Equal("bot2ID", agents[0].ID)
@@ -154,23 +165,25 @@ func (s *BotLoggerSuite) TestGetContainerLogsError() {
 
 	s.dockerClient.EXPECT().GetContainerLogs(
 		ctx, "bot1",
-		strconv.Itoa(defaultAgentLogTailLines),
-		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+		"60s",
+		defaultAgentLogTailLines,
 	).Return("", errors.New("test")).Times(1)
 
 	s.dockerClient.EXPECT().GetContainerLogs(
 		ctx, "bot2",
-		strconv.Itoa(defaultAgentLogTailLines),
-		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+		"60s",
+		defaultAgentLogTailLines,
 	).Return("some log", nil).Times(1)
 
-	s.r.NoError(botLogger.SendBotLogs(ctx))
+	s.agentRegistry.EXPECT().GetConfigByID("bot2ID").Return(&config.AgentConfig{ChainID: 2}, nil).Times(1)
+
+	s.r.NoError(botLogger.SendBotLogs(ctx, time.Minute))
 }
 
 // Fails sending agent logs
 func (s *BotLoggerSuite) TestFailsToSendLogs() {
 	botLogger := NewBotLogger(
-		s.botClient, s.dockerClient, s.key,
+		s.botClient, s.dockerClient, s.agentRegistry, s.key,
 		func(agents agentlogs.Agents, authToken string) error {
 			return errors.New("test")
 		},
@@ -192,9 +205,11 @@ func (s *BotLoggerSuite) TestFailsToSendLogs() {
 
 	s.dockerClient.EXPECT().GetContainerLogs(
 		ctx, "bot1",
-		strconv.Itoa(defaultAgentLogTailLines),
-		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+		"60s",
+		defaultAgentLogTailLines,
 	).Return("some log", nil).Times(1)
 
-	s.r.EqualError(botLogger.SendBotLogs(ctx), "failed to send agent logs: test")
+	s.agentRegistry.EXPECT().GetConfigByID("bot1ID").Return(&config.AgentConfig{ChainID: 1}, nil).Times(1)
+
+	s.r.EqualError(botLogger.SendBotLogs(ctx, time.Minute), "failed to send agent logs: test")
 }