feature(checker): Add additional parameter for metric event trigger c…

…ache expiration (#993)
moira-alert · Feb 15, 2024 · 7cdeac6 · 7cdeac6
1 parent 1dd7c8b
commit 7cdeac6
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 31 deletions.
diff --git a/checker/config.go b/checker/config.go
@@ -8,17 +8,17 @@ import (
 
 // Config represent checker config
 type Config struct {
-	Enabled                     bool
-	NoDataCheckInterval         time.Duration
-	LazyTriggersCheckInterval   time.Duration
-	SourceCheckConfigs          map[moira.ClusterKey]SourceCheckConfig
-	StopCheckingIntervalSeconds int64
-	LogFile                     string
-	LogLevel                    string
-	LogTriggersToLevel          map[string]string
-	MetricEventPopBatchSize     int64
-	MetricEventPopDelay         time.Duration
-	CriticalTimeOfCheck         time.Duration
+	Enabled                         bool
+	LazyTriggersCheckInterval       time.Duration
+	SourceCheckConfigs              map[moira.ClusterKey]SourceCheckConfig
+	StopCheckingIntervalSeconds     int64
+	LogFile                         string
+	LogLevel                        string
+	LogTriggersToLevel              map[string]string
+	MetricEventPopBatchSize         int64
+	MetricEventPopDelay             time.Duration
+	CriticalTimeOfCheck             time.Duration
+	MetricEventTriggerCheckInterval time.Duration
 }
 
 // SourceCheckConfig represents check parameters for a single metric source

diff --git a/cmd/checker/config.go b/cmd/checker/config.go
@@ -33,8 +33,8 @@ type localCheckConfig struct {
 }
 
 type checkerConfig struct {
-	// Period for every trigger to perform forced check on
-	NoDataCheckInterval string `yaml:"nodata_check_interval"`
+	// Delay between trigger checks, scheduled due to metric events
+	MetricEventTriggerCheckInterval string `yaml:"metric_event_trigger_check_interval"`
 	// Period for every trigger to cancel forced check (earlier than 'NoDataCheckInterval') if no metrics were received
 	StopCheckingInterval string `yaml:"stop_checking_interval"`
 	// Max period to perform lazy triggers re-check. Note: lazy triggers are triggers which has no subscription for it. Moira will check its state less frequently.
@@ -114,14 +114,14 @@ func (config *config) getSettings(logger moira.Logger) *checker.Config {
 	}
 
 	return &checker.Config{
-		SourceCheckConfigs:          sourceCheckConfigs,
-		LazyTriggersCheckInterval:   to.Duration(config.Checker.LazyTriggersCheckInterval),
-		NoDataCheckInterval:         to.Duration(config.Checker.NoDataCheckInterval),
-		StopCheckingIntervalSeconds: int64(to.Duration(config.Checker.StopCheckingInterval).Seconds()),
-		LogTriggersToLevel:          logTriggersToLevel,
-		MetricEventPopBatchSize:     int64(config.Checker.MetricEventPopBatchSize),
-		MetricEventPopDelay:         to.Duration(config.Checker.MetricEventPopDelay),
-		CriticalTimeOfCheck:         to.Duration(config.Checker.CriticalTimeOfCheck),
+		SourceCheckConfigs:              sourceCheckConfigs,
+		LazyTriggersCheckInterval:       to.Duration(config.Checker.LazyTriggersCheckInterval),
+		StopCheckingIntervalSeconds:     int64(to.Duration(config.Checker.StopCheckingInterval).Seconds()),
+		LogTriggersToLevel:              logTriggersToLevel,
+		MetricEventPopBatchSize:         int64(config.Checker.MetricEventPopBatchSize),
+		MetricEventPopDelay:             to.Duration(config.Checker.MetricEventPopDelay),
+		CriticalTimeOfCheck:             to.Duration(config.Checker.CriticalTimeOfCheck),
+		MetricEventTriggerCheckInterval: to.Duration(config.Checker.MetricEventTriggerCheckInterval),
 	}
 }
 
@@ -138,10 +138,10 @@ func getDefault() config {
 			LogPrettyFormat: false,
 		},
 		Checker: checkerConfig{
-			NoDataCheckInterval:       "60s",
-			LazyTriggersCheckInterval: "10m",
-			StopCheckingInterval:      "30s",
-			CriticalTimeOfCheck:       "1h",
+			LazyTriggersCheckInterval:       "10m",
+			StopCheckingInterval:            "30s",
+			CriticalTimeOfCheck:             "1h",
+			MetricEventTriggerCheckInterval: "30s",
 		},
 		Telemetry: cmd.TelemetryConfig{
 			Listen: ":8092",

diff --git a/cmd/checker/main.go b/cmd/checker/main.go
@@ -96,18 +96,16 @@ func main() {
 		checkSingleTrigger(database, checkerMetrics, checkerSettings, metricSourceProvider)
 	}
 
-	cacheDefaultExpiration := checkerSettings.
-		SourceCheckConfigs[moira.DefaultLocalCluster].
-		CheckInterval
+	cacheExpiration := checkerSettings.MetricEventTriggerCheckInterval
 	checkerWorkerManager := &worker.WorkerManager{
 		Logger:            logger,
 		Database:          database,
 		Config:            checkerSettings,
 		SourceProvider:    metricSourceProvider,
 		Metrics:           checkerMetrics,
-		TriggerCache:      cache.New(cacheDefaultExpiration, time.Minute*60), //nolint
-		LazyTriggersCache: cache.New(time.Minute*10, time.Minute*60),         //nolint
-		PatternCache:      cache.New(cacheDefaultExpiration, time.Minute*60), //nolint
+		TriggerCache:      cache.New(cacheExpiration, time.Minute*60), //nolint
+		LazyTriggersCache: cache.New(time.Minute*10, time.Minute*60),  //nolint
+		PatternCache:      cache.New(cacheExpiration, time.Minute*60), //nolint
 	}
 	err = checkerWorkerManager.StartWorkers()
 	if err != nil {