merge with master

api/controller/triggers.go

@@ -10,7 +10,7 @@ import (
 	"github.com/moira-alert/moira"
 	"github.com/moira-alert/moira/api"
 	"github.com/moira-alert/moira/api/dto"
-	"github.com/moira-alert/moira/database"
+	db "github.com/moira-alert/moira/database"
 )
 
 const pageSizeUnlimited int64 = -1
@@ -50,19 +50,16 @@ func GetAllTriggers(database moira.Database) (*dto.TriggersList, *api.ErrorRespo
 	if err != nil {
 		return nil, api.ErrorInternalServer(err)
 	}
-	triggerChecks, err := database.GetTriggerChecks(triggerIDs)
+
+	triggerChecks, err := getTriggerChecks(database, triggerIDs)
 	if err != nil {
 		return nil, api.ErrorInternalServer(err)
 	}
-	triggersList := dto.TriggersList{
-		List: make([]moira.TriggerCheck, 0),
+	triggersList := &dto.TriggersList{
+		List: triggerChecks,
 	}
-	for _, triggerCheck := range triggerChecks {
-		if triggerCheck != nil {
-			triggersList.List = append(triggersList.List, *triggerCheck)
-		}
-	}
-	return &triggersList, nil
+
+	return triggersList, nil
 }
 
 // SearchTriggers gets trigger page and filter trigger by tags and search request terms
@@ -100,7 +97,10 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, options mo
 			return nil, api.ErrorInternalServer(err)
 		}
 		options.PagerID = uuid4.String()
-		database.SaveTriggersSearchResults(options.PagerID, searchResults) //nolint
+		err = database.SaveTriggersSearchResults(options.PagerID, searchResults)
+		if err != nil {
+			return nil, api.ErrorInternalServer(err)
+		}
 	}
 
 	if options.CreatePager {
@@ -112,7 +112,7 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, options mo
 		searchResults = searchResults[from:to]
 	}
 
-	var triggerIDs []string //nolint
+	triggerIDs := make([]string, 0, len(searchResults))
 	for _, searchResult := range searchResults {
 		triggerIDs = append(triggerIDs, searchResult.ObjectID)
 	}
@@ -150,24 +150,57 @@ func SearchTriggers(database moira.Database, searcher moira.Searcher, options mo
 	return &triggersList, nil
 }
 
-func DeleteTriggersPager(dataBase moira.Database, pagerID string) (dto.TriggersSearchResultDeleteResponse, *api.ErrorResponse) {
-	exists, err := dataBase.IsTriggersSearchResultsExist(pagerID)
+func DeleteTriggersPager(database moira.Database, pagerID string) (dto.TriggersSearchResultDeleteResponse, *api.ErrorResponse) {
+	exists, err := database.IsTriggersSearchResultsExist(pagerID)
 	if err != nil {
 		return dto.TriggersSearchResultDeleteResponse{}, api.ErrorInternalServer(err)
 	}
 	if !exists {
 		return dto.TriggersSearchResultDeleteResponse{}, api.ErrorNotFound(fmt.Sprintf("pager with id %s not found", pagerID))
 	}
-	err = dataBase.DeleteTriggersSearchResults(pagerID)
+	err = database.DeleteTriggersSearchResults(pagerID)
 	if err != nil {
 		return dto.TriggersSearchResultDeleteResponse{}, api.ErrorInternalServer(err)
 	}
 	return dto.TriggersSearchResultDeleteResponse{PagerID: pagerID}, nil
 }
 
-func triggerExists(dataBase moira.Database, triggerID string) (bool, error) {
-	_, err := dataBase.GetTrigger(triggerID)
-	if err == database.ErrNil {
+// GetUnusedTriggerIDs returns unused triggers ids.
+func GetUnusedTriggerIDs(database moira.Database) (*dto.TriggersList, *api.ErrorResponse) {
+	triggerIDs, err := database.GetUnusedTriggerIDs()
+	if err != nil {
+		return nil, api.ErrorInternalServer(err)
+	}
+
+	triggerChecks, err := getTriggerChecks(database, triggerIDs)
+	if err != nil {
+		return nil, api.ErrorInternalServer(err)
+	}
+	triggersList := &dto.TriggersList{
+		List: triggerChecks,
+	}
+
+	return triggersList, nil
+}
+
+func getTriggerChecks(database moira.Database, triggerIDs []string) ([]moira.TriggerCheck, error) {
+	triggerChecks, err := database.GetTriggerChecks(triggerIDs)
+	if err != nil {
+		return nil, err
+	}
+	list := make([]moira.TriggerCheck, 0, len(triggerChecks))
+	for _, triggerCheck := range triggerChecks {
+		if triggerCheck != nil {
+			list = append(list, *triggerCheck)
+		}
+	}
+
+	return list, nil
+}
+
+func triggerExists(database moira.Database, triggerID string) (bool, error) {
+	_, err := database.GetTrigger(triggerID)
+	if err == db.ErrNil {
 		return false, nil
 	}
 	if err != nil {

api/controller/triggers_test.go

@@ -927,3 +927,45 @@ func TestDeleteTriggersPager(t *testing.T) {
 		})
 	})
 }
+
+func TestGetUnusedTriggerIDs(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl)
+
+	Convey("Has triggers", t, func() {
+		triggerIDs := []string{uuid.Must(uuid.NewV4()).String(), uuid.Must(uuid.NewV4()).String()}
+		triggers := []*moira.TriggerCheck{{Trigger: moira.Trigger{ID: triggerIDs[0]}}, {Trigger: moira.Trigger{ID: triggerIDs[1]}}}
+		triggersList := []moira.TriggerCheck{{Trigger: moira.Trigger{ID: triggerIDs[0]}}, {Trigger: moira.Trigger{ID: triggerIDs[1]}}}
+		mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(triggerIDs, nil)
+		mockDatabase.EXPECT().GetTriggerChecks(triggerIDs).Return(triggers, nil)
+		list, err := GetUnusedTriggerIDs(mockDatabase)
+		So(err, ShouldBeNil)
+		So(list, ShouldResemble, &dto.TriggersList{List: triggersList})
+	})
+
+	Convey("No triggers", t, func() {
+		mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(make([]string, 0), nil)
+		mockDatabase.EXPECT().GetTriggerChecks(make([]string, 0)).Return(make([]*moira.TriggerCheck, 0), nil)
+		list, err := GetUnusedTriggerIDs(mockDatabase)
+		So(err, ShouldBeNil)
+		So(list, ShouldResemble, &dto.TriggersList{List: make([]moira.TriggerCheck, 0)})
+	})
+
+	Convey("GetUnusedTriggerIDs error", t, func() {
+		expected := fmt.Errorf("getTriggerIDs error")
+		mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(nil, expected)
+		list, err := GetUnusedTriggerIDs(mockDatabase)
+		So(err, ShouldResemble, api.ErrorInternalServer(expected))
+		So(list, ShouldBeNil)
+	})
+
+	Convey("GetTriggerChecks error", t, func() {
+		expected := fmt.Errorf("getTriggerChecks error")
+		mockDatabase.EXPECT().GetUnusedTriggerIDs().Return(make([]string, 0), nil)
+		mockDatabase.EXPECT().GetTriggerChecks(make([]string, 0)).Return(nil, expected)
+		list, err := GetUnusedTriggerIDs(mockDatabase)
+		So(err, ShouldResemble, api.ErrorInternalServer(expected))
+		So(list, ShouldBeNil)
+	})
+}

api/handler/triggers.go

@@ -28,6 +28,7 @@ func triggers(metricSourceProvider *metricSource.SourceProvider, searcher moira.
 		router.Use(middleware.MetricSourceProvider(metricSourceProvider))
 		router.Use(middleware.SearchIndexContext(searcher))
 		router.Get("/", getAllTriggers)
+		router.Get("/unused", getUnusedTriggers)
 		router.Put("/", createTrigger)
 		router.Put("/check", triggerCheck)
 		router.Route("/{triggerId}", trigger)
@@ -61,6 +62,29 @@ func getAllTriggers(writer http.ResponseWriter, request *http.Request) {
 	}
 }
 
+// nolint: gofmt,goimports
+//
+//	@summary	Get unused triggers
+//	@id			get-unused-triggers
+//	@tags		trigger
+//	@produce	json
+//	@success	200	{object}	dto.TriggersList				"Fetched unused triggers"
+//	@failure	422	{object}	api.ErrorRenderExample			"Render error"
+//	@failure	500	{object}	api.ErrorInternalServerExample	"Internal server error"
+//	@router		/trigger [get]
+func getUnusedTriggers(writer http.ResponseWriter, request *http.Request) {
+	triggersList, errorResponse := controller.GetUnusedTriggerIDs(database)
+	if errorResponse != nil {
+		render.Render(writer, request, errorResponse) //nolint
+		return
+	}
+
+	if err := render.Render(writer, request, triggersList); err != nil {
+		render.Render(writer, request, api.ErrorRender(err)) //nolint
+		return
+	}
+}
+
 // nolint: gofmt,goimports
 // createTrigger handler creates moira.Trigger
 //

cmd/api/config.go

@@ -146,5 +146,11 @@ func getDefault() config {
 			Timeout:    "60s",
 			MetricsTTL: "7d",
 		},
+		Prometheus: cmd.PrometheusConfig{
+			Timeout:      "60s",
+			MetricsTTL:   "7d",
+			Retries:      1,
+			RetryTimeout: "10s",
+		},
 	}
 }

cmd/api/config_test.go

@@ -92,6 +92,12 @@ func Test_webConfig_getDefault(t *testing.T) {
 				Timeout:    "60s",
 				MetricsTTL: "7d",
 			},
+			Prometheus: cmd.PrometheusConfig{
+				Timeout:      "60s",
+				MetricsTTL:   "7d",
+				Retries:      1,
+				RetryTimeout: "10s",
+			},
 			NotificationHistory: cmd.NotificationHistoryConfig{
 				NotificationHistoryTTL:        "48h",
 				NotificationHistoryQueryLimit: -1,

cmd/checker/config.go

@@ -109,6 +109,8 @@ func getDefault() config {
 			CheckInterval: "60s",
 			Timeout:       "60s",
 			MetricsTTL:    "7d",
+			Retries:       1,
+			RetryTimeout:  "10s",
 		},
 	}
 }

cmd/config.go

@@ -157,25 +157,39 @@ func (config *RemoteConfig) GetRemoteSourceSettings() *remoteSource.Config {
 }
 
 type PrometheusConfig struct {
-	URL           string `yaml:"url"`
+	// Url of prometheus API
+	URL string `yaml:"url"`
+	// Min period to perform triggers re-check
 	CheckInterval string `yaml:"check_interval"`
-	MetricsTTL    string `yaml:"metrics_ttl"`
-	Timeout       string `yaml:"timeout"`
-	User          string `yaml:"user"`
-	Password      string `yaml:"password"`
-	Enabled       bool   `yaml:"enabled"`
+	// Moira won't fetch metrics older than this value from prometheus remote storage.
+	// Large values will lead to OOM problems in checker.
+	MetricsTTL string `yaml:"metrics_ttl"`
+	// Timeout for prometheus api requests
+	Timeout string `yaml:"timeout"`
+	// Number of retries for prometheus api requests
+	Retries int `yaml:"retries"`
+	// Timeout between retries for prometheus api requests
+	RetryTimeout string `yaml:"retry_timeout"`
+	// Username for basic auth
+	User string `yaml:"user"`
+	// Password for basic auth
+	Password string `yaml:"password"`
+	// If true, prometheus remote worker will be enabled.
+	Enabled bool `yaml:"enabled"`
 }
 
 // GetRemoteSourceSettings returns remote config parsed from moira config files
 func (config *PrometheusConfig) GetPrometheusSourceSettings() *prometheus.Config {
 	return &prometheus.Config{
-		Enabled:       config.Enabled,
-		URL:           config.URL,
-		CheckInterval: to.Duration(config.CheckInterval),
-		MetricsTTL:    to.Duration(config.MetricsTTL),
-		User:          config.User,
-		Password:      config.Password,
-		Timeout:       to.Duration(config.Timeout),
+		Enabled:        config.Enabled,
+		URL:            config.URL,
+		CheckInterval:  to.Duration(config.CheckInterval),
+		MetricsTTL:     to.Duration(config.MetricsTTL),
+		User:           config.User,
+		Password:       config.Password,
+		RequestTimeout: to.Duration(config.Timeout),
+		Retries:        config.Retries,
+		RetryTimeout:   to.Duration(config.RetryTimeout),
 	}
 }
 

cmd/notifier/config.go

@@ -73,6 +73,8 @@ type selfStateConfig struct {
 	Contacts []map[string]string `yaml:"contacts"`
 	// Self state monitor alerting interval
 	NoticeInterval string `yaml:"notice_interval"`
+	// Self state monitor check interval
+	CheckInterval string `yaml:"check_interval"`
 }
 
 func getDefault() config {
@@ -121,6 +123,12 @@ func getDefault() config {
 			Timeout:    "60s",
 			MetricsTTL: "24h",
 		},
+		Prometheus: cmd.PrometheusConfig{
+			Timeout:      "60s",
+			MetricsTTL:   "7d",
+			Retries:      1,
+			RetryTimeout: "10s",
+		},
 		ImageStores: cmd.ImageStoreConfig{},
 	}
 }
@@ -206,12 +214,19 @@ func checkDateTimeFormat(format string) error {
 }
 
 func (config *selfStateConfig) getSettings() selfstate.Config {
+	// 10 sec is default check value
+	checkInterval := 10 * time.Second
+	if config.CheckInterval != "" {
+		checkInterval = to.Duration(config.CheckInterval)
+	}
+
 	return selfstate.Config{
 		Enabled:                        config.Enabled,
 		RedisDisconnectDelaySeconds:    int64(to.Duration(config.RedisDisconnectDelay).Seconds()),
 		LastMetricReceivedDelaySeconds: int64(to.Duration(config.LastMetricReceivedDelay).Seconds()),
 		LastCheckDelaySeconds:          int64(to.Duration(config.LastCheckDelay).Seconds()),
 		LastRemoteCheckDelaySeconds:    int64(to.Duration(config.LastRemoteCheckDelay).Seconds()),
+		CheckInterval:                  checkInterval,
 		Contacts:                       config.Contacts,
 		NoticeIntervalSeconds:          int64(to.Duration(config.NoticeInterval).Seconds()),
 	}

cmd/notifier/main.go

@@ -69,7 +69,7 @@ func main() {
 	}
 	defer logger.Info().
 		String("moira_version", MoiraVersion).
-		Msg("Moira Notifier stopped. Version")
+		Msg("Moira Notifier stopped.")
 
 	telemetry, err := cmd.ConfigureTelemetry(logger, config.Telemetry, serviceName)
 	if err != nil {
@@ -79,7 +79,6 @@ func main() {
 	}
 	defer telemetry.Stop()
 
-	notifierMetrics := metrics.ConfigureNotifierMetrics(telemetry.Metrics, serviceName)
 	databaseSettings := config.Redis.GetSettings()
 	notificationHistorySettings := config.NotificationHistory.GetSettings()
 	database := redis.NewDatabase(logger, databaseSettings, notificationHistorySettings, redis.Notifier)
@@ -103,7 +102,15 @@ func main() {
 
 	notifierConfig := config.Notifier.getSettings(logger)
 
-	sender := notifier.NewNotifier(database, logger, notifierConfig, notifierMetrics, metricSourceProvider, imageStoreMap)
+	notifierMetrics := metrics.ConfigureNotifierMetrics(telemetry.Metrics, serviceName)
+	sender := notifier.NewNotifier(
+		database,
+		logger,
+		notifierConfig,
+		notifierMetrics,
+		metricSourceProvider,
+		imageStoreMap,
+	)
 
 	// Register moira senders
 	if err := sender.RegisterSenders(database); err != nil {
@@ -114,7 +121,7 @@ func main() {
 
 	// Start moira self state checker
 	if config.Notifier.SelfState.getSettings().Enabled {
-		selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings())
+		selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings(), metrics.ConfigureHeartBeatMetrics(telemetry.Metrics))
 		if err := selfState.Start(); err != nil {
 			logger.Fatal().
 				Error(err).

generate_mocks.sh

@@ -24,5 +24,6 @@ mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost
 
 mockgen -destination=mock/moira-alert/metrics/registry.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Registry
 mockgen -destination=mock/moira-alert/metrics/meter.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Meter
+mockgen -destination=mock/moira-alert/prometheus_api.go -package=mock_moira_alert github.com/moira-alert/moira/metric_source/prometheus PrometheusApi
 
 git add mock/*