From e28f391474b6ed09b951341ef569545b5deb9775 Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sun, 6 Nov 2022 10:36:24 +0000 Subject: [PATCH 01/15] Test receivers command setup Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/root.go | 1 + cli/test_receivers.go | 60 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 cli/test_receivers.go diff --git a/cli/root.go b/cli/root.go index 09043159db..5b0956c845 100644 --- a/cli/root.go +++ b/cli/root.go @@ -153,6 +153,7 @@ func Execute() { configureClusterCmd(app) configureConfigCmd(app) configureTemplateCmd(app) + configureTestReceiversCmd(app) err = resolver.Bind(app, os.Args[1:]) if err != nil { diff --git a/cli/test_receivers.go b/cli/test_receivers.go new file mode 100644 index 0000000000..9e26b243ea --- /dev/null +++ b/cli/test_receivers.go @@ -0,0 +1,60 @@ +// Copyright 2022 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "fmt" + + "github.com/prometheus/alertmanager/config" + "gopkg.in/alecthomas/kingpin.v2" +) + +type testReceiversCmd struct { + configFile string +} + +const testReceiversHelp = `Test alertmanager receivers + +Will test receivers for alertmanager config file. +` + +func configureTestReceiversCmd(app *kingpin.Application) { + var ( + t = &testReceiversCmd{} + testCmd = app.Command("test-receivers", testReceiversHelp) + ) + testCmd.Arg("config-file", "Config file to be validated").ExistingFileVar(&t.configFile) + testCmd.Action(t.testReceivers) +} + +func (t *testReceiversCmd) testReceivers(ctx *kingpin.ParseContext) error { + if len(t.configFile) == 0 { + kingpin.Fatalf("No config file was specified") + } + + fmt.Printf("Checking '%s'\n", t.configFile) + cfg, err := config.LoadFile(t.configFile) + if err != nil { + kingpin.Fatalf("Invalid config file") + } + + //successful := 0 + if cfg != nil { + for _, receiver := range cfg.Receivers { + fmt.Printf("Testing receiver '%s'\n", receiver.Name) + } + } + + return nil +} From 03f8e5a401c46a83438dfc9ae8772bdfb459bbb7 Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sun, 6 Nov 2022 15:31:39 +0000 Subject: [PATCH 02/15] First iteration to test receivers Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/test_receivers.go | 189 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 182 insertions(+), 7 deletions(-) diff --git a/cli/test_receivers.go b/cli/test_receivers.go index 9e26b243ea..39ac6f618a 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -14,9 +14,30 @@ package cli import ( + "context" "fmt" + "net/url" + "time" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/notify/discord" + "github.com/prometheus/alertmanager/notify/email" + "github.com/prometheus/alertmanager/notify/opsgenie" + "github.com/prometheus/alertmanager/notify/pagerduty" + "github.com/prometheus/alertmanager/notify/pushover" + "github.com/prometheus/alertmanager/notify/slack" + "github.com/prometheus/alertmanager/notify/sns" + "github.com/prometheus/alertmanager/notify/telegram" + "github.com/prometheus/alertmanager/notify/victorops" + "github.com/prometheus/alertmanager/notify/webhook" + "github.com/prometheus/alertmanager/notify/wechat" + "github.com/prometheus/alertmanager/template" + "github.com/prometheus/alertmanager/types" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promlog" "gopkg.in/alecthomas/kingpin.v2" ) @@ -34,11 +55,11 @@ func configureTestReceiversCmd(app *kingpin.Application) { t = &testReceiversCmd{} testCmd = app.Command("test-receivers", testReceiversHelp) ) - testCmd.Arg("config-file", "Config file to be validated").ExistingFileVar(&t.configFile) - testCmd.Action(t.testReceivers) + testCmd.Arg("config.file", "Config file to be tested.").ExistingFileVar(&t.configFile) + testCmd.Action(execWithTimeout(t.testReceivers)) } -func (t *testReceiversCmd) testReceivers(ctx *kingpin.ParseContext) error { +func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseContext) error { if len(t.configFile) == 0 { kingpin.Fatalf("No config file was specified") } @@ -46,15 +67,169 @@ func (t *testReceiversCmd) testReceivers(ctx *kingpin.ParseContext) error { fmt.Printf("Checking '%s'\n", t.configFile) cfg, err := config.LoadFile(t.configFile) if err != nil { - kingpin.Fatalf("Invalid config file") + return errors.Wrap(err, "invalid config file") } - //successful := 0 if cfg != nil { - for _, receiver := range cfg.Receivers { - fmt.Printf("Testing receiver '%s'\n", receiver.Name) + tmpl, err := template.FromGlobs(cfg.Templates...) + if err != nil { + return errors.Wrap(err, "failed to parse templates") } + if alertmanagerURL != nil { + tmpl.ExternalURL = alertmanagerURL + } else { + u, err := url.Parse("http://localhost:1234") + if err != nil { + return errors.Wrap(err, "failed to parse fake url") + } + tmpl.ExternalURL = u + } + + return TestReceivers(ctx, cfg.Receivers, tmpl) } return nil } + +func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *template.Template) error { + // now represents the start time of the test + now := time.Now() + testAlert := newTestAlert(now, now) + + // we must set a group key that is unique per test as some receivers use this key to deduplicate alerts + ctx = notify.WithGroupKey(ctx, testAlert.Labels.String()+now.String()) + ctx = notify.WithGroupLabels(ctx, testAlert.Labels) + + logger := promlog.New(&promlog.Config{}) + + // job contains all metadata required to test a receiver + type job struct { + Receiver *config.Receiver + Integration *notify.Integration + } + + // result contains the receiver that was tested and an error that is non-nil if the test failed + type result struct { + Receiver *config.Receiver + Integration *notify.Integration + Error error + } + + // invalid keeps track of all invalid receiver configurations + var invalid []result + // jobs keeps track of all receivers that need to be sent test notifications + var jobs []job + + for _, receiver := range receivers { + integrations, err := buildReceiverIntegrations(receiver, tmpl, logger) + for _, integration := range integrations { + if err != nil { + invalid = append(invalid, result{ + Receiver: receiver, + Integration: &integration, + Error: err, + }) + } else { + jobs = append(jobs, job{ + Receiver: receiver, + Integration: &integration, + }) + } + } + } + + fmt.Printf("Performing %v jobs!\n", len(jobs)) + + for _, job := range jobs { + v := result{ + Receiver: job.Receiver, + Integration: job.Integration, + } + if _, err := job.Integration.Notify(notify.WithReceiverName(ctx, job.Receiver.Name), &testAlert); err != nil { + v.Error = err + } + } + + fmt.Printf("Done!\n") + return nil +} + +func newTestAlert(startsAt, updatedAt time.Time) types.Alert { + var ( + defaultAnnotations = model.LabelSet{ + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]", + } + defaultLabels = model.LabelSet{ + "alertname": "TestAlert", + "instance": "Alertmanager", + } + ) + + alert := types.Alert{ + Alert: model.Alert{ + Labels: defaultLabels, + Annotations: defaultAnnotations, + StartsAt: startsAt, + }, + UpdatedAt: updatedAt, + } + + return alert +} + +// buildReceiverIntegrations builds a list of integration notifiers off of a +// receiver config. +func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) ([]notify.Integration, error) { + var ( + errs types.MultiError + integrations []notify.Integration + add = func(name string, i int, rs notify.ResolvedSender, f func(l log.Logger) (notify.Notifier, error)) { + n, err := f(log.With(logger, "integration", name)) + if err != nil { + errs.Add(err) + return + } + integrations = append(integrations, notify.NewIntegration(n, rs, name, i)) + } + ) + + for i, c := range nc.WebhookConfigs { + add("webhook", i, c, func(l log.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) }) + } + for i, c := range nc.EmailConfigs { + add("email", i, c, func(l log.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil }) + } + for i, c := range nc.PagerdutyConfigs { + add("pagerduty", i, c, func(l log.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) }) + } + for i, c := range nc.OpsGenieConfigs { + add("opsgenie", i, c, func(l log.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) }) + } + for i, c := range nc.WechatConfigs { + add("wechat", i, c, func(l log.Logger) (notify.Notifier, error) { return wechat.New(c, tmpl, l) }) + } + for i, c := range nc.SlackConfigs { + add("slack", i, c, func(l log.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) }) + } + for i, c := range nc.VictorOpsConfigs { + add("victorops", i, c, func(l log.Logger) (notify.Notifier, error) { return victorops.New(c, tmpl, l) }) + } + for i, c := range nc.PushoverConfigs { + add("pushover", i, c, func(l log.Logger) (notify.Notifier, error) { return pushover.New(c, tmpl, l) }) + } + for i, c := range nc.SNSConfigs { + add("sns", i, c, func(l log.Logger) (notify.Notifier, error) { return sns.New(c, tmpl, l) }) + } + for i, c := range nc.TelegramConfigs { + add("telegram", i, c, func(l log.Logger) (notify.Notifier, error) { return telegram.New(c, tmpl, l) }) + } + for i, c := range nc.DiscordConfigs { + add("discord", i, c, func(l log.Logger) (notify.Notifier, error) { return discord.New(c, tmpl, l) }) + } + + if errs.Len() > 0 { + return nil, &errs + } + return integrations, nil +} From 31c7e7188e1d18ccb9c4899a75ab5827479d526c Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sun, 6 Nov 2022 20:07:28 +0000 Subject: [PATCH 03/15] Improvements to test receivers Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/test_receivers.go | 181 +++++++++++++++++++++++++++++++++++------- go.mod | 4 +- go.sum | 2 + 3 files changed, 159 insertions(+), 28 deletions(-) diff --git a/cli/test_receivers.go b/cli/test_receivers.go index 39ac6f618a..340498bccf 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "net/url" + "sort" "time" "github.com/go-kit/log" @@ -38,6 +39,7 @@ import ( "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "github.com/prometheus/common/promlog" + "golang.org/x/sync/errgroup" "gopkg.in/alecthomas/kingpin.v2" ) @@ -85,13 +87,67 @@ func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseCo tmpl.ExternalURL = u } - return TestReceivers(ctx, cfg.Receivers, tmpl) + fmt.Printf("Testing %d receivers...\n", len(cfg.Receivers)) + result := TestReceivers(ctx, cfg.Receivers, tmpl) + printTestReceiversResults(result) } return nil } -func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *template.Template) error { +func printTestReceiversResults(result *TestReceiversResult) { + successful := 0 + successfulCounts := make(map[string]int) + for _, rcv := range result.Receivers { + successfulCounts[rcv.Name] = 0 + for _, cfg := range rcv.ConfigResults { + if cfg.Error == nil { + successful += 1 + successfulCounts[rcv.Name] += 1 + } + } + } + + fmt.Printf("\nSuccessfully notified %d/%d receivers at %v:\n", successful, len(result.Receivers), result.NotifedAt.Format("2006-01-02 15:04:05")) + + for _, rcv := range result.Receivers { + fmt.Printf(" %d/%d - '%s'\n", successfulCounts[rcv.Name], len(rcv.ConfigResults), rcv.Name) + for _, cfg := range rcv.ConfigResults { + if cfg.Error != nil { + fmt.Printf(" - %s - %s: %s\n", cfg.Name, cfg.Status, cfg.Error.Error()) + } else { + fmt.Printf(" - %s - %s\n", cfg.Name, cfg.Status) + } + } + } +} + +const ( + maxTestReceiversWorkers = 10 +) + +var ( + ErrNoReceivers = errors.New("no receivers") +) + +type TestReceiversResult struct { + Alert types.Alert + Receivers []TestReceiverResult + NotifedAt time.Time +} + +type TestReceiverResult struct { + Name string + ConfigResults []TestReceiverConfigResult +} + +type TestReceiverConfigResult struct { + Name string + Status string + Error error +} + +func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *template.Template) *TestReceiversResult { // now represents the start time of the test now := time.Now() testAlert := newTestAlert(now, now) @@ -115,43 +171,108 @@ func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *temp Error error } + newTestReceiversResult := func(alert types.Alert, results []result, notifiedAt time.Time) *TestReceiversResult { + m := make(map[string]TestReceiverResult) + for _, receiver := range receivers { + // set up the result for this receiver + m[receiver.Name] = TestReceiverResult{ + Name: receiver.Name, + ConfigResults: []TestReceiverConfigResult{}, + } + } + for _, result := range results { + tmp := m[result.Receiver.Name] + status := "ok" + if result.Error != nil { + status = "failed" + } + tmp.ConfigResults = append(tmp.ConfigResults, TestReceiverConfigResult{ + Name: result.Integration.Name(), + Status: status, + Error: result.Error, + }) + m[result.Receiver.Name] = tmp + } + v := new(TestReceiversResult) + v.Alert = alert + v.Receivers = make([]TestReceiverResult, 0, len(receivers)) + v.NotifedAt = notifiedAt + for _, result := range m { + v.Receivers = append(v.Receivers, result) + } + + // Make sure the return order is deterministic. + sort.Slice(v.Receivers, func(i, j int) bool { + return v.Receivers[i].Name < v.Receivers[j].Name + }) + + return v + } + // invalid keeps track of all invalid receiver configurations var invalid []result // jobs keeps track of all receivers that need to be sent test notifications var jobs []job for _, receiver := range receivers { - integrations, err := buildReceiverIntegrations(receiver, tmpl, logger) + integrations := buildReceiverIntegrations(receiver, tmpl, logger) for _, integration := range integrations { - if err != nil { + if integration.Error != nil { invalid = append(invalid, result{ Receiver: receiver, - Integration: &integration, - Error: err, + Integration: &integration.Integration, + Error: integration.Error, }) } else { jobs = append(jobs, job{ Receiver: receiver, - Integration: &integration, + Integration: &integration.Integration, }) } } } - fmt.Printf("Performing %v jobs!\n", len(jobs)) + if len(jobs) == 0 { + return newTestReceiversResult(testAlert, invalid, now) + } + + numWorkers := maxTestReceiversWorkers + if numWorkers > len(jobs) { + numWorkers = len(jobs) + } + resultCh := make(chan result, len(jobs)) + jobCh := make(chan job, len(jobs)) for _, job := range jobs { - v := result{ - Receiver: job.Receiver, - Integration: job.Integration, - } - if _, err := job.Integration.Notify(notify.WithReceiverName(ctx, job.Receiver.Name), &testAlert); err != nil { - v.Error = err - } + jobCh <- job } + close(jobCh) - fmt.Printf("Done!\n") - return nil + g, ctx := errgroup.WithContext(ctx) + for i := 0; i < numWorkers; i++ { + g.Go(func() error { + for job := range jobCh { + v := result{ + Receiver: job.Receiver, + Integration: job.Integration, + } + if _, err := job.Integration.Notify(notify.WithReceiverName(ctx, job.Receiver.Name), &testAlert); err != nil { + v.Error = err + } + resultCh <- v + } + return nil + }) + } + g.Wait() // nolint + close(resultCh) + + results := make([]result, 0, len(jobs)) + for next := range resultCh { + results = append(results, next) + } + + return newTestReceiversResult(testAlert, append(invalid, results...), now) } func newTestAlert(startsAt, updatedAt time.Time) types.Alert { @@ -178,19 +299,28 @@ func newTestAlert(startsAt, updatedAt time.Time) types.Alert { return alert } +type ReceiverIntegration struct { + Integration notify.Integration + Error error +} + // buildReceiverIntegrations builds a list of integration notifiers off of a // receiver config. -func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) ([]notify.Integration, error) { +func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) []ReceiverIntegration { var ( - errs types.MultiError - integrations []notify.Integration + integrations []ReceiverIntegration add = func(name string, i int, rs notify.ResolvedSender, f func(l log.Logger) (notify.Notifier, error)) { n, err := f(log.With(logger, "integration", name)) if err != nil { - errs.Add(err) - return + integrations = append(integrations, ReceiverIntegration{ + Integration: notify.NewIntegration(nil, rs, name, i), + Error: err, + }) + } else { + integrations = append(integrations, ReceiverIntegration{ + Integration: notify.NewIntegration(n, rs, name, i), + }) } - integrations = append(integrations, notify.NewIntegration(n, rs, name, i)) } ) @@ -228,8 +358,5 @@ func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, log add("discord", i, c, func(l log.Logger) (notify.Notifier, error) { return discord.New(c, tmpl, l) }) } - if errs.Len() > 0 { - return nil, &errs - } - return integrations, nil + return integrations } diff --git a/go.mod b/go.mod index 51ec62710e..c33a51b4e3 100644 --- a/go.mod +++ b/go.mod @@ -42,13 +42,16 @@ require ( go.uber.org/atomic v1.11.0 golang.org/x/mod v0.12.0 golang.org/x/net v0.17.0 + golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 golang.org/x/tools v0.13.0 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/telebot.v3 v3.1.3 gopkg.in/yaml.v2 v2.4.0 ) require ( + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/armon/go-metrics v0.3.10 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -86,7 +89,6 @@ require ( go.opentelemetry.io/otel/trace v1.14.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 38db50b0cf..088f748046 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,7 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -1084,6 +1085,7 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From ee0589f42d5619b26e682dfffd6ecc14c3a0fae4 Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Mon, 7 Nov 2022 22:07:06 +0000 Subject: [PATCH 04/15] Refactor code Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/receivers.go | 284 ++++++++++++++++++++++++++++++++++++++++++ cli/test_receivers.go | 267 +-------------------------------------- 2 files changed, 290 insertions(+), 261 deletions(-) create mode 100644 cli/receivers.go diff --git a/cli/receivers.go b/cli/receivers.go new file mode 100644 index 0000000000..07f807b426 --- /dev/null +++ b/cli/receivers.go @@ -0,0 +1,284 @@ +// Copyright 2022 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "context" + "errors" + "sort" + "time" + + "github.com/go-kit/log" + "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/notify/discord" + "github.com/prometheus/alertmanager/notify/email" + "github.com/prometheus/alertmanager/notify/opsgenie" + "github.com/prometheus/alertmanager/notify/pagerduty" + "github.com/prometheus/alertmanager/notify/pushover" + "github.com/prometheus/alertmanager/notify/slack" + "github.com/prometheus/alertmanager/notify/sns" + "github.com/prometheus/alertmanager/notify/telegram" + "github.com/prometheus/alertmanager/notify/victorops" + "github.com/prometheus/alertmanager/notify/webhook" + "github.com/prometheus/alertmanager/notify/wechat" + "github.com/prometheus/alertmanager/template" + "github.com/prometheus/alertmanager/types" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promlog" + "golang.org/x/sync/errgroup" +) + +const ( + maxTestReceiversWorkers = 10 +) + +var ( + ErrNoReceivers = errors.New("no receivers with configuration set") +) + +type TestReceiversResult struct { + Alert types.Alert + Receivers []TestReceiverResult + NotifedAt time.Time +} + +type TestReceiverResult struct { + Name string + ConfigResults []TestReceiverConfigResult +} + +type TestReceiverConfigResult struct { + Name string + Status string + Error error +} + +func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *template.Template) (*TestReceiversResult, error) { + // now represents the start time of the test + now := time.Now() + testAlert := newTestAlert(now, now) + + // we must set a group key that is unique per test as some receivers use this key to deduplicate alerts + ctx = notify.WithGroupKey(ctx, testAlert.Labels.String()+now.String()) + ctx = notify.WithGroupLabels(ctx, testAlert.Labels) + + logger := promlog.New(&promlog.Config{}) + + // job contains all metadata required to test a receiver + type job struct { + Receiver *config.Receiver + Integration *notify.Integration + } + + // result contains the receiver that was tested and an error that is non-nil if the test failed + type result struct { + Receiver *config.Receiver + Integration *notify.Integration + Error error + } + + newTestReceiversResult := func(alert types.Alert, results []result, notifiedAt time.Time) *TestReceiversResult { + m := make(map[string]TestReceiverResult) + for _, receiver := range receivers { + // set up the result for this receiver + m[receiver.Name] = TestReceiverResult{ + Name: receiver.Name, + ConfigResults: []TestReceiverConfigResult{}, + } + } + for _, result := range results { + tmp := m[result.Receiver.Name] + status := "ok" + if result.Error != nil { + status = "failed" + } + tmp.ConfigResults = append(tmp.ConfigResults, TestReceiverConfigResult{ + Name: result.Integration.Name(), + Status: status, + Error: result.Error, + }) + m[result.Receiver.Name] = tmp + } + v := new(TestReceiversResult) + v.Alert = alert + v.Receivers = make([]TestReceiverResult, 0, len(receivers)) + v.NotifedAt = notifiedAt + for _, result := range m { + v.Receivers = append(v.Receivers, result) + } + + // Make sure the return order is deterministic. + sort.Slice(v.Receivers, func(i, j int) bool { + return v.Receivers[i].Name < v.Receivers[j].Name + }) + + return v + } + + // invalid keeps track of all invalid receiver configurations + var invalid []result + // jobs keeps track of all receivers that need to be sent test notifications + var jobs []job + + for _, receiver := range receivers { + integrations := buildReceiverIntegrations(receiver, tmpl, logger) + for _, integration := range integrations { + if integration.Error != nil { + invalid = append(invalid, result{ + Receiver: receiver, + Integration: &integration.Integration, + Error: integration.Error, + }) + } else { + jobs = append(jobs, job{ + Receiver: receiver, + Integration: &integration.Integration, + }) + } + } + } + + if len(invalid)+len(jobs) == 0 { + return nil, ErrNoReceivers + } + + if len(jobs) == 0 { + return newTestReceiversResult(testAlert, invalid, now), nil + } + + numWorkers := maxTestReceiversWorkers + if numWorkers > len(jobs) { + numWorkers = len(jobs) + } + + resultCh := make(chan result, len(jobs)) + jobCh := make(chan job, len(jobs)) + for _, job := range jobs { + jobCh <- job + } + close(jobCh) + + g, ctx := errgroup.WithContext(ctx) + for i := 0; i < numWorkers; i++ { + g.Go(func() error { + for job := range jobCh { + v := result{ + Receiver: job.Receiver, + Integration: job.Integration, + } + if _, err := job.Integration.Notify(notify.WithReceiverName(ctx, job.Receiver.Name), &testAlert); err != nil { + v.Error = err + } + resultCh <- v + } + return nil + }) + } + g.Wait() // nolint + close(resultCh) + + results := make([]result, 0, len(jobs)) + for next := range resultCh { + results = append(results, next) + } + + return newTestReceiversResult(testAlert, append(invalid, results...), now), nil +} + +func newTestAlert(startsAt, updatedAt time.Time) types.Alert { + var ( + defaultAnnotations = model.LabelSet{ + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]", + } + defaultLabels = model.LabelSet{ + "alertname": "TestAlert", + "instance": "Alertmanager", + } + ) + + alert := types.Alert{ + Alert: model.Alert{ + Labels: defaultLabels, + Annotations: defaultAnnotations, + StartsAt: startsAt, + }, + UpdatedAt: updatedAt, + } + + return alert +} + +type ReceiverIntegration struct { + Integration notify.Integration + Error error +} + +// buildReceiverIntegrations builds a list of integration notifiers off of a +// receiver config. +func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) []ReceiverIntegration { + var ( + integrations []ReceiverIntegration + add = func(name string, i int, rs notify.ResolvedSender, f func(l log.Logger) (notify.Notifier, error)) { + n, err := f(log.With(logger, "integration", name)) + if err != nil { + integrations = append(integrations, ReceiverIntegration{ + Integration: notify.NewIntegration(nil, rs, name, i), + Error: err, + }) + } else { + integrations = append(integrations, ReceiverIntegration{ + Integration: notify.NewIntegration(n, rs, name, i), + }) + } + } + ) + + for i, c := range nc.WebhookConfigs { + add("webhook", i, c, func(l log.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) }) + } + for i, c := range nc.EmailConfigs { + add("email", i, c, func(l log.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil }) + } + for i, c := range nc.PagerdutyConfigs { + add("pagerduty", i, c, func(l log.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) }) + } + for i, c := range nc.OpsGenieConfigs { + add("opsgenie", i, c, func(l log.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) }) + } + for i, c := range nc.WechatConfigs { + add("wechat", i, c, func(l log.Logger) (notify.Notifier, error) { return wechat.New(c, tmpl, l) }) + } + for i, c := range nc.SlackConfigs { + add("slack", i, c, func(l log.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) }) + } + for i, c := range nc.VictorOpsConfigs { + add("victorops", i, c, func(l log.Logger) (notify.Notifier, error) { return victorops.New(c, tmpl, l) }) + } + for i, c := range nc.PushoverConfigs { + add("pushover", i, c, func(l log.Logger) (notify.Notifier, error) { return pushover.New(c, tmpl, l) }) + } + for i, c := range nc.SNSConfigs { + add("sns", i, c, func(l log.Logger) (notify.Notifier, error) { return sns.New(c, tmpl, l) }) + } + for i, c := range nc.TelegramConfigs { + add("telegram", i, c, func(l log.Logger) (notify.Notifier, error) { return telegram.New(c, tmpl, l) }) + } + for i, c := range nc.DiscordConfigs { + add("discord", i, c, func(l log.Logger) (notify.Notifier, error) { return discord.New(c, tmpl, l) }) + } + + return integrations +} diff --git a/cli/test_receivers.go b/cli/test_receivers.go index 340498bccf..ca7f1ef477 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -17,29 +17,10 @@ import ( "context" "fmt" "net/url" - "sort" - "time" - "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/alertmanager/config" - "github.com/prometheus/alertmanager/notify" - "github.com/prometheus/alertmanager/notify/discord" - "github.com/prometheus/alertmanager/notify/email" - "github.com/prometheus/alertmanager/notify/opsgenie" - "github.com/prometheus/alertmanager/notify/pagerduty" - "github.com/prometheus/alertmanager/notify/pushover" - "github.com/prometheus/alertmanager/notify/slack" - "github.com/prometheus/alertmanager/notify/sns" - "github.com/prometheus/alertmanager/notify/telegram" - "github.com/prometheus/alertmanager/notify/victorops" - "github.com/prometheus/alertmanager/notify/webhook" - "github.com/prometheus/alertmanager/notify/wechat" "github.com/prometheus/alertmanager/template" - "github.com/prometheus/alertmanager/types" - "github.com/prometheus/common/model" - "github.com/prometheus/common/promlog" - "golang.org/x/sync/errgroup" "gopkg.in/alecthomas/kingpin.v2" ) @@ -69,7 +50,7 @@ func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseCo fmt.Printf("Checking '%s'\n", t.configFile) cfg, err := config.LoadFile(t.configFile) if err != nil { - return errors.Wrap(err, "invalid config file") + kingpin.Fatalf("Invalid config file") } if cfg != nil { @@ -82,13 +63,16 @@ func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseCo } else { u, err := url.Parse("http://localhost:1234") if err != nil { - return errors.Wrap(err, "failed to parse fake url") + return errors.Wrap(err, "failed to parse mock url") } tmpl.ExternalURL = u } fmt.Printf("Testing %d receivers...\n", len(cfg.Receivers)) - result := TestReceivers(ctx, cfg.Receivers, tmpl) + result, err := TestReceivers(ctx, cfg.Receivers, tmpl) + if err != nil { + return err + } printTestReceiversResults(result) } @@ -121,242 +105,3 @@ func printTestReceiversResults(result *TestReceiversResult) { } } } - -const ( - maxTestReceiversWorkers = 10 -) - -var ( - ErrNoReceivers = errors.New("no receivers") -) - -type TestReceiversResult struct { - Alert types.Alert - Receivers []TestReceiverResult - NotifedAt time.Time -} - -type TestReceiverResult struct { - Name string - ConfigResults []TestReceiverConfigResult -} - -type TestReceiverConfigResult struct { - Name string - Status string - Error error -} - -func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *template.Template) *TestReceiversResult { - // now represents the start time of the test - now := time.Now() - testAlert := newTestAlert(now, now) - - // we must set a group key that is unique per test as some receivers use this key to deduplicate alerts - ctx = notify.WithGroupKey(ctx, testAlert.Labels.String()+now.String()) - ctx = notify.WithGroupLabels(ctx, testAlert.Labels) - - logger := promlog.New(&promlog.Config{}) - - // job contains all metadata required to test a receiver - type job struct { - Receiver *config.Receiver - Integration *notify.Integration - } - - // result contains the receiver that was tested and an error that is non-nil if the test failed - type result struct { - Receiver *config.Receiver - Integration *notify.Integration - Error error - } - - newTestReceiversResult := func(alert types.Alert, results []result, notifiedAt time.Time) *TestReceiversResult { - m := make(map[string]TestReceiverResult) - for _, receiver := range receivers { - // set up the result for this receiver - m[receiver.Name] = TestReceiverResult{ - Name: receiver.Name, - ConfigResults: []TestReceiverConfigResult{}, - } - } - for _, result := range results { - tmp := m[result.Receiver.Name] - status := "ok" - if result.Error != nil { - status = "failed" - } - tmp.ConfigResults = append(tmp.ConfigResults, TestReceiverConfigResult{ - Name: result.Integration.Name(), - Status: status, - Error: result.Error, - }) - m[result.Receiver.Name] = tmp - } - v := new(TestReceiversResult) - v.Alert = alert - v.Receivers = make([]TestReceiverResult, 0, len(receivers)) - v.NotifedAt = notifiedAt - for _, result := range m { - v.Receivers = append(v.Receivers, result) - } - - // Make sure the return order is deterministic. - sort.Slice(v.Receivers, func(i, j int) bool { - return v.Receivers[i].Name < v.Receivers[j].Name - }) - - return v - } - - // invalid keeps track of all invalid receiver configurations - var invalid []result - // jobs keeps track of all receivers that need to be sent test notifications - var jobs []job - - for _, receiver := range receivers { - integrations := buildReceiverIntegrations(receiver, tmpl, logger) - for _, integration := range integrations { - if integration.Error != nil { - invalid = append(invalid, result{ - Receiver: receiver, - Integration: &integration.Integration, - Error: integration.Error, - }) - } else { - jobs = append(jobs, job{ - Receiver: receiver, - Integration: &integration.Integration, - }) - } - } - } - - if len(jobs) == 0 { - return newTestReceiversResult(testAlert, invalid, now) - } - - numWorkers := maxTestReceiversWorkers - if numWorkers > len(jobs) { - numWorkers = len(jobs) - } - - resultCh := make(chan result, len(jobs)) - jobCh := make(chan job, len(jobs)) - for _, job := range jobs { - jobCh <- job - } - close(jobCh) - - g, ctx := errgroup.WithContext(ctx) - for i := 0; i < numWorkers; i++ { - g.Go(func() error { - for job := range jobCh { - v := result{ - Receiver: job.Receiver, - Integration: job.Integration, - } - if _, err := job.Integration.Notify(notify.WithReceiverName(ctx, job.Receiver.Name), &testAlert); err != nil { - v.Error = err - } - resultCh <- v - } - return nil - }) - } - g.Wait() // nolint - close(resultCh) - - results := make([]result, 0, len(jobs)) - for next := range resultCh { - results = append(results, next) - } - - return newTestReceiversResult(testAlert, append(invalid, results...), now) -} - -func newTestAlert(startsAt, updatedAt time.Time) types.Alert { - var ( - defaultAnnotations = model.LabelSet{ - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]", - } - defaultLabels = model.LabelSet{ - "alertname": "TestAlert", - "instance": "Alertmanager", - } - ) - - alert := types.Alert{ - Alert: model.Alert{ - Labels: defaultLabels, - Annotations: defaultAnnotations, - StartsAt: startsAt, - }, - UpdatedAt: updatedAt, - } - - return alert -} - -type ReceiverIntegration struct { - Integration notify.Integration - Error error -} - -// buildReceiverIntegrations builds a list of integration notifiers off of a -// receiver config. -func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) []ReceiverIntegration { - var ( - integrations []ReceiverIntegration - add = func(name string, i int, rs notify.ResolvedSender, f func(l log.Logger) (notify.Notifier, error)) { - n, err := f(log.With(logger, "integration", name)) - if err != nil { - integrations = append(integrations, ReceiverIntegration{ - Integration: notify.NewIntegration(nil, rs, name, i), - Error: err, - }) - } else { - integrations = append(integrations, ReceiverIntegration{ - Integration: notify.NewIntegration(n, rs, name, i), - }) - } - } - ) - - for i, c := range nc.WebhookConfigs { - add("webhook", i, c, func(l log.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) }) - } - for i, c := range nc.EmailConfigs { - add("email", i, c, func(l log.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil }) - } - for i, c := range nc.PagerdutyConfigs { - add("pagerduty", i, c, func(l log.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) }) - } - for i, c := range nc.OpsGenieConfigs { - add("opsgenie", i, c, func(l log.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) }) - } - for i, c := range nc.WechatConfigs { - add("wechat", i, c, func(l log.Logger) (notify.Notifier, error) { return wechat.New(c, tmpl, l) }) - } - for i, c := range nc.SlackConfigs { - add("slack", i, c, func(l log.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) }) - } - for i, c := range nc.VictorOpsConfigs { - add("victorops", i, c, func(l log.Logger) (notify.Notifier, error) { return victorops.New(c, tmpl, l) }) - } - for i, c := range nc.PushoverConfigs { - add("pushover", i, c, func(l log.Logger) (notify.Notifier, error) { return pushover.New(c, tmpl, l) }) - } - for i, c := range nc.SNSConfigs { - add("sns", i, c, func(l log.Logger) (notify.Notifier, error) { return sns.New(c, tmpl, l) }) - } - for i, c := range nc.TelegramConfigs { - add("telegram", i, c, func(l log.Logger) (notify.Notifier, error) { return telegram.New(c, tmpl, l) }) - } - for i, c := range nc.DiscordConfigs { - add("discord", i, c, func(l log.Logger) (notify.Notifier, error) { return discord.New(c, tmpl, l) }) - } - - return integrations -} From 0b84da2a439c1a2b12f29bd736f1cc003cc628ed Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sat, 12 Nov 2022 22:37:57 +0000 Subject: [PATCH 05/15] Add alert.file option to mock alert Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/receivers.go | 35 +++++++++++++++++++----- cli/test_receivers.go | 63 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 80 insertions(+), 18 deletions(-) diff --git a/cli/receivers.go b/cli/receivers.go index 07f807b426..71355adbb2 100644 --- a/cli/receivers.go +++ b/cli/receivers.go @@ -48,6 +48,16 @@ var ( ErrNoReceivers = errors.New("no receivers with configuration set") ) +type TestReceiversParams struct { + Alert *TestReceiversAlertParams + Receivers []*config.Receiver +} + +type TestReceiversAlertParams struct { + Annotations model.LabelSet `yaml:"annotations,omitempty" json:"annotations,omitempty"` + Labels model.LabelSet `yaml:"labels,omitempty" json:"labels,omitempty"` +} + type TestReceiversResult struct { Alert types.Alert Receivers []TestReceiverResult @@ -65,10 +75,10 @@ type TestReceiverConfigResult struct { Error error } -func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *template.Template) (*TestReceiversResult, error) { +func TestReceivers(ctx context.Context, c TestReceiversParams, tmpl *template.Template) (*TestReceiversResult, error) { // now represents the start time of the test now := time.Now() - testAlert := newTestAlert(now, now) + testAlert := newTestAlert(c, now, now) // we must set a group key that is unique per test as some receivers use this key to deduplicate alerts ctx = notify.WithGroupKey(ctx, testAlert.Labels.String()+now.String()) @@ -91,7 +101,7 @@ func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *temp newTestReceiversResult := func(alert types.Alert, results []result, notifiedAt time.Time) *TestReceiversResult { m := make(map[string]TestReceiverResult) - for _, receiver := range receivers { + for _, receiver := range c.Receivers { // set up the result for this receiver m[receiver.Name] = TestReceiverResult{ Name: receiver.Name, @@ -113,7 +123,7 @@ func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *temp } v := new(TestReceiversResult) v.Alert = alert - v.Receivers = make([]TestReceiverResult, 0, len(receivers)) + v.Receivers = make([]TestReceiverResult, 0, len(c.Receivers)) v.NotifedAt = notifiedAt for _, result := range m { v.Receivers = append(v.Receivers, result) @@ -132,7 +142,7 @@ func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *temp // jobs keeps track of all receivers that need to be sent test notifications var jobs []job - for _, receiver := range receivers { + for _, receiver := range c.Receivers { integrations := buildReceiverIntegrations(receiver, tmpl, logger) for _, integration := range integrations { if integration.Error != nil { @@ -197,7 +207,7 @@ func TestReceivers(ctx context.Context, receivers []*config.Receiver, tmpl *temp return newTestReceiversResult(testAlert, append(invalid, results...), now), nil } -func newTestAlert(startsAt, updatedAt time.Time) types.Alert { +func newTestAlert(c TestReceiversParams, startsAt, updatedAt time.Time) types.Alert { var ( defaultAnnotations = model.LabelSet{ "summary": "Notification test", @@ -218,6 +228,19 @@ func newTestAlert(startsAt, updatedAt time.Time) types.Alert { UpdatedAt: updatedAt, } + if c.Alert != nil { + if c.Alert.Annotations != nil { + for k, v := range c.Alert.Annotations { + alert.Annotations[k] = v + } + } + if c.Alert.Labels != nil { + for k, v := range c.Alert.Labels { + alert.Labels[k] = v + } + } + } + return alert } diff --git a/cli/test_receivers.go b/cli/test_receivers.go index ca7f1ef477..eb4fbab228 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -17,20 +17,23 @@ import ( "context" "fmt" "net/url" + "os" "github.com/pkg/errors" "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/template" "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/yaml.v2" ) type testReceiversCmd struct { configFile string + alertFile string } const testReceiversHelp = `Test alertmanager receivers -Will test receivers for alertmanager config file. +Send test notifications to every receiver for an alertmanager config file. ` func configureTestReceiversCmd(app *kingpin.Application) { @@ -39,6 +42,7 @@ func configureTestReceiversCmd(app *kingpin.Application) { testCmd = app.Command("test-receivers", testReceiversHelp) ) testCmd.Arg("config.file", "Config file to be tested.").ExistingFileVar(&t.configFile) + testCmd.Flag("alert.file", "Mock alert file with annotations and labels to add to test alert.").ExistingFileVar(&t.alertFile) testCmd.Action(execWithTimeout(t.testReceivers)) } @@ -47,29 +51,32 @@ func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseCo kingpin.Fatalf("No config file was specified") } - fmt.Printf("Checking '%s'\n", t.configFile) + fmt.Printf("Checking alertmanager config '%s'...\n", t.configFile) cfg, err := config.LoadFile(t.configFile) if err != nil { - kingpin.Fatalf("Invalid config file") + kingpin.Fatalf("Invalid alertmanager config file") } if cfg != nil { - tmpl, err := template.FromGlobs(cfg.Templates...) + tmpl, err := getTemplate(cfg) if err != nil { - return errors.Wrap(err, "failed to parse templates") + return err + } + + c := TestReceiversParams{ + Receivers: cfg.Receivers, } - if alertmanagerURL != nil { - tmpl.ExternalURL = alertmanagerURL - } else { - u, err := url.Parse("http://localhost:1234") + + if t.alertFile != "" { + alert, err := loadAlertConfigFile(t.alertFile) if err != nil { - return errors.Wrap(err, "failed to parse mock url") + kingpin.Fatalf("Invalid alert config file") } - tmpl.ExternalURL = u + c.Alert = alert } fmt.Printf("Testing %d receivers...\n", len(cfg.Receivers)) - result, err := TestReceivers(ctx, cfg.Receivers, tmpl) + result, err := TestReceivers(ctx, c, tmpl) if err != nil { return err } @@ -105,3 +112,35 @@ func printTestReceiversResults(result *TestReceiversResult) { } } } + +func getTemplate(cfg *config.Config) (*template.Template, error) { + tmpl, err := template.FromGlobs(cfg.Templates...) + if err != nil { + return nil, errors.Wrap(err, "failed to parse templates") + } + if alertmanagerURL != nil { + tmpl.ExternalURL = alertmanagerURL + } else { + u, err := url.Parse("http://localhost:1234") + if err != nil { + return nil, errors.Wrap(err, "failed to parse mock url") + } + tmpl.ExternalURL = u + } + return tmpl, nil +} + +func loadAlertConfigFile(filename string) (*TestReceiversAlertParams, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + alert := &TestReceiversAlertParams{} + err = yaml.UnmarshalStrict(b, alert) + if err != nil { + return nil, err + } + + return alert, nil +} From 4864402748f7db441848f29501f0eae3db93b4ee Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sun, 13 Nov 2022 16:13:12 +0000 Subject: [PATCH 06/15] Add tests for test receivers command Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/receivers.go | 1 + cli/test_receivers.go | 53 +++++++----------------------- cli/test_receivers_test.go | 58 +++++++++++++++++++++++++++++++++ cli/testdata/conf.bad-alert.yml | 1 + cli/testdata/conf.receiver.yml | 14 ++++++++ cli/utils.go | 34 +++++++++++++++++++ 6 files changed, 120 insertions(+), 41 deletions(-) create mode 100644 cli/test_receivers_test.go create mode 100644 cli/testdata/conf.bad-alert.yml create mode 100644 cli/testdata/conf.receiver.yml diff --git a/cli/receivers.go b/cli/receivers.go index 71355adbb2..1177c2382f 100644 --- a/cli/receivers.go +++ b/cli/receivers.go @@ -82,6 +82,7 @@ func TestReceivers(ctx context.Context, c TestReceiversParams, tmpl *template.Te // we must set a group key that is unique per test as some receivers use this key to deduplicate alerts ctx = notify.WithGroupKey(ctx, testAlert.Labels.String()+now.String()) + // we must set group labels to avoid issues with templating ctx = notify.WithGroupLabels(ctx, testAlert.Labels) logger := promlog.New(&promlog.Config{}) diff --git a/cli/test_receivers.go b/cli/test_receivers.go index eb4fbab228..007c1a6a50 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -15,15 +15,10 @@ package cli import ( "context" + "errors" "fmt" - "net/url" - "os" - - "github.com/pkg/errors" "github.com/prometheus/alertmanager/config" - "github.com/prometheus/alertmanager/template" "gopkg.in/alecthomas/kingpin.v2" - "gopkg.in/yaml.v2" ) type testReceiversCmd struct { @@ -36,6 +31,14 @@ const testReceiversHelp = `Test alertmanager receivers Send test notifications to every receiver for an alertmanager config file. ` +var ( + ErrNoConfigFile = errors.New("no config file was specified") + ErrInvalidConfigFile = errors.New("invalid alertmanager config file") + ErrInvalidAlertFile = errors.New("invalid alert config file") + ErrInvalidTemplate = errors.New("failed to parse templates") + ErrInternal = errors.New("internal error parsing mock url") +) + func configureTestReceiversCmd(app *kingpin.Application) { var ( t = &testReceiversCmd{} @@ -48,13 +51,13 @@ func configureTestReceiversCmd(app *kingpin.Application) { func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseContext) error { if len(t.configFile) == 0 { - kingpin.Fatalf("No config file was specified") + return ErrNoConfigFile } fmt.Printf("Checking alertmanager config '%s'...\n", t.configFile) cfg, err := config.LoadFile(t.configFile) if err != nil { - kingpin.Fatalf("Invalid alertmanager config file") + return ErrInvalidConfigFile } if cfg != nil { @@ -70,7 +73,7 @@ func (t *testReceiversCmd) testReceivers(ctx context.Context, _ *kingpin.ParseCo if t.alertFile != "" { alert, err := loadAlertConfigFile(t.alertFile) if err != nil { - kingpin.Fatalf("Invalid alert config file") + return ErrInvalidAlertFile } c.Alert = alert } @@ -112,35 +115,3 @@ func printTestReceiversResults(result *TestReceiversResult) { } } } - -func getTemplate(cfg *config.Config) (*template.Template, error) { - tmpl, err := template.FromGlobs(cfg.Templates...) - if err != nil { - return nil, errors.Wrap(err, "failed to parse templates") - } - if alertmanagerURL != nil { - tmpl.ExternalURL = alertmanagerURL - } else { - u, err := url.Parse("http://localhost:1234") - if err != nil { - return nil, errors.Wrap(err, "failed to parse mock url") - } - tmpl.ExternalURL = u - } - return tmpl, nil -} - -func loadAlertConfigFile(filename string) (*TestReceiversAlertParams, error) { - b, err := os.ReadFile(filename) - if err != nil { - return nil, err - } - - alert := &TestReceiversAlertParams{} - err = yaml.UnmarshalStrict(b, alert) - if err != nil { - return nil, err - } - - return alert, nil -} diff --git a/cli/test_receivers_test.go b/cli/test_receivers_test.go new file mode 100644 index 0000000000..9a8987d739 --- /dev/null +++ b/cli/test_receivers_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/alecthomas/kingpin.v2" +) + +func Test_TestReceivers_Error(t *testing.T) { + ctx := context.Background() + parseContext := kingpin.ParseContext{} + + t.Run("invalid alertmanager config", func(t *testing.T) { + test := testReceiversCmd{ + configFile: "testdata/conf.bad.yml", + } + + err := test.testReceivers(ctx, &parseContext) + require.Error(t, err) + require.Equal(t, ErrInvalidConfigFile.Error(), err.Error()) + }) + + t.Run("invalid alert", func(t *testing.T) { + test := testReceiversCmd{ + configFile: "testdata/conf.receiver.yml", + alertFile: "testdata/conf.bad-alert.yml", + } + + err := test.testReceivers(ctx, &parseContext) + require.Error(t, err) + require.Equal(t, ErrInvalidAlertFile.Error(), err.Error()) + }) + + t.Run("no receivers", func(t *testing.T) { + test := testReceiversCmd{ + configFile: "testdata/conf.good.yml", + } + + err := test.testReceivers(ctx, &parseContext) + require.Error(t, err) + require.Equal(t, ErrNoReceivers.Error(), err.Error()) + }) +} diff --git a/cli/testdata/conf.bad-alert.yml b/cli/testdata/conf.bad-alert.yml new file mode 100644 index 0000000000..a3e2920015 --- /dev/null +++ b/cli/testdata/conf.bad-alert.yml @@ -0,0 +1 @@ +BAD \ No newline at end of file diff --git a/cli/testdata/conf.receiver.yml b/cli/testdata/conf.receiver.yml new file mode 100644 index 0000000000..e768723471 --- /dev/null +++ b/cli/testdata/conf.receiver.yml @@ -0,0 +1,14 @@ +global: + slack_api_url: "https://hooks.slack.com/services/random/random" + +route: + group_by: ["alertname"] + group_wait: 30s + group_interval: 5m + repeat_interval: 5m + receiver: "slack-alerts" + +receivers: + - name: "slack-alerts" + slack_configs: + - channel: "#dev" \ No newline at end of file diff --git a/cli/utils.go b/cli/utils.go index 60221e239a..2b5d55b260 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -22,11 +22,13 @@ import ( kingpin "github.com/alecthomas/kingpin/v2" "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" "github.com/prometheus/alertmanager/api/v2/client/general" "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/alertmanager/template" ) // parseMatchers parses a list of matchers (cli arguments). @@ -146,3 +148,35 @@ func execWithTimeout(fn func(context.Context, *kingpin.ParseContext) error) func return fn(ctx, x) } } + +func loadAlertConfigFile(filename string) (*TestReceiversAlertParams, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + alert := &TestReceiversAlertParams{} + err = yaml.UnmarshalStrict(b, alert) + if err != nil { + return nil, err + } + + return alert, nil +} + +func getTemplate(cfg *config.Config) (*template.Template, error) { + tmpl, err := template.FromGlobs(cfg.Templates...) + if err != nil { + return nil, ErrInvalidTemplate + } + if alertmanagerURL != nil { + tmpl.ExternalURL = alertmanagerURL + } else { + u, err := url.Parse("http://localhost:1234") + if err != nil { + return nil, ErrInternal + } + tmpl.ExternalURL = u + } + return tmpl, nil +} From 016abe35c69eecf7a9ad5e47db58525be933d590 Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sun, 13 Nov 2022 16:32:06 +0000 Subject: [PATCH 07/15] update mock url Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/utils.go b/cli/utils.go index 2b5d55b260..08e7ee7d05 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -172,7 +172,7 @@ func getTemplate(cfg *config.Config) (*template.Template, error) { if alertmanagerURL != nil { tmpl.ExternalURL = alertmanagerURL } else { - u, err := url.Parse("http://localhost:1234") + u, err := url.Parse("https://example.com") if err != nil { return nil, ErrInternal } From 674391883a43e281ac210064fb3070a18ae2a69c Mon Sep 17 00:00:00 2001 From: Oktarian TB Date: Sun, 13 Nov 2022 16:34:01 +0000 Subject: [PATCH 08/15] fix linting Signed-off-by: Oktarian TB Signed-off-by: Alex Weaver --- cli/test_receivers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/test_receivers_test.go b/cli/test_receivers_test.go index 9a8987d739..175da71f3f 100644 --- a/cli/test_receivers_test.go +++ b/cli/test_receivers_test.go @@ -38,7 +38,7 @@ func Test_TestReceivers_Error(t *testing.T) { t.Run("invalid alert", func(t *testing.T) { test := testReceiversCmd{ configFile: "testdata/conf.receiver.yml", - alertFile: "testdata/conf.bad-alert.yml", + alertFile: "testdata/conf.bad-alert.yml", } err := test.testReceivers(ctx, &parseContext) From c24d3398b8cdfc9e95562a2d92c37fed4e1eff62 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Mon, 28 Aug 2023 15:11:21 -0500 Subject: [PATCH 09/15] Fix updated FromGlobs signature Signed-off-by: Alex Weaver --- cli/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/utils.go b/cli/utils.go index 08e7ee7d05..41051b2da2 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -165,7 +165,7 @@ func loadAlertConfigFile(filename string) (*TestReceiversAlertParams, error) { } func getTemplate(cfg *config.Config) (*template.Template, error) { - tmpl, err := template.FromGlobs(cfg.Templates...) + tmpl, err := template.FromGlobs(cfg.Templates) if err != nil { return nil, ErrInvalidTemplate } From c2690bd9d3c171321e0d408cb599685f6f59a099 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Mon, 28 Aug 2023 15:17:24 -0500 Subject: [PATCH 10/15] Update to latest config definition, use latest package path for kingpin Signed-off-by: Alex Weaver --- cli/receivers.go | 8 ++++---- cli/test_receivers.go | 3 ++- cli/test_receivers_test.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cli/receivers.go b/cli/receivers.go index 1177c2382f..0dc216679e 100644 --- a/cli/receivers.go +++ b/cli/receivers.go @@ -50,7 +50,7 @@ var ( type TestReceiversParams struct { Alert *TestReceiversAlertParams - Receivers []*config.Receiver + Receivers []config.Receiver } type TestReceiversAlertParams struct { @@ -89,13 +89,13 @@ func TestReceivers(ctx context.Context, c TestReceiversParams, tmpl *template.Te // job contains all metadata required to test a receiver type job struct { - Receiver *config.Receiver + Receiver config.Receiver Integration *notify.Integration } // result contains the receiver that was tested and an error that is non-nil if the test failed type result struct { - Receiver *config.Receiver + Receiver config.Receiver Integration *notify.Integration Error error } @@ -252,7 +252,7 @@ type ReceiverIntegration struct { // buildReceiverIntegrations builds a list of integration notifiers off of a // receiver config. -func buildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) []ReceiverIntegration { +func buildReceiverIntegrations(nc config.Receiver, tmpl *template.Template, logger log.Logger) []ReceiverIntegration { var ( integrations []ReceiverIntegration add = func(name string, i int, rs notify.ResolvedSender, f func(l log.Logger) (notify.Notifier, error)) { diff --git a/cli/test_receivers.go b/cli/test_receivers.go index 007c1a6a50..a2a7739c2d 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -17,8 +17,9 @@ import ( "context" "errors" "fmt" + + "github.com/alecthomas/kingpin/v2" "github.com/prometheus/alertmanager/config" - "gopkg.in/alecthomas/kingpin.v2" ) type testReceiversCmd struct { diff --git a/cli/test_receivers_test.go b/cli/test_receivers_test.go index 175da71f3f..abd6008c22 100644 --- a/cli/test_receivers_test.go +++ b/cli/test_receivers_test.go @@ -17,8 +17,8 @@ import ( "context" "testing" + "github.com/alecthomas/kingpin/v2" "github.com/stretchr/testify/require" - "gopkg.in/alecthomas/kingpin.v2" ) func Test_TestReceivers_Error(t *testing.T) { From 7071b28c94a45addd95704495f3b2d477450e140 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Mon, 28 Aug 2023 15:22:16 -0500 Subject: [PATCH 11/15] Remove unused old path for kingpin Signed-off-by: Alex Weaver --- go.mod | 2 -- go.sum | 2 -- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index c33a51b4e3..2c025f887f 100644 --- a/go.mod +++ b/go.mod @@ -45,13 +45,11 @@ require ( golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 golang.org/x/tools v0.13.0 - gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/telebot.v3 v3.1.3 gopkg.in/yaml.v2 v2.4.0 ) require ( - github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/armon/go-metrics v0.3.10 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 088f748046..38db50b0cf 100644 --- a/go.sum +++ b/go.sum @@ -63,7 +63,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -1085,7 +1084,6 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0f68c1ede992159881ddb428b28fafb375a14d74 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Fri, 1 Sep 2023 15:22:35 -0500 Subject: [PATCH 12/15] Lint imports Signed-off-by: Alex Weaver --- cli/receivers.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/receivers.go b/cli/receivers.go index 0dc216679e..e4aafffd26 100644 --- a/cli/receivers.go +++ b/cli/receivers.go @@ -20,6 +20,10 @@ import ( "time" "github.com/go-kit/log" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promlog" + "golang.org/x/sync/errgroup" + "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/notify/discord" @@ -35,9 +39,6 @@ import ( "github.com/prometheus/alertmanager/notify/wechat" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" - "github.com/prometheus/common/model" - "github.com/prometheus/common/promlog" - "golang.org/x/sync/errgroup" ) const ( From 8974d9ca868da55d0746becc9a50a43013ab0e85 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Fri, 1 Sep 2023 15:23:28 -0500 Subject: [PATCH 13/15] Lint again Signed-off-by: Alex Weaver --- cli/test_receivers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/test_receivers.go b/cli/test_receivers.go index a2a7739c2d..6af96f2a7c 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/alecthomas/kingpin/v2" + "github.com/prometheus/alertmanager/config" ) From e73ffa3ccd25c501ef1145e2e9dae8e9d930cdc0 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Fri, 1 Sep 2023 15:28:19 -0500 Subject: [PATCH 14/15] One more time... Signed-off-by: Alex Weaver --- cli/receivers.go | 4 +--- cli/test_receivers.go | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cli/receivers.go b/cli/receivers.go index e4aafffd26..65d3e9db8c 100644 --- a/cli/receivers.go +++ b/cli/receivers.go @@ -45,9 +45,7 @@ const ( maxTestReceiversWorkers = 10 ) -var ( - ErrNoReceivers = errors.New("no receivers with configuration set") -) +var ErrNoReceivers = errors.New("no receivers with configuration set") type TestReceiversParams struct { Alert *TestReceiversAlertParams diff --git a/cli/test_receivers.go b/cli/test_receivers.go index 6af96f2a7c..ba56253df3 100644 --- a/cli/test_receivers.go +++ b/cli/test_receivers.go @@ -98,8 +98,8 @@ func printTestReceiversResults(result *TestReceiversResult) { successfulCounts[rcv.Name] = 0 for _, cfg := range rcv.ConfigResults { if cfg.Error == nil { - successful += 1 - successfulCounts[rcv.Name] += 1 + successful++ + successfulCounts[rcv.Name]++ } } } From 16e33413e807a9dc847c4df6ef81da3b0e4fc4b3 Mon Sep 17 00:00:00 2001 From: Alex Weaver Date: Wed, 4 Oct 2023 15:09:06 -0500 Subject: [PATCH 15/15] Add two missing receiver types --- cli/receivers.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cli/receivers.go b/cli/receivers.go index 65d3e9db8c..50a6839efb 100644 --- a/cli/receivers.go +++ b/cli/receivers.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/notify/discord" "github.com/prometheus/alertmanager/notify/email" + "github.com/prometheus/alertmanager/notify/msteams" "github.com/prometheus/alertmanager/notify/opsgenie" "github.com/prometheus/alertmanager/notify/pagerduty" "github.com/prometheus/alertmanager/notify/pushover" @@ -35,6 +36,7 @@ import ( "github.com/prometheus/alertmanager/notify/sns" "github.com/prometheus/alertmanager/notify/telegram" "github.com/prometheus/alertmanager/notify/victorops" + "github.com/prometheus/alertmanager/notify/webex" "github.com/prometheus/alertmanager/notify/webhook" "github.com/prometheus/alertmanager/notify/wechat" "github.com/prometheus/alertmanager/template" @@ -302,6 +304,12 @@ func buildReceiverIntegrations(nc config.Receiver, tmpl *template.Template, logg for i, c := range nc.DiscordConfigs { add("discord", i, c, func(l log.Logger) (notify.Notifier, error) { return discord.New(c, tmpl, l) }) } + for i, c := range nc.WebexConfigs { + add("webex", i, c, func(l log.Logger) (notify.Notifier, error) { return webex.New(c, tmpl, l) }) + } + for i, c := range nc.MSTeamsConfigs { + add("msteams", i, c, func(l log.Logger) (notify.Notifier, error) { return msteams.New(c, tmpl, l) }) + } return integrations }