From 091af996d86e3d76e9b05b6e76fcbc044c5b156c Mon Sep 17 00:00:00 2001 From: Shaun Warman Date: Fri, 23 Aug 2024 12:55:24 -0500 Subject: [PATCH] feat: add forwardemail as email option --- cmd/bounce.go | 17 +++++ cmd/init.go | 20 +++--- cmd/settings.go | 5 ++ docs/docs/content/bounces.md | 11 +-- docs/swagger/collections.yaml | 8 +++ frontend/src/views/Settings.vue | 6 ++ frontend/src/views/settings/bounces.vue | 14 ++++ frontend/src/views/settings/smtp.vue | 4 ++ i18n/en.json | 2 + internal/bounce/bounce.go | 40 ++++++----- internal/bounce/webhooks/forwardemail.go | 87 ++++++++++++++++++++++++ models/settings.go | 4 +- schema.sql | 2 + 13 files changed, 190 insertions(+), 30 deletions(-) create mode 100644 internal/bounce/webhooks/forwardemail.go diff --git a/cmd/bounce.go b/cmd/bounce.go index 7bf820115..c75b2fa90 100644 --- a/cmd/bounce.go +++ b/cmd/bounce.go @@ -191,6 +191,23 @@ func handleBounceWebhook(c echo.Context) error { } bounces = append(bounces, bs...) + // ForwardEmail. + case service == "forwardemail" && app.constants.BounceForwardemailEnabled: + var ( + sig = c.Request().Header.Get("X-Webhook-Signature") + ) + + bs, err := app.bounce.Forwardemail.ProcessBounce([]byte(sig), rawReq) + if err != nil { + app.log.Printf("error processing forwardemail notification: %v", err) + if _, ok := err.(*echo.HTTPError); ok { + return err + } + + return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData")) + } + bounces = append(bounces, bs...) + // Postmark. case service == "postmark" && app.constants.BouncePostmarkEnabled: bs, err := app.bounce.Postmark.ProcessBounce(rawReq, c) diff --git a/cmd/init.go b/cmd/init.go index 049f959e6..c88d719fa 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -106,10 +106,11 @@ type constants struct { Extensions []string } - BounceWebhooksEnabled bool - BounceSESEnabled bool - BounceSendgridEnabled bool - BouncePostmarkEnabled bool + BounceWebhooksEnabled bool + BounceSESEnabled bool + BounceSendgridEnabled bool + BounceForwardemailEnabled bool + BouncePostmarkEnabled bool } type notifTpls struct { @@ -418,6 +419,7 @@ func initConstants() *constants { c.BounceWebhooksEnabled = ko.Bool("bounce.webhooks_enabled") c.BounceSESEnabled = ko.Bool("bounce.ses_enabled") c.BounceSendgridEnabled = ko.Bool("bounce.sendgrid_enabled") + c.BounceForwardemailEnabled = ko.Bool("bounce.forwardemail_enabled") c.BouncePostmarkEnabled = ko.Bool("bounce.postmark.enabled") b := md5.Sum([]byte(time.Now().String())) @@ -666,10 +668,12 @@ func initNotifTemplates(path string, fs stuffbin.FileSystem, i *i18n.I18n, cs *c // for incoming bounce events. func initBounceManager(app *App) *bounce.Manager { opt := bounce.Opt{ - WebhooksEnabled: ko.Bool("bounce.webhooks_enabled"), - SESEnabled: ko.Bool("bounce.ses_enabled"), - SendgridEnabled: ko.Bool("bounce.sendgrid_enabled"), - SendgridKey: ko.String("bounce.sendgrid_key"), + WebhooksEnabled: ko.Bool("bounce.webhooks_enabled"), + SESEnabled: ko.Bool("bounce.ses_enabled"), + SendgridEnabled: ko.Bool("bounce.sendgrid_enabled"), + SendgridKey: ko.String("bounce.sendgrid_key"), + ForwardemailEnabled: ko.Bool("bounce.forwardemail_enabled"), + ForwardemailKey: ko.String("bounce.forwardemail_key"), Postmark: struct { Enabled bool Username string diff --git a/cmd/settings.go b/cmd/settings.go index e6e03e2d8..4fc917af8 100644 --- a/cmd/settings.go +++ b/cmd/settings.go @@ -67,6 +67,8 @@ func handleGetSettings(c echo.Context) error { for i := 0; i < len(s.Messengers); i++ { s.Messengers[i].Password = strings.Repeat(pwdMask, utf8.RuneCountInString(s.Messengers[i].Password)) } + + s.ForwardemailKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.ForwardemailKey)) s.UploadS3AwsSecretAccessKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.UploadS3AwsSecretAccessKey)) s.SendgridKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.SendgridKey)) s.SecurityCaptchaSecret = strings.Repeat(pwdMask, utf8.RuneCountInString(s.SecurityCaptchaSecret)) @@ -195,6 +197,9 @@ func handleUpdateSettings(c echo.Context) error { if set.SendgridKey == "" { set.SendgridKey = cur.SendgridKey } + if set.ForwardemailKey == "" { + set.ForwardemailKey = cur.ForwardemailKey + } if set.BouncePostmark.Password == "" { set.BouncePostmark.Password = cur.BouncePostmark.Password } diff --git a/docs/docs/content/bounces.md b/docs/docs/content/bounces.md index f8c995759..3df2947cf 100644 --- a/docs/docs/content/bounces.md +++ b/docs/docs/content/bounces.md @@ -42,11 +42,12 @@ curl -u 'username:password' -X POST 'http://localhost:9000/webhooks/bounce' \ ## External webhooks listmonk supports receiving bounce webhook events from the following SMTP providers. -| Endpoint | Description | More info | -|:----------------------------------------------------------|:---------------------------------------|:----------------------------------------------------------------------------------------------------------------------| -| `https://listmonk.yoursite.com/webhooks/service/ses` | Amazon (AWS) SES | See below | -| `https://listmonk.yoursite.com/webhooks/service/sendgrid` | Sendgrid / Twilio Signed event webhook | [More info](https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features) | -| `https://listmonk.yoursite.com/webhooks/service/postmark` | Postmark webhook | [More info](https://postmarkapp.com/developer/webhooks/webhooks-overview) | +| Endpoint | Description | More info | +|:--------------------------------------------------------------|:---------------------------------------|:----------------------------------------------------------------------------------------------------------------------| +| `https://listmonk.yoursite.com/webhooks/service/ses` | Amazon (AWS) SES | See below | +| `https://listmonk.yoursite.com/webhooks/service/sendgrid` | Sendgrid / Twilio Signed event webhook | [More info](https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features) | +| `https://listmonk.yoursite.com/webhooks/service/postmark` | Postmark webhook | [More info](https://postmarkapp.com/developer/webhooks/webhooks-overview) | +| `https://listmonk.yoursite.com/webhooks/service/forwardemail` | Forward Email webhook | [More info](https://forwardemail.net/en/faq#do-you-support-bounce-webhooks) | ## Amazon Simple Email Service (SES) diff --git a/docs/swagger/collections.yaml b/docs/swagger/collections.yaml index d68df2489..b8577de8b 100644 --- a/docs/swagger/collections.yaml +++ b/docs/swagger/collections.yaml @@ -2703,6 +2703,8 @@ components: type: string settings.bounces.enableSendgrid: type: string + settings.bounces.enableForwardemail: + type: string settings.bounces.enablePostmark: type: string settings.bounces.enableWebhooks: @@ -2723,6 +2725,8 @@ components: type: string settings.bounces.sendgridKey: type: string + settings.bounces.forwardemailKey: + type: string settings.bounces.postmarkUsername: type: string settings.bounces.postmarkUsernameHelp: @@ -3406,6 +3410,10 @@ components: type: boolean bounce.sendgrid_key: type: string + bounce.forwardemail_enabled: + type: boolean + bounce.forwardemail_key: + type: string bounce.postmark_enabled: type: boolean bounce.postmark_username: diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue index bc412328c..1d695f4fe 100644 --- a/frontend/src/views/Settings.vue +++ b/frontend/src/views/Settings.vue @@ -166,6 +166,12 @@ export default Vue.extend({ hasDummy = 'postmark'; } + if (this.isDummy(form['bounce.forwardemail_key'])) { + form['bounce.forwardemail_key'] = ''; + } else if (this.hasDummy(form['bounce.forwardemail_key'])) { + hasDummy = 'forwardemail'; + } + for (let i = 0; i < form.messengers.length; i += 1) { // If it's the dummy UI password placeholder, ignore it. if (this.isDummy(form.messengers[i].password)) { diff --git a/frontend/src/views/settings/bounces.vue b/frontend/src/views/settings/bounces.vue index 2df8fb27a..886944ab8 100644 --- a/frontend/src/views/settings/bounces.vue +++ b/frontend/src/views/settings/bounces.vue @@ -59,6 +59,20 @@ +
+
+ + + +
+
+ + + +
+
diff --git a/frontend/src/views/settings/smtp.vue b/frontend/src/views/settings/smtp.vue index fe5c5b28f..6d6905426 100644 --- a/frontend/src/views/settings/smtp.vue +++ b/frontend/src/views/settings/smtp.vue @@ -67,6 +67,7 @@
+ Forward Email Gmail Amazon SES Mailgun @@ -220,6 +221,9 @@ const smtpTemplates = { sendgrid: { host: 'smtp.sendgrid.net', port: 465, auth_protocol: 'login', tls_type: 'TLS', }, + forwardemail: { + host: 'smtp.forwardemail.net', port: 465, auth_protocol: 'login', tls_type: 'TLS', + }, postmark: { host: 'smtp.postmarkapp.com', port: 587, auth_protocol: 'cram', tls_type: 'STARTTLS', }, diff --git a/i18n/en.json b/i18n/en.json index 91e4bcb8c..fd3df2fa4 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -369,12 +369,14 @@ "settings.bounces.enable": "Enable bounce processing", "settings.bounces.enableMailbox": "Enable bounce mailbox", "settings.bounces.enablePostmark": "Enable Postmark", + "settings.bounces.enableForwardemail": "Enable Forward Email", "settings.bounces.enableSES": "Enable SES", "settings.bounces.enableSendgrid": "Enable SendGrid", "settings.bounces.enableWebhooks": "Enable bounce webhooks", "settings.bounces.enabled": "Enabled", "settings.bounces.folder": "Folder", "settings.bounces.folderHelp": "Name of the IMAP folder to scan. Eg: Inbox.", + "settings.bounces.forwardemailKey": "Forward Email Key", "settings.bounces.invalidScanInterval": "Bounce scan interval should be minimum 1 minute.", "settings.bounces.name": "Bounces", "settings.bounces.none": "None", diff --git a/internal/bounce/bounce.go b/internal/bounce/bounce.go index 857a50a70..1a3303a97 100644 --- a/internal/bounce/bounce.go +++ b/internal/bounce/bounce.go @@ -26,14 +26,16 @@ type Mailbox interface { // Opt represents bounce processing options. type Opt struct { - MailboxEnabled bool `json:"mailbox_enabled"` - MailboxType string `json:"mailbox_type"` - Mailbox mailbox.Opt `json:"mailbox"` - WebhooksEnabled bool `json:"webhooks_enabled"` - SESEnabled bool `json:"ses_enabled"` - SendgridEnabled bool `json:"sendgrid_enabled"` - SendgridKey string `json:"sendgrid_key"` - Postmark struct { + MailboxEnabled bool `json:"mailbox_enabled"` + MailboxType string `json:"mailbox_type"` + Mailbox mailbox.Opt `json:"mailbox"` + WebhooksEnabled bool `json:"webhooks_enabled"` + SESEnabled bool `json:"ses_enabled"` + SendgridEnabled bool `json:"sendgrid_enabled"` + SendgridKey string `json:"sendgrid_key"` + ForwardemailEnabled bool `json:"forwardemail_enabled"` + ForwardemailKey string `json:"forwardemail_key"` + Postmark struct { Enabled bool Username string Password string @@ -44,14 +46,15 @@ type Opt struct { // Manager handles e-mail bounces. type Manager struct { - queue chan models.Bounce - mailbox Mailbox - SES *webhooks.SES - Sendgrid *webhooks.Sendgrid - Postmark *webhooks.Postmark - queries *Queries - opt Opt - log *log.Logger + queue chan models.Bounce + mailbox Mailbox + SES *webhooks.SES + Sendgrid *webhooks.Sendgrid + Postmark *webhooks.Postmark + Forwardemail *webhooks.Forwardemail + queries *Queries + opt Opt + log *log.Logger } // Queries contains the queries. @@ -93,6 +96,11 @@ func New(opt Opt, q *Queries, lo *log.Logger) (*Manager, error) { } } + if opt.ForwardemailEnabled { + fe := webhooks.NewForwardemail([]byte(opt.ForwardemailKey)) + m.Forwardemail = fe + } + if opt.Postmark.Enabled { m.Postmark = webhooks.NewPostmark(opt.Postmark.Username, opt.Postmark.Password) } diff --git a/internal/bounce/webhooks/forwardemail.go b/internal/bounce/webhooks/forwardemail.go new file mode 100644 index 000000000..dd17dd518 --- /dev/null +++ b/internal/bounce/webhooks/forwardemail.go @@ -0,0 +1,87 @@ +package webhooks + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/knadh/listmonk/models" +) + +type BounceDetails struct { + Action string `json:"action"` + Message string `json:"message"` + Category string `json:"category"` + Code int `json:"code"` + Status string `json:"status"` + Line int `json:"line"` +} + +type forwardemailNotif struct { + EmailID string `json:"email_id"` + ListID string `json:"list_id"` + ListUnsubscribe string `json:"list_unsubscribe"` + FeedbackID string `json:"feedback_id"` + Recipient string `json:"recipient"` + Message string `json:"message"` + Response string `json:"response"` + ResponseCode int `json:"response_code"` + TruthSource string `json:"truth_source"` + Headers map[string]string `json:"headers"` + Bounce BounceDetails `json:"bounce"` + BouncedAt time.Time `json:"bounced_at"` +} + +// Forwardemail handles webhook notifications (mainly bounce notifications). +type Forwardemail struct { + hmacKey []byte +} + +func NewForwardemail(key []byte) *Forwardemail { + return &Forwardemail{hmacKey: key} +} + +// ProcessBounce processes Forward Email bounce notifications and returns one object. +func (p *Forwardemail) ProcessBounce(sig, b []byte) ([]models.Bounce, error) { + key := []byte(p.hmacKey) + + mac := hmac.New(sha256.New, key) + + mac.Write(b) + + signature := mac.Sum(nil) + + if subtle.ConstantTimeCompare(signature, []byte(sig)) != 1 { + return nil, fmt.Errorf("invalid signature") + } + + var n forwardemailNotif + if err := json.Unmarshal(b, &n); err != nil { + return nil, fmt.Errorf("error unmarshalling Forwardemail notification: %v", err) + } + + typ := models.BounceTypeSoft + // TODO: support `typ = models.BounceTypeComplaint` in future + switch n.Bounce.Category { + case "block", "recipient", "virus", "spam": + typ = models.BounceTypeHard + } + + campUUID := "" + if v, ok := n.Headers["X-Listmonk-Campaign"]; ok { + campUUID = v + } + + return []models.Bounce{{ + Email: strings.ToLower(n.Recipient), + CampaignUUID: campUUID, + Type: typ, + Source: "forwardemail", + Meta: json.RawMessage(b), + CreatedAt: n.BouncedAt, + }}, nil +} diff --git a/models/settings.go b/models/settings.go index 577ea48ee..f0fd3663a 100644 --- a/models/settings.go +++ b/models/settings.go @@ -99,7 +99,9 @@ type Settings struct { Username string `json:"username"` Password string `json:"password"` } `json:"bounce.postmark"` - BounceBoxes []struct { + ForwardemailEnabled bool `json:"bounce.forwardemail_enabled"` + ForwardemailKey string `json:"bounce.forwardemail_key"` + BounceBoxes []struct { UUID string `json:"uuid"` Enabled bool `json:"enabled"` Type string `json:"type"` diff --git a/schema.sql b/schema.sql index 12276e4dd..44e33042c 100644 --- a/schema.sql +++ b/schema.sql @@ -268,6 +268,8 @@ INSERT INTO settings (key, value) VALUES ('bounce.enabled', 'false'), ('bounce.webhooks_enabled', 'false'), ('bounce.actions', '{"soft": {"count": 2, "action": "none"}, "hard": {"count": 1, "action": "blocklist"}, "complaint" : {"count": 1, "action": "blocklist"}}'), + ('bounce.forwardemail_enabled', 'false'), + ('bounce.forwardemail_key', '""'), ('bounce.ses_enabled', 'false'), ('bounce.sendgrid_enabled', 'false'), ('bounce.sendgrid_key', '""'),