diff --git a/notify/discord/discord.go b/notify/discord/discord.go index 2cc6d0170b..9a6d6cc9b8 100644 --- a/notify/discord/discord.go +++ b/notify/discord/discord.go @@ -101,7 +101,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) alerts := types.Alerts(as...) data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmpl := notify.TmplText(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) if err != nil { return false, err } diff --git a/notify/email/email.go b/notify/email/email.go index 6ffb99ba3f..3006943c92 100644 --- a/notify/email/email.go +++ b/notify/email/email.go @@ -215,7 +215,7 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { var ( tmplErr error data = notify.GetTemplateData(ctx, n.tmpl, as, n.logger) - tmpl = notify.TmplText(n.tmpl, data, &tmplErr) + tmpl = notify.TmplTextWithLogger(n.tmpl, data, &tmplErr, n.logger) ) from := tmpl(n.conf.From) if tmplErr != nil { diff --git a/notify/jira/jira.go b/notify/jira/jira.go index 428ac89a1d..5f3bba3b8d 100644 --- a/notify/jira/jira.go +++ b/notify/jira/jira.go @@ -77,7 +77,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) tmplTextErr error data = notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmplText = notify.TmplText(n.tmpl, data, &tmplTextErr) + tmplText = notify.TmplTextWithLogger(n.tmpl, data, &tmplTextErr, logger) tmplTextFunc = func(tmpl string) (string, error) { return tmplText(tmpl), tmplTextErr } diff --git a/notify/mattermost/mattermost.go b/notify/mattermost/mattermost.go index 9522763ff7..bb2c5601a0 100644 --- a/notify/mattermost/mattermost.go +++ b/notify/mattermost/mattermost.go @@ -118,7 +118,8 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err return false, errors.New("webhook url missing") } - req := n.createRequest(notify.TmplText(n.tmpl, data, &err)) + tmplText := notify.TmplTextWithLogger(n.tmpl, data, &err, n.logger) + req := n.createRequest(tmplText) if err != nil { return false, err } diff --git a/notify/msteams/msteams.go b/notify/msteams/msteams.go index cf7748e27f..4bde49738e 100644 --- a/notify/msteams/msteams.go +++ b/notify/msteams/msteams.go @@ -90,7 +90,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) logger.Debug("extracted group key") data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmpl := notify.TmplText(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) if err != nil { return false, err } diff --git a/notify/msteamsv2/msteamsv2.go b/notify/msteamsv2/msteamsv2.go index 5602d503dc..ada5d427e0 100644 --- a/notify/msteamsv2/msteamsv2.go +++ b/notify/msteamsv2/msteamsv2.go @@ -114,7 +114,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) logger.Debug("extracted group key") data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmpl := notify.TmplText(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) if err != nil { return false, err } diff --git a/notify/opsgenie/opsgenie.go b/notify/opsgenie/opsgenie.go index 96a84894e6..593721209f 100644 --- a/notify/opsgenie/opsgenie.go +++ b/notify/opsgenie/opsgenie.go @@ -137,7 +137,7 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmpl := notify.TmplText(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) details := make(map[string]string) diff --git a/notify/pagerduty/pagerduty.go b/notify/pagerduty/pagerduty.go index 8532723992..271cbaf0ab 100644 --- a/notify/pagerduty/pagerduty.go +++ b/notify/pagerduty/pagerduty.go @@ -151,7 +151,7 @@ func (n *Notifier) notifyV1( details map[string]any, ) (bool, error) { var tmplErr error - tmpl := notify.TmplText(n.tmpl, data, &tmplErr) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &tmplErr, n.logger) description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes) if truncated { @@ -211,7 +211,7 @@ func (n *Notifier) notifyV2( details map[string]any, ) (bool, error) { var tmplErr error - tmpl := notify.TmplText(n.tmpl, data, &tmplErr) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &tmplErr, n.logger) if n.conf.Severity == "" { n.conf.Severity = "error" @@ -367,7 +367,7 @@ func (n *Notifier) renderDetails( ) (map[string]any, error) { var ( tmplTextErr error - tmplText = notify.TmplText(n.tmpl, data, &tmplTextErr) + tmplText = notify.TmplTextWithLogger(n.tmpl, data, &tmplTextErr, n.logger) tmplTextFunc = func(tmpl string) (string, error) { return tmplText(tmpl), tmplTextErr } diff --git a/notify/pagerduty/pagerduty_test.go b/notify/pagerduty/pagerduty_test.go index 302b4c3d3b..96d0f90f23 100644 --- a/notify/pagerduty/pagerduty_test.go +++ b/notify/pagerduty/pagerduty_test.go @@ -875,7 +875,8 @@ func TestRenderDetails(t *testing.T) { conf: &config.PagerdutyConfig{ Details: tt.args.details, }, - tmpl: test.CreateTmpl(t), + tmpl: test.CreateTmpl(t), + logger: promslog.NewNopLogger(), } got, err := n.renderDetails(tt.args.data) if (err != nil) != tt.wantErr { diff --git a/notify/pushover/pushover.go b/notify/pushover/pushover.go index 3f5155668c..b19015fd13 100644 --- a/notify/pushover/pushover.go +++ b/notify/pushover/pushover.go @@ -81,8 +81,8 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) err error message string ) - tmpl := notify.TmplText(n.tmpl, data, &err) - tmplHTML := notify.TmplHTML(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) + tmplHTML := notify.TmplHTMLWithLogger(n.tmpl, data, &err, logger) var ( token string diff --git a/notify/rocketchat/rocketchat.go b/notify/rocketchat/rocketchat.go index 8d1a6f4085..3a95cbd2fb 100644 --- a/notify/rocketchat/rocketchat.go +++ b/notify/rocketchat/rocketchat.go @@ -148,7 +148,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) logger.Debug("extracted group key") data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmplText := notify.TmplText(n.tmpl, data, &err) + tmplText := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) if err != nil { return false, err } diff --git a/notify/slack/slack.go b/notify/slack/slack.go index f96efe61fa..ddea425ba3 100644 --- a/notify/slack/slack.go +++ b/notify/slack/slack.go @@ -65,7 +65,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) var ( data = notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmplText = notify.TmplText(n.tmpl, data, &err) + tmplText = notify.TmplTextWithLogger(n.tmpl, data, &err, logger) ) var markdownIn []string diff --git a/notify/sns/sns.go b/notify/sns/sns.go index 873e3328d1..7152a16f48 100644 --- a/notify/sns/sns.go +++ b/notify/sns/sns.go @@ -104,7 +104,7 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err var ( tmplErr error data = notify.GetTemplateData(ctx, n.tmpl, alert, n.logger) - tmpl = notify.TmplText(n.tmpl, data, &tmplErr) + tmpl = notify.TmplTextWithLogger(n.tmpl, data, &tmplErr, n.logger) ) client, err := n.createSNSClient(ctx, tmpl, &tmplErr) diff --git a/notify/telegram/telegram.go b/notify/telegram/telegram.go index 82919cf5f6..5c8a19d292 100644 --- a/notify/telegram/telegram.go +++ b/notify/telegram/telegram.go @@ -75,14 +75,14 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err var ( err error data = notify.GetTemplateData(ctx, n.tmpl, alert, logger) - tmpl = notify.TmplText(n.tmpl, data, &err) + tmpl = notify.TmplTextWithLogger(n.tmpl, data, &err, logger) messageText string truncated bool ) switch n.conf.ParseMode { case "HTML": - tmpl = notify.TmplHTML(n.tmpl, data, &err) + tmpl = notify.TmplHTMLWithLogger(n.tmpl, data, &err, logger) messageText = tmpl(n.conf.Message) if err != nil { return false, err diff --git a/notify/util.go b/notify/util.go index fe4c9ea508..303ff05721 100644 --- a/notify/util.go +++ b/notify/util.go @@ -168,6 +168,44 @@ func TmplHTML(tmpl *template.Template, data *template.Data, err *error) func(str } } +// TmplTextWithLogger is like TmplText but additionally logs a warning whenever +// a template fails to render. This makes template errors visible in Alertmanager +// logs even when the calling notifier continues to deliver the notification with +// the partially rendered output. +func TmplTextWithLogger(tmpl *template.Template, data *template.Data, err *error, logger *slog.Logger) func(string) string { + return func(name string) (s string) { + if *err != nil { + return s + } + var tmplErr error + s, tmplErr = tmpl.ExecuteTextString(name, data) + if tmplErr != nil { + logger.Warn("template execution failed", "template", name, "err", tmplErr) + *err = tmplErr + } + return s + } +} + +// TmplHTMLWithLogger is like TmplHTML but additionally logs a warning whenever +// a template fails to render. This makes template errors visible in Alertmanager +// logs even when the calling notifier continues to deliver the notification with +// the partially rendered output. +func TmplHTMLWithLogger(tmpl *template.Template, data *template.Data, err *error, logger *slog.Logger) func(string) string { + return func(name string) (s string) { + if *err != nil { + return s + } + var tmplErr error + s, tmplErr = tmpl.ExecuteHTMLString(name, data) + if tmplErr != nil { + logger.Warn("template execution failed", "template", name, "err", tmplErr) + *err = tmplErr + } + return s + } +} + // Key is a string that can be hashed. type Key string diff --git a/notify/util_test.go b/notify/util_test.go index 2c2d4922e8..3c5ce3fe65 100644 --- a/notify/util_test.go +++ b/notify/util_test.go @@ -273,3 +273,53 @@ func TestGetFailureReasonFromStatusCode(t *testing.T) { }) } } + +func TestTmplTextWithLogger_LogsOnError(t *testing.T) { + tmpl, err := template.FromGlobs([]string{}) + require.NoError(t, err) + + data := &template.Data{} + var tmplErr error + + var buf bytes.Buffer + logger := promslog.New(&promslog.Config{Writer: &buf}) + + tmplFn := TmplTextWithLogger(tmpl, data, &tmplErr, logger) + + // An unclosed action causes a parse/execute error. + result := tmplFn("{{ .Missing") + + require.Error(t, tmplErr) + require.Empty(t, result) + logOutput := buf.String() + require.Contains(t, logOutput, "template execution failed") + require.Contains(t, logOutput, "{{ .Missing") // template name is logged + require.Contains(t, logOutput, tmplErr.Error()) // rendering error is logged + + // Subsequent calls must short-circuit without logging again. + buf.Reset() + _ = tmplFn("{{ .AnotherField }}") + require.Empty(t, buf.String(), "no second log expected after first error") +} + +func TestTmplHTMLWithLogger_LogsOnError(t *testing.T) { + tmpl, err := template.FromGlobs([]string{}) + require.NoError(t, err) + + data := &template.Data{} + var tmplErr error + + var buf bytes.Buffer + logger := promslog.New(&promslog.Config{Writer: &buf}) + + tmplFn := TmplHTMLWithLogger(tmpl, data, &tmplErr, logger) + + result := tmplFn("{{ .Missing") + + require.Error(t, tmplErr) + require.Empty(t, result) + logOutput := buf.String() + require.Contains(t, logOutput, "template execution failed") + require.Contains(t, logOutput, "{{ .Missing") // template name is logged + require.Contains(t, logOutput, tmplErr.Error()) // rendering error is logged +} diff --git a/notify/victorops/victorops.go b/notify/victorops/victorops.go index 746a10b55d..846cc9c963 100644 --- a/notify/victorops/victorops.go +++ b/notify/victorops/victorops.go @@ -71,7 +71,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) var err error var ( data = notify.GetTemplateData(ctx, n.tmpl, as, n.logger) - tmpl = notify.TmplText(n.tmpl, data, &err) + tmpl = notify.TmplTextWithLogger(n.tmpl, data, &err, n.logger) apiURL = n.conf.APIURL.Copy() ) @@ -125,7 +125,7 @@ func (n *Notifier) createVictorOpsPayload(ctx context.Context, as ...*types.Aler var ( alerts = types.Alerts(as...) data = notify.GetTemplateData(ctx, n.tmpl, as, n.logger) - tmpl = notify.TmplText(n.tmpl, data, &err) + tmpl = notify.TmplTextWithLogger(n.tmpl, data, &err, n.logger) messageType = tmpl(n.conf.MessageType) stateMessage = tmpl(n.conf.StateMessage) diff --git a/notify/webex/webex.go b/notify/webex/webex.go index 949bb614f8..9d70f4e9de 100644 --- a/notify/webex/webex.go +++ b/notify/webex/webex.go @@ -76,7 +76,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) logger.Debug("extracted group key") data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmpl := notify.TmplText(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) if err != nil { return false, err } diff --git a/notify/webhook/webhook.go b/notify/webhook/webhook.go index b0f4913ca7..d37c8c2835 100644 --- a/notify/webhook/webhook.go +++ b/notify/webhook/webhook.go @@ -110,7 +110,7 @@ func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, er var url string var tmplErr error - tmpl := notify.TmplText(n.tmpl, data, &tmplErr) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &tmplErr, logger) if n.conf.URL != "" { url = tmpl(string(n.conf.URL)) @@ -157,7 +157,7 @@ func (n *Notifier) renderPayload( ) (bytes.Buffer, error) { var ( tmplTextErr error - tmplText = notify.TmplText(n.tmpl, data.Data, &tmplTextErr) + tmplText = notify.TmplTextWithLogger(n.tmpl, data.Data, &tmplTextErr, n.logger) tmplTextFunc = func(tmpl string) (string, error) { return tmplText(tmpl), tmplTextErr } diff --git a/notify/wechat/wechat.go b/notify/wechat/wechat.go index 35d418931d..3536975b55 100644 --- a/notify/wechat/wechat.go +++ b/notify/wechat/wechat.go @@ -93,7 +93,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) data := notify.GetTemplateData(ctx, n.tmpl, as, logger) - tmpl := notify.TmplText(n.tmpl, data, &err) + tmpl := notify.TmplTextWithLogger(n.tmpl, data, &err, logger) if err != nil { return false, err }