Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
## main / (unreleased)

* [CHANGE] eventrecorder: All outputs now require a configured `name`, used instead of paths, URLs, brokers, topics, or other destination configuration in metric label values. Webhook URLs and Kafka configuration are also omitted from logs; file paths remain available in file-output error logs.

## 0.34.0 / 2026-08-16

* [CHANGE] notify: The `reason` label on `alertmanager_notifications_failed_total` now distinguishes `authError` (HTTP 401/403) and `rateLimited` (HTTP 429) from the generic `clientError`. Dashboards/alerts matching `reason="clientError"` for these codes must be updated. #5332
Expand Down
4 changes: 3 additions & 1 deletion config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,16 @@ receivers:
- name: default
event_recorder:
webhook_outputs:
- url: https://stream-id.ingest.cloudflare.com
- name: pipelines
url: https://stream-id.ingest.cloudflare.com
batch: true
http_config:
authorization:
credentials_file: pipeline-token
`)
require.NoError(t, err)
require.Len(t, cfg.EventRecorder.WebhookOutputs, 1)
require.Equal(t, "pipelines", cfg.EventRecorder.WebhookOutputs[0].Name)
require.True(t, cfg.EventRecorder.WebhookOutputs[0].Batch)

resolveFilepaths("/etc/alertmanager", cfg)
Expand Down
26 changes: 23 additions & 3 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -2146,7 +2146,14 @@ Event recording is configured under the top-level `event_recorder` key.

Outputs are grouped by type, one list per destination kind (mirroring the
way receivers group their integrations). Every recorded event is sent to
every output across all lists.
every output across all lists. Every output requires a name, which is used
with its type as the output identifier in metrics and logs (for example,
`webhook:primary`). Destination configuration such as paths, URLs, brokers,
and topics is not included in metric labels. URLs, brokers, and topics are
also omitted from logs; file paths remain in file-output error logs for
troubleshooting. Names must be unique within each output type, no longer than
128 characters, and contain only letters, digits, hyphens, underscores, and
periods.

```yaml
# JSONL file outputs.
Expand All @@ -2173,6 +2180,9 @@ when the parent directory observes a rename/remove/create on the target
path (for compatibility with `logrotate` and similar tools).

```yaml
# Name used to identify this output in metrics and logs.
name: <string>

# Path to the JSONL output file. Will be created if it does not exist.
path: <filepath>
```
Expand All @@ -2188,6 +2198,9 @@ duplicate events after ambiguous failures. With multiple workers, requests
may complete out of order; set `workers: 1` when request ordering matters.

```yaml
# Name used to identify this output in metrics and logs.
name: <string>

# URL to POST events to.
url: <secret>

Expand Down Expand Up @@ -2230,7 +2243,8 @@ as a batched webhook output:
```yaml
event_recorder:
webhook_outputs:
- url: https://<stream-id>.ingest.cloudflare.com
- name: pipelines
url: https://<stream-id>.ingest.cloudflare.com
batch: true
http_config:
# The token must have the "Workers Pipeline Send" permission when
Expand All @@ -2255,6 +2269,9 @@ The target topic must already exist (or the brokers must be configured to
auto-create topics); Alertmanager does not create it.

```yaml
# Name used to identify this output in metrics and logs.
name: <string>

# Seed broker list (host:port). At least one entry is required.
brokers:
[ - <string> ... ]
Expand Down Expand Up @@ -2302,4 +2319,7 @@ driver (Docker, Kubernetes, etc.) captures stdout automatically.
> distinct formats on the same stream that may complicate downstream
> log parsing.

This output type takes no additional configuration fields.
```yaml
# Name used to identify this output in metrics and logs.
name: <string>
```
75 changes: 75 additions & 0 deletions eventrecorder/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@

package eventrecorder

import (
"fmt"
)

const maxOutputNameLength = 128

// Config configures the event recorder feature.
//
// Outputs are grouped by type, one list per destination kind, mirroring
Expand All @@ -26,6 +32,75 @@ type Config struct {
StdoutOutputs []StdoutOutputConfig `yaml:"stdout_outputs,omitempty" json:"stdout_outputs,omitempty"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface, validating that
// each output identifier is unique.
func (c *Config) UnmarshalYAML(unmarshal func(any) error) error {
type plain Config
if err := unmarshal((*plain)(c)); err != nil {
return err
}
return c.validate()
}

func (c Config) validate() error {
seen := make(map[string]struct{}, c.totalOutputs())
add := func(kind, name string) error {
id, err := outputIdentifier(kind, name)
if err != nil {
return err
}
if _, ok := seen[id]; ok {
return fmt.Errorf("event_recorder output name %q is duplicated for type %s", name, kind)
}
seen[id] = struct{}{}
return nil
}
for _, out := range c.FileOutputs {
if err := add("file", out.Name); err != nil {
return err
}
}
for _, out := range c.WebhookOutputs {
if err := add("webhook", out.Name); err != nil {
return err
}
}
for _, out := range c.KafkaOutputs {
if err := add("kafka", out.Name); err != nil {
return err
}
}
for _, out := range c.StdoutOutputs {
if err := add("stdout", out.Name); err != nil {
return err
}
}
return nil
}

func outputIdentifier(kind, name string) (string, error) {
if name == "" {
return "", fmt.Errorf("event_recorder %s output requires a name", kind)
}
if len(name) > maxOutputNameLength {
return "", fmt.Errorf("event_recorder %s output name must not exceed %d characters", kind, maxOutputNameLength)
}
for _, r := range name {
if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '-' && r != '_' && r != '.' {
return "", fmt.Errorf("event_recorder %s output name must contain only letters, digits, hyphens, underscores, and periods", kind)
}
}
return kind + ":" + name, nil
}

func safeOutputIdentifier(kind, name string) string {
id, err := outputIdentifier(kind, name)
if err != nil {
return kind + ":<invalid>"
}
return id
}

// totalOutputs returns the number of configured outputs across all
// destination kinds.
func (c Config) totalOutputs() int {
Expand Down
32 changes: 23 additions & 9 deletions eventrecorder/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import (

// FileOutputConfig configures a JSONL file event recorder output.
type FileOutputConfig struct {
// Name identifies this output in metrics and logs.
Name string `yaml:"name" json:"name"`
// Path is the JSONL file to append events to. Created if absent.
Path string `yaml:"path" json:"path"`
}
Expand All @@ -38,6 +40,9 @@ type FileOutputConfig struct {
func (c *FileOutputConfig) UnmarshalYAML(unmarshal func(any) error) error {
type plain FileOutputConfig
if err := unmarshal((*plain)(c)); err != nil {
return errors.New("invalid event_recorder file output configuration")
}
if _, err := outputIdentifier("file", c.Name); err != nil {
return err
}
if c.Path == "" {
Expand All @@ -48,14 +53,15 @@ func (c *FileOutputConfig) UnmarshalYAML(unmarshal func(any) error) error {

// equal reports whether two file output configs are semantically equal.
func (c FileOutputConfig) equal(o FileOutputConfig) bool {
return c.Path == o.Path
return c.Name == o.Name && c.Path == o.Path
}

// FileOutput writes pre-serialized JSON event bytes to a JSONL file.
// The file is reopened when fsnotify detects a rename or remove (e.g.
// logrotate).
type FileOutput struct {
path string
name string
mu sync.Mutex
f *os.File
closed bool
Expand All @@ -66,20 +72,28 @@ type FileOutput struct {

// Name returns a stable identifier for this output.
func (fo *FileOutput) Name() string {
return fmt.Sprintf("file:%s", fo.path)
return fo.name
}

// NewFileOutput creates a new file-based event recorder output at the given
// path. The file is watched with fsnotify so that external log
// NewFileOutput creates a new file-based event recorder output. The file is
// watched with fsnotify so that external log
// rotation tools (e.g., logrotate) trigger an immediate reopen.
func NewFileOutput(path string, logger *slog.Logger) (*FileOutput, error) {
f, err := openAppend(path)
func NewFileOutput(cfg FileOutputConfig, logger *slog.Logger) (*FileOutput, error) {
name, err := outputIdentifier("file", cfg.Name)
if err != nil {
return nil, err
}
if cfg.Path == "" {
return nil, errors.New("file output requires a path")
}
f, err := openAppend(cfg.Path)
if err != nil {
return nil, err
}

fo := &FileOutput{
path: path,
path: cfg.Path,
name: name,
f: f,
logger: logger,
done: make(chan struct{}),
Expand Down Expand Up @@ -111,7 +125,7 @@ func (fo *FileOutput) reopen() {
}
f, err := openAppend(fo.path)
if err != nil {
fo.logger.Error("Failed to reopen event recorder file", "path", fo.path, "err", err)
fo.logger.Error("Failed to reopen event recorder file", "output", fo.name, "path", fo.path, "err", err)
return
}
fo.f = f
Expand Down Expand Up @@ -163,7 +177,7 @@ func (fo *FileOutput) watchLoop(ready chan<- error) {
if !ok {
return
}
fo.logger.Error("fsnotify error on event recorder directory", "err", err)
fo.logger.Error("fsnotify error on event recorder directory", "output", fo.name, "path", fo.path, "err", err)
case <-fo.done:
return
}
Expand Down
29 changes: 19 additions & 10 deletions eventrecorder/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ func TestFileOutput_SendEvent(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "events.jsonl")

fo, err := NewFileOutput(path, slog.Default())
fo, err := NewFileOutput(FileOutputConfig{Name: "primary", Path: path}, slog.Default())
require.NoError(t, err)
defer fo.Close()

require.Equal(t, "file:"+path, fo.Name())
require.Equal(t, "file:primary", fo.Name())

n1, err := fo.SendEvent(sampleEvent())
require.NoError(t, err)
Expand All @@ -57,7 +57,7 @@ func TestFileOutput_ReopenAfterRename(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "events.jsonl")

fo, err := NewFileOutput(path, slog.Default())
fo, err := NewFileOutput(FileOutputConfig{Name: "rotate", Path: path}, slog.Default())
require.NoError(t, err)
defer fo.Close()

Expand Down Expand Up @@ -93,7 +93,7 @@ func TestFileOutput_ReopenAfterRemove(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "events.jsonl")

fo, err := NewFileOutput(path, slog.Default())
fo, err := NewFileOutput(FileOutputConfig{Name: "remove", Path: path}, slog.Default())
require.NoError(t, err)
defer fo.Close()

Expand All @@ -120,7 +120,7 @@ func TestFileOutput_Close(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "events.jsonl")

fo, err := NewFileOutput(path, slog.Default())
fo, err := NewFileOutput(FileOutputConfig{Name: "close", Path: path}, slog.Default())
require.NoError(t, err)

_, err = fo.SendEvent(sampleEvent())
Expand All @@ -133,7 +133,7 @@ func TestFileOutput_Close(t *testing.T) {
}

func TestFileOutput_InvalidPath(t *testing.T) {
_, err := NewFileOutput("/nonexistent/dir/events.jsonl", slog.Default())
_, err := NewFileOutput(FileOutputConfig{Name: "invalid", Path: "/nonexistent/dir/events.jsonl"}, slog.Default())
require.Error(t, err)
}

Expand All @@ -148,14 +148,20 @@ func TestFileOutputConfig_UnmarshalYAML(t *testing.T) {
}{
{
name: "valid",
yaml: "path: /tmp/events.jsonl\n",
yaml: "name: primary\npath: /tmp/events.jsonl\n",
check: func(t *testing.T, c FileOutputConfig) {
require.Equal(t, "primary", c.Name)
require.Equal(t, "/tmp/events.jsonl", c.Path)
},
},
{
name: "missing path",
yaml: "{}\n",
yaml: "name: primary\n",
wantErr: true,
},
{
name: "missing name",
yaml: "path: /tmp/events.jsonl\n",
wantErr: true,
},
}
Expand All @@ -176,10 +182,13 @@ func TestFileOutputConfig_UnmarshalYAML(t *testing.T) {
}

func TestEventRecorderConfigEqual_File(t *testing.T) {
a := Config{FileOutputs: []FileOutputConfig{{Path: "/tmp/events.jsonl"}}}
b := Config{FileOutputs: []FileOutputConfig{{Path: "/tmp/events.jsonl"}}}
a := Config{FileOutputs: []FileOutputConfig{{Name: "primary", Path: "/tmp/events.jsonl"}}}
b := Config{FileOutputs: []FileOutputConfig{{Name: "primary", Path: "/tmp/events.jsonl"}}}
require.True(t, configEqual(a, b))

b.FileOutputs[0].Path = "/tmp/other.jsonl"
require.False(t, configEqual(a, b))
b.FileOutputs[0].Path = a.FileOutputs[0].Path
b.FileOutputs[0].Name = "secondary"
require.False(t, configEqual(a, b))
}
Loading
Loading