feat(limit): drop resolved alerts without an admitted firing counterpart

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## main / (unreleased)
 
+* [CHANGE] limit: `alertmanager_alerts_limited_total` now has a `state` label (`firing`/`resolved`) to distinguish dropped firing alerts from dropped resolved notifications. Dashboards/alerts using this metric must be updated.
+* [CHANGE] limit: With `--alerts.per-alertname-limit` set, a resolved notification is only forwarded if its firing counterpart was previously admitted (which frees its slot); resolved notifications with no admitted firing alert are now dropped.
+
 ## 0.34.0 / 2026-08-16
 
 * [CHANGE] notify: The `reason` label on `alertmanager_notifications_failed_total` now distinguishes `authError` (HTTP 401/403) and `rateLimited` (HTTP 429) from the generic `clientError`. Dashboards/alerts matching `reason="clientError"` for these codes must be updated. #5332

docs/alertmanager.md

@@ -78,11 +78,19 @@ It's important not to load balance traffic between Prometheus and its Alertmanag
 Alertmanager supports configuration to limit the number of active alerts per alertname.
 This can be configured using the [--alerts.per-alertname-limit] flag.
 
-When the limit is reached any new alerts are dropped, heartbeats from already know alerts are processed.
-The known alert (fingerprint) automatically expire to make room for new alerts.
+When the limit is reached any new firing alerts are dropped, while heartbeats from
+already known alerts are still processed. Known alerts (fingerprints) automatically
+expire to make room for new alerts.
+
+Resolved notifications are handled specially. A resolved notification is only
+forwarded if its firing counterpart was previously admitted (its fingerprint is
+still tracked); forwarding it frees the slot the firing alert was holding. A
+resolved notification with no previously admitted firing alert is dropped as
+noise, since nothing downstream ever received a firing alert for it.
 
 This feature is useful when an unexpected high number of instances of the same alert are sent to Alertmanager.
 Limiting the number of alerts per alertname can prevent reliability issues and avoid alert receivers from being flooded.
 
 The `alertmanager_alerts_limited_total` metric shows the total number of alerts that were dropped due to per alert name limit.
+The `state` label distinguishes dropped `firing` alerts from dropped `resolved` notifications.
 Enabling the `alert-names-in-metrics` feature flag will add the `alertname` label to the metric.

docs/configuration.md

@@ -29,7 +29,13 @@ use the `--silences.max-silences` flag.
 You can limit the maximum size of individual silences with `--silences.max-silence-size-bytes`,
 where the unit is in bytes.
 
-Both limits are disabled by default.
+To limit the maximum number of active alerts per alertname, use the
+`--alerts.per-alertname-limit` flag. When the limit is reached, new firing alerts
+for that alertname are dropped. Resolved notifications are only forwarded when
+their firing counterpart was previously admitted; otherwise they are dropped. See
+[Alert limits](alertmanager.md#alert-limits-optional) for details.
+
+All limits are disabled by default.
 
 ## Configuration file introduction
 

limit/bucket.go

@@ -97,6 +97,21 @@ func NewBucket[V comparable](capacity int) *Bucket[V] {
 	}
 }
 
+// Remove deletes the value from the bucket, freeing its slot.
+// It returns true if the value was present.
+func (b *Bucket[V]) Remove(value V) bool {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+
+	item, ok := b.index[value]
+	if !ok {
+		return false
+	}
+	heap.Remove(&b.items, item.index)
+	delete(b.index, value)
+	return true
+}
+
 // IsStale returns true if the latest item in the bucket is expired.
 func (b *Bucket[V]) IsStale() (stale bool) {
 	b.mtx.Lock()

limit/bucket_test.go

@@ -238,6 +238,116 @@ func TestBucketAddEdgeCases(t *testing.T) {
 	})
 }
 
+func TestBucketRemove(t *testing.T) {
+	t.Run("Remove present fingerprint returns true and frees its slot", func(t *testing.T) {
+		bucket := NewBucket[model.Fingerprint](2)
+		alert := model.Alert{Labels: model.LabelSet{"alertname": "Alert1"}, EndsAt: time.Now().Add(1 * time.Hour)}
+
+		require.True(t, bucket.Upsert(alert.Fingerprint(), alert.EndsAt), "alert should be added")
+
+		require.True(t, bucket.Remove(alert.Fingerprint()), "removing a present fingerprint should return true")
+		require.Empty(t, bucket.index, "index should be empty after removal")
+		require.Zero(t, bucket.items.Len(), "heap should be empty after removal")
+	})
+
+	t.Run("Remove absent fingerprint returns false", func(t *testing.T) {
+		bucket := NewBucket[model.Fingerprint](2)
+		present := model.Alert{Labels: model.LabelSet{"alertname": "Present"}, EndsAt: time.Now().Add(1 * time.Hour)}
+		bucket.Upsert(present.Fingerprint(), present.EndsAt)
+
+		absent := model.Alert{Labels: model.LabelSet{"alertname": "Absent"}}
+		require.False(t, bucket.Remove(absent.Fingerprint()), "removing an absent fingerprint should return false")
+		require.Len(t, bucket.index, 1, "present item should remain")
+		require.Equal(t, 1, bucket.items.Len(), "present item should remain in heap")
+	})
+
+	t.Run("Remove from empty bucket returns false", func(t *testing.T) {
+		bucket := NewBucket[model.Fingerprint](2)
+		alert := model.Alert{Labels: model.LabelSet{"alertname": "Alert1"}}
+		require.False(t, bucket.Remove(alert.Fingerprint()), "removing from an empty bucket should return false")
+	})
+
+	t.Run("Removing twice returns false the second time", func(t *testing.T) {
+		bucket := NewBucket[model.Fingerprint](2)
+		alert := model.Alert{Labels: model.LabelSet{"alertname": "Alert1"}, EndsAt: time.Now().Add(1 * time.Hour)}
+		bucket.Upsert(alert.Fingerprint(), alert.EndsAt)
+
+		require.True(t, bucket.Remove(alert.Fingerprint()), "first removal should return true")
+		require.False(t, bucket.Remove(alert.Fingerprint()), "second removal should return false")
+	})
+
+	t.Run("Remove frees a slot in a full bucket so a new alert is admitted", func(t *testing.T) {
+		bucket := NewBucket[model.Fingerprint](2)
+		a := model.Alert{Labels: model.LabelSet{"alertname": "A"}, EndsAt: time.Now().Add(1 * time.Hour)}
+		b := model.Alert{Labels: model.LabelSet{"alertname": "B"}, EndsAt: time.Now().Add(1 * time.Hour)}
+		c := model.Alert{Labels: model.LabelSet{"alertname": "C"}, EndsAt: time.Now().Add(1 * time.Hour)}
+
+		require.True(t, bucket.Upsert(a.Fingerprint(), a.EndsAt), "A should be added")
+		require.True(t, bucket.Upsert(b.Fingerprint(), b.EndsAt), "B should be added")
+		require.False(t, bucket.Upsert(c.Fingerprint(), c.EndsAt), "C should be rejected while bucket is full of active items")
+
+		require.True(t, bucket.Remove(a.Fingerprint()), "A should be removed")
+		require.True(t, bucket.Upsert(c.Fingerprint(), c.EndsAt), "C should be admitted after A freed a slot")
+
+		_, hasA := bucket.index[a.Fingerprint()]
+		_, hasC := bucket.index[c.Fingerprint()]
+		require.False(t, hasA, "A should no longer be present")
+		require.True(t, hasC, "C should now be present")
+		require.Len(t, bucket.index, 2, "bucket should hold B and C")
+	})
+
+	t.Run("Remove keeps heap eviction order intact", func(t *testing.T) {
+		bucket := NewBucket[model.Fingerprint](3)
+		oldest := model.Alert{Labels: model.LabelSet{"alertname": "Oldest"}, EndsAt: time.Now().Add(-2 * time.Hour)}
+		middle := model.Alert{Labels: model.LabelSet{"alertname": "Middle"}, EndsAt: time.Now().Add(-1 * time.Hour)}
+		newest := model.Alert{Labels: model.LabelSet{"alertname": "Newest"}, EndsAt: time.Now().Add(1 * time.Hour)}
+
+		bucket.Upsert(oldest.Fingerprint(), oldest.EndsAt)
+		bucket.Upsert(middle.Fingerprint(), middle.EndsAt)
+		bucket.Upsert(newest.Fingerprint(), newest.EndsAt)
+
+		// Remove the current heap root (oldest); the next-oldest expired item
+		// must then sit at the root and be the one evicted when full.
+		require.True(t, bucket.Remove(oldest.Fingerprint()), "oldest should be removed")
+		require.Equal(t, middle.Fingerprint(), bucket.items[0].value, "middle should be the new heap root")
+
+		// Bucket has room for one more; fill it, then force an eviction.
+		other := model.Alert{Labels: model.LabelSet{"alertname": "Other"}, EndsAt: time.Now().Add(2 * time.Hour)}
+		bucket.Upsert(other.Fingerprint(), other.EndsAt)
+
+		evictor := model.Alert{Labels: model.LabelSet{"alertname": "Evictor"}, EndsAt: time.Now().Add(3 * time.Hour)}
+		require.True(t, bucket.Upsert(evictor.Fingerprint(), evictor.EndsAt), "evictor should replace the expired middle item")
+
+		_, hasMiddle := bucket.index[middle.Fingerprint()]
+		require.False(t, hasMiddle, "expired middle item should have been evicted")
+		require.Len(t, bucket.index, 3, "index and heap should stay consistent")
+		require.Equal(t, 3, bucket.items.Len(), "index and heap should stay consistent")
+	})
+}
+
+func TestBucketRemoveConcurrency(t *testing.T) {
+	bucket := NewBucket[model.Fingerprint](2)
+	alert1 := model.Alert{Labels: model.LabelSet{"alertname": "Alert1"}, EndsAt: time.Now().Add(1 * time.Hour)}
+	alert2 := model.Alert{Labels: model.LabelSet{"alertname": "Alert2"}, EndsAt: time.Now().Add(1 * time.Hour)}
+	bucket.Upsert(alert1.Fingerprint(), alert1.EndsAt)
+	bucket.Upsert(alert2.Fingerprint(), alert2.EndsAt)
+
+	done := make(chan bool, 2)
+	go func() {
+		bucket.Remove(alert1.Fingerprint())
+		done <- true
+	}()
+	go func() {
+		bucket.Remove(alert2.Fingerprint())
+		done <- true
+	}()
+	<-done
+	<-done
+
+	require.Empty(t, bucket.index, "both alerts should be removed after concurrent removes")
+	require.Zero(t, bucket.items.Len(), "heap should be empty after concurrent removes")
+}
+
 // Benchmark tests for Bucket.Upsert() performance.
 func BenchmarkBucketUpsert(b *testing.B) {
 	b.Run("EmptyBucket", func(b *testing.B) {

provider/mem/mem.go

@@ -28,13 +28,13 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/trace"
 
+	"github.com/prometheus/alertmanager/alert"
 	"github.com/prometheus/alertmanager/eventrecorder"
 	"github.com/prometheus/alertmanager/eventrecorder/eventrecorderpb"
 	"github.com/prometheus/alertmanager/featurecontrol"
 	"github.com/prometheus/alertmanager/provider"
 	"github.com/prometheus/alertmanager/store"
 	"github.com/prometheus/alertmanager/tracing"
-	"github.com/prometheus/alertmanager/types"
 )
 
 const alertChannelLength = 200
@@ -69,13 +69,13 @@ type AlertStoreCallback interface {
 	// alert is not stored.
 	// Existing flag indicates whether alert has existed before (and is only updated) or not.
 	// If alert has existed before, then alert passed to PreStore is result of merging existing alert with new alert.
-	PreStore(alert *types.Alert, existing bool) error
+	PreStore(alert *alert.Alert, existing bool) error
 
 	// PostStore is called after alert has been put into store.
-	PostStore(alert *types.Alert, existing bool)
+	PostStore(alert *alert.Alert, existing bool)
 
 	// PostDelete is called after alert have been removed from the store due to alert garbage collection.
-	PostDelete(alert *types.Alert)
+	PostDelete(alert *alert.Alert)
 
 	// PostGC is called after alerts have been removed from the store due to alert garbage collection.
 	PostGC(fingerprints model.Fingerprints)
@@ -97,6 +97,7 @@ func (a *Alerts) registerMetrics(r prometheus.Registerer) {
 	if a.flagger.EnableAlertNamesInMetrics() {
 		labels = append(labels, "alertname")
 	}
+	labels = append(labels, "state")
 	a.alertsLimitedTotal = promauto.With(r).NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "alertmanager_alerts_limited_total",
@@ -194,7 +195,7 @@ func (a *Alerts) gc() {
 	a.callback.PostGC(ff)
 }
 
-func (a *Alerts) gcAlerts() []*types.Alert {
+func (a *Alerts) gcAlerts() []*alert.Alert {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 	return a.alerts.GC()
@@ -247,7 +248,7 @@ func (a *Alerts) Subscribe(name string) provider.AlertIterator {
 	return provider.NewAlertIterator(ch, done, nil)
 }
 
-func (a *Alerts) SlurpAndSubscribe(name string) ([]*types.Alert, provider.AlertIterator) {
+func (a *Alerts) SlurpAndSubscribe(name string) ([]*alert.Alert, provider.AlertIterator) {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 
@@ -292,14 +293,14 @@ func (a *Alerts) GetPending() provider.AlertIterator {
 }
 
 // Get returns the alert for a given fingerprint.
-func (a *Alerts) Get(fp model.Fingerprint) (*types.Alert, error) {
+func (a *Alerts) Get(fp model.Fingerprint) (*alert.Alert, error) {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 	return a.alerts.Get(fp)
 }
 
 // Put adds the given alert to the set.
-func (a *Alerts) Put(ctx context.Context, alerts ...*types.Alert) error {
+func (a *Alerts) Put(ctx context.Context, alerts ...*alert.Alert) error {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 
@@ -340,6 +341,11 @@ func (a *Alerts) Put(ctx context.Context, alerts ...*types.Alert) error {
 				if a.flagger.EnableAlertNamesInMetrics() {
 					labels = append(labels, alert.Name())
 				}
+				state := "firing"
+				if alert.Resolved() {
+					state = "resolved"
+				}
+				labels = append(labels, state)
 				a.alertsLimitedTotal.WithLabelValues(labels...).Inc()
 			}
 			continue
@@ -374,7 +380,7 @@ func (a *Alerts) Put(ctx context.Context, alerts ...*types.Alert) error {
 
 type noopCallback struct{}
 
-func (n noopCallback) PreStore(_ *types.Alert, _ bool) error { return nil }
-func (n noopCallback) PostStore(_ *types.Alert, _ bool)      {}
-func (n noopCallback) PostDelete(_ *types.Alert)             {}
+func (n noopCallback) PreStore(_ *alert.Alert, _ bool) error { return nil }
+func (n noopCallback) PostStore(_ *alert.Alert, _ bool)      {}
+func (n noopCallback) PostDelete(_ *alert.Alert)             {}
 func (n noopCallback) PostGC(_ model.Fingerprints)           {}