diff --git a/deadlines/src/main/java/com/palantir/deadlines/ExpirationObservation.java b/deadlines/src/main/java/com/palantir/deadlines/ExpirationObservation.java
new file mode 100644
index 0000000..e41f585
--- /dev/null
+++ b/deadlines/src/main/java/com/palantir/deadlines/ExpirationObservation.java
@@ -0,0 +1,174 @@
+/*
+ * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.deadlines;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.palantir.deadlines.DeadlineMetrics.Expired_Cause;
+import com.palantir.deadlines.DeadlineMetrics.Expired_Intent;
+import com.palantir.tritium.metrics.registry.SharedTaggedMetricRegistries;
+
+/**
+ * Provides the ability to observe the number of deadline expirations that happen between points in time.
+ *
+ * Usage example:
+ *
+ * ExpirationObservation start = ExpirationObservation.start();
+ * // ... operation that may cause deadline expiration ...
+ * ExpirationObservation end = start.observeFrom();
+ * if (end.totalExpirations() > 0) {
+ * // at least one deadline expiration happened
+ * }
+ *
+ *
+ * Calling {@link #start()} will create a new observation with no recorded expirations.
+ *
+ * Calling {@link #observeFrom()} will return an observation comparing the current point in time to the count of
+ * expirations when the original observation was created.
+ *
+ * Calling {@link #totalExpirations()} will return the number of deadline expirations that occurred
+ * between two observations.
+ */
+public final class ExpirationObservation {
+
+ private static final DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
+
+ private final MeterValues start;
+ private final MeterValues end;
+
+ private ExpirationObservation(MeterValues start, MeterValues end) {
+ this.start = start;
+ this.end = end;
+ }
+
+ /**
+ * Begin a new observation.
+ */
+ public static ExpirationObservation start() {
+ MeterValues now = readMeters();
+ return new ExpirationObservation(now, now);
+ }
+
+ /**
+ * Return a new observation with the number of deadline expirations that have occurred since this observation was
+ * initialized.
+ *
+ * The current observation should have been previously initialized either via a call to this method, or
+ * a call to {@link #start()},
+ */
+ public ExpirationObservation observeFrom() {
+ MeterValues now = readMeters();
+ return new ExpirationObservation(end, now);
+ }
+
+ /**
+ * Return the total number of expirations that have occurred since this observation was initialized.
+ */
+ public long totalExpirations() {
+ return nExternalPropagate()
+ + nExternalPropagateAlreadyExpired()
+ + nExternalIgnore()
+ + nInternalPropagate()
+ + nInternalPropagateAlreadyExpired()
+ + nInternalIgnore();
+ }
+
+ @VisibleForTesting
+ long totalWithExternalCause() {
+ return nExternalPropagate() + nExternalPropagateAlreadyExpired() + nExternalIgnore();
+ }
+
+ @VisibleForTesting
+ long totalWithInternalCause() {
+ return nInternalPropagate() + nInternalPropagateAlreadyExpired() + nInternalIgnore();
+ }
+
+ @VisibleForTesting
+ long totalWithPropagateIntent() {
+ return nExternalPropagate() + nInternalPropagate();
+ }
+
+ @VisibleForTesting
+ long totalWithPropagateAlreadyExpiredIntent() {
+ return nExternalPropagateAlreadyExpired() + nInternalPropagateAlreadyExpired();
+ }
+
+ @VisibleForTesting
+ long totalWithIgnoreIntent() {
+ return nExternalIgnore() + nInternalIgnore();
+ }
+
+ @VisibleForTesting
+ long nExternalPropagate() {
+ return end.externalPropagate - start.externalPropagate;
+ }
+
+ @VisibleForTesting
+ long nExternalPropagateAlreadyExpired() {
+ return end.externalPropagateAlreadyExpired - start.externalPropagateAlreadyExpired;
+ }
+
+ @VisibleForTesting
+ long nExternalIgnore() {
+ return end.externalIgnore - start.externalIgnore;
+ }
+
+ @VisibleForTesting
+ long nInternalPropagate() {
+ return end.internalPropagate - start.internalPropagate;
+ }
+
+ @VisibleForTesting
+ long nInternalPropagateAlreadyExpired() {
+ return end.internalPropagateAlreadyExpired - start.internalPropagateAlreadyExpired;
+ }
+
+ @VisibleForTesting
+ long nInternalIgnore() {
+ return end.internalIgnore - start.internalIgnore;
+ }
+
+ private static MeterValues readMeters() {
+ long externalPropagate = getCountFor(Expired_Cause.EXTERNAL, Expired_Intent.PROPAGATE);
+ long externalPropagateAlreadyExpired =
+ getCountFor(Expired_Cause.EXTERNAL, Expired_Intent.PROPAGATE_ALREADY_EXPIRED);
+ long externalIgnore = getCountFor(Expired_Cause.EXTERNAL, Expired_Intent.IGNORE);
+ long internalPropagate = getCountFor(Expired_Cause.INTERNAL, Expired_Intent.PROPAGATE);
+ long internalPropagateAlreadyExpired =
+ getCountFor(Expired_Cause.INTERNAL, Expired_Intent.PROPAGATE_ALREADY_EXPIRED);
+ long internalIgnore = getCountFor(Expired_Cause.INTERNAL, Expired_Intent.IGNORE);
+
+ return new MeterValues(
+ externalPropagate,
+ externalPropagateAlreadyExpired,
+ externalIgnore,
+ internalPropagate,
+ internalPropagateAlreadyExpired,
+ internalIgnore);
+ }
+
+ private static long getCountFor(Expired_Cause cause, Expired_Intent intent) {
+ return metrics.expired().cause(cause).intent(intent).build().getCount();
+ }
+
+ private record MeterValues(
+ long externalPropagate,
+ long externalPropagateAlreadyExpired,
+ long externalIgnore,
+ long internalPropagate,
+ long internalPropagateAlreadyExpired,
+ long internalIgnore) {}
+}
diff --git a/deadlines/src/test/java/com/palantir/deadlines/DeadlinesTest.java b/deadlines/src/test/java/com/palantir/deadlines/DeadlinesTest.java
index 90cf72d..a82248d 100644
--- a/deadlines/src/test/java/com/palantir/deadlines/DeadlinesTest.java
+++ b/deadlines/src/test/java/com/palantir/deadlines/DeadlinesTest.java
@@ -20,16 +20,12 @@
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import com.codahale.metrics.Meter;
-import com.palantir.deadlines.DeadlineMetrics.Expired_Cause;
-import com.palantir.deadlines.DeadlineMetrics.Expired_Intent;
import com.palantir.deadlines.Deadlines.Enforcement;
import com.palantir.deadlines.Deadlines.RequestDecodingAdapter;
import com.palantir.deadlines.Deadlines.RequestEncodingAdapter;
import com.palantir.tracing.CloseableSpan;
import com.palantir.tracing.CloseableTracer;
import com.palantir.tracing.DetachedSpan;
-import com.palantir.tritium.metrics.registry.SharedTaggedMetricRegistries;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
@@ -335,23 +331,15 @@ public void test_encode_to_request_expiration_external_deadline() {
Optional remaining = Deadlines.getRemainingDeadline();
assertThat(remaining).hasValueSatisfying(d -> assertThat(d).isEqualTo(Duration.ZERO));
- DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
- Meter externalMeter = metrics.expired()
- .cause(Expired_Cause.EXTERNAL)
- .intent(Expired_Intent.PROPAGATE)
- .build();
- Meter internalMeter = metrics.expired()
- .cause(Expired_Cause.INTERNAL)
- .intent(Expired_Intent.PROPAGATE)
- .build();
- long originalExternalValue = externalMeter.getCount();
- long originalInternalValue = internalMeter.getCount();
+ ExpirationObservation start = ExpirationObservation.start();
Map outbound = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound, DummyRequestEncoder.INSTANCE);
- assertThat(externalMeter.getCount()).isGreaterThan(originalExternalValue);
- assertThat(internalMeter.getCount()).isEqualTo(originalInternalValue);
+ ExpirationObservation end = start.observeFrom();
+
+ assertThat(end.nExternalPropagate()).isGreaterThan(0);
+ assertThat(end.nInternalPropagate()).isEqualTo(0);
}
}
@@ -371,23 +359,15 @@ public void test_encode_to_request_expiration_internal_deadline() {
Optional remaining = Deadlines.getRemainingDeadline();
assertThat(remaining).hasValueSatisfying(d -> assertThat(d).isEqualTo(Duration.ZERO));
- DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
- Meter externalMeter = metrics.expired()
- .cause(Expired_Cause.EXTERNAL)
- .intent(Expired_Intent.PROPAGATE)
- .build();
- Meter internalMeter = metrics.expired()
- .cause(Expired_Cause.INTERNAL)
- .intent(Expired_Intent.PROPAGATE)
- .build();
- long originalExternalValue = externalMeter.getCount();
- long originalInternalValue = internalMeter.getCount();
+ ExpirationObservation start = ExpirationObservation.start();
Map outbound = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound, DummyRequestEncoder.INSTANCE);
- assertThat(internalMeter.getCount()).isGreaterThan(originalInternalValue);
- assertThat(externalMeter.getCount()).isEqualTo(originalExternalValue);
+ ExpirationObservation end = start.observeFrom();
+
+ assertThat(end.nInternalPropagate()).isGreaterThan(0);
+ assertThat(end.nExternalPropagate()).isEqualTo(0);
}
}
@@ -407,26 +387,16 @@ public void disabled_propagation_reports_metrics_on_expiration() {
Optional remaining = Deadlines.getRemainingDeadline();
assertThat(remaining).hasValueSatisfying(d -> assertThat(d).isEqualTo(Duration.ZERO));
- DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
- Meter externalMeterWillPropagate = metrics.expired()
- .cause(Expired_Cause.EXTERNAL)
- .intent(Expired_Intent.PROPAGATE)
- .build();
- Meter externalMeterWontPropagate = metrics.expired()
- .cause(Expired_Cause.EXTERNAL)
- .intent(Expired_Intent.IGNORE)
- .build();
- long originalWillPropagateValue = externalMeterWillPropagate.getCount();
- long originalWontPropagateValue = externalMeterWontPropagate.getCount();
+ ExpirationObservation start = ExpirationObservation.start();
// first request is allowed to propagate the deadline, make sure the correct meter is marked
Map outbound1 = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound1, DummyRequestEncoder.INSTANCE);
- assertThat(externalMeterWillPropagate.getCount()).isGreaterThan(originalWillPropagateValue);
- assertThat(externalMeterWontPropagate.getCount()).isEqualTo(originalWontPropagateValue);
- originalWillPropagateValue = externalMeterWillPropagate.getCount();
- originalWontPropagateValue = externalMeterWontPropagate.getCount();
+ ExpirationObservation obs1 = start.observeFrom();
+
+ assertThat(obs1.nExternalPropagate()).isGreaterThan(0);
+ assertThat(obs1.nExternalIgnore()).isEqualTo(0);
// and now disable propagation
Deadlines.disableFurtherDeadlinePropagation();
@@ -434,8 +404,11 @@ public void disabled_propagation_reports_metrics_on_expiration() {
// second request is not allowed to propagate the deadline, make sure the correct meter is marked
Map outbound2 = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound2, DummyRequestEncoder.INSTANCE);
- assertThat(externalMeterWontPropagate.getCount()).isGreaterThan(originalWontPropagateValue);
- assertThat(externalMeterWillPropagate.getCount()).isEqualTo(originalWillPropagateValue);
+
+ ExpirationObservation obs2 = obs1.observeFrom();
+
+ assertThat(obs2.nExternalIgnore()).isGreaterThan(0);
+ assertThat(obs2.nExternalPropagate()).isEqualTo(0);
}
}
@@ -481,18 +454,7 @@ public void multihop_expired_received_deadline_marks_propagate_already_expired_m
DetachedSpan server2Span = DetachedSpan.start("server2");
try (CloseableSpan ignored = server1Span.attach()) {
- DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
- Meter expiredMeterPropagateIntent = metrics.expired()
- .cause(Expired_Cause.EXTERNAL)
- .intent(Expired_Intent.PROPAGATE)
- .build();
- Meter expiredMeterPropagateAlreadyExpiredIntent = metrics.expired()
- .cause(Expired_Cause.EXTERNAL)
- .intent(Expired_Intent.PROPAGATE_ALREADY_EXPIRED)
- .build();
-
- long expiredMeterPropagateIntentValue = expiredMeterPropagateIntent.getCount();
- long expiredMeterPropagateAlreadyExpiredIntentValue = expiredMeterPropagateAlreadyExpiredIntent.getCount();
+ ExpirationObservation start = ExpirationObservation.start();
// the first hop receives a valid, non-zero deadline on the wire
Map request = new HashMap<>();
@@ -500,10 +462,12 @@ public void multihop_expired_received_deadline_marks_propagate_already_expired_m
request.put(
DeadlinesHttpHeaders.EXPECT_WITHIN, Deadlines.durationToHeaderValue(providedDeadline.toNanos()));
Deadlines.parseFromRequest(Optional.empty(), request, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);
+
// nothing yet...
- assertThat(expiredMeterPropagateIntent.getCount()).isEqualTo(expiredMeterPropagateIntentValue);
- assertThat(expiredMeterPropagateAlreadyExpiredIntent.getCount())
- .isEqualTo(expiredMeterPropagateAlreadyExpiredIntentValue);
+ ExpirationObservation obs1 = start.observeFrom();
+
+ assertThat(obs1.nExternalPropagate()).isEqualTo(0);
+ assertThat(obs1.nExternalPropagateAlreadyExpired()).isEqualTo(0);
// force expiration within the first hop
clock.elapsed += 2_000_000;
@@ -515,23 +479,25 @@ public void multihop_expired_received_deadline_marks_propagate_already_expired_m
assertThat(outbound1.get(DeadlinesHttpHeaders.EXPECT_WITHIN))
.isNotNull()
.isEqualTo("0");
- assertThat(expiredMeterPropagateIntent.getCount()).isGreaterThan(expiredMeterPropagateIntentValue);
- assertThat(expiredMeterPropagateAlreadyExpiredIntent.getCount()).isZero();
+
+ ExpirationObservation obs2 = obs1.observeFrom();
+
+ assertThat(obs2.nExternalPropagate()).isGreaterThan(0);
+ assertThat(obs2.nExternalPropagateAlreadyExpired()).isEqualTo(0);
// next hop parses a zero deadline
try (CloseableSpan ignored2 = server2Span.attach()) {
- expiredMeterPropagateIntentValue = expiredMeterPropagateIntent.getCount();
Deadlines.parseFromRequest(
Optional.empty(), outbound1, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);
// sending another request when the deadline has already expired should
// mark the meter with the "propagate-already-expired" intent
- expiredMeterPropagateAlreadyExpiredIntentValue = expiredMeterPropagateAlreadyExpiredIntent.getCount();
Map outbound2 = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound2, DummyRequestEncoder.INSTANCE);
- assertThat(expiredMeterPropagateAlreadyExpiredIntent.getCount())
- .isGreaterThan(expiredMeterPropagateAlreadyExpiredIntentValue);
- // meter with the "propagate" intent is unchanged
- assertThat(expiredMeterPropagateIntent.getCount()).isEqualTo(expiredMeterPropagateIntentValue);
+
+ ExpirationObservation obs3 = obs2.observeFrom();
+
+ assertThat(obs3.nExternalPropagateAlreadyExpired()).isGreaterThan(0);
+ assertThat(obs3.nExternalPropagate()).isEqualTo(0);
}
}
}
diff --git a/deadlines/src/test/java/com/palantir/deadlines/ExpirationObservationTest.java b/deadlines/src/test/java/com/palantir/deadlines/ExpirationObservationTest.java
new file mode 100644
index 0000000..a6f096e
--- /dev/null
+++ b/deadlines/src/test/java/com/palantir/deadlines/ExpirationObservationTest.java
@@ -0,0 +1,79 @@
+/*
+ * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.deadlines;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.palantir.deadlines.DeadlineMetrics.Expired_Cause;
+import com.palantir.deadlines.DeadlineMetrics.Expired_Intent;
+import com.palantir.tritium.metrics.registry.SharedTaggedMetricRegistries;
+import org.junit.jupiter.api.Test;
+
+class ExpirationObservationTest {
+
+ @Test
+ public void test_observe_no_changes() {
+ ExpirationObservation start = ExpirationObservation.start();
+ ExpirationObservation end = start.observeFrom();
+ assertThat(end.totalExpirations()).isEqualTo(start.totalExpirations());
+ }
+
+ @Test
+ public void test_observe_change_external_propagate() {
+ ExpirationObservation start = ExpirationObservation.start();
+
+ DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
+ metrics.expired()
+ .cause(Expired_Cause.EXTERNAL)
+ .intent(Expired_Intent.PROPAGATE)
+ .build()
+ .mark();
+
+ ExpirationObservation end = start.observeFrom();
+
+ assertThat(end.totalExpirations()).isEqualTo(1);
+ assertThat(end.nExternalPropagate()).isEqualTo(1);
+ }
+
+ @Test
+ public void test_observe_change_multiple_observations() {
+ ExpirationObservation start = ExpirationObservation.start();
+
+ DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
+ metrics.expired()
+ .cause(Expired_Cause.EXTERNAL)
+ .intent(Expired_Intent.PROPAGATE)
+ .build()
+ .mark();
+
+ ExpirationObservation obs1 = start.observeFrom();
+
+ assertThat(obs1.totalExpirations()).isEqualTo(1);
+ assertThat(obs1.nExternalPropagate()).isEqualTo(1);
+
+ metrics.expired()
+ .cause(Expired_Cause.INTERNAL)
+ .intent(Expired_Intent.IGNORE)
+ .build()
+ .mark();
+
+ ExpirationObservation obs2 = obs1.observeFrom();
+
+ assertThat(obs2.totalExpirations()).isEqualTo(1);
+ assertThat(obs2.nInternalIgnore()).isEqualTo(1);
+ }
+}