store and expose details of deadline expiration when it happens

deadlines/src/main/java/com/palantir/deadlines/Deadlines.java

@@ -42,6 +42,7 @@ public final class Deadlines {
     private Deadlines() {}
 
     private static final TraceLocal<ProvidedDeadline> deadlineState = TraceLocal.of();
+    private static final TraceLocal<ExpirationObservation> firstExpiration = TraceLocal.of();
     private static final DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
     private static final CharMatcher decimalMatcher =
             CharMatcher.inRange('0', '9').or(CharMatcher.is('.')).precomputed();
@@ -72,6 +73,43 @@ public static Optional<Duration> getRemainingDeadline() {
         return Optional.of(remaining <= 0 ? Duration.ZERO : Duration.ofNanos(remaining));
     }
 
+    /**
+     * Contains data about the circumstances of a deadline expiration.
+     *
+     * @param cause the cause of the expiration when it happened, e.g. "external" or "internal"
+     * @param intent the intent on how the expiration will be propagated, e.g. "propagate", "ignore", etc.
+     */
+    public record ExpirationObservation(String cause, String intent) {}
+
+    /**
+     * Contains data about the current deadline state.
+     *
+     * @param remainingDeadline the amount of time remaining towards the deadline; may be zero (or negative) if the
+     * deadline has expired.
+     * @param expired details about the first expiration, if the deadline has expired; will be empty otherwise
+     */
+    public record DeadlineObservation(Duration remainingDeadline, Optional<ExpirationObservation> expired) {}
+
+    /**
+     * Similar to {@link #getRemainingDeadline()} but allows an observer to see details on the cause of expiration.
+     *
+     * This method is slightly different from {@link #getRemainingDeadline()}, as it allows callers to both observe
+     * how much time is remaining towards the deadline, and see details of how a deadline expiration came about
+     * and how it was handled if such an expiration happened.
+     *
+     * @return a {@link DeadlineObservation} containing details of the amount of time remaining towards the deadline,
+     * as well as details of the expiration if it has already expired. If no deadline state is available for the current
+     * trace, an {@link Optional#empty()} is returned instead.
+     */
+    public static Optional<DeadlineObservation> observeRemainingDeadline() {
+        Optional<Duration> remainingDeadline = getRemainingDeadline();
+        if (remainingDeadline.isEmpty()) {
+            return Optional.empty();
+        }
+        Optional<ExpirationObservation> expired = Optional.ofNullable(firstExpiration.get());
+        return Optional.of(new DeadlineObservation(remainingDeadline.get(), expired));
+    }
+
     /**
      * Disables propagation of deadline values any further for the current trace.
      *
@@ -347,6 +385,12 @@ private static void checkExpiration(
                 intent = Expired_Intent.PROPAGATE_ALREADY_EXPIRED;
             }
             metrics.expired().cause(cause).intent(intent).build().mark();
+
+            // check and store data about the first observed expiration
+            if (firstExpiration.get() == null) {
+                firstExpiration.set(new ExpirationObservation(cause.toString(), intent.toString()));
+            }
+
             if (enforced) {
                 throw internal ? DeadlineExpiredException.internal() : DeadlineExpiredException.external();
             }

deadlines/src/test/java/com/palantir/deadlines/DeadlinesTest.java

@@ -23,6 +23,7 @@
 import com.codahale.metrics.Meter;
 import com.palantir.deadlines.DeadlineMetrics.Expired_Cause;
 import com.palantir.deadlines.DeadlineMetrics.Expired_Intent;
+import com.palantir.deadlines.Deadlines.DeadlineObservation;
 import com.palantir.deadlines.Deadlines.Enforcement;
 import com.palantir.deadlines.Deadlines.RequestDecodingAdapter;
 import com.palantir.deadlines.Deadlines.RequestEncodingAdapter;
@@ -319,6 +320,77 @@ public void test_expiration_get_remaining_deadline() {
         }
     }
 
+    @Test
+    public void test_observe_remaining_deadline_not_expired() {
+        TestClock clock = new TestClock();
+        Deadlines.setClock(clock);
+        try (CloseableTracer tracer = CloseableTracer.startSpan("test")) {
+            Map<String, String> request = new HashMap<>();
+            Duration providedDeadline = Duration.ofMillis(1);
+            request.put(
+                    DeadlinesHttpHeaders.EXPECT_WITHIN, Deadlines.durationToHeaderValue(providedDeadline.toNanos()));
+            Deadlines.parseFromRequest(Optional.empty(), request, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);
+
+            Map<String, String> outbound = new HashMap<>();
+            Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound, DummyRequestEncoder.INSTANCE);
+
+            Optional<DeadlineObservation> remaining = Deadlines.observeRemainingDeadline();
+            assertThat(remaining).isPresent();
+            assertThat(remaining.get().remainingDeadline()).isEqualTo(providedDeadline);
+            assertThat(remaining.get().expired()).isEmpty();
+        }
+    }
+
+    @Test
+    public void test_observe_remaining_deadline_expired() {
+        TestClock clock = new TestClock();
+        Deadlines.setClock(clock);
+        try (CloseableTracer tracer = CloseableTracer.startSpan("test")) {
+            Map<String, String> request = new HashMap<>();
+            Duration providedDeadline = Duration.ofMillis(1);
+            request.put(
+                    DeadlinesHttpHeaders.EXPECT_WITHIN, Deadlines.durationToHeaderValue(providedDeadline.toNanos()));
+            Deadlines.parseFromRequest(Optional.empty(), request, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);
+
+            clock.elapsed += 2_000_000;
+
+            Map<String, String> outbound = new HashMap<>();
+            Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound, DummyRequestEncoder.INSTANCE);
+
+            Optional<DeadlineObservation> remaining = Deadlines.observeRemainingDeadline();
+            assertThat(remaining).isPresent();
+            assertThat(remaining.get().remainingDeadline()).isLessThanOrEqualTo(Duration.ZERO);
+            assertThat(remaining.get().expired()).hasValueSatisfying(exp -> {
+                assertThat(exp.cause()).isEqualTo(Expired_Cause.EXTERNAL.toString());
+                assertThat(exp.intent()).isEqualTo(Expired_Intent.PROPAGATE.toString());
+            });
+        }
+    }
+
+    @Test
+    public void test_observe_remaining_deadline_no_deadline_state() {
+        Optional<DeadlineObservation> remaining = Deadlines.observeRemainingDeadline();
+        assertThat(remaining).isEmpty();
+
+        TestClock clock = new TestClock();
+        Deadlines.setClock(clock);
+        try (CloseableTracer tracer = CloseableTracer.startSpan("test")) {
+            Map<String, String> request = new HashMap<>();
+            Duration providedDeadline = Duration.ofMillis(1);
+            request.put(
+                    DeadlinesHttpHeaders.EXPECT_WITHIN, Deadlines.durationToHeaderValue(providedDeadline.toNanos()));
+            Deadlines.parseFromRequest(Optional.empty(), request, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);
+
+            remaining = Deadlines.observeRemainingDeadline();
+            assertThat(remaining).hasValue(new DeadlineObservation(providedDeadline, Optional.empty()));
+
+            Deadlines.disableFurtherDeadlinePropagation();
+
+            remaining = Deadlines.observeRemainingDeadline();
+            assertThat(remaining).isEmpty();
+        }
+    }
+
     @Test
     public void test_encode_to_request_expiration_external_deadline() {
         TestClock clock = new TestClock();