Improve timeout error handling and maxFailures behavior

packages/probitas-runner/errors.ts

@@ -31,35 +31,128 @@ export class StepTimeoutError extends Error {
     this.attemptNumber = attemptNumber;
     this.elapsedMs = elapsedMs;
   }
+
+  /**
+   * Creates an AbortSignal that aborts with StepTimeoutError after the specified timeout.
+   *
+   * @param timeout - Timeout duration in milliseconds
+   * @param params - Step information for the error
+   * @returns AbortSignal with Disposable interface for cleanup
+   */
+  static timeoutSignal(
+    timeout: number,
+    params: {
+      stepName: string;
+      attemptNumber: number;
+    },
+  ): AbortSignal & Disposable {
+    const controller = new AbortController();
+    const startTime = performance.now();
+    const timeoutId = setTimeout(() => {
+      const elapsedMs = Math.round(performance.now() - startTime);
+      const error = new StepTimeoutError(
+        params.stepName,
+        timeout,
+        params.attemptNumber,
+        elapsedMs,
+      );
+      controller.abort(error);
+    }, timeout);
+
+    // Cleanup timeout when signal is aborted by other means
+    controller.signal.addEventListener("abort", () => {
+      clearTimeout(timeoutId);
+    }, { once: true });
+
+    // Add Disposable interface for manual cleanup
+    return Object.assign(controller.signal, {
+      [Symbol.dispose]: () => {
+        clearTimeout(timeoutId);
+      },
+    });
+  }
 }
 
 /**
  * Error thrown when a scenario execution times out.
  *
  * Contains structured information about the timeout including
- * the scenario name, timeout duration, and total elapsed time.
+ * the scenario name, timeout duration, total elapsed time,
+ * and optionally the step being executed when timeout occurred.
  */
 export class ScenarioTimeoutError extends Error {
   override readonly name = "ScenarioTimeoutError";
   readonly scenarioName: string;
   readonly timeoutMs: number;
   readonly elapsedMs: number;
+  readonly currentStepName?: string;
+  readonly currentStepIndex?: number;
 
   constructor(
     scenarioName: string,
     timeoutMs: number,
     elapsedMs: number,
-    options?: ErrorOptions,
+    options?: ErrorOptions & {
+      currentStepName?: string;
+      currentStepIndex?: number;
+    },
   ) {
+    const stepMsg = options?.currentStepName
+      ? ` while executing step "${options.currentStepName}"${
+        options.currentStepIndex !== undefined
+          ? ` (step ${options.currentStepIndex + 1})`
+          : ""
+      }`
+      : "";
     const elapsedMsg = elapsedMs !== timeoutMs
       ? `, total elapsed: ${elapsedMs}ms`
       : "";
     super(
-      `Scenario "${scenarioName}" timed out after ${timeoutMs}ms${elapsedMsg}`,
+      `Scenario "${scenarioName}" timed out after ${timeoutMs}ms${stepMsg}${elapsedMsg}`,
       options,
     );
     this.scenarioName = scenarioName;
     this.timeoutMs = timeoutMs;
     this.elapsedMs = elapsedMs;
+    this.currentStepName = options?.currentStepName;
+    this.currentStepIndex = options?.currentStepIndex;
+  }
+
+  /**
+   * Creates an AbortSignal that aborts with ScenarioTimeoutError after the specified timeout.
+   *
+   * @param timeout - Timeout duration in milliseconds
+   * @param params - Scenario information for the error
+   * @returns AbortSignal with Disposable interface for cleanup
+   */
+  static timeoutSignal(
+    timeout: number,
+    params: {
+      scenarioName: string;
+    },
+  ): AbortSignal & Disposable {
+    const controller = new AbortController();
+    const startTime = performance.now();
+    const timeoutId = setTimeout(() => {
+      const elapsedMs = Math.round(performance.now() - startTime);
+      const error = new ScenarioTimeoutError(
+        params.scenarioName,
+        timeout,
+        elapsedMs,
+      );
+      controller.abort(error);
+    }, timeout);
+
+    // Cleanup timeout when signal is aborted by other means
+    controller.signal.addEventListener("abort", () => {
+      clearTimeout(timeoutId);
+    }, { once: true });
+
+    // Add Disposable interface for manual cleanup
+    return Object.assign(controller.signal, {
+      [Symbol.dispose]: () => {
+        clearTimeout(timeoutId);
+      },
+    });
   }
 }

packages/probitas-runner/errors_test.ts

@@ -0,0 +1,166 @@
+import { expect } from "@std/expect";
+import { delay } from "@std/async/delay";
+import { ScenarioTimeoutError, StepTimeoutError } from "./errors.ts";
+
+Deno.test("ScenarioTimeoutError - constructor creates error with correct properties", () => {
+  const error = new ScenarioTimeoutError("test-scenario", 5000, 5100);
+
+  expect(error.name).toBe("ScenarioTimeoutError");
+  expect(error.scenarioName).toBe("test-scenario");
+  expect(error.timeoutMs).toBe(5000);
+  expect(error.elapsedMs).toBe(5100);
+  expect(error.currentStepName).toBeUndefined();
+  expect(error.currentStepIndex).toBeUndefined();
+  expect(error.message).toBe(
+    'Scenario "test-scenario" timed out after 5000ms, total elapsed: 5100ms',
+  );
+});
+
+Deno.test("ScenarioTimeoutError - constructor with step information", () => {
+  const error = new ScenarioTimeoutError("test-scenario", 5000, 5100, {
+    currentStepName: "slow-step",
+    currentStepIndex: 2,
+  });
+
+  expect(error.currentStepName).toBe("slow-step");
+  expect(error.currentStepIndex).toBe(2);
+  expect(error.message).toBe(
+    'Scenario "test-scenario" timed out after 5000ms while executing step "slow-step" (step 3), total elapsed: 5100ms',
+  );
+});
+
+Deno.test("ScenarioTimeoutError - constructor with only step name", () => {
+  const error = new ScenarioTimeoutError("test-scenario", 5000, 5000, {
+    currentStepName: "slow-step",
+  });
+
+  expect(error.currentStepName).toBe("slow-step");
+  expect(error.currentStepIndex).toBeUndefined();
+  expect(error.message).toBe(
+    'Scenario "test-scenario" timed out after 5000ms while executing step "slow-step"',
+  );
+});
+
+Deno.test("ScenarioTimeoutError.timeoutSignal - creates signal that aborts after timeout", async () => {
+  using signal = ScenarioTimeoutError.timeoutSignal(50, {
+    scenarioName: "test-scenario",
+  });
+
+  expect(signal.aborted).toBe(false);
+
+  await delay(100);
+
+  expect(signal.aborted).toBe(true);
+  expect(signal.reason).toBeInstanceOf(ScenarioTimeoutError);
+
+  const error = signal.reason as ScenarioTimeoutError;
+  expect(error.scenarioName).toBe("test-scenario");
+  expect(error.timeoutMs).toBe(50);
+  expect(error.elapsedMs).toBeGreaterThanOrEqual(50);
+  expect(error.elapsedMs).toBeLessThan(200);
+});
+
+Deno.test("ScenarioTimeoutError.timeoutSignal - cleanup via dispose", async () => {
+  const signal = ScenarioTimeoutError.timeoutSignal(1000, {
+    scenarioName: "test-scenario",
+  });
+
+  expect(signal.aborted).toBe(false);
+
+  // Dispose immediately to cleanup timer
+  signal[Symbol.dispose]();
+
+  await delay(50);
+
+  // Should not abort because timer was cleaned up
+  expect(signal.aborted).toBe(false);
+});
+
+Deno.test("ScenarioTimeoutError.timeoutSignal - cleanup via using declaration", async () => {
+  {
+    using signal = ScenarioTimeoutError.timeoutSignal(1000, {
+      scenarioName: "test-scenario",
+    });
+
+    expect(signal.aborted).toBe(false);
+  } // signal is disposed here
+
+  await delay(50);
+
+  // Timer should be cleaned up, no leak
+});
+
+Deno.test("StepTimeoutError - constructor creates error with correct properties", () => {
+  const error = new StepTimeoutError("test-step", 3000, 2, 3200);
+
+  expect(error.name).toBe("StepTimeoutError");
+  expect(error.stepName).toBe("test-step");
+  expect(error.timeoutMs).toBe(3000);
+  expect(error.attemptNumber).toBe(2);
+  expect(error.elapsedMs).toBe(3200);
+  expect(error.message).toBe(
+    'Step "test-step" timed out after 3000ms (attempt 2), total elapsed: 3200ms',
+  );
+});
+
+Deno.test("StepTimeoutError - constructor for first attempt", () => {
+  const error = new StepTimeoutError("test-step", 3000, 1, 3000);
+
+  expect(error.attemptNumber).toBe(1);
+  expect(error.message).toBe(
+    'Step "test-step" timed out after 3000ms',
+  );
+});
+
+Deno.test("StepTimeoutError.timeoutSignal - creates signal that aborts after timeout", async () => {
+  using signal = StepTimeoutError.timeoutSignal(50, {
+    stepName: "test-step",
+    attemptNumber: 1,
+  });
+
+  expect(signal.aborted).toBe(false);
+
+  await delay(100);
+
+  expect(signal.aborted).toBe(true);
+  expect(signal.reason).toBeInstanceOf(StepTimeoutError);
+
+  const error = signal.reason as StepTimeoutError;
+  expect(error.stepName).toBe("test-step");
+  expect(error.timeoutMs).toBe(50);
+  expect(error.attemptNumber).toBe(1);
+  expect(error.elapsedMs).toBeGreaterThanOrEqual(50);
+  expect(error.elapsedMs).toBeLessThan(200);
+});
+
+Deno.test("StepTimeoutError.timeoutSignal - cleanup via dispose", async () => {
+  const signal = StepTimeoutError.timeoutSignal(1000, {
+    stepName: "test-step",
+    attemptNumber: 1,
+  });
+
+  expect(signal.aborted).toBe(false);
+
+  // Dispose immediately to cleanup timer
+  signal[Symbol.dispose]();
+
+  await delay(50);
+
+  // Should not abort because timer was cleaned up
+  expect(signal.aborted).toBe(false);
+});
+
+Deno.test("StepTimeoutError.timeoutSignal - measures elapsed time correctly", async () => {
+  using signal = StepTimeoutError.timeoutSignal(50, {
+    stepName: "test-step",
+    attemptNumber: 1,
+  });
+
+  await delay(100);
+
+  const error = signal.reason as StepTimeoutError;
+  // Elapsed time should be approximately 50ms (actual timeout duration)
+  // with some tolerance for timer precision
+  expect(error.elapsedMs).toBeGreaterThanOrEqual(50);
+  expect(error.elapsedMs).toBeLessThan(150);
+});