diff --git a/.env.example b/.env.example index 56bbd20..f679020 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,11 @@ FAL_EDIT_TIER=balanced # next try, so those are retried in place (3 attempts total). `false` reverts # to fail-fast. A deterministic decliner costs up to 3 attempts before erroring. # RETRY_NO_MEDIA=true +# Per-attempt wall-clock deadline on every fal call. Live-caught: one edit +# call hung 12+ minutes while the SSE heartbeat kept the dead generation +# open forever (endless shimmer, no banner). A timeout is NOT retried — +# it fails fast to the friendly "took too long — hit retry" frame. +# FAL_CALL_TIMEOUT_S=240 # Errors reaching the browser are mapped to short human messages (the raw fal # body echoes the WHOLE prompt back — it must not render in the UI). Full # exception stays in logs + the additive `detail` field. `false` = raw again. diff --git a/apps/modal-backend/providers/image.py b/apps/modal-backend/providers/image.py index 26012fd..4081d9a 100644 --- a/apps/modal-backend/providers/image.py +++ b/apps/modal-backend/providers/image.py @@ -15,6 +15,7 @@ from __future__ import annotations +import asyncio import base64 import os from dataclasses import dataclass @@ -668,6 +669,17 @@ async def _fal_subscribe( no_media_generated. Default False keeps non-image callers (segmenter, video) byte-identical. """ + # Wall-clock deadline per attempt (live-caught 2026-07-20: one fal edit + # call hung 12+ minutes with no error — and the SSE heartbeat happily + # kept the dead generation open forever, endless shimmer, no banner). + # A timeout is NON-retryable by design: fail fast to the friendly + # "took too long — hit retry" frame instead of stacking 3x the deadline. + # 240s clears the slowest legitimate fal path (gpt-image-2 edits ran + # ~170s in calibration); openrouter-hosted models don't ride this seam. + try: + deadline_s = max(30.0, float(os.environ.get("FAL_CALL_TIMEOUT_S", "") or 240.0)) + except ValueError: + deadline_s = 240.0 async for attempt in AsyncRetrying( stop=stop_after_attempt(3), wait=wait_exponential(multiplier=0.5, min=0.5, max=4), @@ -675,8 +687,11 @@ async def _fal_subscribe( reraise=True, ): with attempt: - result = await fal_client.subscribe_async( - model, arguments=arguments, with_logs=False + result = await asyncio.wait_for( + fal_client.subscribe_async( + model, arguments=arguments, with_logs=False + ), + timeout=deadline_s, ) # Two success shapes exist: `images: [...]` (nano/kontext/fill) and # BRIA Expand's singular `image` object — accept either, so a diff --git a/apps/modal-backend/tests/conftest.py b/apps/modal-backend/tests/conftest.py index 75a4462..8bda351 100644 --- a/apps/modal-backend/tests/conftest.py +++ b/apps/modal-backend/tests/conftest.py @@ -69,6 +69,7 @@ # for hermeticity). "RETRY_NO_MEDIA", "FRIENDLY_ERRORS", + "FAL_CALL_TIMEOUT_S", # Candidate empty-roll retry (default ON — same reasoning). "PRECOMPUTE_EMPTY_RETRY", # Interior enters (default ON — indoor register + interior judge; scrub diff --git a/apps/modal-backend/tests/test_image_provider.py b/apps/modal-backend/tests/test_image_provider.py index 7f900f2..da9ab15 100644 --- a/apps/modal-backend/tests/test_image_provider.py +++ b/apps/modal-backend/tests/test_image_provider.py @@ -704,3 +704,57 @@ async def fake_subscribe(model: str, arguments: dict, with_logs: bool) -> dict: monkeypatch.setattr(image.fal_client, "subscribe_async", fake_subscribe) result = await image._fal_subscribe("fal-ai/sam-3", {}) assert result == {"masks": []} + + +async def test_fal_subscribe_deadline_fails_fast(monkeypatch) -> None: + # Live-caught 2026-07-20: a fal edit call hung 12+ min; heartbeats kept + # the dead stream open forever. The per-attempt deadline turns a hang + # into TimeoutError (→ the friendly "took too long" frame) and is NOT + # retried — one deadline, not three stacked. + import asyncio + + from providers import image as image_provider + + calls = {"n": 0} + + async def hung_subscribe(model, arguments=None, with_logs=False): + calls["n"] += 1 + await asyncio.sleep(30) + + monkeypatch.setenv("FAL_CALL_TIMEOUT_S", "30") # floor clamps to 30s + monkeypatch.setattr(image_provider.fal_client, "subscribe_async", hung_subscribe) + # Shrink the effective deadline below the floor via a direct patch of the + # env read is not possible (floor 30s) — instead patch wait_for's timeout + # source: use a tiny sleep vs a tiny deadline by patching asyncio.wait_for + # would test the stdlib, not us. So: patch subscribe to outlive a 30s + # deadline is too slow for CI — instead verify the wiring by patching + # asyncio.wait_for to observe the timeout value and raise immediately. + seen = {} + + async def spy_wait_for(coro, timeout=None): + seen["timeout"] = timeout + coro.close() + raise TimeoutError("deadline") + + monkeypatch.setattr(image_provider.asyncio, "wait_for", spy_wait_for) + with pytest.raises(TimeoutError): + await image_provider._fal_subscribe("fal-ai/x", {}) + assert seen["timeout"] == 30.0 # env honored (with the 30s floor) + assert calls["n"] == 0 # wait_for raised before subscribe ran (spy short-circuit) + + +async def test_fal_subscribe_deadline_not_retried(monkeypatch) -> None: + + from providers import image as image_provider + + attempts = {"n": 0} + + async def spy_wait_for(coro, timeout=None): + attempts["n"] += 1 + coro.close() + raise TimeoutError("deadline") + + monkeypatch.setattr(image_provider.asyncio, "wait_for", spy_wait_for) + with pytest.raises(TimeoutError): + await image_provider._fal_subscribe("fal-ai/x", {}) + assert attempts["n"] == 1 # a hang fails FAST — no 3x deadline stack