From bd47e7c86cc3c234f9a32282d98d099082512a6a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:37:17 -0700 Subject: [PATCH 1/2] Stop usage chips flashing and back off retries for failing providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two behaviors made the status bar unusable when any provider was persistently failing (bad auth, unsupported plan): 1. Every refetch repainted all providers as 'fetching', so a settled error chip flashed to a loading "…" chip and back on every cycle. withFetchingStatus now keeps settled states (ok/error/unavailable) visible until the new result lands; only providers with no settled state (first load, explicit account-switch clear) show loading. 2. Error providers on the fast activation-retry lane (claude/codex/ grok) were retried every 30s on any focus/show/restore event — forever. Repeated hits drove Claude's tight-budget usage endpoint into 429s, flipping the chip between 'Limited' and its actual error. Retries now back off exponentially per consecutive applied failure (30s, 60s, 120s, … capped at the 15-minute poll cadence) and reset on success or account/target switch. --- src/main/rate-limits/service.test.ts | 109 ++++++++++++++++++++++++++- src/main/rate-limits/service.ts | 77 ++++++++++++++++++- 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/src/main/rate-limits/service.test.ts b/src/main/rate-limits/service.test.ts index 261a94f36be..f40f11aa3ce 100644 --- a/src/main/rate-limits/service.test.ts +++ b/src/main/rate-limits/service.test.ts @@ -393,7 +393,7 @@ describe('RateLimitService', () => { } }) - it('throttles repeated active-window retries while Claude is still failing', async () => { + it('backs off repeated active-window retries while Claude is still failing', async () => { vi.useFakeTimers() try { vi.mocked(fetchClaudeRateLimits).mockResolvedValue(errorProvider('claude', 'still failing')) @@ -407,6 +407,7 @@ describe('RateLimitService', () => { await vi.advanceTimersByTimeAsync(1000) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1) + // First activation recovers immediately (retry timestamps start at 0). window.emit('focus') await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) @@ -415,10 +416,116 @@ describe('RateLimitService', () => { await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + // Two consecutive failures: the retry window doubled to 60s, so an + // activation at +30s must not hammer the endpoint again. + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('restore') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Three consecutive failures: 120s window. + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('show') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + await vi.advanceTimersByTimeAsync(60 * 1000) window.emit('restore') await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('resets the retry backoff once Claude recovers', async () => { + vi.useFakeTimers() + try { + vi.mocked(fetchClaudeRateLimits) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockResolvedValueOnce(okProvider('claude', 12)) + .mockResolvedValue(errorProvider('claude', 'failing again')) + vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 24)) + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + // Recovery fetch succeeds and must clear the failure streak. + await vi.advanceTimersByTimeAsync(60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + expect(service.getState().claude?.status).toBe('ok') + + // Next failure starts back at the 30s window, not the doubled one. + await vi.advanceTimersByTimeAsync(5 * 60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + expect(service.getState().claude?.status).toBe('error') + + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('keeps a settled error chip settled during background refetches instead of flashing fetching', async () => { + vi.useFakeTimers() + try { + const secondClaude = deferred() + vi.mocked(fetchClaudeRateLimits) + .mockResolvedValueOnce(errorProvider('claude', 'still failing')) + .mockImplementationOnce(() => secondClaude.promise) + vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 24)) + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + const claudeStatuses: string[] = [] + service.onStateChange((state) => { + if (state.claude) { + claudeStatuses.push(state.claude.status) + } + }) + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + expect(service.getState().claude?.status).toBe('error') + + // Why: the retry must not repaint the settled error chip as a loading + // "…" chip while the refetch is in flight — that is the flash users see + // every cycle when a provider is stuck failing. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(service.getState().claude?.status).toBe('error') + + secondClaude.resolve(okProvider('claude', 12)) + await vi.advanceTimersByTimeAsync(0) + expect(service.getState().claude?.status).toBe('ok') + + const statusesAfterFirstSettle = claudeStatuses.slice(claudeStatuses.indexOf('error')) + expect(statusesAfterFirstSettle).not.toContain('fetching') service.stop() } finally { diff --git a/src/main/rate-limits/service.ts b/src/main/rate-limits/service.ts index cd301cfe516..26a7a85b2d0 100644 --- a/src/main/rate-limits/service.ts +++ b/src/main/rate-limits/service.ts @@ -77,6 +77,12 @@ const MIN_POLL_MS = 30 * 1000 // 30 seconds — renderer input should never crea const MAX_POLL_MS = 2_147_483_647 // Max safe setInterval delay before Node clamps back to 1ms. const MIN_REFETCH_MS = 5 * 60 * 1000 // 5 minutes — debounce resume/manual refresh bursts const ACTIVE_FAILURE_REFETCH_MS = MIN_POLL_MS +// Why: a persistent failure (bad auth, unsupported plan) retried at the 30s +// floor hammers provider endpoints — Claude's tight-budget usage endpoint +// starts returning 429s — without ever recovering. Back off per consecutive +// failure, capped at the background poll cadence. +const MAX_ACTIVE_FAILURE_REFETCH_MS = DEFAULT_POLL_MS +const MAX_ACTIVE_FAILURE_STREAK = 8 // Why: these providers have a dedicated fetch cycle, so an activation retry can // refresh just the failing one. Providers without one force a full fetchAll, so // their error retries stay on the 5-minute cadence to protect Claude's budget. @@ -152,6 +158,18 @@ export class RateLimitService { grok: 0, antigravity: 0 } + // Why: consecutive applied failures per provider drive exponential backoff of + // the fast activation-retry lane; reset on any successful/unavailable result. + private activeFailureStreakByProvider: Record = { + claude: 0, + codex: 0, + gemini: 0, + 'opencode-go': 0, + kimi: 0, + minimax: 0, + grok: 0, + antigravity: 0 + } private mainWindow: BrowserWindow | null = null private detachWindowListeners: (() => void) | null = null private isFetching = false @@ -358,6 +376,8 @@ export class RateLimitService { } this.codexFetchTarget = nextTarget this.codexFetchGeneration += 1 + // Why: a new account/target starts with a clean retry schedule. + this.activeFailureStreakByProvider.codex = 0 this.inactiveCodexAccountsGeneration += 1 this.pruneInactiveCodexState() this.lastInactiveCodexFetchAt = 0 @@ -377,6 +397,7 @@ export class RateLimitService { const targetChanged = !this.isSameCodexTarget(this.codexFetchTarget, nextTarget) this.codexFetchTarget = nextTarget this.codexFetchGeneration += 1 + this.activeFailureStreakByProvider.codex = 0 this.updateState({ ...this.state, codex: this.withFetchingStatus(targetChanged ? null : this.state.codex, 'codex') @@ -426,6 +447,8 @@ export class RateLimitService { this.inactiveClaudeAccountsGeneration += 1 this.pruneInactiveClaudeState() this.claudeFetchGeneration += 1 + // Why: a new account/target starts with a clean retry schedule. + this.activeFailureStreakByProvider.claude = 0 this.lastInactiveClaudeFetchAt = 0 this.updateState({ ...this.state, @@ -440,6 +463,7 @@ export class RateLimitService { const targetChanged = !this.isSameClaudeTarget(this.claudeFetchTarget, nextTarget) this.claudeFetchTarget = nextTarget this.claudeFetchGeneration += 1 + this.activeFailureStreakByProvider.claude = 0 this.updateState({ ...this.state, claude: this.withFetchingStatus(targetChanged ? null : this.state.claude, 'claude') @@ -764,7 +788,11 @@ export class RateLimitService { if (limits.status === 'error') { const lastRetryAt = this.lastActiveFailureRetryAtByProvider[provider] const throttleMs = INDIVIDUALLY_REFRESHABLE_PROVIDERS.has(provider) - ? ACTIVE_FAILURE_REFETCH_MS + ? Math.min( + ACTIVE_FAILURE_REFETCH_MS * + 2 ** Math.max(0, this.activeFailureStreakByProvider[provider] - 1), + MAX_ACTIVE_FAILURE_REFETCH_MS + ) : MIN_REFETCH_MS if (now - lastRetryAt >= throttleMs) { retryableFailures.push(provider) @@ -1242,6 +1270,22 @@ export class RateLimitService { } } + private trackActiveFailureStreak( + provider: ActiveRateLimitProvider, + fresh: ProviderRateLimits + ): void { + if (fresh.status === 'error') { + this.activeFailureStreakByProvider[provider] = Math.min( + this.activeFailureStreakByProvider[provider] + 1, + MAX_ACTIVE_FAILURE_STREAK + ) + return + } + if (fresh.status === 'ok' || fresh.status === 'unavailable') { + this.activeFailureStreakByProvider[provider] = 0 + } + } + private withFetchingStatus( current: ProviderRateLimits | null, provider: @@ -1264,6 +1308,13 @@ export class RateLimitService { status: 'fetching' } } + // Why: repainting a settled chip as "fetching" on every background refetch + // makes the status bar flash "…" → error each retry cycle when a provider + // is persistently failing. Keep the settled state visible until the new + // result lands; only providers with no settled state show a loading chip. + if (current.status === 'ok' || current.status === 'error' || current.status === 'unavailable') { + return current + } return { ...current, status: 'fetching' } } @@ -1481,6 +1532,22 @@ export class RateLimitService { const shouldApplyOpencode = opencodeGeneration === this.opencodeFetchGeneration const shouldApplyMiniMax = miniMaxGeneration === this.minimaxFetchGeneration + if (shouldApplyClaude) { + this.trackActiveFailureStreak('claude', claude) + } + if (shouldApplyCodex) { + this.trackActiveFailureStreak('codex', codex) + } + this.trackActiveFailureStreak('gemini', gemini) + this.trackActiveFailureStreak('antigravity', antigravity) + if (shouldApplyOpencode) { + this.trackActiveFailureStreak('opencode-go', opencodeGo) + } + this.trackActiveFailureStreak('kimi', kimi) + if (shouldApplyMiniMax) { + this.trackActiveFailureStreak('minimax', miniMax) + } + // Why: account switches can race in-flight Codex fetches. Only apply a // Codex result if both the selected-account provenance and the request // generation still match, otherwise an old account could overwrite the @@ -1523,6 +1590,7 @@ export class RateLimitService { error: grokResult.reason instanceof Error ? grokResult.reason.message : 'Unknown error', status: 'error' } satisfies ProviderRateLimits) + this.trackActiveFailureStreak('grok', grok) this.updateState({ ...this.state, grok: this.applyStalePolicy(grok, previousState.grok) @@ -1575,6 +1643,9 @@ export class RateLimitService { const shouldApplyCodex = codexGeneration === this.codexFetchGeneration && codexProvenance === latestCodexProvenance + if (shouldApplyCodex) { + this.trackActiveFailureStreak('codex', codex) + } this.updateState({ ...this.state, codex: shouldApplyCodex ? this.applyStalePolicy(codex, previousState.codex) : this.state.codex @@ -1630,6 +1701,9 @@ export class RateLimitService { claudeProvenance === latestClaudeProvenance && this.isSameClaudeTarget(claudeTarget, this.claudeFetchTarget) + if (shouldApplyClaude) { + this.trackActiveFailureStreak('claude', claude) + } this.updateState({ ...this.state, claude: shouldApplyClaude @@ -1669,6 +1743,7 @@ export class RateLimitService { return } + this.trackActiveFailureStreak('grok', grok) this.updateState({ ...this.state, grok: this.applyStalePolicy(grok, previousState.grok) From 24307853879c56912e09b32e0609d834e7272180 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:09:04 -0700 Subject: [PATCH 2/2] Count full fetches as failure-lane retries and keep Grok pane refresh feedback - Stamp failing providers' activation-retry clocks when a stale-driven full fetch runs, so the individual failure lane does not fire a redundant retry right after the full fetch already retried them. - GrokUsagePane: manual refresh spinner/disable is now renderer-local, since settled snapshots no longer repaint as 'fetching' during refetches. - Strengthen the backoff-reset test so it distinguishes a reset streak from a stale retry timestamp (CodeRabbit), and add a regression test for the full-fetch retry stamping. --- src/main/rate-limits/service.test.ts | 75 ++++++++++++++++++- src/main/rate-limits/service.ts | 12 +++ .../src/components/stats/GrokUsagePane.tsx | 17 ++++- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/main/rate-limits/service.test.ts b/src/main/rate-limits/service.test.ts index f40f11aa3ce..9bd66cb806a 100644 --- a/src/main/rate-limits/service.test.ts +++ b/src/main/rate-limits/service.test.ts @@ -120,6 +120,19 @@ function unavailableProvider( } } +// Why: beforeEach caches snapshot objects whose updatedAt is pinned at suite +// start, so after 5 fake minutes every healthy provider looks stale and every +// activation degrades to a full fetch. Backoff tests that reason about the +// individual retry lane need healthy providers minted fresh at fetch time. +function mockFreshBackgroundProviderFetches(): void { + vi.mocked(fetchCodexRateLimits).mockImplementation(async () => okProvider('codex', 24)) + vi.mocked(fetchGeminiRateLimits).mockImplementation(async () => okProvider('gemini', 0)) + vi.mocked(fetchOpenCodeGoRateLimits).mockImplementation(async () => okProvider('opencode-go', 0)) + vi.mocked(fetchKimiRateLimits).mockImplementation(async () => okProvider('kimi', 0)) + vi.mocked(fetchMiniMaxRateLimits).mockImplementation(async () => okProvider('minimax', 0)) + vi.mocked(fetchGrokRateLimits).mockImplementation(async () => unavailableProvider('grok')) +} + function serviceInternals(service: RateLimitService): { fetchAll: () => Promise } { return service as unknown as { fetchAll: () => Promise } } @@ -453,7 +466,7 @@ describe('RateLimitService', () => { .mockResolvedValueOnce(errorProvider('claude', 'still failing')) .mockResolvedValueOnce(okProvider('claude', 12)) .mockResolvedValue(errorProvider('claude', 'failing again')) - vi.mocked(fetchCodexRateLimits).mockResolvedValue(okProvider('codex', 24)) + mockFreshBackgroundProviderFetches() const service = new RateLimitService() const window = new FakeRateLimitWindow() @@ -472,18 +485,76 @@ describe('RateLimitService', () => { expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) expect(service.getState().claude?.status).toBe('ok') - // Next failure starts back at the 30s window, not the doubled one. + // The stale-ok snapshot forces a full refresh, which fails again. await vi.advanceTimersByTimeAsync(5 * 60 * 1000) window.emit('focus') await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) expect(service.getState().claude?.status).toBe('error') + // Consume the stale active-retry timestamp so the next windows measure + // the post-recovery streak rather than time elapsed before recovery. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + + // Post-recovery streak is 2 (not the pre-recovery 4): the window must be + // 60s, so +30s stays throttled and +60s retries. await vi.advanceTimersByTimeAsync(30 * 1000) window.emit('focus') await vi.advanceTimersByTimeAsync(0) expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(5) + await vi.advanceTimersByTimeAsync(30 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(6) + + service.stop() + } finally { + vi.useRealTimers() + } + }) + + it('counts a stale-driven full fetch as the failing provider retry attempt', async () => { + vi.useFakeTimers() + try { + vi.mocked(fetchClaudeRateLimits).mockImplementation(async () => + errorProvider('claude', 'still failing') + ) + mockFreshBackgroundProviderFetches() + + const service = new RateLimitService() + const window = new FakeRateLimitWindow() + service.attach(asRateLimitWindow(window)) + service.start({ fetchImmediately: false }) + + await vi.advanceTimersByTimeAsync(1000) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(1) + + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(2) + + // Healthy providers go stale after 5 minutes, so this activation runs a + // full fetch that also retries failing Claude (streak now 3 → 120s). + await vi.advanceTimersByTimeAsync(5 * 60 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Why: the full fetch was itself a retry. An activation moments later + // must not fire the individual failure lane ahead of the backoff window. + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(3) + + // Once the 120s window elapses, the individual retry lane fires again. + await vi.advanceTimersByTimeAsync(120 * 1000) + window.emit('focus') + await vi.advanceTimersByTimeAsync(0) + expect(fetchClaudeRateLimits).toHaveBeenCalledTimes(4) + service.stop() } finally { vi.useRealTimers() diff --git a/src/main/rate-limits/service.ts b/src/main/rate-limits/service.ts index 26a7a85b2d0..84fb016fec8 100644 --- a/src/main/rate-limits/service.ts +++ b/src/main/rate-limits/service.ts @@ -811,6 +811,18 @@ export class RateLimitService { return } if (plan.kind === 'full') { + // Why: a full fetch retries failing providers too. Restart their retry + // clocks so the individual failure lane doesn't fire again right after, + // ahead of its backoff window. Skip when a fetch is already in flight — + // fetchAll would no-op and the throttle must not be consumed for free. + if (!this.isFetching) { + const now = Date.now() + for (const { provider, limits } of this.getActiveProviderState()) { + if (limits?.status === 'error') { + this.lastActiveFailureRetryAtByProvider[provider] = now + } + } + } await this.fetchAll() return } diff --git a/src/renderer/src/components/stats/GrokUsagePane.tsx b/src/renderer/src/components/stats/GrokUsagePane.tsx index 8e51049e3d4..08600621552 100644 --- a/src/renderer/src/components/stats/GrokUsagePane.tsx +++ b/src/renderer/src/components/stats/GrokUsagePane.tsx @@ -1,3 +1,4 @@ +import { useState } from 'react' import { CalendarClock, ExternalLink, RefreshCw, Sparkles } from 'lucide-react' import { translate } from '@/i18n/i18n' import { useAppStore } from '../../store' @@ -13,6 +14,18 @@ export function GrokUsagePane(): React.JSX.Element { const openSettingsPage = useAppStore((s) => s.openSettingsPage) const openSettingsTarget = useAppStore((s) => s.openSettingsTarget) const recordFeatureInteraction = useAppStore((s) => s.recordFeatureInteraction) + // Why: settled snapshots keep their status during refetches (no 'fetching' + // repaint), so manual-refresh feedback must be renderer-local, matching the + // StatusBar refresh button. + const [isRefreshing, setIsRefreshing] = useState(false) + + const handleRefresh = (): void => { + if (isRefreshing) { + return + } + setIsRefreshing(true) + void refreshGrokRateLimits().finally(() => setIsRefreshing(false)) + } const openGrokAccounts = (): void => { openSettingsTarget({ pane: 'accounts', repoId: null, sectionId: 'accounts-grok' }) @@ -57,7 +70,7 @@ export function GrokUsagePane(): React.JSX.Element { grok?.weekly && typeof grok.weekly.usedPercent === 'number' ? Math.round(grok.weekly.usedPercent) : null - const isFetching = grok?.status === 'fetching' + const isFetching = isRefreshing || grok?.status === 'fetching' return (
void refreshGrokRateLimits()} + onClick={handleRefresh} disabled={isFetching} aria-label={translate( 'auto.components.stats.GrokUsagePane.i0j1k2l3m4',