Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions server/src/__tests__/services/router-bandit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,18 +211,27 @@ describe('bandit router', () => {
});

it('getRoutingScores returns a per-axis breakdown ranked by score', () => {
addModel({ platform: 'google', modelId: 'm1', name: 'M1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
addHistory('google', 'm1', { successes: 30, failures: 0, outTokens: 500, latencyMs: 1000, ttfbMs: 200 });
setRoutingStrategy('balanced');
refreshStatsCache(getDb(), true);
const { strategy, weights, scores } = getRoutingScores();
expect(strategy).toBe('balanced');
expect(weights).toEqual({ reliability: 0.5, speed: 0.25, intelligence: 0.25 });
expect(scores).toHaveLength(1);
expect(scores[0]).toMatchObject({ modelId: 'm1', enabled: true });
expect(scores[0].reliability).toBeGreaterThan(0.9);
expect(scores[0].score).toBeGreaterThan(0);
expect(scores[0].score).toBeLessThanOrEqual(1);
// The time-of-day adjustment (#760) rewrites bandit weights during local
// peak hours (18:00–06:00), so pin the clock to off-peak noon to make the
// balanced-preset assertion deterministic regardless of when CI runs.
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-15T12:00:00'));
try {
addModel({ platform: 'google', modelId: 'm1', name: 'M1', intelligenceRank: 1, sizeLabel: 'Frontier', budget: '~50M', priority: 1 });
addHistory('google', 'm1', { successes: 30, failures: 0, outTokens: 500, latencyMs: 1000, ttfbMs: 200 });
setRoutingStrategy('balanced');
refreshStatsCache(getDb(), true);
const { strategy, weights, scores } = getRoutingScores();
expect(strategy).toBe('balanced');
expect(weights).toEqual({ reliability: 0.5, speed: 0.25, intelligence: 0.25 });
expect(scores).toHaveLength(1);
expect(scores[0]).toMatchObject({ modelId: 'm1', enabled: true });
expect(scores[0].reliability).toBeGreaterThan(0.9);
expect(scores[0].score).toBeGreaterThan(0);
expect(scores[0].score).toBeLessThanOrEqual(1);
} finally {
vi.useRealTimers();
}
});

it('exploration toggle persists and defaults to off', () => {
Expand Down
51 changes: 51 additions & 0 deletions server/src/__tests__/services/scoring.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
BANDIT_PRESETS, combineScore, speedScore, intelligenceScore, intelligenceComposite,
headroomFactor, rateLimitFactor, sampleBeta, reliabilityPosterior,
expectedReliability, SPEED_PRIOR, HEADROOM_FLOOR,
isPeakHours, timeOfDayWeights, PEAK_START_HOUR, PEAK_END_HOUR, PEAK_SPEED_TO_RELIABILITY,
} from '../../services/scoring.js';

describe('scoring: reliability posterior', () => {
Expand Down Expand Up @@ -201,3 +202,53 @@ describe('scoring: Beta sampler (Thompson exploration)', () => {
expect(weakerWonAtLeastOnce).toBe(true);
});
});

describe('scoring: time-of-day dynamic ranking (#760)', () => {
function at(hour: number, minute = 0): Date {
return new Date(2026, 7, 18, hour, minute); // local time, Aug 18
}

it('marks peak hours 18:00–06:00 (inclusive start, exclusive end)', () => {
expect(isPeakHours(at(PEAK_START_HOUR))).toBe(true); // 18:00 → peak
expect(isPeakHours(at(23, 59))).toBe(true); // late night → peak
expect(isPeakHours(at(0))).toBe(true); // midnight → peak
expect(isPeakHours(at(5, 59))).toBe(true); // just before 06:00 → peak
expect(isPeakHours(at(PEAK_END_HOUR))).toBe(false); // 06:00 → off-peak
expect(isPeakHours(at(12))).toBe(false); // noon → off-peak
});

it('keeps off-peak weights unchanged', () => {
const base = BANDIT_PRESETS.balanced;
expect(timeOfDayWeights(base, at(12))).toEqual(base);
expect(timeOfDayWeights(base, at(PEAK_END_HOUR))).toEqual(base);
});

it('shifts speed→reliability during peak hours, intelligence untouched', () => {
const base = BANDIT_PRESETS.balanced; // { reliability: 0.5, speed: 0.25, intelligence: 0.25 }
const peak = timeOfDayWeights(base, at(20));
const shift = base.speed * PEAK_SPEED_TO_RELIABILITY;
expect(peak.reliability).toBeCloseTo(base.reliability + shift, 5);
expect(peak.speed).toBeCloseTo(base.speed - shift, 5);
expect(peak.intelligence).toBe(base.intelligence);
// Weights still sum to 1.
expect(peak.reliability + peak.speed + peak.intelligence).toBeCloseTo(1, 5);
});

it('never returns a negative speed weight for any preset', () => {
for (const preset of Object.values(BANDIT_PRESETS)) {
const peak = timeOfDayWeights(preset, at(21));
expect(peak.speed).toBeGreaterThanOrEqual(0);
expect(peak.reliability + peak.speed + peak.intelligence).toBeCloseTo(1, 5);
}
});

it('defaults to the current wall-clock time', () => {
const now = new Date();
const base = BANDIT_PRESETS.balanced;
if (isPeakHours(now)) {
expect(timeOfDayWeights(base)).not.toEqual(base);
} else {
expect(timeOfDayWeights(base)).toEqual(base);
}
});
});
7 changes: 6 additions & 1 deletion server/src/services/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
reliabilityPosterior, expectedReliability, sampleBeta,
speedScore, intelligenceScore, intelligenceComposite, headroomFactor, rateLimitFactor, combineScore,
timeOfDayWeights,
observedSpeedRank, TIMEOUT_LATENCY_CAP_MS,
} from './scoring.js';
import { TIMEOUT_ERROR_MARKERS } from '../lib/error-classify.js';
Expand Down Expand Up @@ -512,7 +513,11 @@ export function setCommunityPriors(priors: CommunityPriorMap): number {
function weightsFor(strategy: RoutingStrategy): RoutingWeights | null {
if (strategy === 'priority') return null;
if (strategy === 'custom') return getCustomWeights();
return BANDIT_PRESETS[strategy];
// Bandit presets get the time-of-day adjustment (#760): during local peak
// hours (18:00–06:00) free relays are congested, so speed weight is shifted
// onto reliability. priority/custom are the operator's explicit choice and
// are returned untouched.
return timeOfDayWeights(BANDIT_PRESETS[strategy]);
}

// ── Analytics stats cache (decay-weighted) ──────────────────────────────────
Expand Down
29 changes: 29 additions & 0 deletions server/src/services/scoring.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,35 @@ export const BANDIT_PRESETS: Record<Exclude<RoutingStrategy, 'priority' | 'custo
// dashboard or PUT /api/fallback/routing.
export const DEFAULT_STRATEGY: RoutingStrategy = 'balanced';

// ── Time-of-day dynamic ranking (#760) ─────────────────────────────────────
// Local peak hours (18:00–06:00): free relays are congested, so a model's raw
// throughput is a weaker signal than its reliability. Shift part of the speed
// weight onto reliability for bandit strategies; off-peak behaviour is
// unchanged. Custom and priority strategies are the operator's explicit choice
// and are never rewritten.
export const PEAK_START_HOUR = 18;
export const PEAK_END_HOUR = 6; // spans midnight: [18, 24) ∪ [0, 6)
/** Fraction of the speed weight moved onto reliability during peak hours. */
export const PEAK_SPEED_TO_RELIABILITY = 0.6;

/** True when `now` (local time) falls inside the congested peak window. */
export function isPeakHours(now = new Date()): boolean {
const h = now.getHours();
return h >= PEAK_START_HOUR || h < PEAK_END_HOUR;
}

/** Time-of-day adjusted weights: during peak hours move PEAK_SPEED_TO_RELIABILITY
* of the speed weight onto reliability; otherwise return the base unchanged. */
export function timeOfDayWeights(base: RoutingWeights, now = new Date()): RoutingWeights {
if (!isPeakHours(now)) return base;
const shift = base.speed * PEAK_SPEED_TO_RELIABILITY;
return {
reliability: base.reliability + shift,
speed: base.speed - shift,
intelligence: base.intelligence,
};
}

// ── Reliability ───────────────────────────────────────────────────────────
// Beta(1,1) prior = uniform: an unseen model is genuinely uncertain, not assumed
// good or bad. With decay-weighted pseudo-counts the alpha/beta are continuous.
Expand Down
Loading