diff --git a/scripts/lib/brief-compose.mjs b/scripts/lib/brief-compose.mjs index 52c637666..f032de3ff 100644 --- a/scripts/lib/brief-compose.mjs +++ b/scripts/lib/brief-compose.mjs @@ -151,7 +151,36 @@ export function userDisplayNameFromId(userId) { // ── Compose a full brief for a single rule ────────────────────────────────── -const MAX_STORIES_PER_USER = 12; +// Cap on stories shown per user per brief. +// +// Default 12 — kept at the historical value because the offline sweep +// harness (scripts/sweep-topic-thresholds.mjs) showed bumping the cap +// to 16 against 2026-04-24 production replay data DROPPED visible +// quality at the active 0.45 threshold (visible_quality 0.916 → 0.716; +// positions 13-16 are mostly singletons or members of "should-separate" +// clusters at this threshold, so they dilute without helping adjacency). +// +// Env-tunable via DIGEST_MAX_STORIES_PER_USER so future sweep evidence +// (different threshold, different label set, different pool composition) +// can be acted on with a Railway env flip without a redeploy. Any +// invalid / non-positive value falls back to the 12 default. +// +// "Are we getting better" signal: re-run scripts/sweep-topic-thresholds.mjs +// with --cap N before flipping the env, and the daily +// scripts/brief-quality-report.mjs after. +function readMaxStoriesPerUser() { + const raw = process.env.DIGEST_MAX_STORIES_PER_USER; + if (raw == null || raw === '') return 12; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : 12; +} +// Exported so brief-llm.mjs (buildDigestPrompt + hashDigestInput) can +// slice to the same cap. Hard-coding `slice(0, 12)` there would mean +// the LLM prose only references the first 12 stories even when the +// brief envelope carries more — a quiet mismatch between what the +// reader sees as story cards vs the AI summary above them. Reviewer +// P1 on PR #3389. +export const MAX_STORIES_PER_USER = readMaxStoriesPerUser(); /** * Filter + assemble a BriefEnvelope for one alert rule from a diff --git a/scripts/lib/brief-llm.mjs b/scripts/lib/brief-llm.mjs index 5e49503da..0e1a680b0 100644 --- a/scripts/lib/brief-llm.mjs +++ b/scripts/lib/brief-llm.mjs @@ -34,6 +34,10 @@ import { parseWhyMatters, } from '../../shared/brief-llm-core.js'; import { sanitizeForPrompt } from '../../server/_shared/llm-sanitize.js'; +// Single source of truth for the brief story cap. Both buildDigestPrompt +// and hashDigestInput must slice to this value or the LLM prose drifts +// from the rendered story cards (PR #3389 reviewer P1). +import { MAX_STORIES_PER_USER } from './brief-compose.mjs'; /** * Sanitize the story fields that flow into buildWhyMattersUserPrompt and @@ -323,7 +327,7 @@ const DIGEST_PROSE_SYSTEM = * @returns {{ system: string; user: string }} */ export function buildDigestPrompt(stories, sensitivity) { - const lines = stories.slice(0, 12).map((s, i) => { + const lines = stories.slice(0, MAX_STORIES_PER_USER).map((s, i) => { const n = String(i + 1).padStart(2, '0'); return `${n}. [${s.threatLevel}] ${s.headline} — ${s.category} · ${s.country} · ${s.source}`; }); @@ -422,10 +426,11 @@ function hashDigestInput(userId, stories, sensitivity) { // Canonicalise as JSON of the fields the prompt actually references, // in the prompt's ranked order. Stable stringification via an array // of tuples keeps field ordering deterministic without relying on - // JS object-key iteration order. + // JS object-key iteration order. Slice MUST match buildDigestPrompt's + // slice or the cache key drifts from the prompt content. const material = JSON.stringify([ sensitivity ?? '', - ...stories.slice(0, 12).map((s) => [ + ...stories.slice(0, MAX_STORIES_PER_USER).map((s) => [ s.headline ?? '', s.threatLevel ?? '', s.category ?? '', diff --git a/tests/brief-from-digest-stories.test.mjs b/tests/brief-from-digest-stories.test.mjs index 1628bbc9c..d0cc01cb5 100644 --- a/tests/brief-from-digest-stories.test.mjs +++ b/tests/brief-from-digest-stories.test.mjs @@ -192,7 +192,14 @@ describe('composeBriefFromDigestStories — continued', () => { assert.deepEqual(env.data.stories.map((s) => s.headline), ['A', 'B']); }); - it('caps at 12 stories per brief', () => { + it('caps at 12 stories per brief by default (env-tunable via DIGEST_MAX_STORIES_PER_USER)', () => { + // Default kept at 12. Offline sweep harness against 2026-04-24 + // production replay showed cap=16 dropped visible_quality from + // 0.916 → 0.716 at the active 0.45 threshold (positions 13-16 + // are mostly singletons or "should-separate" members at this + // threshold, so they dilute without helping adjacency). The + // constant is env-tunable so a Railway flip can experiment with + // cap values once new sweep evidence justifies them. const many = Array.from({ length: 30 }, (_, i) => digestStory({ hash: `h${i}`, title: `Story ${i}` }), );