test(digest): backfill A6.h/i/l/m acceptance tests via helper extraction

This commit is contained in:
Elie Habib
2026-04-25 14:43:55 +04:00
parent 6ff2c3d9e4
commit df3563080e
3 changed files with 496 additions and 81 deletions

View File

@@ -0,0 +1,150 @@
// Pure helpers for the digest cron's per-user compose loop.
//
// Extracted from scripts/seed-digest-notifications.mjs so they can be
// unit-tested without dragging the cron's env-checking side effects
// (DIGEST_CRON_ENABLED check, Upstash REST helper, Convex relay
// auth) into the test runtime. The cron imports back from here.
import { compareRules, MAX_STORIES_PER_USER } from './brief-compose.mjs';
import { generateDigestProse } from './brief-llm.mjs';
/**
* Build the email subject string. Extracted so the synthesis-level
* → subject ternary can be unit-tested without standing up the whole
* cron loop. (Plan acceptance criterion A6.i.)
*
* Rules:
* - synthesisLevel 1 or 2 + non-empty briefLead → "Intelligence Brief"
* - synthesisLevel 3 OR empty/null briefLead → "Digest"
*
* Mirrors today's UX where the editorial subject only appeared when
* a real LLM-produced lead was available; the L3 stub falls back to
* the plain "Digest" subject to set reader expectations correctly.
*
* @param {{ briefLead: string | null | undefined; synthesisLevel: number; shortDate: string }} input
* @returns {string}
*/
export function subjectForBrief({ briefLead, synthesisLevel, shortDate }) {
if (briefLead && synthesisLevel >= 1 && synthesisLevel <= 2) {
return `WorldMonitor Intelligence Brief — ${shortDate}`;
}
return `WorldMonitor Digest — ${shortDate}`;
}
/**
* Walk an annotated rule list and return the winning candidate +
* its non-empty story pool. Two-pass: due rules first (so the
* synthesis comes from a rule that's actually sending), then ALL
* eligible rules (compose-only tick — keeps the dashboard brief
* fresh for weekly/twice_daily users). Within each pass, walk by
* compareRules priority and pick the FIRST candidate whose pool is
* non-empty.
*
* Returns null when every candidate has an empty pool — caller
* skips the user (same as today's behavior).
*
* Plan acceptance criteria A6.l (compose-only tick still works for
* weekly user) + A6.m (winner walks past empty-pool top-priority
* candidate). Codex Round-3 High #1 + Round-4 High #1 + Round-4
* Medium #2.
*
* `log` is the per-empty-pool log emitter — passed in so tests can
* capture lines without reaching for console.log.
*
* @param {Array<{ rule: object; lastSentAt: number | null; due: boolean }>} annotated
* @param {(rule: object) => Promise<unknown[] | null | undefined>} digestFor
* @param {(line: string) => void} log
* @param {string} userId
* @returns {Promise<{ winner: { rule: object; lastSentAt: number | null; due: boolean }; stories: unknown[] } | null>}
*/
export async function pickWinningCandidateWithPool(annotated, digestFor, log, userId) {
if (!Array.isArray(annotated) || annotated.length === 0) return null;
const sortedDue = annotated.filter((a) => a.due).sort((a, b) => compareRules(a.rule, b.rule));
const sortedAll = [...annotated].sort((a, b) => compareRules(a.rule, b.rule));
// Build the walk order, deduping by rule reference so the same
// rule isn't tried twice (a due rule appears in both sortedDue
// and sortedAll).
const seen = new Set();
const walkOrder = [];
for (const cand of [...sortedDue, ...sortedAll]) {
if (seen.has(cand.rule)) continue;
seen.add(cand.rule);
walkOrder.push(cand);
}
for (const cand of walkOrder) {
const stories = await digestFor(cand.rule);
if (!stories || stories.length === 0) {
log(
`[digest] brief filter drops user=${userId} ` +
`sensitivity=${cand.rule.sensitivity ?? 'high'} ` +
`variant=${cand.rule.variant ?? 'full'} ` +
`due=${cand.due} ` +
`outcome=empty-pool ` +
`in=0 dropped_severity=0 dropped_url=0 dropped_headline=0 dropped_shape=0 dropped_cap=0 out=0`,
);
continue;
}
return { winner: cand, stories };
}
return null;
}
/**
* Run the three-level canonical synthesis fallback chain.
* L1: full pre-cap pool + ctx (profile, greeting, !public) — canonical.
* L2: envelope-sized slice + empty ctx — degraded fallback (mirrors
* today's enrichBriefEnvelopeWithLLM behaviour).
* L3: null synthesis — caller composes from stub.
*
* Returns { synthesis, level } with `synthesis` matching
* generateDigestProse's output shape (or null on L3) and `level`
* one of {1, 2, 3}.
*
* Pure helper — no I/O beyond the deps.callLLM the inner functions
* already perform. Errors at L1 propagate to L2; L2 errors propagate
* to L3 (null/stub). `trace` callback fires per level transition so
* callers can quantify failure-mode distribution in production logs.
*
* Plan acceptance criterion A6.h (3-level fallback triggers).
*
* @param {string} userId
* @param {Array} stories — full pre-cap pool
* @param {string} sensitivity
* @param {{ profile: string | null; greeting: string | null }} ctx
* @param {{ callLLM: Function; cacheGet: Function; cacheSet: Function }} deps
* @param {(level: 1 | 2 | 3, kind: 'success' | 'fall' | 'throw', err?: unknown) => void} [trace]
* @returns {Promise<{ synthesis: object | null; level: 1 | 2 | 3 }>}
*/
export async function runSynthesisWithFallback(userId, stories, sensitivity, ctx, deps, trace) {
const noteTrace = typeof trace === 'function' ? trace : () => {};
// L1 — canonical
try {
const l1 = await generateDigestProse(userId, stories, sensitivity, deps, {
profile: ctx?.profile ?? null,
greeting: ctx?.greeting ?? null,
isPublic: false,
});
if (l1) {
noteTrace(1, 'success');
return { synthesis: l1, level: 1 };
}
noteTrace(1, 'fall');
} catch (err) {
noteTrace(1, 'throw', err);
}
// L2 — degraded fallback
try {
const cappedSlice = (Array.isArray(stories) ? stories : []).slice(0, MAX_STORIES_PER_USER);
const l2 = await generateDigestProse(userId, cappedSlice, sensitivity, deps);
if (l2) {
noteTrace(2, 'success');
return { synthesis: l2, level: 2 };
}
noteTrace(2, 'fall');
} catch (err) {
noteTrace(2, 'throw', err);
}
// L3 — stub
noteTrace(3, 'success');
return { synthesis: null, level: 3 };
}

View File

@@ -39,6 +39,11 @@ import {
MAX_STORIES_PER_USER,
shouldExitNonZero as shouldExitOnBriefFailures,
} from './lib/brief-compose.mjs';
import {
pickWinningCandidateWithPool,
runSynthesisWithFallback,
subjectForBrief,
} from './lib/digest-orchestration-helpers.mjs';
import { issueSlotInTz } from '../shared/brief-filter.js';
import {
enrichBriefEnvelopeWithLLM,
@@ -1322,46 +1327,12 @@ async function composeBriefsForRun(rules, nowMs) {
* @param {number} nowMs
*/
async function composeAndStoreBriefForUser(userId, annotated, insightsNumbers, digestFor, nowMs) {
// Two-pass walk: prefer DUE rules first (so the synthesis pool +
// ctx come from a rule that's about to send), fall back to ANY
// eligible rule (compose-only tick — keeps the dashboard brief
// refreshed for weekly/twice_daily users on non-due ticks). Within
// each pass, `compareRules` priority order. Pick first candidate
// with non-empty pool — mirrors today's behavior at the legacy
// composeAndStoreBriefForUser walk.
// Codex Round-3 High #1 + Round-4 High #1 + Round-4 Medium #2.
const sortedDue = annotated.filter((a) => a.due).sort((a, b) => compareRules(a.rule, b.rule));
const sortedAll = [...annotated].sort((a, b) => compareRules(a.rule, b.rule));
let winner = null;
let winnerStories = null;
for (const cand of [...sortedDue, ...sortedAll]) {
if (winner) break;
const stories = await digestFor(cand.rule);
const dropStats = { severity: 0, headline: 0, url: 0, shape: 0, cap: 0, in: stories?.length ?? 0 };
if (!stories || stories.length === 0) {
// Per-attempt filter-drop line for the empty-pool case so ops
// can distinguish "pool empty upstream" from "all stories
// filtered out post-compose". outcome=empty-pool is a third
// value beyond shipped/rejected.
console.log(
`[digest] brief filter drops user=${userId} ` +
`sensitivity=${cand.rule.sensitivity ?? 'high'} ` +
`variant=${cand.rule.variant ?? 'full'} ` +
`due=${cand.due} ` +
`outcome=empty-pool ` +
`in=0 dropped_severity=0 dropped_url=0 dropped_headline=0 dropped_shape=0 dropped_cap=0 out=0`,
);
continue;
}
winner = cand;
winnerStories = stories;
// No console.log here — the per-attempt log is emitted inside the
// composeBriefFromDigestStories call below via onDrop, so the row
// carries the actual `out` count after filtering.
}
if (!winner) return null;
// Two-pass walk extracted to a pure helper so it can be unit-tested
// (A6.l + A6.m). When no candidate has a non-empty pool, returns
// null — same outcome as today's behavior.
const winnerResult = await pickWinningCandidateWithPool(annotated, digestFor, (line) => console.log(line), userId);
if (!winnerResult) return null;
const { winner, stories: winnerStories } = winnerResult;
// ── Canonical synthesis (3-level fallback chain) ────────────────────
//
@@ -1381,41 +1352,27 @@ async function composeAndStoreBriefForUser(userId, annotated, insightsNumbers, d
let synthesisLevel = 3; // pessimistic default; bumped on success
if (BRIEF_LLM_ENABLED) {
const ctx = await buildSynthesisCtx(winner.rule, nowMs);
// L1 — canonical
try {
const l1 = await generateDigestProse(
const result = await runSynthesisWithFallback(
userId,
winnerStories,
sensitivity,
ctx,
briefLlmDeps,
{ profile: ctx.profile, greeting: ctx.greeting, isPublic: false },
(level, kind, err) => {
if (kind === 'throw') {
console.warn(
`[digest] brief: synthesis L${level} threw for ${userId} — falling to L${level + 1}:`,
err?.message,
);
if (l1) {
synthesis = l1;
synthesisLevel = 1;
}
} catch (err) {
console.warn(`[digest] brief: synthesis L1 threw for ${userId} — falling to L2:`, err?.message);
}
// L2 — degraded fallback (no profile/greeting; envelope-sized pool slice)
if (!synthesis) {
try {
const cappedSlice = winnerStories.slice(0, MAX_STORIES_PER_USER);
const l2 = await generateDigestProse(userId, cappedSlice, sensitivity, briefLlmDeps);
if (l2) {
synthesis = l2;
synthesisLevel = 2;
} else if (kind === 'success' && level === 2) {
console.log(`[digest] synthesis level=2_degraded user=${userId}`);
}
} catch (err) {
console.warn(`[digest] brief: synthesis L2 threw for ${userId} — falling to L3 stub:`, err?.message);
}
}
// L3 — stub. synthesis stays null; composer's assembleStubbedBriefEnvelope
// path produces the stub lead.
if (!synthesis) {
} else if (kind === 'success' && level === 3) {
console.log(`[digest] synthesis level=3_stub user=${userId}`);
}
},
);
synthesis = result.synthesis;
synthesisLevel = result.level;
// publicLead — parallel call. Profile-stripped; cache-shared
// across all users for the same (date, sensitivity, story-pool).
// Failure is non-fatal — the renderer's public-mode fail-safe
@@ -1698,14 +1655,7 @@ async function main() {
const html = injectBriefCta(htmlWithSummary, magazineUrl);
const shortDate = new Intl.DateTimeFormat('en-US', { month: 'short', day: 'numeric' }).format(new Date(nowMs));
// Subject: "Intelligence Brief" when the canonical synthesis (L1
// or L2) succeeded; "Digest" when only the L3 stub is in the
// envelope. Mirrors today's behavior (briefLead non-null = signal
// strong enough for the editorial subject) but reads from the
// cron-local synthesisLevel rather than re-checking the LLM call.
const subject = (briefLead && synthesisLevel <= 2)
? `WorldMonitor Intelligence Brief — ${shortDate}`
: `WorldMonitor Digest — ${shortDate}`;
const subject = subjectForBrief({ briefLead, synthesisLevel, shortDate });
let anyDelivered = false;

View File

@@ -0,0 +1,315 @@
// Pure-function unit tests for the canonical-synthesis orchestration
// helpers extracted from scripts/seed-digest-notifications.mjs.
//
// Covers plan acceptance criteria:
// A6.h — three-level synthesis fallback chain
// A6.i — subject-line correctness ("Intelligence Brief" vs "Digest")
// A6.l — compose-only tick still works for weekly user (sortedAll fallback)
// A6.m — winner walks past empty-pool top-priority candidate
//
// Acceptance criteria A6.a-d (multi-rule, twice_daily, weekly window
// parity, all-channel reads) require a full mock of the cron's main()
// loop with Upstash + Convex stubs — out of scope for this PR's
// pure-function coverage. They are exercised via the parity log line
// (A5) in production observability instead.
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
pickWinningCandidateWithPool,
runSynthesisWithFallback,
subjectForBrief,
} from '../scripts/lib/digest-orchestration-helpers.mjs';
// ── subjectForBrief — A6.i ────────────────────────────────────────────────
describe('subjectForBrief — synthesis-level → email subject', () => {
it('synthesis level 1 + non-empty briefLead → Intelligence Brief', () => {
assert.equal(
subjectForBrief({ briefLead: 'A real lead', synthesisLevel: 1, shortDate: 'Apr 25' }),
'WorldMonitor Intelligence Brief — Apr 25',
);
});
it('synthesis level 2 + non-empty briefLead → Intelligence Brief (L2 still editorial)', () => {
assert.equal(
subjectForBrief({ briefLead: 'A degraded lead', synthesisLevel: 2, shortDate: 'Apr 25' }),
'WorldMonitor Intelligence Brief — Apr 25',
);
});
it('synthesis level 3 → Digest (stub fallback ships less editorial subject)', () => {
assert.equal(
subjectForBrief({ briefLead: 'a stub', synthesisLevel: 3, shortDate: 'Apr 25' }),
'WorldMonitor Digest — Apr 25',
);
});
it('null briefLead → Digest regardless of level (no signal for editorial subject)', () => {
assert.equal(
subjectForBrief({ briefLead: null, synthesisLevel: 1, shortDate: 'Apr 25' }),
'WorldMonitor Digest — Apr 25',
);
});
it('empty-string briefLead → Digest', () => {
assert.equal(
subjectForBrief({ briefLead: '', synthesisLevel: 1, shortDate: 'Apr 25' }),
'WorldMonitor Digest — Apr 25',
);
});
});
// ── pickWinningCandidateWithPool — A6.l + A6.m ────────────────────────────
function rule(overrides) {
return {
userId: 'u1',
variant: 'full',
sensitivity: 'all',
aiDigestEnabled: true,
updatedAt: 1,
...overrides,
};
}
function annotated(rule, due, lastSentAt = null) {
return { rule, lastSentAt, due };
}
describe('pickWinningCandidateWithPool — winner walk', () => {
it('A6.l — picks ANY eligible rule when none are due (compose-only tick)', async () => {
// Weekly user on a non-due tick: no rules due, but the dashboard
// contract says we still compose a brief from the user's
// preferred rule. sortedAll fallback covers this.
const weeklyRule = rule({ variant: 'full', digestMode: 'weekly' });
const annotatedList = [annotated(weeklyRule, false)];
const digestFor = async () => [{ hash: 'h1', title: 'A story' }];
const lines = [];
const result = await pickWinningCandidateWithPool(
annotatedList,
digestFor,
(l) => lines.push(l),
'u1',
);
assert.ok(result, 'compose-only tick must still pick a winner');
assert.equal(result.winner.rule, weeklyRule);
assert.equal(result.winner.due, false);
assert.equal(result.stories.length, 1);
});
it('A6.m — walks past empty-pool top-priority due rule to lower-priority due rule with stories', async () => {
// A user with two due rules: full:critical (top priority by
// compareRules) has empty pool; regional:high (lower priority)
// has stories. Winner must be regional:high — not null.
const fullCritical = rule({ variant: 'full', sensitivity: 'critical', updatedAt: 100 });
const regionalHigh = rule({ variant: 'regional', sensitivity: 'high', updatedAt: 50 });
const annotatedList = [annotated(fullCritical, true), annotated(regionalHigh, true)];
const digestFor = async (r) => {
if (r === fullCritical) return []; // empty pool
if (r === regionalHigh) return [{ hash: 'h2', title: 'Story from regional' }];
return [];
};
const lines = [];
const result = await pickWinningCandidateWithPool(
annotatedList,
digestFor,
(l) => lines.push(l),
'u1',
);
assert.ok(result, 'lower-priority candidate with stories must still win');
assert.equal(result.winner.rule, regionalHigh);
// Empty-pool log emitted for the skipped top-priority candidate
assert.ok(
lines.some((l) => l.includes('outcome=empty-pool') && l.includes('variant=full')),
'empty-pool line must be logged for the skipped candidate',
);
});
it('prefers DUE rules over not-due rules even when not-due is higher priority', async () => {
// Higher-priority rule isn't due; lower-priority rule IS due.
// Plan rule: pick from due candidates first. Codex Round-3 High #1.
const higherPriorityNotDue = rule({ variant: 'full', sensitivity: 'critical', updatedAt: 100 });
const lowerPriorityDue = rule({ variant: 'regional', sensitivity: 'high', updatedAt: 50 });
const annotatedList = [
annotated(higherPriorityNotDue, false), // higher priority, NOT due
annotated(lowerPriorityDue, true), // lower priority, DUE
];
const digestFor = async () => [{ hash: 'h', title: 'X' }];
const result = await pickWinningCandidateWithPool(
annotatedList,
digestFor,
() => {},
'u1',
);
assert.ok(result);
assert.equal(result.winner.rule, lowerPriorityDue, 'due rule wins over higher-priority not-due');
});
it('returns null when EVERY candidate has an empty pool', async () => {
const annotatedList = [annotated(rule({ variant: 'a' }), true), annotated(rule({ variant: 'b' }), false)];
const digestFor = async () => [];
const result = await pickWinningCandidateWithPool(
annotatedList,
digestFor,
() => {},
'u1',
);
assert.equal(result, null);
});
it('returns null on empty annotated list (no rules for user)', async () => {
const result = await pickWinningCandidateWithPool([], async () => [{ hash: 'h' }], () => {}, 'u1');
assert.equal(result, null);
});
it('does not call digestFor twice for the same rule (dedup across passes)', async () => {
// A rule that's due appears in BOTH sortedDue and sortedAll —
// walk must dedupe so digestFor (Upstash GET) only fires once.
const dueRule = rule({ variant: 'full' });
const annotatedList = [annotated(dueRule, true)];
let calls = 0;
const digestFor = async () => { calls++; return [{ hash: 'h' }]; };
await pickWinningCandidateWithPool(annotatedList, digestFor, () => {}, 'u1');
assert.equal(calls, 1, 'same rule must not be tried twice');
});
});
// ── runSynthesisWithFallback — A6.h ───────────────────────────────────────
const validProse = {
lead: 'A long-enough executive lead about Hormuz and the Gaza humanitarian crisis, written in editorial tone.',
threads: [{ tag: 'Energy', teaser: 'Hormuz tensions resurface today.' }],
signals: ['Watch for naval redeployment.'],
};
function makeDeps(callLLM) {
const cache = new Map();
return {
callLLM,
cacheGet: async (k) => cache.has(k) ? cache.get(k) : null,
cacheSet: async (k, v) => { cache.set(k, v); },
};
}
describe('runSynthesisWithFallback — three-level chain', () => {
it('L1 success — canonical synthesis returned, level=1', async () => {
const deps = makeDeps(async () => JSON.stringify(validProse));
const trace = [];
const result = await runSynthesisWithFallback(
'u1',
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
'all',
{ profile: 'Watching: oil', greeting: 'Good morning' },
deps,
(level, kind) => trace.push({ level, kind }),
);
assert.ok(result.synthesis);
assert.equal(result.level, 1);
assert.match(result.synthesis.lead, /editorial tone/);
assert.deepEqual(trace, [{ level: 1, kind: 'success' }]);
});
it('L1 LLM down → L2 succeeds, level=2', async () => {
// Note: generateDigestProse internally absorbs callLLM throws and
// returns null (its return-null-on-failure contract). So
// runSynthesisWithFallback sees the L1 attempt as a "fall" event,
// not a "throw". Test verifies the BEHAVIOR (L2 wins) rather than
// the trace event kind.
let firstCall = true;
const deps = makeDeps(async () => {
if (firstCall) { firstCall = false; throw new Error('L1 LLM down'); }
return JSON.stringify(validProse);
});
const trace = [];
const result = await runSynthesisWithFallback(
'u1',
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
'all',
{ profile: 'Watching: oil', greeting: 'Good morning' },
deps,
(level, kind) => trace.push({ level, kind }),
);
assert.ok(result.synthesis);
assert.equal(result.level, 2);
// Trace: L1 fell (callLLM throw absorbed → null), L2 succeeded.
assert.equal(trace[0].level, 1);
assert.equal(trace[0].kind, 'fall');
assert.equal(trace[1].level, 2);
assert.equal(trace[1].kind, 'success');
});
it('L1 returns null + L2 returns null → L3 stub, level=3', async () => {
const deps = makeDeps(async () => null); // both calls return null
const trace = [];
const result = await runSynthesisWithFallback(
'u1',
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
'all',
{ profile: null, greeting: null },
deps,
(level, kind) => trace.push({ level, kind }),
);
assert.equal(result.synthesis, null);
assert.equal(result.level, 3);
// Trace shows L1 fell, L2 fell, L3 success (synthesis=null is the
// stub path's contract).
assert.deepEqual(trace.map((t) => `${t.level}:${t.kind}`), [
'1:fall',
'2:fall',
'3:success',
]);
});
it('cache.cacheGet throws — generateDigestProse swallows it, L1 still succeeds via LLM call', async () => {
// generateDigestProse's cache try/catch catches ALL throws (not
// just misses), so a cache-layer outage falls through to a fresh
// LLM call and returns successfully. Documented contract: cache
// is best-effort. This test locks the contract — if a future
// refactor narrows the catch, fallback semantics change.
const deps = {
callLLM: async () => JSON.stringify(validProse),
cacheGet: async () => { throw new Error('cache outage'); },
cacheSet: async () => {},
};
const result = await runSynthesisWithFallback(
'u1',
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
'all',
{ profile: null, greeting: null },
deps,
);
assert.ok(result.synthesis);
assert.equal(result.level, 1);
});
it('callLLM down on every call → L3 stub, no exception escapes', async () => {
const deps = makeDeps(async () => { throw new Error('LLM totally down'); });
const result = await runSynthesisWithFallback(
'u1',
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
'all',
{ profile: null, greeting: null },
deps,
);
// generateDigestProse absorbs each callLLM throw → returns null;
// fallback chain reaches L3 stub. The brief still ships.
assert.equal(result.synthesis, null);
assert.equal(result.level, 3);
});
it('omits trace callback safely (defensive — production callers may not pass one)', async () => {
const deps = makeDeps(async () => JSON.stringify(validProse));
// No trace argument
const result = await runSynthesisWithFallback(
'u1',
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
'all',
{ profile: null, greeting: null },
deps,
);
assert.equal(result.level, 1);
assert.ok(result.synthesis);
});
});