mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
test(digest): backfill A6.h/i/l/m acceptance tests via helper extraction
This commit is contained in:
150
scripts/lib/digest-orchestration-helpers.mjs
Normal file
150
scripts/lib/digest-orchestration-helpers.mjs
Normal file
@@ -0,0 +1,150 @@
|
||||
// Pure helpers for the digest cron's per-user compose loop.
|
||||
//
|
||||
// Extracted from scripts/seed-digest-notifications.mjs so they can be
|
||||
// unit-tested without dragging the cron's env-checking side effects
|
||||
// (DIGEST_CRON_ENABLED check, Upstash REST helper, Convex relay
|
||||
// auth) into the test runtime. The cron imports back from here.
|
||||
|
||||
import { compareRules, MAX_STORIES_PER_USER } from './brief-compose.mjs';
|
||||
import { generateDigestProse } from './brief-llm.mjs';
|
||||
|
||||
/**
|
||||
* Build the email subject string. Extracted so the synthesis-level
|
||||
* → subject ternary can be unit-tested without standing up the whole
|
||||
* cron loop. (Plan acceptance criterion A6.i.)
|
||||
*
|
||||
* Rules:
|
||||
* - synthesisLevel 1 or 2 + non-empty briefLead → "Intelligence Brief"
|
||||
* - synthesisLevel 3 OR empty/null briefLead → "Digest"
|
||||
*
|
||||
* Mirrors today's UX where the editorial subject only appeared when
|
||||
* a real LLM-produced lead was available; the L3 stub falls back to
|
||||
* the plain "Digest" subject to set reader expectations correctly.
|
||||
*
|
||||
* @param {{ briefLead: string | null | undefined; synthesisLevel: number; shortDate: string }} input
|
||||
* @returns {string}
|
||||
*/
|
||||
export function subjectForBrief({ briefLead, synthesisLevel, shortDate }) {
|
||||
if (briefLead && synthesisLevel >= 1 && synthesisLevel <= 2) {
|
||||
return `WorldMonitor Intelligence Brief — ${shortDate}`;
|
||||
}
|
||||
return `WorldMonitor Digest — ${shortDate}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk an annotated rule list and return the winning candidate +
|
||||
* its non-empty story pool. Two-pass: due rules first (so the
|
||||
* synthesis comes from a rule that's actually sending), then ALL
|
||||
* eligible rules (compose-only tick — keeps the dashboard brief
|
||||
* fresh for weekly/twice_daily users). Within each pass, walk by
|
||||
* compareRules priority and pick the FIRST candidate whose pool is
|
||||
* non-empty.
|
||||
*
|
||||
* Returns null when every candidate has an empty pool — caller
|
||||
* skips the user (same as today's behavior).
|
||||
*
|
||||
* Plan acceptance criteria A6.l (compose-only tick still works for
|
||||
* weekly user) + A6.m (winner walks past empty-pool top-priority
|
||||
* candidate). Codex Round-3 High #1 + Round-4 High #1 + Round-4
|
||||
* Medium #2.
|
||||
*
|
||||
* `log` is the per-empty-pool log emitter — passed in so tests can
|
||||
* capture lines without reaching for console.log.
|
||||
*
|
||||
* @param {Array<{ rule: object; lastSentAt: number | null; due: boolean }>} annotated
|
||||
* @param {(rule: object) => Promise<unknown[] | null | undefined>} digestFor
|
||||
* @param {(line: string) => void} log
|
||||
* @param {string} userId
|
||||
* @returns {Promise<{ winner: { rule: object; lastSentAt: number | null; due: boolean }; stories: unknown[] } | null>}
|
||||
*/
|
||||
export async function pickWinningCandidateWithPool(annotated, digestFor, log, userId) {
|
||||
if (!Array.isArray(annotated) || annotated.length === 0) return null;
|
||||
const sortedDue = annotated.filter((a) => a.due).sort((a, b) => compareRules(a.rule, b.rule));
|
||||
const sortedAll = [...annotated].sort((a, b) => compareRules(a.rule, b.rule));
|
||||
// Build the walk order, deduping by rule reference so the same
|
||||
// rule isn't tried twice (a due rule appears in both sortedDue
|
||||
// and sortedAll).
|
||||
const seen = new Set();
|
||||
const walkOrder = [];
|
||||
for (const cand of [...sortedDue, ...sortedAll]) {
|
||||
if (seen.has(cand.rule)) continue;
|
||||
seen.add(cand.rule);
|
||||
walkOrder.push(cand);
|
||||
}
|
||||
for (const cand of walkOrder) {
|
||||
const stories = await digestFor(cand.rule);
|
||||
if (!stories || stories.length === 0) {
|
||||
log(
|
||||
`[digest] brief filter drops user=${userId} ` +
|
||||
`sensitivity=${cand.rule.sensitivity ?? 'high'} ` +
|
||||
`variant=${cand.rule.variant ?? 'full'} ` +
|
||||
`due=${cand.due} ` +
|
||||
`outcome=empty-pool ` +
|
||||
`in=0 dropped_severity=0 dropped_url=0 dropped_headline=0 dropped_shape=0 dropped_cap=0 out=0`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
return { winner: cand, stories };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the three-level canonical synthesis fallback chain.
|
||||
* L1: full pre-cap pool + ctx (profile, greeting, !public) — canonical.
|
||||
* L2: envelope-sized slice + empty ctx — degraded fallback (mirrors
|
||||
* today's enrichBriefEnvelopeWithLLM behaviour).
|
||||
* L3: null synthesis — caller composes from stub.
|
||||
*
|
||||
* Returns { synthesis, level } with `synthesis` matching
|
||||
* generateDigestProse's output shape (or null on L3) and `level`
|
||||
* one of {1, 2, 3}.
|
||||
*
|
||||
* Pure helper — no I/O beyond the deps.callLLM the inner functions
|
||||
* already perform. Errors at L1 propagate to L2; L2 errors propagate
|
||||
* to L3 (null/stub). `trace` callback fires per level transition so
|
||||
* callers can quantify failure-mode distribution in production logs.
|
||||
*
|
||||
* Plan acceptance criterion A6.h (3-level fallback triggers).
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {Array} stories — full pre-cap pool
|
||||
* @param {string} sensitivity
|
||||
* @param {{ profile: string | null; greeting: string | null }} ctx
|
||||
* @param {{ callLLM: Function; cacheGet: Function; cacheSet: Function }} deps
|
||||
* @param {(level: 1 | 2 | 3, kind: 'success' | 'fall' | 'throw', err?: unknown) => void} [trace]
|
||||
* @returns {Promise<{ synthesis: object | null; level: 1 | 2 | 3 }>}
|
||||
*/
|
||||
export async function runSynthesisWithFallback(userId, stories, sensitivity, ctx, deps, trace) {
|
||||
const noteTrace = typeof trace === 'function' ? trace : () => {};
|
||||
// L1 — canonical
|
||||
try {
|
||||
const l1 = await generateDigestProse(userId, stories, sensitivity, deps, {
|
||||
profile: ctx?.profile ?? null,
|
||||
greeting: ctx?.greeting ?? null,
|
||||
isPublic: false,
|
||||
});
|
||||
if (l1) {
|
||||
noteTrace(1, 'success');
|
||||
return { synthesis: l1, level: 1 };
|
||||
}
|
||||
noteTrace(1, 'fall');
|
||||
} catch (err) {
|
||||
noteTrace(1, 'throw', err);
|
||||
}
|
||||
// L2 — degraded fallback
|
||||
try {
|
||||
const cappedSlice = (Array.isArray(stories) ? stories : []).slice(0, MAX_STORIES_PER_USER);
|
||||
const l2 = await generateDigestProse(userId, cappedSlice, sensitivity, deps);
|
||||
if (l2) {
|
||||
noteTrace(2, 'success');
|
||||
return { synthesis: l2, level: 2 };
|
||||
}
|
||||
noteTrace(2, 'fall');
|
||||
} catch (err) {
|
||||
noteTrace(2, 'throw', err);
|
||||
}
|
||||
// L3 — stub
|
||||
noteTrace(3, 'success');
|
||||
return { synthesis: null, level: 3 };
|
||||
}
|
||||
@@ -39,6 +39,11 @@ import {
|
||||
MAX_STORIES_PER_USER,
|
||||
shouldExitNonZero as shouldExitOnBriefFailures,
|
||||
} from './lib/brief-compose.mjs';
|
||||
import {
|
||||
pickWinningCandidateWithPool,
|
||||
runSynthesisWithFallback,
|
||||
subjectForBrief,
|
||||
} from './lib/digest-orchestration-helpers.mjs';
|
||||
import { issueSlotInTz } from '../shared/brief-filter.js';
|
||||
import {
|
||||
enrichBriefEnvelopeWithLLM,
|
||||
@@ -1322,46 +1327,12 @@ async function composeBriefsForRun(rules, nowMs) {
|
||||
* @param {number} nowMs
|
||||
*/
|
||||
async function composeAndStoreBriefForUser(userId, annotated, insightsNumbers, digestFor, nowMs) {
|
||||
// Two-pass walk: prefer DUE rules first (so the synthesis pool +
|
||||
// ctx come from a rule that's about to send), fall back to ANY
|
||||
// eligible rule (compose-only tick — keeps the dashboard brief
|
||||
// refreshed for weekly/twice_daily users on non-due ticks). Within
|
||||
// each pass, `compareRules` priority order. Pick first candidate
|
||||
// with non-empty pool — mirrors today's behavior at the legacy
|
||||
// composeAndStoreBriefForUser walk.
|
||||
// Codex Round-3 High #1 + Round-4 High #1 + Round-4 Medium #2.
|
||||
const sortedDue = annotated.filter((a) => a.due).sort((a, b) => compareRules(a.rule, b.rule));
|
||||
const sortedAll = [...annotated].sort((a, b) => compareRules(a.rule, b.rule));
|
||||
|
||||
let winner = null;
|
||||
let winnerStories = null;
|
||||
for (const cand of [...sortedDue, ...sortedAll]) {
|
||||
if (winner) break;
|
||||
const stories = await digestFor(cand.rule);
|
||||
const dropStats = { severity: 0, headline: 0, url: 0, shape: 0, cap: 0, in: stories?.length ?? 0 };
|
||||
if (!stories || stories.length === 0) {
|
||||
// Per-attempt filter-drop line for the empty-pool case so ops
|
||||
// can distinguish "pool empty upstream" from "all stories
|
||||
// filtered out post-compose". outcome=empty-pool is a third
|
||||
// value beyond shipped/rejected.
|
||||
console.log(
|
||||
`[digest] brief filter drops user=${userId} ` +
|
||||
`sensitivity=${cand.rule.sensitivity ?? 'high'} ` +
|
||||
`variant=${cand.rule.variant ?? 'full'} ` +
|
||||
`due=${cand.due} ` +
|
||||
`outcome=empty-pool ` +
|
||||
`in=0 dropped_severity=0 dropped_url=0 dropped_headline=0 dropped_shape=0 dropped_cap=0 out=0`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
winner = cand;
|
||||
winnerStories = stories;
|
||||
// No console.log here — the per-attempt log is emitted inside the
|
||||
// composeBriefFromDigestStories call below via onDrop, so the row
|
||||
// carries the actual `out` count after filtering.
|
||||
}
|
||||
|
||||
if (!winner) return null;
|
||||
// Two-pass walk extracted to a pure helper so it can be unit-tested
|
||||
// (A6.l + A6.m). When no candidate has a non-empty pool, returns
|
||||
// null — same outcome as today's behavior.
|
||||
const winnerResult = await pickWinningCandidateWithPool(annotated, digestFor, (line) => console.log(line), userId);
|
||||
if (!winnerResult) return null;
|
||||
const { winner, stories: winnerStories } = winnerResult;
|
||||
|
||||
// ── Canonical synthesis (3-level fallback chain) ────────────────────
|
||||
//
|
||||
@@ -1381,41 +1352,27 @@ async function composeAndStoreBriefForUser(userId, annotated, insightsNumbers, d
|
||||
let synthesisLevel = 3; // pessimistic default; bumped on success
|
||||
if (BRIEF_LLM_ENABLED) {
|
||||
const ctx = await buildSynthesisCtx(winner.rule, nowMs);
|
||||
// L1 — canonical
|
||||
try {
|
||||
const l1 = await generateDigestProse(
|
||||
const result = await runSynthesisWithFallback(
|
||||
userId,
|
||||
winnerStories,
|
||||
sensitivity,
|
||||
ctx,
|
||||
briefLlmDeps,
|
||||
{ profile: ctx.profile, greeting: ctx.greeting, isPublic: false },
|
||||
(level, kind, err) => {
|
||||
if (kind === 'throw') {
|
||||
console.warn(
|
||||
`[digest] brief: synthesis L${level} threw for ${userId} — falling to L${level + 1}:`,
|
||||
err?.message,
|
||||
);
|
||||
if (l1) {
|
||||
synthesis = l1;
|
||||
synthesisLevel = 1;
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[digest] brief: synthesis L1 threw for ${userId} — falling to L2:`, err?.message);
|
||||
}
|
||||
// L2 — degraded fallback (no profile/greeting; envelope-sized pool slice)
|
||||
if (!synthesis) {
|
||||
try {
|
||||
const cappedSlice = winnerStories.slice(0, MAX_STORIES_PER_USER);
|
||||
const l2 = await generateDigestProse(userId, cappedSlice, sensitivity, briefLlmDeps);
|
||||
if (l2) {
|
||||
synthesis = l2;
|
||||
synthesisLevel = 2;
|
||||
} else if (kind === 'success' && level === 2) {
|
||||
console.log(`[digest] synthesis level=2_degraded user=${userId}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[digest] brief: synthesis L2 threw for ${userId} — falling to L3 stub:`, err?.message);
|
||||
}
|
||||
}
|
||||
// L3 — stub. synthesis stays null; composer's assembleStubbedBriefEnvelope
|
||||
// path produces the stub lead.
|
||||
if (!synthesis) {
|
||||
} else if (kind === 'success' && level === 3) {
|
||||
console.log(`[digest] synthesis level=3_stub user=${userId}`);
|
||||
}
|
||||
},
|
||||
);
|
||||
synthesis = result.synthesis;
|
||||
synthesisLevel = result.level;
|
||||
// publicLead — parallel call. Profile-stripped; cache-shared
|
||||
// across all users for the same (date, sensitivity, story-pool).
|
||||
// Failure is non-fatal — the renderer's public-mode fail-safe
|
||||
@@ -1698,14 +1655,7 @@ async function main() {
|
||||
const html = injectBriefCta(htmlWithSummary, magazineUrl);
|
||||
|
||||
const shortDate = new Intl.DateTimeFormat('en-US', { month: 'short', day: 'numeric' }).format(new Date(nowMs));
|
||||
// Subject: "Intelligence Brief" when the canonical synthesis (L1
|
||||
// or L2) succeeded; "Digest" when only the L3 stub is in the
|
||||
// envelope. Mirrors today's behavior (briefLead non-null = signal
|
||||
// strong enough for the editorial subject) but reads from the
|
||||
// cron-local synthesisLevel rather than re-checking the LLM call.
|
||||
const subject = (briefLead && synthesisLevel <= 2)
|
||||
? `WorldMonitor Intelligence Brief — ${shortDate}`
|
||||
: `WorldMonitor Digest — ${shortDate}`;
|
||||
const subject = subjectForBrief({ briefLead, synthesisLevel, shortDate });
|
||||
|
||||
let anyDelivered = false;
|
||||
|
||||
|
||||
315
tests/digest-orchestration-helpers.test.mjs
Normal file
315
tests/digest-orchestration-helpers.test.mjs
Normal file
@@ -0,0 +1,315 @@
|
||||
// Pure-function unit tests for the canonical-synthesis orchestration
|
||||
// helpers extracted from scripts/seed-digest-notifications.mjs.
|
||||
//
|
||||
// Covers plan acceptance criteria:
|
||||
// A6.h — three-level synthesis fallback chain
|
||||
// A6.i — subject-line correctness ("Intelligence Brief" vs "Digest")
|
||||
// A6.l — compose-only tick still works for weekly user (sortedAll fallback)
|
||||
// A6.m — winner walks past empty-pool top-priority candidate
|
||||
//
|
||||
// Acceptance criteria A6.a-d (multi-rule, twice_daily, weekly window
|
||||
// parity, all-channel reads) require a full mock of the cron's main()
|
||||
// loop with Upstash + Convex stubs — out of scope for this PR's
|
||||
// pure-function coverage. They are exercised via the parity log line
|
||||
// (A5) in production observability instead.
|
||||
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
pickWinningCandidateWithPool,
|
||||
runSynthesisWithFallback,
|
||||
subjectForBrief,
|
||||
} from '../scripts/lib/digest-orchestration-helpers.mjs';
|
||||
|
||||
// ── subjectForBrief — A6.i ────────────────────────────────────────────────
|
||||
|
||||
describe('subjectForBrief — synthesis-level → email subject', () => {
|
||||
it('synthesis level 1 + non-empty briefLead → Intelligence Brief', () => {
|
||||
assert.equal(
|
||||
subjectForBrief({ briefLead: 'A real lead', synthesisLevel: 1, shortDate: 'Apr 25' }),
|
||||
'WorldMonitor Intelligence Brief — Apr 25',
|
||||
);
|
||||
});
|
||||
|
||||
it('synthesis level 2 + non-empty briefLead → Intelligence Brief (L2 still editorial)', () => {
|
||||
assert.equal(
|
||||
subjectForBrief({ briefLead: 'A degraded lead', synthesisLevel: 2, shortDate: 'Apr 25' }),
|
||||
'WorldMonitor Intelligence Brief — Apr 25',
|
||||
);
|
||||
});
|
||||
|
||||
it('synthesis level 3 → Digest (stub fallback ships less editorial subject)', () => {
|
||||
assert.equal(
|
||||
subjectForBrief({ briefLead: 'a stub', synthesisLevel: 3, shortDate: 'Apr 25' }),
|
||||
'WorldMonitor Digest — Apr 25',
|
||||
);
|
||||
});
|
||||
|
||||
it('null briefLead → Digest regardless of level (no signal for editorial subject)', () => {
|
||||
assert.equal(
|
||||
subjectForBrief({ briefLead: null, synthesisLevel: 1, shortDate: 'Apr 25' }),
|
||||
'WorldMonitor Digest — Apr 25',
|
||||
);
|
||||
});
|
||||
|
||||
it('empty-string briefLead → Digest', () => {
|
||||
assert.equal(
|
||||
subjectForBrief({ briefLead: '', synthesisLevel: 1, shortDate: 'Apr 25' }),
|
||||
'WorldMonitor Digest — Apr 25',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── pickWinningCandidateWithPool — A6.l + A6.m ────────────────────────────
|
||||
|
||||
function rule(overrides) {
|
||||
return {
|
||||
userId: 'u1',
|
||||
variant: 'full',
|
||||
sensitivity: 'all',
|
||||
aiDigestEnabled: true,
|
||||
updatedAt: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function annotated(rule, due, lastSentAt = null) {
|
||||
return { rule, lastSentAt, due };
|
||||
}
|
||||
|
||||
describe('pickWinningCandidateWithPool — winner walk', () => {
|
||||
it('A6.l — picks ANY eligible rule when none are due (compose-only tick)', async () => {
|
||||
// Weekly user on a non-due tick: no rules due, but the dashboard
|
||||
// contract says we still compose a brief from the user's
|
||||
// preferred rule. sortedAll fallback covers this.
|
||||
const weeklyRule = rule({ variant: 'full', digestMode: 'weekly' });
|
||||
const annotatedList = [annotated(weeklyRule, false)];
|
||||
const digestFor = async () => [{ hash: 'h1', title: 'A story' }];
|
||||
const lines = [];
|
||||
const result = await pickWinningCandidateWithPool(
|
||||
annotatedList,
|
||||
digestFor,
|
||||
(l) => lines.push(l),
|
||||
'u1',
|
||||
);
|
||||
assert.ok(result, 'compose-only tick must still pick a winner');
|
||||
assert.equal(result.winner.rule, weeklyRule);
|
||||
assert.equal(result.winner.due, false);
|
||||
assert.equal(result.stories.length, 1);
|
||||
});
|
||||
|
||||
it('A6.m — walks past empty-pool top-priority due rule to lower-priority due rule with stories', async () => {
|
||||
// A user with two due rules: full:critical (top priority by
|
||||
// compareRules) has empty pool; regional:high (lower priority)
|
||||
// has stories. Winner must be regional:high — not null.
|
||||
const fullCritical = rule({ variant: 'full', sensitivity: 'critical', updatedAt: 100 });
|
||||
const regionalHigh = rule({ variant: 'regional', sensitivity: 'high', updatedAt: 50 });
|
||||
const annotatedList = [annotated(fullCritical, true), annotated(regionalHigh, true)];
|
||||
|
||||
const digestFor = async (r) => {
|
||||
if (r === fullCritical) return []; // empty pool
|
||||
if (r === regionalHigh) return [{ hash: 'h2', title: 'Story from regional' }];
|
||||
return [];
|
||||
};
|
||||
const lines = [];
|
||||
const result = await pickWinningCandidateWithPool(
|
||||
annotatedList,
|
||||
digestFor,
|
||||
(l) => lines.push(l),
|
||||
'u1',
|
||||
);
|
||||
assert.ok(result, 'lower-priority candidate with stories must still win');
|
||||
assert.equal(result.winner.rule, regionalHigh);
|
||||
// Empty-pool log emitted for the skipped top-priority candidate
|
||||
assert.ok(
|
||||
lines.some((l) => l.includes('outcome=empty-pool') && l.includes('variant=full')),
|
||||
'empty-pool line must be logged for the skipped candidate',
|
||||
);
|
||||
});
|
||||
|
||||
it('prefers DUE rules over not-due rules even when not-due is higher priority', async () => {
|
||||
// Higher-priority rule isn't due; lower-priority rule IS due.
|
||||
// Plan rule: pick from due candidates first. Codex Round-3 High #1.
|
||||
const higherPriorityNotDue = rule({ variant: 'full', sensitivity: 'critical', updatedAt: 100 });
|
||||
const lowerPriorityDue = rule({ variant: 'regional', sensitivity: 'high', updatedAt: 50 });
|
||||
const annotatedList = [
|
||||
annotated(higherPriorityNotDue, false), // higher priority, NOT due
|
||||
annotated(lowerPriorityDue, true), // lower priority, DUE
|
||||
];
|
||||
const digestFor = async () => [{ hash: 'h', title: 'X' }];
|
||||
const result = await pickWinningCandidateWithPool(
|
||||
annotatedList,
|
||||
digestFor,
|
||||
() => {},
|
||||
'u1',
|
||||
);
|
||||
assert.ok(result);
|
||||
assert.equal(result.winner.rule, lowerPriorityDue, 'due rule wins over higher-priority not-due');
|
||||
});
|
||||
|
||||
it('returns null when EVERY candidate has an empty pool', async () => {
|
||||
const annotatedList = [annotated(rule({ variant: 'a' }), true), annotated(rule({ variant: 'b' }), false)];
|
||||
const digestFor = async () => [];
|
||||
const result = await pickWinningCandidateWithPool(
|
||||
annotatedList,
|
||||
digestFor,
|
||||
() => {},
|
||||
'u1',
|
||||
);
|
||||
assert.equal(result, null);
|
||||
});
|
||||
|
||||
it('returns null on empty annotated list (no rules for user)', async () => {
|
||||
const result = await pickWinningCandidateWithPool([], async () => [{ hash: 'h' }], () => {}, 'u1');
|
||||
assert.equal(result, null);
|
||||
});
|
||||
|
||||
it('does not call digestFor twice for the same rule (dedup across passes)', async () => {
|
||||
// A rule that's due appears in BOTH sortedDue and sortedAll —
|
||||
// walk must dedupe so digestFor (Upstash GET) only fires once.
|
||||
const dueRule = rule({ variant: 'full' });
|
||||
const annotatedList = [annotated(dueRule, true)];
|
||||
let calls = 0;
|
||||
const digestFor = async () => { calls++; return [{ hash: 'h' }]; };
|
||||
await pickWinningCandidateWithPool(annotatedList, digestFor, () => {}, 'u1');
|
||||
assert.equal(calls, 1, 'same rule must not be tried twice');
|
||||
});
|
||||
});
|
||||
|
||||
// ── runSynthesisWithFallback — A6.h ───────────────────────────────────────
|
||||
|
||||
const validProse = {
|
||||
lead: 'A long-enough executive lead about Hormuz and the Gaza humanitarian crisis, written in editorial tone.',
|
||||
threads: [{ tag: 'Energy', teaser: 'Hormuz tensions resurface today.' }],
|
||||
signals: ['Watch for naval redeployment.'],
|
||||
};
|
||||
|
||||
function makeDeps(callLLM) {
|
||||
const cache = new Map();
|
||||
return {
|
||||
callLLM,
|
||||
cacheGet: async (k) => cache.has(k) ? cache.get(k) : null,
|
||||
cacheSet: async (k, v) => { cache.set(k, v); },
|
||||
};
|
||||
}
|
||||
|
||||
describe('runSynthesisWithFallback — three-level chain', () => {
|
||||
it('L1 success — canonical synthesis returned, level=1', async () => {
|
||||
const deps = makeDeps(async () => JSON.stringify(validProse));
|
||||
const trace = [];
|
||||
const result = await runSynthesisWithFallback(
|
||||
'u1',
|
||||
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
|
||||
'all',
|
||||
{ profile: 'Watching: oil', greeting: 'Good morning' },
|
||||
deps,
|
||||
(level, kind) => trace.push({ level, kind }),
|
||||
);
|
||||
assert.ok(result.synthesis);
|
||||
assert.equal(result.level, 1);
|
||||
assert.match(result.synthesis.lead, /editorial tone/);
|
||||
assert.deepEqual(trace, [{ level: 1, kind: 'success' }]);
|
||||
});
|
||||
|
||||
it('L1 LLM down → L2 succeeds, level=2', async () => {
|
||||
// Note: generateDigestProse internally absorbs callLLM throws and
|
||||
// returns null (its return-null-on-failure contract). So
|
||||
// runSynthesisWithFallback sees the L1 attempt as a "fall" event,
|
||||
// not a "throw". Test verifies the BEHAVIOR (L2 wins) rather than
|
||||
// the trace event kind.
|
||||
let firstCall = true;
|
||||
const deps = makeDeps(async () => {
|
||||
if (firstCall) { firstCall = false; throw new Error('L1 LLM down'); }
|
||||
return JSON.stringify(validProse);
|
||||
});
|
||||
const trace = [];
|
||||
const result = await runSynthesisWithFallback(
|
||||
'u1',
|
||||
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
|
||||
'all',
|
||||
{ profile: 'Watching: oil', greeting: 'Good morning' },
|
||||
deps,
|
||||
(level, kind) => trace.push({ level, kind }),
|
||||
);
|
||||
assert.ok(result.synthesis);
|
||||
assert.equal(result.level, 2);
|
||||
// Trace: L1 fell (callLLM throw absorbed → null), L2 succeeded.
|
||||
assert.equal(trace[0].level, 1);
|
||||
assert.equal(trace[0].kind, 'fall');
|
||||
assert.equal(trace[1].level, 2);
|
||||
assert.equal(trace[1].kind, 'success');
|
||||
});
|
||||
|
||||
it('L1 returns null + L2 returns null → L3 stub, level=3', async () => {
|
||||
const deps = makeDeps(async () => null); // both calls return null
|
||||
const trace = [];
|
||||
const result = await runSynthesisWithFallback(
|
||||
'u1',
|
||||
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
|
||||
'all',
|
||||
{ profile: null, greeting: null },
|
||||
deps,
|
||||
(level, kind) => trace.push({ level, kind }),
|
||||
);
|
||||
assert.equal(result.synthesis, null);
|
||||
assert.equal(result.level, 3);
|
||||
// Trace shows L1 fell, L2 fell, L3 success (synthesis=null is the
|
||||
// stub path's contract).
|
||||
assert.deepEqual(trace.map((t) => `${t.level}:${t.kind}`), [
|
||||
'1:fall',
|
||||
'2:fall',
|
||||
'3:success',
|
||||
]);
|
||||
});
|
||||
|
||||
it('cache.cacheGet throws — generateDigestProse swallows it, L1 still succeeds via LLM call', async () => {
|
||||
// generateDigestProse's cache try/catch catches ALL throws (not
|
||||
// just misses), so a cache-layer outage falls through to a fresh
|
||||
// LLM call and returns successfully. Documented contract: cache
|
||||
// is best-effort. This test locks the contract — if a future
|
||||
// refactor narrows the catch, fallback semantics change.
|
||||
const deps = {
|
||||
callLLM: async () => JSON.stringify(validProse),
|
||||
cacheGet: async () => { throw new Error('cache outage'); },
|
||||
cacheSet: async () => {},
|
||||
};
|
||||
const result = await runSynthesisWithFallback(
|
||||
'u1',
|
||||
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
|
||||
'all',
|
||||
{ profile: null, greeting: null },
|
||||
deps,
|
||||
);
|
||||
assert.ok(result.synthesis);
|
||||
assert.equal(result.level, 1);
|
||||
});
|
||||
|
||||
it('callLLM down on every call → L3 stub, no exception escapes', async () => {
|
||||
const deps = makeDeps(async () => { throw new Error('LLM totally down'); });
|
||||
const result = await runSynthesisWithFallback(
|
||||
'u1',
|
||||
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
|
||||
'all',
|
||||
{ profile: null, greeting: null },
|
||||
deps,
|
||||
);
|
||||
// generateDigestProse absorbs each callLLM throw → returns null;
|
||||
// fallback chain reaches L3 stub. The brief still ships.
|
||||
assert.equal(result.synthesis, null);
|
||||
assert.equal(result.level, 3);
|
||||
});
|
||||
|
||||
it('omits trace callback safely (defensive — production callers may not pass one)', async () => {
|
||||
const deps = makeDeps(async () => JSON.stringify(validProse));
|
||||
// No trace argument
|
||||
const result = await runSynthesisWithFallback(
|
||||
'u1',
|
||||
[{ hash: 'h1', headline: 'Story 1', threatLevel: 'critical' }],
|
||||
'all',
|
||||
{ profile: null, greeting: null },
|
||||
deps,
|
||||
);
|
||||
assert.equal(result.level, 1);
|
||||
assert.ok(result.synthesis);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user