From 80b8071356bce094b20e403ead86a6da60dbe2fc Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Wed, 4 Mar 2026 20:42:51 +0400 Subject: [PATCH] feat: server-side AI insights via Railway cron + bootstrap hydration (#1003) Move the heavy AI insights pipeline (clustering, scoring, LLM brief) from client-side (15-40s per user) to a 5-min Railway cron job. The frontend reads pre-computed insights instantly via bootstrap hydration, with graceful fallback to the existing client-side pipeline. - Add _clustering.mjs: Jaccard clustering + importance scoring (pure JS) - Add seed-insights.mjs: Railway cron reads digest, clusters, calls Groq/OpenRouter for brief, writes to Redis with LKG preservation - Register insights key in bootstrap.js FAST_KEYS tier - Add insights-loader.ts: module-level cached bootstrap reader - Modify InsightsPanel.ts: server-first path (2-step progress) with client fallback (4-step, unchanged behavior) - Add unit tests for clustering (12) and insights-loader (7) --- api/bootstrap.js | 2 + scripts/_clustering.mjs | 197 +++++++++++++++++++++ scripts/seed-insights.mjs | 296 ++++++++++++++++++++++++++++++++ src/components/InsightsPanel.ts | 149 ++++++++++++++++ src/services/insights-loader.ts | 53 ++++++ tests/clustering.test.mjs | 108 ++++++++++++ tests/insights-loader.test.mjs | 65 +++++++ 7 files changed, 870 insertions(+) create mode 100644 scripts/_clustering.mjs create mode 100644 scripts/seed-insights.mjs create mode 100644 src/services/insights-loader.ts create mode 100644 tests/clustering.test.mjs create mode 100644 tests/insights-loader.test.mjs diff --git a/api/bootstrap.js b/api/bootstrap.js index 6a0c0b5f1..8f6e4aeb4 100644 --- a/api/bootstrap.js +++ b/api/bootstrap.js @@ -27,6 +27,7 @@ const BOOTSTRAP_CACHE_KEYS = { theaterPosture: 'theater-posture:sebuf:stale:v1', riskScores: 'risk:scores:sebuf:stale:v1', naturalEvents: 'natural:events:v1', + insights: 'news:insights:v1', }; const SLOW_KEYS = new Set([ @@ -37,6 +38,7 @@ const SLOW_KEYS = new Set([ const FAST_KEYS = new Set([ 'earthquakes', 'outages', 'serviceStatuses', 'macroSignals', 'chokepoints', 'marketQuotes', 'commodityQuotes', 'positiveGeoEvents', 'riskScores', + 'insights', ]); const TIER_CACHE = { diff --git a/scripts/_clustering.mjs b/scripts/_clustering.mjs new file mode 100644 index 000000000..2521013c2 --- /dev/null +++ b/scripts/_clustering.mjs @@ -0,0 +1,197 @@ +#!/usr/bin/env node + +const SIMILARITY_THRESHOLD = 0.5; + +const STOP_WORDS = new Set([ + 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', + 'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been', + 'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', + 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need', + 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'you', 'he', + 'she', 'we', 'they', 'what', 'which', 'who', 'whom', 'how', 'when', + 'where', 'why', 'all', 'each', 'every', 'both', 'few', 'more', 'most', + 'other', 'some', 'such', 'no', 'not', 'only', 'same', 'so', 'than', + 'too', 'very', 'just', 'also', 'now', 'new', 'says', 'said', 'after', +]); + +const MILITARY_KEYWORDS = [ + 'war', 'armada', 'invasion', 'airstrike', 'strike', 'missile', 'troops', + 'deployed', 'offensive', 'artillery', 'bomb', 'combat', 'fleet', 'warship', + 'carrier', 'navy', 'airforce', 'deployment', 'mobilization', 'attack', +]; + +const VIOLENCE_KEYWORDS = [ + 'killed', 'dead', 'death', 'shot', 'blood', 'massacre', 'slaughter', + 'fatalities', 'casualties', 'wounded', 'injured', 'murdered', 'execution', + 'crackdown', 'violent', 'clashes', 'gunfire', 'shooting', +]; + +const UNREST_KEYWORDS = [ + 'protest', 'protests', 'uprising', 'revolt', 'revolution', 'riot', 'riots', + 'demonstration', 'unrest', 'dissent', 'rebellion', 'insurgent', 'overthrow', + 'coup', 'martial law', 'curfew', 'shutdown', 'blackout', +]; + +const FLASHPOINT_KEYWORDS = [ + 'iran', 'tehran', 'russia', 'moscow', 'china', 'beijing', 'taiwan', 'ukraine', 'kyiv', + 'north korea', 'pyongyang', 'israel', 'gaza', 'west bank', 'syria', 'damascus', + 'yemen', 'hezbollah', 'hamas', 'kremlin', 'pentagon', 'nato', 'wagner', +]; + +const CRISIS_KEYWORDS = [ + 'crisis', 'emergency', 'catastrophe', 'disaster', 'collapse', 'humanitarian', + 'sanctions', 'ultimatum', 'threat', 'retaliation', 'escalation', 'tensions', + 'breaking', 'urgent', 'developing', 'exclusive', +]; + +const DEMOTE_KEYWORDS = [ + 'ceo', 'earnings', 'stock', 'startup', 'data center', 'datacenter', 'revenue', + 'quarterly', 'profit', 'investor', 'ipo', 'funding', 'valuation', +]; + +function tokenize(text) { + const words = text + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .split(/\s+/) + .filter(w => w.length > 2 && !STOP_WORDS.has(w)); + return new Set(words); +} + +function jaccardSimilarity(a, b) { + if (a.size === 0 && b.size === 0) return 0; + let intersection = 0; + for (const x of a) { + if (b.has(x)) intersection++; + } + const union = a.size + b.size - intersection; + return intersection / union; +} + +export function clusterItems(items) { + if (items.length === 0) return []; + + const tokenList = items.map(item => tokenize(item.title || '')); + + const invertedIndex = new Map(); + for (let i = 0; i < tokenList.length; i++) { + for (const token of tokenList[i]) { + const bucket = invertedIndex.get(token); + if (bucket) bucket.push(i); + else invertedIndex.set(token, [i]); + } + } + + const clusters = []; + const assigned = new Set(); + + for (let i = 0; i < items.length; i++) { + if (assigned.has(i)) continue; + + const cluster = [i]; + assigned.add(i); + const tokensI = tokenList[i]; + + const candidates = new Set(); + for (const token of tokensI) { + const bucket = invertedIndex.get(token); + if (!bucket) continue; + for (const idx of bucket) { + if (idx > i) candidates.add(idx); + } + } + + for (const j of Array.from(candidates).sort((a, b) => a - b)) { + if (assigned.has(j)) continue; + if (jaccardSimilarity(tokensI, tokenList[j]) >= SIMILARITY_THRESHOLD) { + cluster.push(j); + assigned.add(j); + } + } + + clusters.push(cluster.map(idx => items[idx])); + } + + return clusters.map(group => { + const sorted = [...group].sort((a, b) => { + const tierDiff = (a.tier ?? 99) - (b.tier ?? 99); + if (tierDiff !== 0) return tierDiff; + return new Date(b.pubDate).getTime() - new Date(a.pubDate).getTime(); + }); + + const primary = sorted[0]; + return { + primaryTitle: primary.title, + primarySource: primary.source, + primaryLink: primary.link, + sourceCount: group.length, + isAlert: group.some(i => i.isAlert), + }; + }); +} + +function countMatches(text, keywords) { + return keywords.filter(kw => text.includes(kw)).length; +} + +export function scoreImportance(cluster) { + let score = 0; + const titleLower = (cluster.primaryTitle || '').toLowerCase(); + + score += (cluster.sourceCount || 1) * 10; + + const violenceN = countMatches(titleLower, VIOLENCE_KEYWORDS); + if (violenceN > 0) score += 100 + violenceN * 25; + + const militaryN = countMatches(titleLower, MILITARY_KEYWORDS); + if (militaryN > 0) score += 80 + militaryN * 20; + + const unrestN = countMatches(titleLower, UNREST_KEYWORDS); + if (unrestN > 0) score += 70 + unrestN * 18; + + const flashpointN = countMatches(titleLower, FLASHPOINT_KEYWORDS); + if (flashpointN > 0) score += 60 + flashpointN * 15; + + if ((violenceN > 0 || unrestN > 0) && flashpointN > 0) { + score *= 1.5; + } + + const crisisN = countMatches(titleLower, CRISIS_KEYWORDS); + if (crisisN > 0) score += 30 + crisisN * 10; + + const demoteN = countMatches(titleLower, DEMOTE_KEYWORDS); + if (demoteN > 0) score *= 0.3; + + if (cluster.isAlert) score += 50; + + return score; +} + +// Note: velocity filter omitted (vs frontend selectTopStories) because digest +// items lack velocity data. Phase B may add velocity when RPC provides it. +export function selectTopStories(clusters, maxCount = 8) { + const scored = clusters + .map(c => ({ cluster: c, score: scoreImportance(c) })) + .filter(({ cluster: c, score }) => + (c.sourceCount || 1) >= 2 || + c.isAlert || + score > 100 + ) + .sort((a, b) => b.score - a.score); + + const selected = []; + const sourceCount = new Map(); + const MAX_PER_SOURCE = 3; + + for (const { cluster, score } of scored) { + const source = cluster.primarySource; + const count = sourceCount.get(source) || 0; + if (count < MAX_PER_SOURCE) { + selected.push({ ...cluster, importanceScore: score }); + sourceCount.set(source, count + 1); + } + if (selected.length >= maxCount) break; + } + + return selected; +} diff --git a/scripts/seed-insights.mjs b/scripts/seed-insights.mjs new file mode 100644 index 000000000..e57957327 --- /dev/null +++ b/scripts/seed-insights.mjs @@ -0,0 +1,296 @@ +#!/usr/bin/env node + +import { loadEnvFile, CHROME_UA, getRedisCredentials, runSeed } from './_seed-utils.mjs'; +import { clusterItems, selectTopStories } from './_clustering.mjs'; + +loadEnvFile(import.meta.url); + +const CANONICAL_KEY = 'news:insights:v1'; +const DIGEST_KEY = 'news:digest:v1:full:en'; +const CACHE_TTL = 600; // 10 min (2x the 5-min cron interval) +const MAX_HEADLINES = 10; +const MAX_HEADLINE_LEN = 500; +const GROQ_MODEL = 'llama-3.1-8b-instant'; + +const TASK_NARRATION = /^(we need to|i need to|let me|i'll |i should|i will |the task is|the instructions|according to the rules|so we need to|okay[,.]\s*(i'll|let me|so|we need|the task|i should|i will)|sure[,.]\s*(i'll|let me|so|we need|the task|i should|i will|here)|first[, ]+(i|we|let)|to summarize (the headlines|the task|this)|my task (is|was|:)|step \d)/i; +const PROMPT_ECHO = /^(summarize the top story|summarize the key|rules:|here are the rules|the top story is likely)/i; + +function stripReasoningPreamble(text) { + const trimmed = text.trim(); + if (TASK_NARRATION.test(trimmed) || PROMPT_ECHO.test(trimmed)) { + const lines = trimmed.split('\n').filter(l => l.trim()); + const clean = lines.filter(l => !TASK_NARRATION.test(l.trim()) && !PROMPT_ECHO.test(l.trim())); + return clean.join('\n').trim() || trimmed; + } + return trimmed; +} + +function sanitizeTitle(title) { + if (typeof title !== 'string') return ''; + return title + .replace(/<[^>]*>/g, '') + .replace(/[\x00-\x1f\x7f]/g, '') + .slice(0, MAX_HEADLINE_LEN) + .trim(); +} + +async function readDigestFromRedis() { + const { url, token } = getRedisCredentials(); + const resp = await fetch(`${url}/get/${encodeURIComponent(DIGEST_KEY)}`, { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(5_000), + }); + if (!resp.ok) return null; + const data = await resp.json(); + return data.result ? JSON.parse(data.result) : null; +} + +async function readExistingInsights() { + const { url, token } = getRedisCredentials(); + const resp = await fetch(`${url}/get/${encodeURIComponent(CANONICAL_KEY)}`, { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(5_000), + }); + if (!resp.ok) return null; + const data = await resp.json(); + return data.result ? JSON.parse(data.result) : null; +} + +async function callGroq(headlines) { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) return null; + + const headlineText = headlines.map((h, i) => `${i + 1}. ${h}`).join('\n'); + const dateContext = `Current date: ${new Date().toISOString().split('T')[0]}. Provide geopolitical context appropriate for the current date.`; + + const systemPrompt = `${dateContext} + +Summarize the single most important headline in 2 concise sentences MAX (under 60 words total). +Rules: +- Each numbered headline below is a SEPARATE, UNRELATED story +- Pick the ONE most significant headline and summarize ONLY that story +- NEVER combine or merge people, places, or facts from different headlines into one sentence +- Lead with WHAT happened and WHERE - be specific +- NEVER start with "Breaking news", "Good evening", "Tonight", or TV-style openings +- Start directly with the subject of the chosen headline +- No bullet points, no meta-commentary, no elaboration beyond the core facts`; + + const userPrompt = `Each headline below is a separate story. Pick the most important ONE and summarize only that story:\n${headlineText}`; + + try { + const resp = await fetch('https://api.groq.com/openai/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'User-Agent': CHROME_UA, + }, + body: JSON.stringify({ + model: GROQ_MODEL, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userPrompt }, + ], + max_tokens: 300, + temperature: 0.3, + }), + signal: AbortSignal.timeout(15_000), + }); + + if (!resp.ok) { + console.warn(` Groq API error: ${resp.status}`); + return null; + } + + const json = await resp.json(); + const rawText = json.choices?.[0]?.message?.content?.trim(); + if (!rawText) return null; + const text = stripReasoningPreamble(rawText); + + return { text, model: json.model || GROQ_MODEL, provider: 'groq' }; + } catch (err) { + console.warn(` Groq call failed: ${err.message}`); + return null; + } +} + +async function callOpenRouter(headlines) { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey) return null; + + const headlineText = headlines.map((h, i) => `${i + 1}. ${h}`).join('\n'); + const dateContext = `Current date: ${new Date().toISOString().split('T')[0]}.`; + + try { + const resp = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://worldmonitor.app', + 'X-Title': 'WorldMonitor', + 'User-Agent': CHROME_UA, + }, + body: JSON.stringify({ + model: 'openrouter/free', + messages: [ + { role: 'system', content: `${dateContext} Summarize the single most important headline in 2 concise sentences MAX (under 60 words). Each headline is a SEPARATE story. Pick ONE. NEVER combine facts from different headlines. Lead with WHAT happened and WHERE.` }, + { role: 'user', content: `Pick the most important story:\n${headlineText}` }, + ], + max_tokens: 300, + temperature: 0.3, + }), + signal: AbortSignal.timeout(20_000), + }); + + if (!resp.ok) { + console.warn(` OpenRouter API error: ${resp.status}`); + return null; + } + + const json = await resp.json(); + const rawText = json.choices?.[0]?.message?.content?.trim(); + if (!rawText) return null; + const text = stripReasoningPreamble(rawText); + + return { text, model: json.model || 'openrouter/free', provider: 'openrouter' }; + } catch (err) { + console.warn(` OpenRouter call failed: ${err.message}`); + return null; + } +} + +function categorizeStory(title) { + const lower = (title || '').toLowerCase(); + const categories = [ + { keywords: ['war', 'attack', 'missile', 'troops', 'airstrike', 'combat', 'military'], cat: 'conflict', threat: 'critical' }, + { keywords: ['killed', 'dead', 'casualties', 'massacre', 'shooting'], cat: 'violence', threat: 'high' }, + { keywords: ['protest', 'uprising', 'riot', 'unrest', 'coup'], cat: 'unrest', threat: 'high' }, + { keywords: ['sanctions', 'tensions', 'escalation', 'threat'], cat: 'geopolitical', threat: 'elevated' }, + { keywords: ['crisis', 'emergency', 'disaster', 'collapse'], cat: 'crisis', threat: 'high' }, + { keywords: ['earthquake', 'flood', 'hurricane', 'wildfire', 'tsunami'], cat: 'natural_disaster', threat: 'elevated' }, + { keywords: ['election', 'vote', 'parliament', 'legislation'], cat: 'political', threat: 'moderate' }, + { keywords: ['market', 'economy', 'trade', 'tariff', 'inflation'], cat: 'economic', threat: 'moderate' }, + ]; + + for (const { keywords, cat, threat } of categories) { + if (keywords.some(kw => lower.includes(kw))) { + return { category: cat, threatLevel: threat }; + } + } + return { category: 'general', threatLevel: 'moderate' }; +} + +async function fetchInsights() { + const digest = await readDigestFromRedis(); + if (!digest) throw new Error('No news digest found in Redis'); + + const items = Array.isArray(digest) ? digest : + (digest.items || digest.articles || digest.headlines || []); + + if (items.length === 0) { + const keys = typeof digest === 'object' && digest !== null ? Object.keys(digest).join(', ') : typeof digest; + throw new Error(`Digest has no items (shape: ${keys})`); + } + + console.log(` Digest items: ${items.length}`); + + const normalizedItems = items.map(item => ({ + title: sanitizeTitle(item.title || item.headline || ''), + source: item.source || item.feed || '', + link: item.link || item.url || '', + pubDate: item.pubDate || item.publishedAt || item.date || new Date().toISOString(), + isAlert: item.isAlert || false, + tier: item.tier, + })).filter(item => item.title.length > 10); + + const clusters = clusterItems(normalizedItems); + console.log(` Clusters: ${clusters.length}`); + + const topStories = selectTopStories(clusters, 8); + console.log(` Top stories: ${topStories.length}`); + + if (topStories.length === 0) throw new Error('No top stories after scoring'); + + const headlines = topStories + .slice(0, MAX_HEADLINES) + .map(s => sanitizeTitle(s.primaryTitle)); + + let worldBrief = ''; + let briefProvider = ''; + let briefModel = ''; + let status = 'ok'; + + const groqResult = await callGroq(headlines); + if (groqResult) { + worldBrief = groqResult.text; + briefProvider = groqResult.provider; + briefModel = groqResult.model; + console.log(` Brief generated via ${briefProvider} (${briefModel})`); + } else { + const orResult = await callOpenRouter(headlines); + if (orResult) { + worldBrief = orResult.text; + briefProvider = orResult.provider; + briefModel = orResult.model; + console.log(` Brief generated via ${briefProvider} (${briefModel})`); + } else { + status = 'degraded'; + console.warn(' No LLM available — publishing degraded (stories without brief)'); + } + } + + const multiSourceCount = clusters.filter(c => c.sourceCount >= 2).length; + const fastMovingCount = 0; // velocity not available in digest items + + const enrichedStories = topStories.map(story => { + const { category, threatLevel } = categorizeStory(story.primaryTitle); + return { + primaryTitle: story.primaryTitle, + primarySource: story.primarySource, + primaryLink: story.primaryLink, + sourceCount: story.sourceCount, + importanceScore: story.importanceScore, + velocity: { level: 'normal', sourcesPerHour: 0 }, + isAlert: story.isAlert, + category, + threatLevel, + }; + }); + + const payload = { + worldBrief, + briefProvider, + briefModel, + status, + topStories: enrichedStories, + generatedAt: new Date().toISOString(), + clusterCount: clusters.length, + multiSourceCount, + fastMovingCount, + }; + + // LKG preservation: don't overwrite "ok" with "degraded" + if (status === 'degraded') { + const existing = await readExistingInsights(); + if (existing?.status === 'ok') { + console.log(' LKG preservation: existing payload is "ok", skipping degraded overwrite'); + return existing; + } + } + + return payload; +} + +function validate(data) { + return Array.isArray(data?.topStories) && data.topStories.length >= 1; +} + +runSeed('news', 'insights', CANONICAL_KEY, fetchInsights, { + validateFn: validate, + ttlSeconds: CACHE_TTL, + sourceVersion: 'digest-clustering-v1', +}).catch((err) => { + console.error('FATAL:', err.message || err); + process.exit(1); +}); diff --git a/src/components/InsightsPanel.ts b/src/components/InsightsPanel.ts index 77f98fc8c..1382883b8 100644 --- a/src/components/InsightsPanel.ts +++ b/src/components/InsightsPanel.ts @@ -14,6 +14,7 @@ import { deletePersistentCache, getPersistentCache, setPersistentCache } from '@ import { t } from '@/services/i18n'; import { isDesktopRuntime } from '@/services/runtime'; import { getAiFlowSettings, isAnyAiProviderEnabled, subscribeAiFlowChange } from '@/services/ai-flow-settings'; +import { getServerInsights, type ServerInsights, type ServerInsightStory } from '@/services/insights-loader'; import type { ClusteredEvent, FocalPoint, MilitaryFlight } from '@/types'; export class InsightsPanel extends Panel { @@ -269,6 +270,70 @@ export class InsightsPanel extends Panel { return; } + // Try server-side pre-computed insights first (instant) + const serverInsights = getServerInsights(); + if (serverInsights) { + await this.updateFromServer(serverInsights, clusters, thisGeneration); + return; + } + + // Fallback: full client-side pipeline + await this.updateFromClient(clusters, thisGeneration); + } + + private async updateFromServer( + serverInsights: ServerInsights, + clusters: ClusteredEvent[], + thisGeneration: number, + ): Promise { + const totalSteps = 2; + + try { + // Step 1: Signal aggregation (client-side, depends on real-time map data) + this.setProgress(1, totalSteps, 'Loading server insights...'); + + let signalSummary: ReturnType; + let focalSummary: ReturnType; + + if (SITE_VARIANT === 'full') { + if (this.lastMilitaryFlights.length > 0) { + const postures = getTheaterPostureSummaries(this.lastMilitaryFlights); + signalAggregator.ingestTheaterPostures(postures); + } + signalSummary = signalAggregator.getSummary(); + this.lastConvergenceZones = signalSummary.convergenceZones; + focalSummary = focalPointDetector.analyze(clusters, signalSummary); + this.lastFocalPoints = focalSummary.focalPoints; + if (focalSummary.focalPoints.length > 0) { + ingestNewsForCII(clusters); + window.dispatchEvent(new CustomEvent('focal-points-ready')); + } + } else { + this.lastConvergenceZones = []; + this.lastFocalPoints = []; + } + + if (this.updateGeneration !== thisGeneration) return; + + // Step 2: Sentiment analysis on server story titles (fast browser ML) + this.setProgress(2, totalSteps, t('components.insights.analyzingSentiment')); + const titles = serverInsights.topStories.slice(0, 5).map(s => s.primaryTitle); + let sentiments: Array<{ label: string; score: number }> | null = null; + if (mlWorker.isAvailable) { + sentiments = await mlWorker.classifySentiment(titles).catch(() => null); + } + + if (this.updateGeneration !== thisGeneration) return; + + this.setDataBadge(serverInsights.status === 'ok' ? 'live' : 'cached'); + this.renderServerInsights(serverInsights, sentiments); + } catch (error) { + console.error('[InsightsPanel] Server path error, falling back:', error); + await this.updateFromClient(clusters, thisGeneration); + } + } + + private async updateFromClient(clusters: ClusteredEvent[], thisGeneration: number): Promise { // Web-only: if no AI providers enabled, show disabled state if (!isDesktopRuntime() && !isAnyAiProviderEnabled()) { this.setDataBadge('unavailable'); @@ -440,6 +505,90 @@ export class InsightsPanel extends Panel { `); } + private renderServerInsights( + insights: ServerInsights, + sentiments: Array<{ label: string; score: number }> | null, + ): void { + const briefHtml = insights.worldBrief ? this.renderWorldBrief(insights.worldBrief) : ''; + const focalPointsHtml = this.renderFocalPoints(); + const convergenceHtml = this.renderConvergenceZones(); + const sentimentOverview = this.renderSentimentOverview(sentiments); + const storiesHtml = this.renderServerStories(insights.topStories, sentiments); + const statsHtml = this.renderServerStats(insights); + const missedHtml = this.renderMissedStories(); + + this.setContent(` + ${briefHtml} + ${focalPointsHtml} + ${convergenceHtml} + ${sentimentOverview} + ${statsHtml} +
+
BREAKING & CONFIRMED
+ ${storiesHtml} +
+ ${missedHtml} + `); + } + + private renderServerStories( + stories: ServerInsightStory[], + sentiments: Array<{ label: string; score: number }> | null, + ): string { + return stories.map((story, i) => { + const sentiment = sentiments?.[i]; + const sentimentClass = sentiment?.label === 'negative' ? 'negative' : + sentiment?.label === 'positive' ? 'positive' : 'neutral'; + + const badges: string[] = []; + + if (story.sourceCount >= 3) { + badges.push(`✓ ${story.sourceCount} sources`); + } else if (story.sourceCount >= 2) { + badges.push(`${story.sourceCount} sources`); + } + + if (story.isAlert) { + badges.push('⚠ ALERT'); + } + + const VALID_THREAT_LEVELS = ['critical', 'high', 'elevated', 'moderate']; + if (story.threatLevel === 'critical' || story.threatLevel === 'high') { + const safeThreat = VALID_THREAT_LEVELS.includes(story.threatLevel) ? story.threatLevel : 'moderate'; + badges.push(`${escapeHtml(story.category)}`); + } + + return ` +
+
+ + ${escapeHtml(story.primaryTitle.slice(0, 100))}${story.primaryTitle.length > 100 ? '...' : ''} +
+ ${badges.length > 0 ? `
${badges.join('')}
` : ''} +
+ `; + }).join(''); + } + + private renderServerStats(insights: ServerInsights): string { + return ` +
+
+ ${insights.multiSourceCount} + Multi-source +
+
+ ${insights.fastMovingCount} + Fast-moving +
+
+ ${insights.clusterCount} + Clusters +
+
+ `; + } + private renderWorldBrief(brief: string): string { return `
diff --git a/src/services/insights-loader.ts b/src/services/insights-loader.ts new file mode 100644 index 000000000..298327d9f --- /dev/null +++ b/src/services/insights-loader.ts @@ -0,0 +1,53 @@ +import { getHydratedData } from '@/services/bootstrap'; + +export interface ServerInsightStory { + primaryTitle: string; + primarySource: string; + primaryLink: string; + sourceCount: number; + importanceScore: number; + velocity: { level: string; sourcesPerHour: number }; + isAlert: boolean; + category: string; + threatLevel: string; +} + +export interface ServerInsights { + worldBrief: string; + briefProvider: string; + status: 'ok' | 'degraded'; + topStories: ServerInsightStory[]; + generatedAt: string; + clusterCount: number; + multiSourceCount: number; + fastMovingCount: number; +} + +let cached: ServerInsights | null = null; +const MAX_AGE_MS = 15 * 60 * 1000; + +function isFresh(data: ServerInsights): boolean { + const age = Date.now() - new Date(data.generatedAt).getTime(); + return age < MAX_AGE_MS; +} + +export function getServerInsights(): ServerInsights | null { + if (cached && isFresh(cached)) { + return cached; + } + cached = null; + + const raw = getHydratedData('insights'); + if (!raw || typeof raw !== 'object') return null; + const data = raw as ServerInsights; + if (!Array.isArray(data.topStories) || data.topStories.length === 0) return null; + if (typeof data.generatedAt !== 'string') return null; + if (!isFresh(data)) return null; + + cached = data; + return data; +} + +export function setServerInsights(data: ServerInsights): void { + cached = data; +} diff --git a/tests/clustering.test.mjs b/tests/clustering.test.mjs new file mode 100644 index 000000000..3e637f70a --- /dev/null +++ b/tests/clustering.test.mjs @@ -0,0 +1,108 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { clusterItems, scoreImportance, selectTopStories } from '../scripts/_clustering.mjs'; + +describe('_clustering.mjs', () => { + describe('clusterItems', () => { + it('groups similar titles into one cluster', () => { + const items = [ + { title: 'Iran launches missile strikes on targets in Syria overnight', source: 'Reuters', link: 'http://a' }, + { title: 'Iran launches missile strikes on targets in Syria overnight says officials', source: 'AP', link: 'http://b' }, + ]; + const clusters = clusterItems(items); + assert.equal(clusters.length, 1); + assert.equal(clusters[0].sourceCount, 2); + }); + + it('keeps different titles as separate clusters', () => { + const items = [ + { title: 'Iran launches missile strikes on targets in Syria', source: 'Reuters', link: 'http://a' }, + { title: 'Stock market rallies on tech earnings report', source: 'CNBC', link: 'http://b' }, + ]; + const clusters = clusterItems(items); + assert.equal(clusters.length, 2); + }); + + it('returns empty array for empty input', () => { + assert.deepEqual(clusterItems([]), []); + }); + + it('preserves primaryTitle from highest-tier source', () => { + const items = [ + { title: 'Iran strikes Syria overnight', source: 'Blog', link: 'http://b', tier: 5 }, + { title: 'Iran strikes Syria overnight confirms officials', source: 'Reuters', link: 'http://a', tier: 1 }, + ]; + const clusters = clusterItems(items); + assert.equal(clusters.length, 1); + assert.equal(clusters[0].primarySource, 'Reuters'); + }); + }); + + describe('scoreImportance', () => { + it('scores military/violence headlines higher than business', () => { + const military = { primaryTitle: 'Troops deployed after missile attack in Ukraine', sourceCount: 2 }; + const business = { primaryTitle: 'Tech startup raises funding in quarterly earnings', sourceCount: 2 }; + assert.ok(scoreImportance(military) > scoreImportance(business)); + }); + + it('gives combo bonus for flashpoint + violence', () => { + const flashpointViolence = { primaryTitle: 'Iran crackdown killed dozens in Tehran protests', sourceCount: 1 }; + const violenceOnly = { primaryTitle: 'Crackdown killed dozens in protests', sourceCount: 1 }; + assert.ok(scoreImportance(flashpointViolence) > scoreImportance(violenceOnly)); + }); + + it('demotes business context', () => { + const pure = { primaryTitle: 'Strike hits military targets', sourceCount: 1 }; + const business = { primaryTitle: 'Strike hits military targets says CEO in earnings call', sourceCount: 1 }; + assert.ok(scoreImportance(pure) > scoreImportance(business)); + }); + + it('adds alert bonus', () => { + const noAlert = { primaryTitle: 'Earthquake hits region', sourceCount: 1, isAlert: false }; + const alert = { primaryTitle: 'Earthquake hits region', sourceCount: 1, isAlert: true }; + assert.ok(scoreImportance(alert) > scoreImportance(noAlert)); + }); + }); + + describe('selectTopStories', () => { + it('returns at most maxCount stories', () => { + const clusters = Array.from({ length: 20 }, (_, i) => ({ + primaryTitle: `War conflict attack story number ${i}`, + primarySource: `Source${i % 5}`, + primaryLink: `http://${i}`, + sourceCount: 3, + isAlert: false, + })); + const top = selectTopStories(clusters, 5); + assert.ok(top.length <= 5); + }); + + it('filters out low-scoring single-source non-alert stories', () => { + const clusters = [ + { primaryTitle: 'Nice weather today', primarySource: 'Blog', primaryLink: 'http://a', sourceCount: 1, isAlert: false }, + ]; + const top = selectTopStories(clusters, 8); + assert.equal(top.length, 0); + }); + + it('includes high-scoring single-source stories', () => { + const clusters = [ + { primaryTitle: 'Iran missile attack kills dozens in massive airstrike', primarySource: 'Reuters', primaryLink: 'http://a', sourceCount: 1, isAlert: false }, + ]; + const top = selectTopStories(clusters, 8); + assert.equal(top.length, 1); + }); + + it('limits per-source diversity', () => { + const clusters = Array.from({ length: 10 }, (_, i) => ({ + primaryTitle: `War attack missile strike story ${i}`, + primarySource: 'SameSource', + primaryLink: `http://${i}`, + sourceCount: 2, + isAlert: false, + })); + const top = selectTopStories(clusters, 8); + assert.ok(top.length <= 3); + }); + }); +}); diff --git a/tests/insights-loader.test.mjs b/tests/insights-loader.test.mjs new file mode 100644 index 000000000..c16202962 --- /dev/null +++ b/tests/insights-loader.test.mjs @@ -0,0 +1,65 @@ +import { describe, it, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; + +describe('insights-loader', () => { + describe('getServerInsights (logic validation)', () => { + const MAX_AGE_MS = 15 * 60 * 1000; + + function isFresh(generatedAt) { + const age = Date.now() - new Date(generatedAt).getTime(); + return age < MAX_AGE_MS; + } + + it('rejects data older than 15 minutes', () => { + const old = new Date(Date.now() - 16 * 60 * 1000).toISOString(); + assert.equal(isFresh(old), false); + }); + + it('accepts data younger than 15 minutes', () => { + const fresh = new Date(Date.now() - 5 * 60 * 1000).toISOString(); + assert.equal(isFresh(fresh), true); + }); + + it('accepts data from now', () => { + assert.equal(isFresh(new Date().toISOString()), true); + }); + + it('rejects exactly 15 minutes old data', () => { + const exact = new Date(Date.now() - MAX_AGE_MS).toISOString(); + assert.equal(isFresh(exact), false); + }); + }); + + describe('ServerInsights payload shape', () => { + it('validates required fields', () => { + const valid = { + worldBrief: 'Test brief', + briefProvider: 'groq', + status: 'ok', + topStories: [{ primaryTitle: 'Test', sourceCount: 2 }], + generatedAt: new Date().toISOString(), + clusterCount: 10, + multiSourceCount: 5, + fastMovingCount: 3, + }; + assert.ok(valid.topStories.length >= 1); + assert.ok(['ok', 'degraded'].includes(valid.status)); + }); + + it('allows degraded status with empty brief', () => { + const degraded = { + worldBrief: '', + status: 'degraded', + topStories: [{ primaryTitle: 'Test' }], + generatedAt: new Date().toISOString(), + }; + assert.equal(degraded.worldBrief, ''); + assert.equal(degraded.status, 'degraded'); + }); + + it('rejects empty topStories', () => { + const empty = { topStories: [] }; + assert.equal(empty.topStories.length >= 1, false); + }); + }); +});