From 3af9ca8d96bc3b2a93fd831800d8c06a65766969 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Mon, 23 Mar 2026 16:10:49 +0400 Subject: [PATCH] refactor(intelligence): preserve digest threat fields through normalization (#2050) (#2093) * refactor(intelligence): preserve digest threat fields through normalization (#2050) * fix(intelligence): shallow-copy cluster threat, add medium to GlobeMap color map and VALID_THREAT_LEVELS - _clustering.mjs: spread-copy threat object instead of passing reference to prevent downstream mutation from corrupting cluster data - GlobeMap.ts: add 'medium' to threatLevel color guards (marker render + tooltip) so new digest taxonomy shows yellow/orange instead of falling through to the info-blue default - InsightsPanel.ts: extend VALID_THREAT_LEVELS to include new taxonomy values (medium/low/info) so the safeThreat guard stays in sync * fix(intelligence): normalize proto threat levels and use tier-sorted cluster threat P1: digest items store THREAT_LEVEL_HIGH/MEDIUM/etc (proto enum strings from toProtoItem). Copying item.threat verbatim caused threatLevel: 'THREAT_LEVEL_HIGH' to land in output, missing every downstream check (THREAT_RGB, badge render, GlobeMap color switch). Add normalizeThreat() with PROTO_TO_LEVEL map at read time so values arrive as 'high'/'medium'/etc. P2: cluster threatItem was found via group.find() on the unsorted insertion-order array. A low-tier item first in the array would win over a Reuters item. Switch to sorted.find() (already tier/date sorted for primary selection) to prefer the highest-quality source's threat classification. --- scripts/_clustering.mjs | 3 +++ scripts/seed-insights.mjs | 25 ++++++++++++++++++++++++- src/components/GlobeMap.ts | 4 ++-- src/components/InsightsPanel.ts | 2 +- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/_clustering.mjs b/scripts/_clustering.mjs index c025e092b..13bcb214b 100644 --- a/scripts/_clustering.mjs +++ b/scripts/_clustering.mjs @@ -120,6 +120,8 @@ export function clusterItems(items) { }); const primary = sorted[0]; + // Prefer highest-tier non-keyword threat; `sorted` is already tier/date ordered so reuse it. + const threatItem = sorted.find(i => i.threat?.level && i.threat?.source !== 'keyword'); return { primaryTitle: primary.title, primarySource: primary.source, @@ -127,6 +129,7 @@ export function clusterItems(items) { pubDate: primary.pubDate, sourceCount: group.length, isAlert: group.some(i => i.isAlert), + threat: threatItem?.threat ? { ...threatItem.threat } : (primary.threat ? { ...primary.threat } : undefined), }; }); } diff --git a/scripts/seed-insights.mjs b/scripts/seed-insights.mjs index bd304ab28..03b562a2b 100644 --- a/scripts/seed-insights.mjs +++ b/scripts/seed-insights.mjs @@ -7,6 +7,23 @@ loadEnvFile(import.meta.url); const CANONICAL_KEY = 'news:insights:v1'; const DIGEST_KEY = 'news:digest:v1:full:en'; + +// Digest items store proto enum strings (THREAT_LEVEL_HIGH etc.) from toProtoItem(). +// Normalize to client-side lowercase values before propagating into insights output. +const PROTO_TO_LEVEL = { + THREAT_LEVEL_CRITICAL: 'critical', + THREAT_LEVEL_HIGH: 'high', + THREAT_LEVEL_MEDIUM: 'medium', + THREAT_LEVEL_LOW: 'low', + THREAT_LEVEL_UNSPECIFIED: 'info', +}; + +function normalizeThreat(threat) { + if (!threat) return undefined; + const level = PROTO_TO_LEVEL[threat.level] ?? threat.level; + return { ...threat, level }; +} + const CACHE_TTL = 10800; // 3h — 6x the 30 min cron interval (was 1x = key expired on any missed run) const MAX_HEADLINES = 10; const MAX_HEADLINE_LEN = 500; @@ -246,6 +263,7 @@ async function fetchInsights() { pubDate: item.pubDate || item.publishedAt || item.date || new Date().toISOString(), isAlert: item.isAlert || false, tier: item.tier, + threat: normalizeThreat(item.threat), })).filter(item => item.title.length > 10); const clusters = clusterItems(normalizedItems); @@ -280,7 +298,12 @@ async function fetchInsights() { const fastMovingCount = 0; // velocity not available in digest items const enrichedStories = topStories.map(story => { - const { category, threatLevel } = categorizeStory(story.primaryTitle); + // Use digest threat when present and not keyword-sourced (keyword threat uses old taxonomy). + // Fall back to categorizeStory() for legacy/incomplete payloads. + const hasDigestThreat = story.threat?.level && story.threat?.source !== 'keyword'; + const { category, threatLevel } = hasDigestThreat + ? { category: story.threat.category ?? 'general', threatLevel: story.threat.level } + : categorizeStory(story.primaryTitle); return { primaryTitle: story.primaryTitle, primarySource: story.primarySource, diff --git a/src/components/GlobeMap.ts b/src/components/GlobeMap.ts index 9279bb9b0..99702142b 100644 --- a/src/components/GlobeMap.ts +++ b/src/components/GlobeMap.ts @@ -1163,7 +1163,7 @@ export class GlobeMap { } else if (d._kind === 'newsLocation') { const tc = d.threatLevel === 'critical' ? '#ff2020' : d.threatLevel === 'high' ? '#ff6600' - : d.threatLevel === 'elevated' ? '#ffaa00' + : (d.threatLevel === 'elevated' || d.threatLevel === 'medium') ? '#ffaa00' : '#44aaff'; el.innerHTML = `
@@ -1519,7 +1519,7 @@ export class GlobeMap { `
${esc(d.name)}` + `
${esc(d.severity)} · ${esc(d.description.slice(0, 60))}`; } else if (d._kind === 'newsLocation') { - const tc = d.threatLevel === 'critical' ? '#ff2020' : d.threatLevel === 'high' ? '#ff6600' : d.threatLevel === 'elevated' ? '#ffaa00' : '#44aaff'; + const tc = d.threatLevel === 'critical' ? '#ff2020' : d.threatLevel === 'high' ? '#ff6600' : (d.threatLevel === 'elevated' || d.threatLevel === 'medium') ? '#ffaa00' : '#44aaff'; html = `📰 ${esc(d.title.slice(0, 60))}` + `
${esc(d.threatLevel)}`; } else if (d._kind === 'satellite') { diff --git a/src/components/InsightsPanel.ts b/src/components/InsightsPanel.ts index ff6d08f91..c06a93b7a 100644 --- a/src/components/InsightsPanel.ts +++ b/src/components/InsightsPanel.ts @@ -553,7 +553,7 @@ export class InsightsPanel extends Panel { badges.push('⚠ ALERT'); } - const VALID_THREAT_LEVELS = ['critical', 'high', 'elevated', 'moderate']; + const VALID_THREAT_LEVELS = ['critical', 'high', 'elevated', 'moderate', 'medium', 'low', 'info']; if (story.threatLevel === 'critical' || story.threatLevel === 'high') { const safeThreat = VALID_THREAT_LEVELS.includes(story.threatLevel) ? story.threatLevel : 'moderate'; badges.push(`${escapeHtml(story.category)}`);