refactor(intelligence): preserve digest threat fields through normalization (#2050) (#2093)

* refactor(intelligence): preserve digest threat fields through normalization (#2050)

* fix(intelligence): shallow-copy cluster threat, add medium to GlobeMap color map and VALID_THREAT_LEVELS

- _clustering.mjs: spread-copy threat object instead of passing reference to prevent downstream mutation from corrupting cluster data
- GlobeMap.ts: add 'medium' to threatLevel color guards (marker render + tooltip) so new digest taxonomy shows yellow/orange instead of falling through to the info-blue default
- InsightsPanel.ts: extend VALID_THREAT_LEVELS to include new taxonomy values (medium/low/info) so the safeThreat guard stays in sync

* fix(intelligence): normalize proto threat levels and use tier-sorted cluster threat

P1: digest items store THREAT_LEVEL_HIGH/MEDIUM/etc (proto enum strings from
toProtoItem). Copying item.threat verbatim caused threatLevel: 'THREAT_LEVEL_HIGH'
to land in output, missing every downstream check (THREAT_RGB, badge render,
GlobeMap color switch). Add normalizeThreat() with PROTO_TO_LEVEL map at read
time so values arrive as 'high'/'medium'/etc.

P2: cluster threatItem was found via group.find() on the unsorted insertion-order
array. A low-tier item first in the array would win over a Reuters item. Switch
to sorted.find() (already tier/date sorted for primary selection) to prefer the
highest-quality source's threat classification.

scripts/_clustering.mjs

@@ -120,6 +120,8 @@ export function clusterItems(items) {
     });
 
     const primary = sorted[0];
+    // Prefer highest-tier non-keyword threat; `sorted` is already tier/date ordered so reuse it.
+    const threatItem = sorted.find(i => i.threat?.level && i.threat?.source !== 'keyword');
     return {
       primaryTitle: primary.title,
       primarySource: primary.source,
@@ -127,6 +129,7 @@ export function clusterItems(items) {
       pubDate: primary.pubDate,
       sourceCount: group.length,
       isAlert: group.some(i => i.isAlert),
+      threat: threatItem?.threat ? { ...threatItem.threat } : (primary.threat ? { ...primary.threat } : undefined),
     };
   });
 }

scripts/seed-insights.mjs

@@ -7,6 +7,23 @@ loadEnvFile(import.meta.url);
 
 const CANONICAL_KEY = 'news:insights:v1';
 const DIGEST_KEY = 'news:digest:v1:full:en';
+
+// Digest items store proto enum strings (THREAT_LEVEL_HIGH etc.) from toProtoItem().
+// Normalize to client-side lowercase values before propagating into insights output.
+const PROTO_TO_LEVEL = {
+  THREAT_LEVEL_CRITICAL: 'critical',
+  THREAT_LEVEL_HIGH: 'high',
+  THREAT_LEVEL_MEDIUM: 'medium',
+  THREAT_LEVEL_LOW: 'low',
+  THREAT_LEVEL_UNSPECIFIED: 'info',
+};
+
+function normalizeThreat(threat) {
+  if (!threat) return undefined;
+  const level = PROTO_TO_LEVEL[threat.level] ?? threat.level;
+  return { ...threat, level };
+}
+
 const CACHE_TTL = 10800; // 3h — 6x the 30 min cron interval (was 1x = key expired on any missed run)
 const MAX_HEADLINES = 10;
 const MAX_HEADLINE_LEN = 500;
@@ -246,6 +263,7 @@ async function fetchInsights() {
     pubDate: item.pubDate || item.publishedAt || item.date || new Date().toISOString(),
     isAlert: item.isAlert || false,
     tier: item.tier,
+    threat: normalizeThreat(item.threat),
   })).filter(item => item.title.length > 10);
 
   const clusters = clusterItems(normalizedItems);
@@ -280,7 +298,12 @@ async function fetchInsights() {
   const fastMovingCount = 0; // velocity not available in digest items
 
   const enrichedStories = topStories.map(story => {
-    const { category, threatLevel } = categorizeStory(story.primaryTitle);
+    // Use digest threat when present and not keyword-sourced (keyword threat uses old taxonomy).
+    // Fall back to categorizeStory() for legacy/incomplete payloads.
+    const hasDigestThreat = story.threat?.level && story.threat?.source !== 'keyword';
+    const { category, threatLevel } = hasDigestThreat
+      ? { category: story.threat.category ?? 'general', threatLevel: story.threat.level }
+      : categorizeStory(story.primaryTitle);
     return {
       primaryTitle: story.primaryTitle,
       primarySource: story.primarySource,

src/components/GlobeMap.ts

@@ -1163,7 +1163,7 @@ export class GlobeMap {
     } else if (d._kind === 'newsLocation') {
       const tc = d.threatLevel === 'critical' ? '#ff2020'
                : d.threatLevel === 'high'     ? '#ff6600'
-               : d.threatLevel === 'elevated' ? '#ffaa00'
+               : (d.threatLevel === 'elevated' || d.threatLevel === 'medium') ? '#ffaa00'
                : '#44aaff';
       el.innerHTML = `
         <div style="position:relative;width:16px;height:16px;">
@@ -1519,7 +1519,7 @@ export class GlobeMap {
              `<br><span style="opacity:.7;">${esc(d.name)}</span>` +
              `<br><span style="opacity:.5;">${esc(d.severity)} · ${esc(d.description.slice(0, 60))}</span>`;
     } else if (d._kind === 'newsLocation') {
-      const tc = d.threatLevel === 'critical' ? '#ff2020' : d.threatLevel === 'high' ? '#ff6600' : d.threatLevel === 'elevated' ? '#ffaa00' : '#44aaff';
+      const tc = d.threatLevel === 'critical' ? '#ff2020' : d.threatLevel === 'high' ? '#ff6600' : (d.threatLevel === 'elevated' || d.threatLevel === 'medium') ? '#ffaa00' : '#44aaff';
       html = `<span style="color:${tc};font-weight:bold;">📰 ${esc(d.title.slice(0, 60))}</span>` +
              `<br><span style="opacity:.5;">${esc(d.threatLevel)}</span>`;
     } else if (d._kind === 'satellite') {

src/components/InsightsPanel.ts

@@ -553,7 +553,7 @@ export class InsightsPanel extends Panel {
         badges.push('<span class="insight-badge alert">⚠ ALERT</span>');
       }
 
-      const VALID_THREAT_LEVELS = ['critical', 'high', 'elevated', 'moderate'];
+      const VALID_THREAT_LEVELS = ['critical', 'high', 'elevated', 'moderate', 'medium', 'low', 'info'];
       if (story.threatLevel === 'critical' || story.threatLevel === 'high') {
         const safeThreat = VALID_THREAT_LEVELS.includes(story.threatLevel) ? story.threatLevel : 'moderate';
         badges.push(`<span class="insight-badge velocity ${safeThreat}">${escapeHtml(story.category)}</span>`);