diff --git a/api/health.js b/api/health.js index 82eafc28b..2cb65f986 100644 --- a/api/health.js +++ b/api/health.js @@ -119,13 +119,14 @@ const SEED_META = { corridorrisk: { key: 'seed-meta:supply_chain:corridorrisk', maxStaleMin: 120 }, chokepointTransits: { key: 'seed-meta:supply_chain:chokepoint_transits', maxStaleMin: 15 }, transitSummaries: { key: 'seed-meta:supply_chain:transit-summaries', maxStaleMin: 15 }, + usniFleet: { key: 'seed-meta:military:usni-fleet', maxStaleMin: 420 }, }; // Standalone keys that are populated on-demand by RPC handlers (not seeds). // Empty = WARN not CRIT since they only exist after first request. const ON_DEMAND_KEYS = new Set([ 'riskScoresLive', - 'usniFleet', 'usniFleetStale', 'positiveEventsLive', 'cableHealth', + 'usniFleetStale', 'positiveEventsLive', 'cableHealth', 'bisPolicy', 'bisExchange', 'bisCredit', 'macroSignals', 'shippingRates', 'chokepoints', 'minerals', 'giving', 'cyberThreatsRpc', 'militaryBases', 'temporalAnomalies', 'displacement', diff --git a/scripts/ais-relay.cjs b/scripts/ais-relay.cjs index a0e89246a..402da294f 100644 --- a/scripts/ais-relay.cjs +++ b/scripts/ais-relay.cjs @@ -3885,6 +3885,270 @@ async function startCorridorRiskSeedLoop() { }, CORRIDOR_RISK_SEED_INTERVAL_MS).unref?.(); } +// ───────────────────────────────────────────────────────────── +// USNI Fleet Tracker seed loop — weekly article → Redis +// ───────────────────────────────────────────────────────────── + +const USNI_SEED_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours +const USNI_CACHE_KEY = 'usni-fleet:sebuf:v1'; +const USNI_STALE_CACHE_KEY = 'usni-fleet:sebuf:stale:v1'; +const USNI_CACHE_TTL = 21600; // 6 hours +const USNI_STALE_TTL = 604800; // 7 days +let usniSeedInFlight = false; + +const USNI_HULL_TYPE_MAP = { + CVN: 'carrier', CV: 'carrier', + DDG: 'destroyer', CG: 'destroyer', + LHD: 'amphibious', LHA: 'amphibious', LPD: 'amphibious', LSD: 'amphibious', LCC: 'amphibious', + SSN: 'submarine', SSBN: 'submarine', SSGN: 'submarine', + FFG: 'frigate', LCS: 'frigate', + MCM: 'patrol', PC: 'patrol', + AS: 'auxiliary', ESB: 'auxiliary', ESD: 'auxiliary', + 'T-AO': 'auxiliary', 'T-AKE': 'auxiliary', 'T-AOE': 'auxiliary', + 'T-ARS': 'auxiliary', 'T-ESB': 'auxiliary', 'T-EPF': 'auxiliary', + 'T-AGOS': 'research', 'T-AGS': 'research', 'T-AGM': 'research', AGOS: 'research', +}; + +const USNI_REGION_COORDS = { + 'Philippine Sea': { lat: 18.0, lon: 130.0 }, + 'South China Sea': { lat: 14.0, lon: 115.0 }, + 'East China Sea': { lat: 28.0, lon: 125.0 }, + 'Sea of Japan': { lat: 40.0, lon: 135.0 }, + 'Arabian Sea': { lat: 18.0, lon: 63.0 }, + 'Red Sea': { lat: 20.0, lon: 38.0 }, + 'Mediterranean Sea': { lat: 35.0, lon: 18.0 }, + 'Eastern Mediterranean': { lat: 34.5, lon: 33.0 }, + 'Western Mediterranean': { lat: 37.0, lon: 3.0 }, + 'Persian Gulf': { lat: 26.5, lon: 52.0 }, + 'Gulf of Oman': { lat: 24.5, lon: 58.5 }, + 'Gulf of Aden': { lat: 12.0, lon: 47.0 }, + 'Caribbean Sea': { lat: 15.0, lon: -73.0 }, + 'North Atlantic': { lat: 45.0, lon: -30.0 }, + 'Atlantic Ocean': { lat: 30.0, lon: -40.0 }, + 'Western Atlantic': { lat: 30.0, lon: -60.0 }, + 'Pacific Ocean': { lat: 20.0, lon: -150.0 }, + 'Eastern Pacific': { lat: 18.0, lon: -125.0 }, + 'Western Pacific': { lat: 20.0, lon: 140.0 }, + 'Indian Ocean': { lat: -5.0, lon: 75.0 }, + Antarctic: { lat: -70.0, lon: 20.0 }, + 'Baltic Sea': { lat: 58.0, lon: 20.0 }, + 'Black Sea': { lat: 43.5, lon: 34.0 }, + 'Bay of Bengal': { lat: 14.0, lon: 87.0 }, + Yokosuka: { lat: 35.29, lon: 139.67 }, + Japan: { lat: 35.29, lon: 139.67 }, + Sasebo: { lat: 33.16, lon: 129.72 }, + Guam: { lat: 13.45, lon: 144.79 }, + 'Pearl Harbor': { lat: 21.35, lon: -157.95 }, + 'San Diego': { lat: 32.68, lon: -117.15 }, + Norfolk: { lat: 36.95, lon: -76.30 }, + Mayport: { lat: 30.39, lon: -81.40 }, + Bahrain: { lat: 26.23, lon: 50.55 }, + Rota: { lat: 36.63, lon: -6.35 }, + 'Diego Garcia': { lat: -7.32, lon: 72.42 }, + Djibouti: { lat: 11.55, lon: 43.15 }, + Singapore: { lat: 1.35, lon: 103.82 }, + 'Souda Bay': { lat: 35.49, lon: 24.08 }, + Naples: { lat: 40.84, lon: 14.25 }, +}; + +function usniStripHtml(html) { + return html.replace(/<[^>]+>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&') + .replace(/</g, '<').replace(/>/g, '>').replace(/’/g, "'") + .replace(/“/g, '"').replace(/”/g, '"').replace(/–/g, '\u2013') + .replace(/\s+/g, ' ').trim(); +} + +function usniHullToType(hull) { + if (!hull) return 'unknown'; + for (const [prefix, type] of Object.entries(USNI_HULL_TYPE_MAP)) { + if (hull.startsWith(prefix)) return type; + } + return 'unknown'; +} + +function usniDetectStatus(text) { + if (!text) return 'unknown'; + const l = text.toLowerCase(); + if (l.includes('deployed') || l.includes('deployment')) return 'deployed'; + if (l.includes('underway') || l.includes('transiting') || l.includes('transit')) return 'underway'; + if (l.includes('homeport') || l.includes('in port') || l.includes('pierside') || l.includes('returned')) return 'in-port'; + return 'unknown'; +} + +function usniExtractHomePort(text) { + const m = text.match(/homeported (?:at|in) ([^.,]+)/i) || text.match(/home[ -]?ported (?:at|in) ([^.,]+)/i); + return m ? m[1].trim() : ''; +} + +function usniGetRegionCoords(regionText) { + const norm = regionText.replace(/^(In the|In|The)\s+/i, '').replace(/\s+/g, ' ').trim(); + if (USNI_REGION_COORDS[norm]) return USNI_REGION_COORDS[norm]; + const lower = norm.toLowerCase(); + for (const [key, coords] of Object.entries(USNI_REGION_COORDS)) { + if (key.toLowerCase() === lower || lower.includes(key.toLowerCase()) || key.toLowerCase().includes(lower)) return coords; + } + return null; +} + +function usniParseLeadingInt(text) { + const m = text.match(/\d{1,3}(?:,\d{3})*/); + return m ? parseInt(m[0].replace(/,/g, ''), 10) : undefined; +} + +function usniExtractBattleForceSummary(tableHtml) { + const rows = Array.from(tableHtml.matchAll(/]*>([\s\S]*?)<\/tr>/gi)); + if (rows.length < 2) return undefined; + const headers = Array.from(rows[0][1].matchAll(/]*>([\s\S]*?)<\/t[dh]>/gi)).map(m => usniStripHtml(m[1]).toLowerCase()); + const values = Array.from(rows[1][1].matchAll(/]*>([\s\S]*?)<\/t[dh]>/gi)).map(m => usniParseLeadingInt(usniStripHtml(m[1]))); + const summary = { totalShips: 0, deployed: 0, underway: 0 }; + let matched = false; + for (let i = 0; i < headers.length; i++) { + const label = headers[i] || ''; + const val = values[i]; + if (!Number.isFinite(val)) continue; + if (label.includes('battle force') || label.includes('total') || label.includes('ships')) { summary.totalShips = val; matched = true; } + else if (label.includes('deployed')) { summary.deployed = val; matched = true; } + else if (label.includes('underway')) { summary.underway = val; matched = true; } + } + return matched ? summary : undefined; +} + +function usniParseArticle(html, articleUrl, articleDate, articleTitle) { + const warnings = []; + const vessels = []; + const vesselByKey = new Map(); + const strikeGroups = []; + const regionsSet = new Set(); + + let battleForceSummary; + const tableMatch = html.match(/]*>([\s\S]*?)<\/table>/i); + if (tableMatch) battleForceSummary = usniExtractBattleForceSummary(tableMatch[1]); + + const h2Parts = html.split(/]*>/i); + for (let i = 1; i < h2Parts.length; i++) { + const part = h2Parts[i]; + const h2End = part.indexOf(''); + if (h2End === -1) continue; + const regionName = usniStripHtml(part.substring(0, h2End)).replace(/^(In the|In|The)\s+/i, '').replace(/\s+/g, ' ').trim(); + if (!regionName) continue; + regionsSet.add(regionName); + const coords = usniGetRegionCoords(regionName); + if (!coords) warnings.push(`Unknown region: "${regionName}"`); + const regionLat = coords?.lat ?? 0; + const regionLon = coords?.lon ?? 0; + const regionContent = part.substring(h2End + 5); + const h3Parts = regionContent.split(/]*>/i); + let currentSG = null; + for (let j = 0; j < h3Parts.length; j++) { + const section = h3Parts[j]; + if (j > 0) { + const h3End = section.indexOf(''); + if (h3End !== -1) { + const sgName = usniStripHtml(section.substring(0, h3End)); + if (sgName) { currentSG = { name: sgName, carrier: '', airWing: '', destroyerSquadron: '', escorts: [] }; strikeGroups.push(currentSG); } + } + } + const shipRegex = /(USS|USNS)\s+(?:<[^>]+>)?([^<(]+?)(?:<\/[^>]+>)?\s*\(([^)]+)\)/gi; + let match; + const sectionText = usniStripHtml(section); + const deploymentStatus = usniDetectStatus(sectionText); + const homePort = usniExtractHomePort(sectionText); + const activityDesc = sectionText.length > 10 ? sectionText.substring(0, 200).trim() : ''; + while ((match = shipRegex.exec(section)) !== null) { + const prefix = match[1].toUpperCase(); + const shipName = match[2].trim(); + const hullNumber = match[3].trim(); + const vesselType = usniHullToType(hullNumber); + if (prefix === 'USS' && vesselType === 'carrier' && currentSG) currentSG.carrier = `USS ${shipName} (${hullNumber})`; + if (currentSG) currentSG.escorts.push(`${prefix} ${shipName} (${hullNumber})`); + const key = `${regionName}|${hullNumber.toUpperCase()}`; + if (!vesselByKey.has(key)) { + const v = { name: `${prefix} ${shipName}`, hullNumber, vesselType, region: regionName, regionLat, regionLon, deploymentStatus, homePort, strikeGroup: currentSG?.name || '', activityDescription: activityDesc, articleUrl, articleDate }; + vessels.push(v); + vesselByKey.set(key, v); + } + } + } + } + + for (const sg of strikeGroups) { + const wingMatch = html.match(new RegExp(sg.name + '[\\s\\S]{0,500}Carrier Air Wing\\s*(\\w+)', 'i')); + if (wingMatch) sg.airWing = `Carrier Air Wing ${wingMatch[1]}`; + const desronMatch = html.match(new RegExp(sg.name + '[\\s\\S]{0,500}Destroyer Squadron\\s*(\\w+)', 'i')); + if (desronMatch) sg.destroyerSquadron = `Destroyer Squadron ${desronMatch[1]}`; + sg.escorts = [...new Set(sg.escorts)]; + } + + return { + articleUrl, articleDate, articleTitle, + battleForceSummary: battleForceSummary || { totalShips: 0, deployed: 0, underway: 0 }, + vessels, strikeGroups, regions: [...regionsSet], + parsingWarnings: warnings, + timestamp: Date.now(), + }; +} + +async function seedUSNIFleet() { + if (usniSeedInFlight) return; + usniSeedInFlight = true; + const t0 = Date.now(); + try { + const wpData = await new Promise((resolve, reject) => { + const req = https.get('https://news.usni.org/wp-json/wp/v2/posts?categories=4137&per_page=1', { + headers: { 'User-Agent': CHROME_UA, Accept: 'application/json', 'Accept-Encoding': 'gzip' }, + timeout: 15000, + }, (resp) => { + const chunks = []; + const stream = resp.headers['content-encoding'] === 'gzip' ? resp.pipe(zlib.createGunzip()) : resp; + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('end', () => { + try { resolve(JSON.parse(Buffer.concat(chunks).toString())); } + catch (e) { reject(e); } + }); + stream.on('error', reject); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('USNI fetch timeout')); }); + }); + + if (!Array.isArray(wpData) || !wpData.length) { + console.warn('[USNI] No fleet tracker articles found'); + return; + } + + const post = wpData[0]; + const articleUrl = post.link || `https://news.usni.org/?p=${post.id}`; + const articleDate = post.date || new Date().toISOString(); + const articleTitle = usniStripHtml(post.title?.rendered || 'USNI Fleet Tracker'); + const htmlContent = post.content?.rendered || ''; + if (!htmlContent) { console.warn('[USNI] Empty article content'); return; } + + const report = usniParseArticle(htmlContent, articleUrl, articleDate, articleTitle); + const ok = await upstashSet(USNI_CACHE_KEY, report, USNI_CACHE_TTL); + await upstashSet(USNI_STALE_CACHE_KEY, report, USNI_STALE_TTL); + await upstashSet('seed-meta:military:usni-fleet', { fetchedAt: Date.now(), recordCount: report.vessels.length }, 604800); + const elapsed = ((Date.now() - t0) / 1000).toFixed(1); + console.log(`[USNI] Seeded ${report.vessels.length} vessels, ${report.strikeGroups.length} CSGs, ${report.regions.length} regions (redis: ${ok ? 'OK' : 'FAIL'}) in ${elapsed}s`); + if (report.parsingWarnings.length > 0) console.warn('[USNI] Warnings:', report.parsingWarnings.join('; ')); + } catch (e) { + console.warn('[USNI] Seed error:', e?.message || e); + } finally { + usniSeedInFlight = false; + } +} + +async function startUSNISeedLoop() { + if (!UPSTASH_ENABLED) { + console.log('[USNI] Disabled (no Upstash Redis)'); + return; + } + console.log(`[USNI] Seed loop starting (interval ${USNI_SEED_INTERVAL_MS / 1000 / 60 / 60}h)`); + seedUSNIFleet().catch(e => console.warn('[USNI] Initial seed error:', e?.message || e)); + setInterval(() => { + seedUSNIFleet().catch(e => console.warn('[USNI] Seed error:', e?.message || e)); + }, USNI_SEED_INTERVAL_MS).unref?.(); +} + function gzipSyncBuffer(body) { try { return zlib.gzipSync(typeof body === 'string' ? Buffer.from(body) : body); @@ -7073,6 +7337,7 @@ server.listen(PORT, () => { startTechEventsSeedLoop(); startPortWatchSeedLoop(); startCorridorRiskSeedLoop(); + startUSNISeedLoop(); }); wss.on('connection', (ws, req) => { diff --git a/server/worldmonitor/military/v1/get-usni-fleet-report.ts b/server/worldmonitor/military/v1/get-usni-fleet-report.ts index 4e220b6f6..cdcabec63 100644 --- a/server/worldmonitor/military/v1/get-usni-fleet-report.ts +++ b/server/worldmonitor/military/v1/get-usni-fleet-report.ts @@ -2,434 +2,41 @@ import type { ServerContext, GetUSNIFleetReportRequest, GetUSNIFleetReportResponse, - USNIVessel, - USNIStrikeGroup, - BattleForceSummary, USNIFleetReport, } from '../../../../src/generated/server/worldmonitor/military/v1/service_server'; -import { getCachedJson, setCachedJson, cachedFetchJsonWithMeta } from '../../../_shared/redis'; -import { CHROME_UA } from '../../../_shared/constants'; +import { getCachedJson } from '../../../_shared/redis'; const USNI_CACHE_KEY = 'usni-fleet:sebuf:v1'; const USNI_STALE_CACHE_KEY = 'usni-fleet:sebuf:stale:v1'; -const USNI_CACHE_TTL = 21600; // 6 hours -const USNI_STALE_TTL = 604800; // 7 days // ======================================================================== -// USNI parsing helpers +// RPC handler (Redis-read-only — Railway relay seeds the data) // ======================================================================== -const HULL_TYPE_MAP: Record = { - CVN: 'carrier', CV: 'carrier', - DDG: 'destroyer', CG: 'destroyer', - LHD: 'amphibious', LHA: 'amphibious', LPD: 'amphibious', LSD: 'amphibious', LCC: 'amphibious', - SSN: 'submarine', SSBN: 'submarine', SSGN: 'submarine', - FFG: 'frigate', LCS: 'frigate', - MCM: 'patrol', PC: 'patrol', - AS: 'auxiliary', ESB: 'auxiliary', ESD: 'auxiliary', - 'T-AO': 'auxiliary', 'T-AKE': 'auxiliary', 'T-AOE': 'auxiliary', - 'T-ARS': 'auxiliary', 'T-ESB': 'auxiliary', 'T-EPF': 'auxiliary', - 'T-AGOS': 'research', 'T-AGS': 'research', 'T-AGM': 'research', AGOS: 'research', -}; - -function hullToVesselType(hull: string): string { - if (!hull) return 'unknown'; - for (const [prefix, type] of Object.entries(HULL_TYPE_MAP)) { - if (hull.startsWith(prefix)) return type; - } - return 'unknown'; -} - -function detectDeploymentStatus(text: string): string { - if (!text) return 'unknown'; - const lower = text.toLowerCase(); - if (lower.includes('deployed') || lower.includes('deployment')) return 'deployed'; - if (lower.includes('underway') || lower.includes('transiting') || lower.includes('transit')) return 'underway'; - if (lower.includes('homeport') || lower.includes('in port') || lower.includes('pierside') || lower.includes('returned')) return 'in-port'; - return 'unknown'; -} - -function extractHomePort(text: string): string | undefined { - const match = text.match(/homeported (?:at|in) ([^.,]+)/i) || text.match(/home[ -]?ported (?:at|in) ([^.,]+)/i); - return match ? match[1]!.trim() : undefined; -} - -function stripHtml(html: string): string { - return html - .replace(/<[^>]+>/g, ' ') - .replace(/ /g, ' ') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/’/g, "'") - .replace(/“/g, '"') - .replace(/”/g, '"') - .replace(/–/g, '\u2013') - .replace(/\s+/g, ' ') - .trim(); -} - -const REGION_COORDS: Record = { - 'Philippine Sea': { lat: 18.0, lon: 130.0 }, - 'South China Sea': { lat: 14.0, lon: 115.0 }, - 'East China Sea': { lat: 28.0, lon: 125.0 }, - 'Sea of Japan': { lat: 40.0, lon: 135.0 }, - 'Arabian Sea': { lat: 18.0, lon: 63.0 }, - 'Red Sea': { lat: 20.0, lon: 38.0 }, - 'Mediterranean Sea': { lat: 35.0, lon: 18.0 }, - 'Eastern Mediterranean': { lat: 34.5, lon: 33.0 }, - 'Western Mediterranean': { lat: 37.0, lon: 3.0 }, - 'Persian Gulf': { lat: 26.5, lon: 52.0 }, - 'Gulf of Oman': { lat: 24.5, lon: 58.5 }, - 'Gulf of Aden': { lat: 12.0, lon: 47.0 }, - 'Caribbean Sea': { lat: 15.0, lon: -73.0 }, - 'North Atlantic': { lat: 45.0, lon: -30.0 }, - 'Atlantic Ocean': { lat: 30.0, lon: -40.0 }, - 'Western Atlantic': { lat: 30.0, lon: -60.0 }, - 'Pacific Ocean': { lat: 20.0, lon: -150.0 }, - 'Eastern Pacific': { lat: 18.0, lon: -125.0 }, - 'Western Pacific': { lat: 20.0, lon: 140.0 }, - 'Indian Ocean': { lat: -5.0, lon: 75.0 }, - Antarctic: { lat: -70.0, lon: 20.0 }, - 'Baltic Sea': { lat: 58.0, lon: 20.0 }, - 'Black Sea': { lat: 43.5, lon: 34.0 }, - 'Bay of Bengal': { lat: 14.0, lon: 87.0 }, - Yokosuka: { lat: 35.29, lon: 139.67 }, - Japan: { lat: 35.29, lon: 139.67 }, - Sasebo: { lat: 33.16, lon: 129.72 }, - Guam: { lat: 13.45, lon: 144.79 }, - 'Pearl Harbor': { lat: 21.35, lon: -157.95 }, - 'San Diego': { lat: 32.68, lon: -117.15 }, - Norfolk: { lat: 36.95, lon: -76.30 }, - Mayport: { lat: 30.39, lon: -81.40 }, - Bahrain: { lat: 26.23, lon: 50.55 }, - Rota: { lat: 36.63, lon: -6.35 }, - 'Diego Garcia': { lat: -7.32, lon: 72.42 }, - Djibouti: { lat: 11.55, lon: 43.15 }, - Singapore: { lat: 1.35, lon: 103.82 }, - 'Souda Bay': { lat: 35.49, lon: 24.08 }, - Naples: { lat: 40.84, lon: 14.25 }, -}; - -function getRegionCoords(regionText: string): { lat: number; lon: number } | null { - const normalized = regionText - .replace(/^(In the|In|The)\s+/i, '') - .replace(/\s+/g, ' ') - .trim(); - if (REGION_COORDS[normalized]) return REGION_COORDS[normalized]; - const lower = normalized.toLowerCase(); - for (const [key, coords] of Object.entries(REGION_COORDS)) { - if (key.toLowerCase() === lower || lower.includes(key.toLowerCase()) || key.toLowerCase().includes(lower)) { - return coords; - } - } - return null; -} - -function parseLeadingInteger(text: string): number | undefined { - const match = text.match(/\d{1,3}(?:,\d{3})*/); - if (!match) return undefined; - return parseInt(match[0].replace(/,/g, ''), 10); -} - -function extractBattleForceSummary(tableHtml: string): BattleForceSummary | undefined { - const rows = Array.from(tableHtml.matchAll(/]*>([\s\S]*?)<\/tr>/gi)); - if (rows.length < 2) return undefined; - - const headerCells = Array.from(rows[0]![1]!.matchAll(/]*>([\s\S]*?)<\/t[dh]>/gi)) - .map((m) => stripHtml(m[1]!).toLowerCase()); - const valueCells = Array.from(rows[1]![1]!.matchAll(/]*>([\s\S]*?)<\/t[dh]>/gi)) - .map((m) => parseLeadingInteger(stripHtml(m[1]!))); - - const summary: BattleForceSummary = { totalShips: 0, deployed: 0, underway: 0 }; - let matched = false; - - for (let idx = 0; idx < headerCells.length; idx++) { - const label = headerCells[idx] || ''; - const value = valueCells[idx]; - if (!Number.isFinite(value)) continue; - - if (label.includes('battle force') || label.includes('total') || label.includes('ships')) { - summary.totalShips = value!; - matched = true; - } else if (label.includes('deployed')) { - summary.deployed = value!; - matched = true; - } else if (label.includes('underway')) { - summary.underway = value!; - matched = true; - } - } - - if (matched) return summary; - - const tableText = stripHtml(tableHtml); - const totalMatch = tableText.match(/(?:battle[- ]?force|ships?|total)[^0-9]{0,40}(\d{1,3}(?:,\d{3})*)/i) - || tableText.match(/(\d{1,3}(?:,\d{3})*)\s*(?:battle[- ]?force|ships?|total)/i); - const deployedMatch = tableText.match(/deployed[^0-9]{0,40}(\d{1,3}(?:,\d{3})*)/i) - || tableText.match(/(\d{1,3}(?:,\d{3})*)\s*deployed/i); - const underwayMatch = tableText.match(/underway[^0-9]{0,40}(\d{1,3}(?:,\d{3})*)/i) - || tableText.match(/(\d{1,3}(?:,\d{3})*)\s*underway/i); - - if (!totalMatch && !deployedMatch && !underwayMatch) return undefined; - return { - totalShips: totalMatch ? parseInt(totalMatch[1]!.replace(/,/g, ''), 10) : 0, - deployed: deployedMatch ? parseInt(deployedMatch[1]!.replace(/,/g, ''), 10) : 0, - underway: underwayMatch ? parseInt(underwayMatch[1]!.replace(/,/g, ''), 10) : 0, - }; -} - -interface ParsedStrikeGroup { - name: string; - carrier?: string; - airWing?: string; - destroyerSquadron?: string; - escorts: string[]; -} - -function parseUSNIArticle( - html: string, - articleUrl: string, - articleDate: string, - articleTitle: string, -): USNIFleetReport { - const warnings: string[] = []; - const vessels: USNIVessel[] = []; - const vesselByRegionHull = new Map(); - const strikeGroups: ParsedStrikeGroup[] = []; - const regionsSet = new Set(); - - let battleForceSummary: BattleForceSummary | undefined; - const tableMatch = html.match(/]*>([\s\S]*?)<\/table>/i); - if (tableMatch) { - battleForceSummary = extractBattleForceSummary(tableMatch[1]!); - } - - const h2Parts = html.split(/]*>/i); - - for (let i = 1; i < h2Parts.length; i++) { - const part = h2Parts[i]!; - const h2EndIdx = part.indexOf(''); - if (h2EndIdx === -1) continue; - const regionRaw = stripHtml(part.substring(0, h2EndIdx)); - const regionContent = part.substring(h2EndIdx + 5); - - const regionName = regionRaw - .replace(/^(In the|In|The)\s+/i, '') - .replace(/\s+/g, ' ') - .trim(); - - if (!regionName) continue; - regionsSet.add(regionName); - - const coords = getRegionCoords(regionName); - if (!coords) { - warnings.push(`Unknown region: "${regionName}"`); - } - const regionLat = coords?.lat ?? 0; - const regionLon = coords?.lon ?? 0; - - const h3Parts = regionContent.split(/]*>/i); - - let currentStrikeGroup: ParsedStrikeGroup | null = null; - - for (let j = 0; j < h3Parts.length; j++) { - const section = h3Parts[j]!; - - if (j > 0) { - const h3EndIdx = section.indexOf(''); - if (h3EndIdx !== -1) { - const sgName = stripHtml(section.substring(0, h3EndIdx)); - if (sgName) { - currentStrikeGroup = { - name: sgName, - carrier: undefined, - airWing: undefined, - destroyerSquadron: undefined, - escorts: [], - }; - strikeGroups.push(currentStrikeGroup); - } - } - } - - // Broadened regex: matches any inline HTML tag (or no tag) wrapping the ship name. - // Handles , , , , , or plain text. - const shipRegex = /(USS|USNS)\s+(?:<[^>]+>)?([^<(]+?)(?:<\/[^>]+>)?\s*\(([^)]+)\)/gi; - let match: RegExpExecArray | null; - const sectionText = stripHtml(section); - const deploymentStatus = detectDeploymentStatus(sectionText); - const homePort = extractHomePort(sectionText); - const activityDesc = sectionText.length > 10 ? sectionText.substring(0, 200).trim() : ''; - let sectionShipCount = 0; - - const upsertVessel = (entry: USNIVessel) => { - const key = `${entry.region}|${entry.hullNumber.toUpperCase()}`; - const existing = vesselByRegionHull.get(key); - if (existing) { - if (!existing.strikeGroup && entry.strikeGroup) existing.strikeGroup = entry.strikeGroup; - if (existing.deploymentStatus === 'unknown' && entry.deploymentStatus !== 'unknown') { - existing.deploymentStatus = entry.deploymentStatus; - } - if (!existing.homePort && entry.homePort) existing.homePort = entry.homePort; - if ((!existing.activityDescription || existing.activityDescription.length < (entry.activityDescription || '').length) && entry.activityDescription) { - existing.activityDescription = entry.activityDescription; - } - return; - } - vessels.push(entry); - vesselByRegionHull.set(key, entry); - }; - - while ((match = shipRegex.exec(section)) !== null) { - const prefix = match[1]!.toUpperCase() as 'USS' | 'USNS'; - const shipName = match[2]!.trim(); - const hullNumber = match[3]!.trim(); - const vesselType = hullToVesselType(hullNumber); - sectionShipCount++; - - if (prefix === 'USS' && vesselType === 'carrier' && currentStrikeGroup) { - currentStrikeGroup.carrier = `USS ${shipName} (${hullNumber})`; - } - if (currentStrikeGroup) { - currentStrikeGroup.escorts.push(`${prefix} ${shipName} (${hullNumber})`); - } - - upsertVessel({ - name: `${prefix} ${shipName}`, - hullNumber, - vesselType, - region: regionName, - regionLat, - regionLon, - deploymentStatus, - homePort: homePort || '', - strikeGroup: currentStrikeGroup?.name || '', - activityDescription: activityDesc, - articleUrl, - articleDate, - }); - } - - // Warn when a strike group section contains text but yields zero ships — - // likely means the HTML format changed and the regex no longer matches. - if (currentStrikeGroup && sectionShipCount === 0 && sectionText.length > 20) { - console.warn( - `[USNI Fleet] Strike group section "${currentStrikeGroup.name}" in region "${regionName}" yielded 0 ships — HTML format may have changed`, - ); - warnings.push(`Strike group "${currentStrikeGroup.name}" yielded 0 ships`); - } - } - } - - for (const sg of strikeGroups) { - const wingMatch = html.match(new RegExp(sg.name + '[\\s\\S]{0,500}Carrier Air Wing\\s*(\\w+)', 'i')); - if (wingMatch) sg.airWing = `Carrier Air Wing ${wingMatch[1]}`; - const desronMatch = html.match(new RegExp(sg.name + '[\\s\\S]{0,500}Destroyer Squadron\\s*(\\w+)', 'i')); - if (desronMatch) sg.destroyerSquadron = `Destroyer Squadron ${desronMatch[1]}`; - sg.escorts = Array.from(new Set(sg.escorts)); - } - - const protoStrikeGroups: USNIStrikeGroup[] = strikeGroups.map((sg) => ({ - name: sg.name, - carrier: sg.carrier || '', - airWing: sg.airWing || '', - destroyerSquadron: sg.destroyerSquadron || '', - escorts: sg.escorts, - })); - - return { - articleUrl, - articleDate, - articleTitle, - battleForceSummary, - vessels, - strikeGroups: protoStrikeGroups, - regions: Array.from(regionsSet), - parsingWarnings: warnings, - timestamp: Date.now(), - }; -} - -// ======================================================================== -// RPC handler -// ======================================================================== - -async function fetchUSNIReport(): Promise { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 15000); - - let wpData: Array>; - try { - const response = await fetch( - 'https://news.usni.org/wp-json/wp/v2/posts?categories=4137&per_page=1', - { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: controller.signal, - }, - ); - if (!response.ok) throw new Error(`USNI API error: ${response.status}`); - wpData = (await response.json()) as Array>; - } finally { - clearTimeout(timeoutId); - } - - if (!wpData || !wpData.length) return null; - - const post = wpData[0]!; - const articleUrl = (post.link as string) || `https://news.usni.org/?p=${post.id}`; - const articleDate = (post.date as string) || new Date().toISOString(); - const articleTitle = stripHtml(((post.title as Record)?.rendered) || 'USNI Fleet Tracker'); - const htmlContent = ((post.content as Record)?.rendered) || ''; - - if (!htmlContent) return null; - - const report = parseUSNIArticle(htmlContent, articleUrl, articleDate, articleTitle); - console.warn(`[USNI Fleet] Parsed: ${report.vessels.length} vessels, ${report.strikeGroups.length} CSGs, ${report.regions.length} regions`); - - if (report.parsingWarnings.length > 0) { - console.warn('[USNI Fleet] Warnings:', report.parsingWarnings.join('; ')); - } - - // Also write to stale backup cache - await setCachedJson(USNI_STALE_CACHE_KEY, report, USNI_STALE_TTL); - - return report; -} - export async function getUSNIFleetReport( _ctx: ServerContext, req: GetUSNIFleetReportRequest, ): Promise { + if (req.forceRefresh) { + return { report: undefined, cached: false, stale: false, error: 'forceRefresh is no longer supported (data is seeded by Railway relay)' }; + } + try { - if (req.forceRefresh) { - // Bypass cachedFetchJson — fetch fresh and write both caches - const report = await fetchUSNIReport(); - if (!report) return { report: undefined, cached: false, stale: false, error: 'No USNI fleet tracker articles found' }; - await setCachedJson(USNI_CACHE_KEY, report, USNI_CACHE_TTL); - return { report, cached: false, stale: false, error: '' }; - } - - // Single atomic call — source tracking inside cachedFetchJsonWithMeta eliminates TOCTOU race - const { data: report, source } = await cachedFetchJsonWithMeta( - USNI_CACHE_KEY, USNI_CACHE_TTL, fetchUSNIReport, - ); + const report = (await getCachedJson(USNI_CACHE_KEY)) as USNIFleetReport | null; if (report) { - return { report, cached: source === 'cache', stale: false, error: '' }; + return { report, cached: true, stale: false, error: '' }; } - return { report: undefined, cached: false, stale: false, error: 'No USNI fleet tracker articles found' }; - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn('[USNI Fleet] Error:', message); - const stale = (await getCachedJson(USNI_STALE_CACHE_KEY)) as USNIFleetReport | null; if (stale) { - console.warn('[USNI Fleet] Returning stale cached data'); return { report: stale, cached: true, stale: true, error: 'Using cached data' }; } + return { report: undefined, cached: false, stale: false, error: 'No USNI fleet data in cache (waiting for seed)' }; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn('[USNI Fleet] Error:', message); return { report: undefined, cached: false, stale: false, error: message }; } } diff --git a/src/services/population-exposure.ts b/src/services/population-exposure.ts index 8e18d0cfe..77f6f5f20 100644 --- a/src/services/population-exposure.ts +++ b/src/services/population-exposure.ts @@ -8,6 +8,13 @@ const client = new DisplacementServiceClient(getRpcBaseUrl(), { fetch: (...args) const countriesBreaker = createCircuitBreaker({ name: 'WorldPop Countries', cacheTtlMs: 30 * 60 * 1000, persistCache: true }); +const exposureBreaker = createCircuitBreaker({ + name: 'PopExposure', + cacheTtlMs: 6 * 60 * 60 * 1000, + persistCache: true, + maxCacheEntries: 64, +}); + export async function fetchCountryPopulations(): Promise { const result = await countriesBreaker.execute(async () => { return client.getPopulationExposure({ mode: 'countries', lat: 0, lon: 0, radius: 0 }); @@ -24,13 +31,15 @@ interface ExposureResponse { } export async function fetchExposure(lat: number, lon: number, radiusKm: number): Promise { - try { - const result = await client.getPopulationExposure({ mode: 'exposure', lat, lon, radius: radiusKm }); - if (!result.exposure) return null; - return result.exposure; - } catch { - return null; - } + const cacheKey = `${lat.toFixed(4)},${lon.toFixed(4)},${radiusKm}`; + return exposureBreaker.execute( + async () => { + const result = await client.getPopulationExposure({ mode: 'exposure', lat, lon, radius: radiusKm }); + return result.exposure ?? null; + }, + null, + { cacheKey }, + ); } interface EventForExposure { diff --git a/src/services/summarization.ts b/src/services/summarization.ts index b76934fa7..18adde3ea 100644 --- a/src/services/summarization.ts +++ b/src/services/summarization.ts @@ -39,6 +39,13 @@ export interface SummarizeOptions { const newsClient = new NewsServiceClient(getRpcBaseUrl(), { fetch: (...args) => globalThis.fetch(...args) }); const summaryBreaker = createCircuitBreaker({ name: 'News Summarization', cacheTtlMs: 0 }); +const summaryResultBreaker = createCircuitBreaker({ + name: 'SummaryResult', + cacheTtlMs: 2 * 60 * 60 * 1000, + persistCache: true, + maxCacheEntries: 32, +}); + const emptySummaryFallback: SummarizeArticleResponse = { summary: '', provider: '', model: '', fallback: true, tokens: 0, error: '', errorType: '', status: 'SUMMARIZE_STATUS_UNSPECIFIED', statusDetail: '' }; // ── Provider definitions ── @@ -167,18 +174,27 @@ export async function generateSummary( return null; } - lastAttemptedProvider = 'none'; - const result = await generateSummaryInternal(headlines, onProgress, geoContext, lang, options); + const optionsSuffix = options?.skipCloudProviders || options?.skipBrowserFallback + ? `:opts${options.skipCloudProviders ? 'C' : ''}${options.skipBrowserFallback ? 'B' : ''}` + : ''; + const cacheKey = buildSummaryCacheKey(headlines, 'brief', geoContext, SITE_VARIANT, lang) + optionsSuffix; - // Track at generateSummary return only (not inside tryApiProvider) to avoid - // double-counting beta comparison traffic. Only the winning provider is recorded. - if (result) { - trackLLMUsage(result.provider, result.model, result.cached); - } else { - trackLLMFailure(lastAttemptedProvider); - } + return summaryResultBreaker.execute( + async () => { + lastAttemptedProvider = 'none'; + const result = await generateSummaryInternal(headlines, onProgress, geoContext, lang, options); - return result; + if (result) { + trackLLMUsage(result.provider, result.model, result.cached); + } else { + trackLLMFailure(lastAttemptedProvider); + } + + return result; + }, + null, + { cacheKey, shouldCache: (result) => result !== null }, + ); } async function generateSummaryInternal( diff --git a/src/services/threat-classifier.ts b/src/services/threat-classifier.ts index 4c78d00a0..c5427a71e 100644 --- a/src/services/threat-classifier.ts +++ b/src/services/threat-classifier.ts @@ -378,9 +378,17 @@ import { ApiError, type ClassifyEventResponse, } from '@/generated/client/worldmonitor/intelligence/v1/service_client'; +import { createCircuitBreaker } from '@/utils'; const classifyClient = new IntelligenceServiceClient(getRpcBaseUrl(), { fetch: (...args) => globalThis.fetch(...args) }); +const classifyBreaker = createCircuitBreaker({ + name: 'AIClassify', + cacheTtlMs: 6 * 60 * 60 * 1000, + persistCache: true, + maxCacheEntries: 256, +}); + const VALID_LEVELS: Record = { critical: 'critical', high: 'high', medium: 'medium', low: 'low', info: 'info', }; @@ -509,7 +517,7 @@ function scheduleBatch(): void { } } -export function classifyWithAI( +function classifyWithAIUncached( title: string, variant: string ): Promise { @@ -524,6 +532,18 @@ export function classifyWithAI( }); } +export function classifyWithAI( + title: string, + variant: string, +): Promise { + const cacheKey = title.trim().toLowerCase().replace(/\s+/g, ' '); + return classifyBreaker.execute( + () => classifyWithAIUncached(title, variant), + null, + { cacheKey, shouldCache: (result) => result !== null }, + ); +} + export function aggregateThreats( items: Array<{ threat?: ThreatClassification; tier?: number }> ): ThreatClassification { diff --git a/src/services/usni-fleet.ts b/src/services/usni-fleet.ts index 11b80a1d8..2b7c8782b 100644 --- a/src/services/usni-fleet.ts +++ b/src/services/usni-fleet.ts @@ -14,12 +14,9 @@ const breaker = createCircuitBreaker({ maxFailures: 3, cooldownMs: 10 * 60 * 1000, cacheTtlMs: 60 * 60 * 1000, // 1hr local cache + persistCache: true, }); -let lastReport: USNIFleetReport | null = null; -let lastFetchTime = 0; -const LOCAL_CACHE_TTL = 60 * 60 * 1000; // 1 hour - function mapProtoToReport(resp: GetUSNIFleetReportResponse): USNIFleetReport | null { const r = resp.report; if (!r) return null; @@ -53,22 +50,11 @@ function mapProtoToReport(resp: GetUSNIFleetReportResponse): USNIFleetReport | n } export async function fetchUSNIFleetReport(): Promise { - if (lastReport && Date.now() - lastFetchTime < LOCAL_CACHE_TTL) { - return lastReport; - } - - const report = await breaker.execute(async () => { + return breaker.execute(async () => { const resp = await client.getUSNIFleetReport({ forceRefresh: false }); if (resp.error && !resp.report) return null; return mapProtoToReport(resp); - }, null); - - if (report) { - lastReport = report; - lastFetchTime = Date.now(); - } - - return report; + }, null, { shouldCache: (result) => result !== null }); } function normalizeHull(hull: string | undefined): string { diff --git a/tests/deploy-config.test.mjs b/tests/deploy-config.test.mjs index 6371d723f..fd0b7eb78 100644 --- a/tests/deploy-config.test.mjs +++ b/tests/deploy-config.test.mjs @@ -16,7 +16,7 @@ const getCacheHeaderValue = (sourcePath) => { describe('deploy/cache configuration guardrails', () => { it('disables caching for HTML entry routes on Vercel', () => { - const spaNoCache = getCacheHeaderValue('/((?!api|assets|blog|docs|favico|map-styles|data|textures|pro|sw\\.js|manifest\\.webmanifest|offline\\.html|robots\\.txt|sitemap\\.xml|llms\\.txt|llms-full\\.txt|\\.well-known).*)'); + const spaNoCache = getCacheHeaderValue('/((?!api|assets|blog|docs|favico|map-styles|data|textures|pro|sw\\.js|workbox-[a-f0-9]+\\.js|manifest\\.webmanifest|offline\\.html|robots\\.txt|sitemap\\.xml|llms\\.txt|llms-full\\.txt|\\.well-known).*)'); assert.equal(spaNoCache, 'no-cache, no-store, must-revalidate'); }); diff --git a/vercel.json b/vercel.json index 61e3649b8..d13e7429a 100644 --- a/vercel.json +++ b/vercel.json @@ -5,7 +5,7 @@ { "source": "/docs", "destination": "https://worldmonitor.mintlify.dev/docs" }, { "source": "/docs/:match*", "destination": "https://worldmonitor.mintlify.dev/docs/:match*" }, { "source": "/pro", "destination": "/pro/index.html" }, - { "source": "/((?!api|assets|blog|docs|favico|map-styles|data|textures|pro|sw\\.js|manifest\\.webmanifest|offline\\.html|robots\\.txt|sitemap\\.xml|llms\\.txt|llms-full\\.txt|\\.well-known).*)", "destination": "/index.html" } + { "source": "/((?!api|assets|blog|docs|favico|map-styles|data|textures|pro|sw\\.js|workbox-[a-f0-9]+\\.js|manifest\\.webmanifest|offline\\.html|robots\\.txt|sitemap\\.xml|llms\\.txt|llms-full\\.txt|\\.well-known).*)", "destination": "/index.html" } ], "headers": [ { @@ -35,7 +35,7 @@ ] }, { - "source": "/((?!api|assets|blog|docs|favico|map-styles|data|textures|pro|sw\\.js|manifest\\.webmanifest|offline\\.html|robots\\.txt|sitemap\\.xml|llms\\.txt|llms-full\\.txt|\\.well-known).*)", + "source": "/((?!api|assets|blog|docs|favico|map-styles|data|textures|pro|sw\\.js|workbox-[a-f0-9]+\\.js|manifest\\.webmanifest|offline\\.html|robots\\.txt|sitemap\\.xml|llms\\.txt|llms-full\\.txt|\\.well-known).*)", "headers": [ { "key": "Cache-Control", "value": "no-cache, no-store, must-revalidate" } ] @@ -100,6 +100,12 @@ { "key": "Cache-Control", "value": "public, max-age=86400" } ] }, + { + "source": "/workbox-:hash.js", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=31536000, immutable" } + ] + }, { "source": "/sw.js", "headers": [ diff --git a/vite.config.ts b/vite.config.ts index c476bf94c..341ac8db2 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -747,6 +747,9 @@ export default defineConfig({ ), }, }, + worker: { + format: 'es', + }, build: { // Geospatial bundles (maplibre/deck) are expected to be large even when split. // Raise warning threshold to reduce noisy false alarms in CI.