#!/usr/bin/env node /** * AIS WebSocket Relay Server * Proxies aisstream.io data to browsers via WebSocket * * Deploy on Railway with: * AISSTREAM_API_KEY=your_key * * Local: node scripts/ais-relay.cjs */ const http = require('http'); const zlib = require('zlib'); const { WebSocketServer, WebSocket } = require('ws'); const AISSTREAM_URL = 'wss://stream.aisstream.io/v0/stream'; const API_KEY = process.env.AISSTREAM_API_KEY || process.env.VITE_AISSTREAM_API_KEY; const PORT = process.env.PORT || 3004; if (!API_KEY) { console.error('[Relay] Error: AISSTREAM_API_KEY environment variable not set'); console.error('[Relay] Get a free key at https://aisstream.io'); process.exit(1); } const MAX_WS_CLIENTS = 10; // Cap WS clients — app uses HTTP snapshots, not WS let upstreamSocket = null; let clients = new Set(); let messageCount = 0; // gzip compress & send a response (reduces egress ~80% for JSON) function sendCompressed(req, res, statusCode, headers, body) { const acceptEncoding = req.headers['accept-encoding'] || ''; if (acceptEncoding.includes('gzip')) { zlib.gzip(typeof body === 'string' ? Buffer.from(body) : body, (err, compressed) => { if (err) { res.writeHead(statusCode, headers); res.end(body); return; } res.writeHead(statusCode, { ...headers, 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding' }); res.end(compressed); }); } else { res.writeHead(statusCode, headers); res.end(body); } } // AIS aggregate state for snapshot API (server-side fanout) const GRID_SIZE = 2; const DENSITY_WINDOW = 30 * 60 * 1000; // 30 minutes const GAP_THRESHOLD = 60 * 60 * 1000; // 1 hour const SNAPSHOT_INTERVAL_MS = Math.max(2000, Number(process.env.AIS_SNAPSHOT_INTERVAL_MS || 5000)); const CANDIDATE_RETENTION_MS = 2 * 60 * 60 * 1000; // 2 hours const MAX_DENSITY_ZONES = 200; const MAX_CANDIDATE_REPORTS = 1500; const vessels = new Map(); const vesselHistory = new Map(); const densityGrid = new Map(); const candidateReports = new Map(); let snapshotSequence = 0; let lastSnapshot = null; let lastSnapshotAt = 0; const CHOKEPOINTS = [ { name: 'Strait of Hormuz', lat: 26.5, lon: 56.5, radius: 2 }, { name: 'Suez Canal', lat: 30.0, lon: 32.5, radius: 1 }, { name: 'Strait of Malacca', lat: 2.5, lon: 101.5, radius: 2 }, { name: 'Bab el-Mandeb', lat: 12.5, lon: 43.5, radius: 1.5 }, { name: 'Panama Canal', lat: 9.0, lon: -79.5, radius: 1 }, { name: 'Taiwan Strait', lat: 24.5, lon: 119.5, radius: 2 }, { name: 'South China Sea', lat: 15.0, lon: 115.0, radius: 5 }, { name: 'Black Sea', lat: 43.5, lon: 34.0, radius: 3 }, ]; const NAVAL_PREFIX_RE = /^(USS|USNS|HMS|HMAS|HMCS|INS|JS|ROKS|TCG|FS|BNS|RFS|PLAN|PLA|CGC|PNS|KRI|ITS|SNS|MMSI)/i; function getGridKey(lat, lon) { const gridLat = Math.floor(lat / GRID_SIZE) * GRID_SIZE; const gridLon = Math.floor(lon / GRID_SIZE) * GRID_SIZE; return `${gridLat},${gridLon}`; } function isLikelyMilitaryCandidate(meta) { const mmsi = String(meta?.MMSI || ''); const shipType = Number(meta?.ShipType); const name = (meta?.ShipName || '').trim().toUpperCase(); if (Number.isFinite(shipType) && (shipType === 35 || shipType === 55 || (shipType >= 50 && shipType <= 59))) { return true; } if (name && NAVAL_PREFIX_RE.test(name)) return true; if (mmsi.length >= 9) { const suffix = mmsi.substring(3); if (suffix.startsWith('00') || suffix.startsWith('99')) return true; } return false; } function processPositionReportForSnapshot(data) { const meta = data?.MetaData; const pos = data?.Message?.PositionReport; if (!meta || !pos) return; const mmsi = String(meta.MMSI || ''); if (!mmsi) return; const lat = Number.isFinite(pos.Latitude) ? pos.Latitude : meta.latitude; const lon = Number.isFinite(pos.Longitude) ? pos.Longitude : meta.longitude; if (!Number.isFinite(lat) || !Number.isFinite(lon)) return; const now = Date.now(); vessels.set(mmsi, { mmsi, name: meta.ShipName || '', lat, lon, timestamp: now, shipType: meta.ShipType, heading: pos.TrueHeading, speed: pos.Sog, course: pos.Cog, }); const history = vesselHistory.get(mmsi) || []; history.push(now); if (history.length > 10) history.shift(); vesselHistory.set(mmsi, history); const gridKey = getGridKey(lat, lon); let cell = densityGrid.get(gridKey); if (!cell) { cell = { lat: Math.floor(lat / GRID_SIZE) * GRID_SIZE + GRID_SIZE / 2, lon: Math.floor(lon / GRID_SIZE) * GRID_SIZE + GRID_SIZE / 2, vessels: new Set(), lastUpdate: now, previousCount: 0, }; densityGrid.set(gridKey, cell); } cell.vessels.add(mmsi); cell.lastUpdate = now; if (isLikelyMilitaryCandidate(meta)) { candidateReports.set(mmsi, { mmsi, name: meta.ShipName || '', lat, lon, shipType: meta.ShipType, heading: pos.TrueHeading, speed: pos.Sog, course: pos.Cog, timestamp: now, }); } } function cleanupAggregates() { const now = Date.now(); const cutoff = now - DENSITY_WINDOW; for (const [mmsi, vessel] of vessels) { if (vessel.timestamp < cutoff) { vessels.delete(mmsi); } } for (const [mmsi, history] of vesselHistory) { const filtered = history.filter((ts) => ts >= cutoff); if (filtered.length === 0) { vesselHistory.delete(mmsi); } else { vesselHistory.set(mmsi, filtered); } } for (const [key, cell] of densityGrid) { cell.previousCount = cell.vessels.size; for (const mmsi of cell.vessels) { const vessel = vessels.get(mmsi); if (!vessel || vessel.timestamp < cutoff) { cell.vessels.delete(mmsi); } } if (cell.vessels.size === 0 && now - cell.lastUpdate > DENSITY_WINDOW * 2) { densityGrid.delete(key); } } for (const [mmsi, report] of candidateReports) { if (report.timestamp < now - CANDIDATE_RETENTION_MS) { candidateReports.delete(mmsi); } } } function detectDisruptions() { const disruptions = []; const now = Date.now(); for (const chokepoint of CHOKEPOINTS) { let vesselCount = 0; for (const vessel of vessels.values()) { const distance = Math.sqrt( Math.pow(vessel.lat - chokepoint.lat, 2) + Math.pow(vessel.lon - chokepoint.lon, 2) ); if (distance <= chokepoint.radius) { vesselCount++; } } if (vesselCount >= 5) { const normalTraffic = chokepoint.radius * 10; const severity = vesselCount > normalTraffic * 1.5 ? 'high' : vesselCount > normalTraffic ? 'elevated' : 'low'; disruptions.push({ id: `chokepoint-${chokepoint.name.toLowerCase().replace(/\s+/g, '-')}`, name: chokepoint.name, type: 'chokepoint_congestion', lat: chokepoint.lat, lon: chokepoint.lon, severity, changePct: normalTraffic > 0 ? Math.round((vesselCount / normalTraffic - 1) * 100) : 0, windowHours: 1, vesselCount, region: chokepoint.name, description: `${vesselCount} vessels in ${chokepoint.name}`, }); } } let darkShipCount = 0; for (const history of vesselHistory.values()) { if (history.length >= 2) { const lastSeen = history[history.length - 1]; const secondLast = history[history.length - 2]; if (lastSeen - secondLast > GAP_THRESHOLD && now - lastSeen < 10 * 60 * 1000) { darkShipCount++; } } } if (darkShipCount >= 1) { disruptions.push({ id: 'global-gap-spike', name: 'AIS Gap Spike Detected', type: 'gap_spike', lat: 0, lon: 0, severity: darkShipCount > 20 ? 'high' : darkShipCount > 10 ? 'elevated' : 'low', changePct: darkShipCount * 10, windowHours: 1, darkShips: darkShipCount, description: `${darkShipCount} vessels returned after extended AIS silence`, }); } return disruptions; } function calculateDensityZones() { const zones = []; const allCells = Array.from(densityGrid.values()).filter((c) => c.vessels.size >= 2); if (allCells.length === 0) return zones; const vesselCounts = allCells.map((c) => c.vessels.size); const maxVessels = Math.max(...vesselCounts); const minVessels = Math.min(...vesselCounts); for (const [key, cell] of densityGrid) { if (cell.vessels.size < 2) continue; const logMax = Math.log(maxVessels + 1); const logMin = Math.log(minVessels + 1); const logCurrent = Math.log(cell.vessels.size + 1); const intensity = logMax > logMin ? 0.2 + (0.8 * (logCurrent - logMin) / (logMax - logMin)) : 0.5; const deltaPct = cell.previousCount > 0 ? Math.round(((cell.vessels.size - cell.previousCount) / cell.previousCount) * 100) : 0; zones.push({ id: `density-${key}`, name: `Zone ${key}`, lat: cell.lat, lon: cell.lon, intensity, deltaPct, shipsPerDay: cell.vessels.size * 48, note: cell.vessels.size >= 10 ? 'High traffic area' : undefined, }); } return zones .sort((a, b) => b.intensity - a.intensity) .slice(0, MAX_DENSITY_ZONES); } function getCandidateReportsSnapshot() { return Array.from(candidateReports.values()) .sort((a, b) => b.timestamp - a.timestamp) .slice(0, MAX_CANDIDATE_REPORTS); } function buildSnapshot() { const now = Date.now(); if (lastSnapshot && now - lastSnapshotAt < Math.floor(SNAPSHOT_INTERVAL_MS / 2)) { return lastSnapshot; } cleanupAggregates(); snapshotSequence++; lastSnapshot = { sequence: snapshotSequence, timestamp: new Date(now).toISOString(), status: { connected: upstreamSocket?.readyState === WebSocket.OPEN, vessels: vessels.size, messages: messageCount, clients: clients.size, }, disruptions: detectDisruptions(), density: calculateDensityZones(), }; lastSnapshotAt = now; return lastSnapshot; } setInterval(() => { if (upstreamSocket?.readyState === WebSocket.OPEN || vessels.size > 0) { buildSnapshot(); } }, SNAPSHOT_INTERVAL_MS); // UCDP GED Events cache (persistent in-memory — Railway advantage) const UCDP_CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours const UCDP_PAGE_SIZE = 1000; const UCDP_MAX_PAGES = 12; const UCDP_FETCH_TIMEOUT = 30000; // 30s per page (no Railway limit) const UCDP_TRAILING_WINDOW_MS = 365 * 24 * 60 * 60 * 1000; let ucdpCache = { data: null, timestamp: 0 }; let ucdpFetchInProgress = false; const UCDP_VIOLENCE_TYPE_MAP = { 1: 'state-based', 2: 'non-state', 3: 'one-sided', }; function ucdpParseDateMs(value) { if (!value) return NaN; return Date.parse(String(value)); } function ucdpGetMaxDateMs(events) { let maxMs = NaN; for (const event of events) { const ms = ucdpParseDateMs(event?.date_start); if (!Number.isFinite(ms)) continue; if (!Number.isFinite(maxMs) || ms > maxMs) maxMs = ms; } return maxMs; } function ucdpBuildVersionCandidates() { const year = new Date().getFullYear() - 2000; return Array.from(new Set([`${year}.1`, `${year - 1}.1`, '25.1', '24.1'])); } async function ucdpFetchPage(version, page) { const https = require('https'); const url = `https://ucdpapi.pcr.uu.se/api/gedevents/${version}?pagesize=${UCDP_PAGE_SIZE}&page=${page}`; return new Promise((resolve, reject) => { const req = https.get(url, { headers: { Accept: 'application/json' }, timeout: UCDP_FETCH_TIMEOUT }, (res) => { if (res.statusCode !== 200) { res.resume(); return reject(new Error(`UCDP API ${res.statusCode} (v${version} p${page})`)); } let data = ''; res.on('data', chunk => data += chunk); res.on('end', () => { try { resolve(JSON.parse(data)); } catch (e) { reject(new Error('UCDP JSON parse error')); } }); }); req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('UCDP timeout')); }); }); } async function ucdpDiscoverVersion() { const candidates = ucdpBuildVersionCandidates(); for (const version of candidates) { try { const page0 = await ucdpFetchPage(version, 0); if (Array.isArray(page0?.Result)) return { version, page0 }; } catch { /* next candidate */ } } throw new Error('No valid UCDP GED version found'); } async function ucdpFetchAllEvents() { const { version, page0 } = await ucdpDiscoverVersion(); const totalPages = Math.max(1, Number(page0?.TotalPages) || 1); const newestPage = totalPages - 1; let allEvents = []; let latestDatasetMs = NaN; for (let offset = 0; offset < UCDP_MAX_PAGES && (newestPage - offset) >= 0; offset++) { const page = newestPage - offset; const rawData = page === 0 ? page0 : await ucdpFetchPage(version, page); const events = Array.isArray(rawData?.Result) ? rawData.Result : []; allEvents = allEvents.concat(events); const pageMaxMs = ucdpGetMaxDateMs(events); if (!Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) { latestDatasetMs = pageMaxMs; } if (Number.isFinite(latestDatasetMs) && Number.isFinite(pageMaxMs)) { if (pageMaxMs < latestDatasetMs - UCDP_TRAILING_WINDOW_MS) break; } console.log(`[UCDP] Fetched v${version} page ${page} (${events.length} events)`); } const sanitized = allEvents .filter(e => { if (!Number.isFinite(latestDatasetMs)) return true; const ms = ucdpParseDateMs(e?.date_start); return Number.isFinite(ms) && ms >= (latestDatasetMs - UCDP_TRAILING_WINDOW_MS); }) .map(e => ({ id: String(e.id || ''), date_start: e.date_start || '', date_end: e.date_end || '', latitude: Number(e.latitude) || 0, longitude: Number(e.longitude) || 0, country: e.country || '', side_a: (e.side_a || '').substring(0, 200), side_b: (e.side_b || '').substring(0, 200), deaths_best: Number(e.best) || 0, deaths_low: Number(e.low) || 0, deaths_high: Number(e.high) || 0, type_of_violence: UCDP_VIOLENCE_TYPE_MAP[e.type_of_violence] || 'state-based', source_original: (e.source_original || '').substring(0, 300), })) .sort((a, b) => { const bMs = ucdpParseDateMs(b.date_start); const aMs = ucdpParseDateMs(a.date_start); return (Number.isFinite(bMs) ? bMs : 0) - (Number.isFinite(aMs) ? aMs : 0); }); return { success: true, count: sanitized.length, data: sanitized, version, cached_at: new Date().toISOString(), }; } async function handleUcdpEventsRequest(req, res) { const now = Date.now(); if (ucdpCache.data && now - ucdpCache.timestamp < UCDP_CACHE_TTL_MS) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=3600', 'X-Cache': 'HIT', }, JSON.stringify(ucdpCache.data)); } if (ucdpCache.data && !ucdpFetchInProgress) { ucdpFetchInProgress = true; ucdpFetchAllEvents() .then(result => { ucdpCache = { data: result, timestamp: Date.now() }; console.log(`[UCDP] Background refresh: ${result.count} events (v${result.version})`); }) .catch(err => console.error('[UCDP] Background refresh error:', err.message)) .finally(() => { ucdpFetchInProgress = false; }); return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=600', 'X-Cache': 'STALE', }, JSON.stringify(ucdpCache.data)); } if (ucdpFetchInProgress) { res.writeHead(202, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ success: false, count: 0, data: [], cached_at: '', message: 'Fetch in progress' })); } try { ucdpFetchInProgress = true; console.log('[UCDP] Cold fetch starting...'); const result = await ucdpFetchAllEvents(); ucdpCache = { data: result, timestamp: Date.now() }; ucdpFetchInProgress = false; console.log(`[UCDP] Cold fetch complete: ${result.count} events (v${result.version})`); sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=3600', 'X-Cache': 'MISS', }, JSON.stringify(result)); } catch (err) { ucdpFetchInProgress = false; console.error('[UCDP] Fetch error:', err.message); res.writeHead(500, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ success: false, error: err.message, count: 0, data: [] })); } } // ── Response caches (eliminates ~1.2TB/day OpenSky + ~30GB/day RSS egress) ── const openskyResponseCache = new Map(); // key: sorted query params → { data, timestamp } const OPENSKY_CACHE_TTL_MS = 30 * 1000; // 30s — OpenSky updates every ~10s but 58 clients hammer it const rssResponseCache = new Map(); // key: feed URL → { data, contentType, timestamp } const RSS_CACHE_TTL_MS = 5 * 60 * 1000; // 5 min — RSS feeds rarely update faster // OpenSky OAuth2 token cache let openskyToken = null; let openskyTokenExpiry = 0; async function getOpenSkyToken() { const clientId = process.env.OPENSKY_CLIENT_ID; const clientSecret = process.env.OPENSKY_CLIENT_SECRET; if (!clientId || !clientSecret) { return null; } // Return cached token if still valid (with 60s buffer) if (openskyToken && Date.now() < openskyTokenExpiry - 60000) { return openskyToken; } try { console.log('[Relay] Fetching new OpenSky OAuth2 token...'); const https = require('https'); return new Promise((resolve, reject) => { const postData = `grant_type=client_credentials&client_id=${encodeURIComponent(clientId)}&client_secret=${encodeURIComponent(clientSecret)}`; const req = https.request({ hostname: 'auth.opensky-network.org', port: 443, path: '/auth/realms/opensky-network/protocol/openid-connect/token', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(postData), }, timeout: 10000 }, (res) => { let data = ''; res.on('data', chunk => data += chunk); res.on('end', () => { try { const json = JSON.parse(data); if (json.access_token) { openskyToken = json.access_token; // Token valid for 30 min, cache with expiry openskyTokenExpiry = Date.now() + (json.expires_in || 1800) * 1000; console.log('[Relay] OpenSky token acquired, expires in', json.expires_in, 'seconds'); resolve(openskyToken); } else { console.error('[Relay] OpenSky token error:', json.error || 'Unknown'); resolve(null); } } catch (e) { console.error('[Relay] OpenSky token parse error:', e.message); resolve(null); } }); }); req.on('error', (err) => { console.error('[Relay] OpenSky token request error:', err.message); resolve(null); }); req.on('timeout', () => { req.destroy(); resolve(null); }); req.write(postData); req.end(); }); } catch (err) { console.error('[Relay] OpenSky token error:', err.message); return null; } } async function handleOpenSkyRequest(req, res, PORT) { try { const url = new URL(req.url, `http://localhost:${PORT}`); const params = url.searchParams; // Build cache key from sorted bounding box params const cacheKey = ['lamin', 'lomin', 'lamax', 'lomax'] .map(k => params.get(k) || '') .join(','); const cached = openskyResponseCache.get(cacheKey); if (cached && Date.now() - cached.timestamp < OPENSKY_CACHE_TTL_MS) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=30', 'X-Cache': 'HIT', }, cached.data); } const token = await getOpenSkyToken(); if (!token) { res.writeHead(503, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'OpenSky not configured or auth failed', time: Date.now(), states: [] })); return; } let openskyUrl = 'https://opensky-network.org/api/states/all'; const queryParams = []; for (const key of ['lamin', 'lomin', 'lamax', 'lomax']) { if (params.has(key)) queryParams.push(`${key}=${params.get(key)}`); } if (queryParams.length > 0) { openskyUrl += '?' + queryParams.join('&'); } console.log('[Relay] OpenSky request (MISS):', openskyUrl); const https = require('https'); const request = https.get(openskyUrl, { headers: { 'Accept': 'application/json', 'User-Agent': 'WorldMonitor/1.0', 'Authorization': `Bearer ${token}`, }, timeout: 15000 }, (response) => { let data = ''; response.on('data', chunk => data += chunk); response.on('end', () => { if (response.statusCode === 401) { openskyToken = null; openskyTokenExpiry = 0; } if (response.statusCode === 200) { openskyResponseCache.set(cacheKey, { data, timestamp: Date.now() }); } sendCompressed(req, res, response.statusCode, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=30', 'X-Cache': 'MISS', }, data); }); }); request.on('error', (err) => { console.error('[Relay] OpenSky error:', err.message); if (cached) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE' }, cached.data); } res.writeHead(500, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: err.message, time: Date.now(), states: null })); }); request.on('timeout', () => { request.destroy(); if (cached) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE' }, cached.data); } res.writeHead(504, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'Request timeout', time: Date.now(), states: null })); }); } catch (err) { res.writeHead(500, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: err.message, time: Date.now(), states: null })); } } // ── World Bank proxy (World Bank blocks Vercel edge IPs with 403) ── const worldbankCache = new Map(); // key: query string → { data, timestamp } const WORLDBANK_CACHE_TTL_MS = 30 * 60 * 1000; // 30 min — data rarely changes function handleWorldBankRequest(req, res) { const url = new URL(req.url, `http://localhost:${PORT}`); const qs = url.search || ''; const cacheKey = qs; const cached = worldbankCache.get(cacheKey); if (cached && Date.now() - cached.timestamp < WORLDBANK_CACHE_TTL_MS) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=1800', 'X-Cache': 'HIT', }, cached.data); } const targetUrl = `https://api.worldbank.org/v2${qs.includes('action=indicators') ? '' : '/country'}${url.pathname.replace('/worldbank', '')}${qs}`; // Passthrough: forward query params to the Vercel edge handler format // The client sends the same params as /api/worldbank, so we re-fetch from upstream const wbParams = new URLSearchParams(url.searchParams); const action = wbParams.get('action'); if (action === 'indicators') { // Static response — return indicator list directly (same as api/worldbank.js) const indicators = { 'IT.NET.USER.ZS': 'Internet Users (% of population)', 'IT.CEL.SETS.P2': 'Mobile Subscriptions (per 100 people)', 'IT.NET.BBND.P2': 'Fixed Broadband Subscriptions (per 100 people)', 'IT.NET.SECR.P6': 'Secure Internet Servers (per million people)', 'GB.XPD.RSDV.GD.ZS': 'R&D Expenditure (% of GDP)', 'IP.PAT.RESD': 'Patent Applications (residents)', 'IP.PAT.NRES': 'Patent Applications (non-residents)', 'IP.TMK.TOTL': 'Trademark Applications', 'TX.VAL.TECH.MF.ZS': 'High-Tech Exports (% of manufactured exports)', 'BX.GSR.CCIS.ZS': 'ICT Service Exports (% of service exports)', 'TM.VAL.ICTG.ZS.UN': 'ICT Goods Imports (% of total goods imports)', 'SE.TER.ENRR': 'Tertiary Education Enrollment (%)', 'SE.XPD.TOTL.GD.ZS': 'Education Expenditure (% of GDP)', 'NY.GDP.MKTP.KD.ZG': 'GDP Growth (annual %)', 'NY.GDP.PCAP.CD': 'GDP per Capita (current US$)', 'NE.EXP.GNFS.ZS': 'Exports of Goods & Services (% of GDP)', }; const defaultCountries = [ 'USA','CHN','JPN','DEU','KOR','GBR','IND','ISR','SGP','TWN', 'FRA','CAN','SWE','NLD','CHE','FIN','IRL','AUS','BRA','IDN', 'ARE','SAU','QAT','BHR','EGY','TUR','MYS','THA','VNM','PHL', 'ESP','ITA','POL','CZE','DNK','NOR','AUT','BEL','PRT','EST', 'MEX','ARG','CHL','COL','ZAF','NGA','KEN', ]; const body = JSON.stringify({ indicators, defaultCountries }); worldbankCache.set(cacheKey, { data: body, timestamp: Date.now() }); return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=86400', 'X-Cache': 'MISS', }, body); } const indicator = wbParams.get('indicator'); if (!indicator) { res.writeHead(400, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ error: 'Missing indicator parameter' })); } const country = wbParams.get('country'); const countries = wbParams.get('countries'); const years = parseInt(wbParams.get('years') || '5', 10); let countryList = country || (countries ? countries.split(',').join(';') : [ 'USA','CHN','JPN','DEU','KOR','GBR','IND','ISR','SGP','TWN', 'FRA','CAN','SWE','NLD','CHE','FIN','IRL','AUS','BRA','IDN', 'ARE','SAU','QAT','BHR','EGY','TUR','MYS','THA','VNM','PHL', 'ESP','ITA','POL','CZE','DNK','NOR','AUT','BEL','PRT','EST', 'MEX','ARG','CHL','COL','ZAF','NGA','KEN', ].join(';')); const currentYear = new Date().getFullYear(); const startYear = currentYear - years; const TECH_INDICATORS = { 'IT.NET.USER.ZS': 'Internet Users (% of population)', 'IT.CEL.SETS.P2': 'Mobile Subscriptions (per 100 people)', 'IT.NET.BBND.P2': 'Fixed Broadband Subscriptions (per 100 people)', 'IT.NET.SECR.P6': 'Secure Internet Servers (per million people)', 'GB.XPD.RSDV.GD.ZS': 'R&D Expenditure (% of GDP)', 'IP.PAT.RESD': 'Patent Applications (residents)', 'IP.PAT.NRES': 'Patent Applications (non-residents)', 'IP.TMK.TOTL': 'Trademark Applications', 'TX.VAL.TECH.MF.ZS': 'High-Tech Exports (% of manufactured exports)', 'BX.GSR.CCIS.ZS': 'ICT Service Exports (% of service exports)', 'TM.VAL.ICTG.ZS.UN': 'ICT Goods Imports (% of total goods imports)', 'SE.TER.ENRR': 'Tertiary Education Enrollment (%)', 'SE.XPD.TOTL.GD.ZS': 'Education Expenditure (% of GDP)', 'NY.GDP.MKTP.KD.ZG': 'GDP Growth (annual %)', 'NY.GDP.PCAP.CD': 'GDP per Capita (current US$)', 'NE.EXP.GNFS.ZS': 'Exports of Goods & Services (% of GDP)', }; const wbUrl = `https://api.worldbank.org/v2/country/${countryList}/indicator/${encodeURIComponent(indicator)}?format=json&date=${startYear}:${currentYear}&per_page=1000`; console.log('[Relay] World Bank request (MISS):', indicator); const https = require('https'); const request = https.get(wbUrl, { headers: { 'Accept': 'application/json', 'User-Agent': 'Mozilla/5.0 (compatible; WorldMonitor/1.0; +https://worldmonitor.app)', }, timeout: 15000, }, (response) => { if (response.statusCode !== 200) { res.writeHead(response.statusCode, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ error: `World Bank API ${response.statusCode}` })); } let rawData = ''; response.on('data', chunk => rawData += chunk); response.on('end', () => { try { const parsed = JSON.parse(rawData); // Transform raw World Bank response to match client-expected format if (!parsed || !Array.isArray(parsed) || parsed.length < 2 || !parsed[1]) { const empty = JSON.stringify({ indicator, indicatorName: TECH_INDICATORS[indicator] || indicator, metadata: { page: 1, pages: 1, total: 0 }, byCountry: {}, latestByCountry: {}, timeSeries: [], }); worldbankCache.set(cacheKey, { data: empty, timestamp: Date.now() }); return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=1800', 'X-Cache': 'MISS', }, empty); } const [metadata, records] = parsed; const transformed = { indicator, indicatorName: TECH_INDICATORS[indicator] || (records[0]?.indicator?.value || indicator), metadata: { page: metadata.page, pages: metadata.pages, total: metadata.total }, byCountry: {}, latestByCountry: {}, timeSeries: [], }; for (const record of records || []) { const cc = record.countryiso3code || record.country?.id; const cn = record.country?.value; const yr = record.date; const val = record.value; if (!cc || val === null) continue; if (!transformed.byCountry[cc]) transformed.byCountry[cc] = { code: cc, name: cn, values: [] }; transformed.byCountry[cc].values.push({ year: yr, value: val }); if (!transformed.latestByCountry[cc] || yr > transformed.latestByCountry[cc].year) { transformed.latestByCountry[cc] = { code: cc, name: cn, year: yr, value: val }; } transformed.timeSeries.push({ countryCode: cc, countryName: cn, year: yr, value: val }); } for (const c of Object.values(transformed.byCountry)) c.values.sort((a, b) => a.year - b.year); transformed.timeSeries.sort((a, b) => b.year - a.year || a.countryCode.localeCompare(b.countryCode)); const body = JSON.stringify(transformed); worldbankCache.set(cacheKey, { data: body, timestamp: Date.now() }); sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=1800', 'X-Cache': 'MISS', }, body); } catch (e) { console.error('[Relay] World Bank parse error:', e.message); res.writeHead(500, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'Parse error' })); } }); }); request.on('error', (err) => { console.error('[Relay] World Bank error:', err.message); if (cached) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE', }, cached.data); } res.writeHead(502, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: err.message })); }); request.on('timeout', () => { request.destroy(); if (cached) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE', }, cached.data); } res.writeHead(504, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'World Bank request timeout' })); }); } // ── Polymarket proxy (Cloudflare JA3 blocks Vercel edge runtime) ── const polymarketCache = new Map(); // key: query string → { data, timestamp } const POLYMARKET_CACHE_TTL_MS = 2 * 60 * 1000; // 2 min — market data changes frequently function handlePolymarketRequest(req, res) { const url = new URL(req.url, `http://localhost:${PORT}`); const cacheKey = url.search || ''; const cached = polymarketCache.get(cacheKey); if (cached && Date.now() - cached.timestamp < POLYMARKET_CACHE_TTL_MS) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=120', 'X-Cache': 'HIT', 'X-Polymarket-Source': 'railway-cache', }, cached.data); } const endpoint = url.searchParams.get('endpoint') || 'markets'; const params = new URLSearchParams(); params.set('closed', url.searchParams.get('closed') || 'false'); params.set('order', url.searchParams.get('order') || 'volume'); params.set('ascending', url.searchParams.get('ascending') || 'false'); const limit = Math.max(1, Math.min(100, parseInt(url.searchParams.get('limit') || '50', 10) || 50)); params.set('limit', String(limit)); const tag = url.searchParams.get('tag') || url.searchParams.get('tag_slug'); if (tag && endpoint === 'events') params.set('tag_slug', tag.replace(/[^a-z0-9-]/gi, '').slice(0, 100)); const gammaUrl = `https://gamma-api.polymarket.com/${endpoint}?${params}`; console.log('[Relay] Polymarket request (MISS):', endpoint, tag || ''); const https = require('https'); const request = https.get(gammaUrl, { headers: { 'Accept': 'application/json' }, timeout: 10000, }, (response) => { if (response.statusCode !== 200) { console.error(`[Relay] Polymarket upstream ${response.statusCode}`); res.writeHead(response.statusCode, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify([])); } let data = ''; response.on('data', chunk => data += chunk); response.on('end', () => { polymarketCache.set(cacheKey, { data, timestamp: Date.now() }); sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=120', 'X-Cache': 'MISS', 'X-Polymarket-Source': 'railway', }, data); }); }); request.on('error', (err) => { console.error('[Relay] Polymarket error:', err.message); if (cached) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE', 'X-Polymarket-Source': 'railway-stale', }, cached.data); } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify([])); }); request.on('timeout', () => { request.destroy(); if (cached) { return sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'X-Cache': 'STALE', 'X-Polymarket-Source': 'railway-stale', }, cached.data); } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify([])); }); } // Periodic cache cleanup to prevent memory leaks setInterval(() => { const now = Date.now(); for (const [key, entry] of openskyResponseCache) { if (now - entry.timestamp > OPENSKY_CACHE_TTL_MS * 2) openskyResponseCache.delete(key); } for (const [key, entry] of rssResponseCache) { if (now - entry.timestamp > RSS_CACHE_TTL_MS * 2) rssResponseCache.delete(key); } for (const [key, entry] of worldbankCache) { if (now - entry.timestamp > WORLDBANK_CACHE_TTL_MS * 2) worldbankCache.delete(key); } for (const [key, entry] of polymarketCache) { if (now - entry.timestamp > POLYMARKET_CACHE_TTL_MS * 2) polymarketCache.delete(key); } }, 60 * 1000); // CORS origin allowlist — only our domains can use this relay const ALLOWED_ORIGINS = [ 'https://worldmonitor.app', 'https://tech.worldmonitor.app', 'http://localhost:5173', // Vite dev 'http://localhost:5174', // Vite dev alt port 'http://localhost:4173', // Vite preview 'https://localhost', // Tauri desktop 'tauri://localhost', // Tauri iOS/macOS ]; function getCorsOrigin(req) { const origin = req.headers.origin || ''; if (ALLOWED_ORIGINS.includes(origin)) return origin; // Allow Vercel preview deployments if (origin.endsWith('.vercel.app')) return origin; return ''; } const server = http.createServer(async (req, res) => { const corsOrigin = getCorsOrigin(req); if (corsOrigin) { res.setHeader('Access-Control-Allow-Origin', corsOrigin); res.setHeader('Vary', 'Origin'); } res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); res.setHeader('Access-Control-Allow-Headers', 'Content-Type'); // Handle CORS preflight if (req.method === 'OPTIONS') { res.writeHead(corsOrigin ? 204 : 403); return res.end(); } if (req.url === '/health' || req.url === '/') { sendCompressed(req, res, 200, { 'Content-Type': 'application/json' }, JSON.stringify({ status: 'ok', clients: clients.size, messages: messageCount, connected: upstreamSocket?.readyState === WebSocket.OPEN, vessels: vessels.size, densityZones: Array.from(densityGrid.values()).filter(c => c.vessels.size >= 2).length, cache: { opensky: openskyResponseCache.size, rss: rssResponseCache.size, ucdp: ucdpCache.data ? 'warm' : 'cold', worldbank: worldbankCache.size, polymarket: polymarketCache.size, }, })); } else if (req.url.startsWith('/ais/snapshot')) { // Aggregated AIS snapshot for server-side fanout connectUpstream(); const url = new URL(req.url, `http://localhost:${PORT}`); const includeCandidates = url.searchParams.get('candidates') === 'true'; const snapshot = buildSnapshot(); const payload = includeCandidates ? { ...snapshot, candidateReports: getCandidateReportsSnapshot() } : { ...snapshot, candidateReports: [] }; sendCompressed(req, res, 200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=2' }, JSON.stringify(payload)); } else if (req.url.startsWith('/rss')) { // Proxy RSS feeds that block Vercel IPs try { const url = new URL(req.url, `http://localhost:${PORT}`); const feedUrl = url.searchParams.get('url'); if (!feedUrl) { res.writeHead(400, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ error: 'Missing url parameter' })); } // Allow domains that block Vercel IPs (must match feeds.ts railwayRss usage) const allowedDomains = [ // Original 'rss.cnn.com', 'www.defensenews.com', 'layoffs.fyi', // International Organizations 'news.un.org', 'www.cisa.gov', 'www.iaea.org', 'www.who.int', 'www.crisisgroup.org', // Middle East & Regional News 'english.alarabiya.net', 'www.arabnews.com', 'www.timesofisrael.com', 'www.scmp.com', 'kyivindependent.com', 'www.themoscowtimes.com', // Africa 'feeds.24.com', 'feeds.capi24.com', // News24 redirect destination 'www.atlanticcouncil.org', ]; const parsed = new URL(feedUrl); if (!allowedDomains.includes(parsed.hostname)) { res.writeHead(403, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ error: 'Domain not allowed on Railway proxy' })); } // Serve from cache if fresh (5 min TTL) const rssCached = rssResponseCache.get(feedUrl); if (rssCached && Date.now() - rssCached.timestamp < RSS_CACHE_TTL_MS) { return sendCompressed(req, res, 200, { 'Content-Type': rssCached.contentType || 'application/xml', 'Cache-Control': 'public, max-age=300', 'X-Cache': 'HIT', }, rssCached.data); } console.log('[Relay] RSS request (MISS):', feedUrl); const https = require('https'); const http = require('http'); let responseHandled = false; const sendError = (statusCode, message) => { if (responseHandled || res.headersSent) return; responseHandled = true; res.writeHead(statusCode, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: message })); }; const fetchWithRedirects = (url, redirectCount = 0) => { if (redirectCount > 3) { return sendError(502, 'Too many redirects'); } const protocol = url.startsWith('https') ? https : http; const request = protocol.get(url, { headers: { 'Accept': 'application/rss+xml, application/xml, text/xml, */*', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept-Language': 'en-US,en;q=0.9', }, timeout: 15000 }, (response) => { if ([301, 302, 303, 307, 308].includes(response.statusCode) && response.headers.location) { const redirectUrl = response.headers.location.startsWith('http') ? response.headers.location : new URL(response.headers.location, url).href; console.log(`[Relay] Following redirect to: ${redirectUrl}`); return fetchWithRedirects(redirectUrl, redirectCount + 1); } const encoding = response.headers['content-encoding']; let stream = response; if (encoding === 'gzip' || encoding === 'deflate') { const zlib = require('zlib'); stream = encoding === 'gzip' ? response.pipe(zlib.createGunzip()) : response.pipe(zlib.createInflate()); } const chunks = []; stream.on('data', chunk => chunks.push(chunk)); stream.on('end', () => { if (responseHandled || res.headersSent) return; responseHandled = true; const data = Buffer.concat(chunks); // Cache successful responses if (response.statusCode >= 200 && response.statusCode < 300) { rssResponseCache.set(feedUrl, { data, contentType: 'application/xml', timestamp: Date.now() }); } sendCompressed(req, res, response.statusCode, { 'Content-Type': 'application/xml', 'Cache-Control': 'public, max-age=300', 'X-Cache': 'MISS', }, data); }); stream.on('error', (err) => { console.error('[Relay] Decompression error:', err.message); sendError(502, 'Decompression failed: ' + err.message); }); }); request.on('error', (err) => { console.error('[Relay] RSS error:', err.message); // Serve stale on error if (rssCached) { if (!responseHandled && !res.headersSent) { responseHandled = true; sendCompressed(req, res, 200, { 'Content-Type': 'application/xml', 'X-Cache': 'STALE' }, rssCached.data); } return; } sendError(502, err.message); }); request.on('timeout', () => { request.destroy(); if (rssCached && !responseHandled && !res.headersSent) { responseHandled = true; return sendCompressed(req, res, 200, { 'Content-Type': 'application/xml', 'X-Cache': 'STALE' }, rssCached.data); } sendError(504, 'Request timeout'); }); }; fetchWithRedirects(feedUrl); } catch (err) { if (!res.headersSent) { res.writeHead(500, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: err.message })); } } } else if (req.url.startsWith('/ucdp-events')) { handleUcdpEventsRequest(req, res); } else if (req.url.startsWith('/opensky')) { handleOpenSkyRequest(req, res, PORT); } else if (req.url.startsWith('/worldbank')) { handleWorldBankRequest(req, res); } else if (req.url.startsWith('/polymarket')) { handlePolymarketRequest(req, res); } else { res.writeHead(404); res.end(); } }); function connectUpstream() { // Skip if already connected or connecting if (upstreamSocket?.readyState === WebSocket.OPEN || upstreamSocket?.readyState === WebSocket.CONNECTING) return; console.log('[Relay] Connecting to aisstream.io...'); const socket = new WebSocket(AISSTREAM_URL); upstreamSocket = socket; socket.on('open', () => { // Verify this socket is still the current one (race condition guard) if (upstreamSocket !== socket) { console.log('[Relay] Stale socket open event, closing'); socket.close(); return; } console.log('[Relay] Connected to aisstream.io'); socket.send(JSON.stringify({ APIKey: API_KEY, BoundingBoxes: [[[-90, -180], [90, 180]]], FilterMessageTypes: ['PositionReport'], })); }); socket.on('message', (data) => { if (upstreamSocket !== socket) return; messageCount++; if (messageCount % 1000 === 0) { console.log(`[Relay] ${messageCount} messages, ${clients.size} clients, cache: opensky=${openskyResponseCache.size} rss=${rssResponseCache.size}`); } const message = data.toString(); try { const parsed = JSON.parse(message); if (parsed?.MessageType === 'PositionReport') { processPositionReportForSnapshot(parsed); } } catch { // Ignore malformed upstream payloads } // Throttled fanout: only forward every 10th message to WS clients // The app uses HTTP snapshot polling, not WS — this is mostly for external consumers if (clients.size > 0 && messageCount % 10 === 0) { for (const client of clients) { if (client.readyState === WebSocket.OPEN) { client.send(message); } } } }); socket.on('close', () => { if (upstreamSocket === socket) { upstreamSocket = null; console.log('[Relay] Disconnected, reconnecting in 5s...'); setTimeout(connectUpstream, 5000); } }); socket.on('error', (err) => { console.error('[Relay] Upstream error:', err.message); }); } const wss = new WebSocketServer({ server }); server.listen(PORT, () => { console.log(`[Relay] WebSocket relay on port ${PORT}`); }); wss.on('connection', (ws, req) => { if (clients.size >= MAX_WS_CLIENTS) { console.log(`[Relay] WS client rejected (max ${MAX_WS_CLIENTS})`); ws.close(1013, 'Max clients reached'); return; } console.log(`[Relay] Client connected (${clients.size + 1}/${MAX_WS_CLIENTS})`); clients.add(ws); connectUpstream(); ws.on('close', () => { clients.delete(ws); }); ws.on('error', (err) => { console.error('[Relay] Client error:', err.message); }); });