diff --git a/scripts/seed-aviation.mjs b/scripts/seed-aviation.mjs index c4ff96298..b349ffa56 100755 --- a/scripts/seed-aviation.mjs +++ b/scripts/seed-aviation.mjs @@ -249,6 +249,9 @@ async function fetchAll() { const opsData = ops.status === 'fulfilled' ? ops.value : null; const newsData = news.status === 'fulfilled' ? news.value : null; + if (ops.status === 'rejected') console.warn(` AirportOps failed: ${ops.reason?.message || ops.reason}`); + if (news.status === 'rejected') console.warn(` AviationNews failed: ${news.reason?.message || news.reason}`); + if (!opsData && !newsData) throw new Error('All aviation fetches failed'); // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write) diff --git a/scripts/seed-conflict-intel.mjs b/scripts/seed-conflict-intel.mjs index d9f9087e6..8a89bad6f 100755 --- a/scripts/seed-conflict-intel.mjs +++ b/scripts/seed-conflict-intel.mjs @@ -260,6 +260,11 @@ async function fetchAll() { const pi = pizzint.status === 'fulfilled' ? pizzint.value : null; const gd = gdelt.status === 'fulfilled' ? gdelt.value : null; + if (acled.status === 'rejected') console.warn(` ACLED failed: ${acled.reason?.message || acled.reason}`); + if (hapi.status === 'rejected') console.warn(` HAPI failed: ${hapi.reason?.message || hapi.reason}`); + if (pizzint.status === 'rejected') console.warn(` PizzINT failed: ${pizzint.reason?.message || pizzint.reason}`); + if (gdelt.status === 'rejected') console.warn(` GDELT failed: ${gdelt.reason?.message || gdelt.reason}`); + if (!ac && !pi) throw new Error('All conflict/intel fetches failed'); // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write) diff --git a/scripts/seed-economy.mjs b/scripts/seed-economy.mjs index f09aaff2c..b198f0cc6 100755 --- a/scripts/seed-economy.mjs +++ b/scripts/seed-economy.mjs @@ -112,24 +112,28 @@ async function fetchEnergyCapacity() { const series = []; for (const source of CAPACITY_SOURCES) { - let yearTotals; - if (source.code === 'COL') { - yearTotals = await fetchCapacityForSource('COL', apiKey, startYear); - if (yearTotals.size === 0) { - const merged = new Map(); - for (const sub of COAL_SUBTYPES) { - const subMap = await fetchCapacityForSource(sub, apiKey, startYear); - for (const [year, mw] of subMap) merged.set(year, (merged.get(year) ?? 0) + mw); + try { + let yearTotals; + if (source.code === 'COL') { + yearTotals = await fetchCapacityForSource('COL', apiKey, startYear); + if (yearTotals.size === 0) { + const merged = new Map(); + for (const sub of COAL_SUBTYPES) { + const subMap = await fetchCapacityForSource(sub, apiKey, startYear); + for (const [year, mw] of subMap) merged.set(year, (merged.get(year) ?? 0) + mw); + } + yearTotals = merged; } - yearTotals = merged; + } else { + yearTotals = await fetchCapacityForSource(source.code, apiKey, startYear); } - } else { - yearTotals = await fetchCapacityForSource(source.code, apiKey, startYear); + const data = Array.from(yearTotals.entries()) + .sort(([a], [b]) => a - b) + .map(([year, mw]) => ({ year, capacityMw: mw })); + series.push({ energySource: source.code, name: source.name, data }); + } catch (e) { + console.warn(` EIA ${source.code}: ${e.message}`); } - const data = Array.from(yearTotals.entries()) - .sort(([a], [b]) => a - b) - .map(([year, mw]) => ({ year, capacityMw: mw })); - series.push({ energySource: source.code, name: source.name, data }); } console.log(` Energy capacity: ${series.length} sources`); return { series }; @@ -371,6 +375,11 @@ async function fetchAll() { const fr = fredResults.status === 'fulfilled' ? fredResults.value : null; const ms = macroSignals.status === 'fulfilled' ? macroSignals.value : null; + if (energyPrices.status === 'rejected') console.warn(` EnergyPrices failed: ${energyPrices.reason?.message || energyPrices.reason}`); + if (energyCapacity.status === 'rejected') console.warn(` EnergyCapacity failed: ${energyCapacity.reason?.message || energyCapacity.reason}`); + if (fredResults.status === 'rejected') console.warn(` FRED failed: ${fredResults.reason?.message || fredResults.reason}`); + if (macroSignals.status === 'rejected') console.warn(` MacroSignals failed: ${macroSignals.reason?.message || macroSignals.reason}`); + if (!ep && !fr && !ms) throw new Error('All economic fetches failed'); // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write) diff --git a/scripts/seed-infra.mjs b/scripts/seed-infra.mjs index 724f918aa..94f2b9814 100755 --- a/scripts/seed-infra.mjs +++ b/scripts/seed-infra.mjs @@ -55,6 +55,8 @@ async function main() { warmPing('Cable Health', '/api/infrastructure/v1/get-cable-health'), ]); + for (const r of results) { if (r.status === 'rejected') console.warn(` Warm-ping failed: ${r.reason?.message || r.reason}`); } + const ok = results.filter(r => r.status === 'fulfilled' && r.value).length; const total = results.length; const duration = Date.now() - start; diff --git a/scripts/seed-military-maritime-news.mjs b/scripts/seed-military-maritime-news.mjs index 5e8446f74..ea89459aa 100755 --- a/scripts/seed-military-maritime-news.mjs +++ b/scripts/seed-military-maritime-news.mjs @@ -58,6 +58,8 @@ async function main() { warmPing('Nav Warnings', '/api/maritime/v1/list-navigational-warnings'), ]); + for (const r of results) { if (r.status === 'rejected') console.warn(` Warm-ping failed: ${r.reason?.message || r.reason}`); } + const ok = results.filter(r => r.status === 'fulfilled' && r.value).length; const total = results.length; const duration = Date.now() - start; diff --git a/scripts/seed-prediction-markets.mjs b/scripts/seed-prediction-markets.mjs index 0c5ea1e6e..1f210ee5d 100644 --- a/scripts/seed-prediction-markets.mjs +++ b/scripts/seed-prediction-markets.mjs @@ -13,7 +13,7 @@ const CANONICAL_KEY = 'prediction:markets-bootstrap:v1'; const CACHE_TTL = 1800; // 30 min — matches client poll interval const GAMMA_BASE = 'https://gamma-api.polymarket.com'; -const KALSHI_BASE = 'https://trading-api.kalshi.com/trade-api/v2'; +const KALSHI_BASE = 'https://api.elections.kalshi.com/trade-api/v2'; const FETCH_TIMEOUT = 10_000; const TAG_DELAY_MS = 300; diff --git a/scripts/seed-research.mjs b/scripts/seed-research.mjs index fcf52cb26..cb9e02e57 100755 --- a/scripts/seed-research.mjs +++ b/scripts/seed-research.mjs @@ -227,38 +227,61 @@ async function fetchTechEvents() { // ─── Trending Repos ─── +const OSSINSIGHT_LANG_MAP = { python: 'Python', javascript: 'JavaScript', typescript: 'TypeScript' }; + +async function fetchTrendingFromOSSInsight(lang) { + const ossLang = OSSINSIGHT_LANG_MAP[lang] || lang; + const resp = await fetch( + `https://api.ossinsight.io/v1/trends/repos/?language=${ossLang}&period=past_24_hours`, + { + headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, + signal: AbortSignal.timeout(10_000), + }, + ); + if (!resp.ok) return null; + const json = await resp.json(); + const rows = json?.data?.rows; + if (!Array.isArray(rows)) return null; + return rows.slice(0, 50).map(r => ({ + fullName: r.repo_name || '', description: r.description || '', + language: r.primary_language || lang, stars: r.stars || 0, + starsToday: 0, forks: r.forks || 0, + url: r.repo_name ? `https://github.com/${r.repo_name}` : '', + })); +} + +async function fetchTrendingFromGitHubSearch(lang) { + const since = new Date(Date.now() - 7 * 86400_000).toISOString().slice(0, 10); + const resp = await fetch( + `https://api.github.com/search/repositories?q=language:${lang}+created:>${since}&sort=stars&order=desc&per_page=50`, + { + headers: { Accept: 'application/vnd.github+json', 'User-Agent': CHROME_UA }, + signal: AbortSignal.timeout(10_000), + }, + ); + if (!resp.ok) return null; + const data = await resp.json(); + if (!Array.isArray(data?.items)) return null; + return data.items.map(r => ({ + fullName: r.full_name, description: r.description || '', + language: r.language || '', stars: r.stargazers_count || 0, + starsToday: 0, forks: r.forks_count || 0, + url: r.html_url, + })); +} + async function fetchTrendingRepos() { const languages = ['python', 'javascript', 'typescript']; const results = {}; for (const lang of languages) { try { - let data; - const primaryUrl = `https://api.gitterapp.com/repositories?language=${lang}&since=daily`; - const resp = await fetch(primaryUrl, { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(10_000), - }); - if (resp.ok) { - data = await resp.json(); - } else { - const fallback = await fetch(`https://gh-trending-api.herokuapp.com/repositories/${lang}?since=daily`, { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(10_000), - }); - if (fallback.ok) data = await fallback.json(); - } - - if (!Array.isArray(data)) { console.warn(` Trending ${lang}: not an array`); continue; } - const repos = data.slice(0, 50).map(r => ({ - fullName: `${r.author}/${r.name}`, description: r.description || '', - language: r.language || '', stars: r.stars || 0, - starsToday: r.currentPeriodStars || 0, forks: r.forks || 0, - url: r.url || `https://github.com/${r.author}/${r.name}`, - })); + let repos = await fetchTrendingFromOSSInsight(lang); + if (!repos) repos = await fetchTrendingFromGitHubSearch(lang); + if (!repos || repos.length === 0) { console.warn(` Trending ${lang}: no data from any source`); continue; } const cacheKey = `research:trending:v1:${lang}:daily:50`; - if (repos.length > 0) results[cacheKey] = { repos, pagination: undefined }; + results[cacheKey] = { repos, pagination: undefined }; console.log(` Trending ${lang}: ${repos.length} repos`); await sleep(500); } catch (e) { @@ -287,6 +310,11 @@ async function fetchAll() { trending: trending.status === 'fulfilled' ? trending.value : null, }; + if (arxiv.status === 'rejected') console.warn(` arXiv failed: ${arxiv.reason?.message || arxiv.reason}`); + if (hn.status === 'rejected') console.warn(` HN failed: ${hn.reason?.message || hn.reason}`); + if (techEvents.status === 'rejected') console.warn(` TechEvents failed: ${techEvents.reason?.message || techEvents.reason}`); + if (trending.status === 'rejected') console.warn(` Trending failed: ${trending.reason?.message || trending.reason}`); + if (!allData.arxiv && !allData.hn && !allData.trending) throw new Error('All research fetches failed'); // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write) diff --git a/scripts/seed-security-advisories.mjs b/scripts/seed-security-advisories.mjs index a25e84ea4..5612e8752 100644 --- a/scripts/seed-security-advisories.mjs +++ b/scripts/seed-security-advisories.mjs @@ -179,8 +179,10 @@ function buildByCountryMap(advisories) { async function fetchAll() { const results = await Promise.allSettled(ADVISORY_FEEDS.map(fetchFeed)); const all = []; - for (const r of results) { + for (let i = 0; i < results.length; i++) { + const r = results[i]; if (r.status === 'fulfilled') all.push(...r.value); + else console.warn(` Feed ${ADVISORY_FEEDS[i]?.name || i} failed: ${r.reason?.message || r.reason}`); } const seen = new Set(); diff --git a/scripts/seed-service-statuses.mjs b/scripts/seed-service-statuses.mjs index e8d99788a..39fd35976 100644 --- a/scripts/seed-service-statuses.mjs +++ b/scripts/seed-service-statuses.mjs @@ -12,7 +12,7 @@ import { loadEnvFile, CHROME_UA, getRedisCredentials, logSeedResult, extendExist loadEnvFile(import.meta.url); -const RPC_URL = 'https://worldmonitor.app/api/infrastructure/v1/list-service-statuses'; +const RPC_URL = 'https://api.worldmonitor.app/api/infrastructure/v1/list-service-statuses'; const CANONICAL_KEY = 'infra:service-statuses:v1'; async function warmPing() { diff --git a/scripts/seed-supply-chain-trade.mjs b/scripts/seed-supply-chain-trade.mjs index c3ca18008..8e97e2b7c 100755 --- a/scripts/seed-supply-chain-trade.mjs +++ b/scripts/seed-supply-chain-trade.mjs @@ -47,28 +47,32 @@ async function fetchShippingRates() { const indices = []; for (const cfg of SHIPPING_SERIES) { - const params = new URLSearchParams({ - series_id: cfg.seriesId, api_key: apiKey, file_type: 'json', - frequency: cfg.frequency, sort_order: 'desc', limit: '24', - }); - const resp = await fetch(`https://api.stlouisfed.org/fred/series/observations?${params}`, { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(10_000), - }); - if (!resp.ok) { console.warn(` FRED ${cfg.seriesId}: HTTP ${resp.status}`); continue; } - const data = await resp.json(); - const observations = (data.observations || []) - .map(o => { const v = parseFloat(o.value); return isNaN(v) || o.value === '.' ? null : { date: o.date, value: v }; }) - .filter(Boolean).reverse(); - if (observations.length === 0) continue; - const current = observations[observations.length - 1].value; - const previous = observations.length > 1 ? observations[observations.length - 2].value : current; - const changePct = previous !== 0 ? ((current - previous) / previous) * 100 : 0; - indices.push({ - indexId: cfg.seriesId, name: cfg.name, currentValue: current, previousValue: previous, - changePct, unit: cfg.unit, history: observations, spikeAlert: detectSpike(observations), - }); - await sleep(200); + try { + const params = new URLSearchParams({ + series_id: cfg.seriesId, api_key: apiKey, file_type: 'json', + frequency: cfg.frequency, sort_order: 'desc', limit: '24', + }); + const resp = await fetch(`https://api.stlouisfed.org/fred/series/observations?${params}`, { + headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, + signal: AbortSignal.timeout(10_000), + }); + if (!resp.ok) { console.warn(` FRED ${cfg.seriesId}: HTTP ${resp.status}`); continue; } + const data = await resp.json(); + const observations = (data.observations || []) + .map(o => { const v = parseFloat(o.value); return isNaN(v) || o.value === '.' ? null : { date: o.date, value: v }; }) + .filter(Boolean).reverse(); + if (observations.length === 0) continue; + const current = observations[observations.length - 1].value; + const previous = observations.length > 1 ? observations[observations.length - 2].value : current; + const changePct = previous !== 0 ? ((current - previous) / previous) * 100 : 0; + indices.push({ + indexId: cfg.seriesId, name: cfg.name, currentValue: current, previousValue: previous, + changePct, unit: cfg.unit, history: observations, spikeAlert: detectSpike(observations), + }); + await sleep(200); + } catch (e) { + console.warn(` FRED ${cfg.seriesId}: ${e.message}`); + } } console.log(` Shipping rates: ${indices.length} indices`); return { indices, fetchedAt: new Date().toISOString(), upstreamUnavailable: false }; @@ -307,6 +311,12 @@ async function fetchAll() { const fl = flows.status === 'fulfilled' ? flows.value : null; const ta = tariffs.status === 'fulfilled' ? tariffs.value : null; + if (shipping.status === 'rejected') console.warn(` Shipping failed: ${shipping.reason?.message || shipping.reason}`); + if (barriers.status === 'rejected') console.warn(` Barriers failed: ${barriers.reason?.message || barriers.reason}`); + if (restrictions.status === 'rejected') console.warn(` Restrictions failed: ${restrictions.reason?.message || restrictions.reason}`); + if (flows.status === 'rejected') console.warn(` Flows failed: ${flows.reason?.message || flows.reason}`); + if (tariffs.status === 'rejected') console.warn(` Tariffs failed: ${tariffs.reason?.message || tariffs.reason}`); + if (!sh && !ba && !re) throw new Error('All supply-chain/trade fetches failed'); // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write) diff --git a/scripts/seed-usni-fleet.mjs b/scripts/seed-usni-fleet.mjs index fc9aec03e..5fa18da4b 100755 --- a/scripts/seed-usni-fleet.mjs +++ b/scripts/seed-usni-fleet.mjs @@ -47,7 +47,8 @@ function fetchDirect(url) { stream.on('data', (c) => chunks.push(c)); stream.on('end', () => { const body = Buffer.concat(chunks).toString(); - if (body.trimStart().startsWith(' chunks.push(c)); stream.on('end', () => { const body = Buffer.concat(chunks).toString(); - if (body.trimStart().startsWith(' = { + python: 'Python', javascript: 'JavaScript', typescript: 'TypeScript', + go: 'Go', rust: 'Rust', java: 'Java', 'c++': 'C++', c: 'C', +}; + +const OSSINSIGHT_PERIOD: Record = { + daily: 'past_24_hours', weekly: 'past_week', monthly: 'past_month', +}; + // ---------- Fetch ---------- +async function fetchFromOSSInsight(language: string, period: string, pageSize: number): Promise { + const ossLang = OSSINSIGHT_LANG[language] || language; + const ossPeriod = OSSINSIGHT_PERIOD[period] || 'past_24_hours'; + const resp = await fetch( + `https://api.ossinsight.io/v1/trends/repos/?language=${ossLang}&period=${ossPeriod}`, + { headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(10_000) }, + ); + if (!resp.ok) return null; + const json = await resp.json() as any; + const rows = json?.data?.rows; + if (!Array.isArray(rows)) return null; + return rows.slice(0, pageSize).map((r: any): GithubRepo => ({ + fullName: r.repo_name || '', description: r.description || '', + language: r.primary_language || language, stars: r.stars || 0, + starsToday: 0, forks: r.forks || 0, + url: r.repo_name ? `https://github.com/${r.repo_name}` : '', + })); +} + +const GH_SEARCH_DAYS: Record = { daily: 1, weekly: 7, monthly: 30 }; + +async function fetchFromGitHubSearch(language: string, period: string, pageSize: number): Promise { + const days = GH_SEARCH_DAYS[period] || 7; + const since = new Date(Date.now() - days * 86400_000).toISOString().slice(0, 10); + const resp = await fetch( + `https://api.github.com/search/repositories?q=language:${language}+created:>${since}&sort=stars&order=desc&per_page=${pageSize}`, + { headers: { Accept: 'application/vnd.github+json', 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(10_000) }, + ); + if (!resp.ok) return null; + const data = await resp.json() as any; + if (!Array.isArray(data?.items)) return null; + return data.items.map((r: any): GithubRepo => ({ + fullName: r.full_name, description: r.description || '', + language: r.language || '', stars: r.stargazers_count || 0, + starsToday: 0, forks: r.forks_count || 0, + url: r.html_url, + })); +} + async function fetchTrendingRepos(req: ListTrendingReposRequest): Promise { const language = req.language || 'python'; const period = req.period || 'daily'; const pageSize = clampInt(req.pageSize, 50, 1, 100); - // Primary API - const primaryUrl = `https://api.gitterapp.com/repositories?language=${language}&since=${period}`; - let data: any[]; + try { + const repos = await fetchFromOSSInsight(language, period, pageSize); + if (repos && repos.length > 0) return repos; + } catch { /* fall through */ } try { - const response = await fetch(primaryUrl, { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(10000), - }); + const repos = await fetchFromGitHubSearch(language, period, pageSize); + if (repos && repos.length > 0) return repos; + } catch { /* fall through */ } - if (!response.ok) throw new Error('Primary API failed'); - data = await response.json() as any[]; - } catch { - // Fallback API - try { - const fallbackUrl = `https://gh-trending-api.herokuapp.com/repositories/${language}?since=${period}`; - const fallbackResponse = await fetch(fallbackUrl, { - headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(10000), - }); - - if (!fallbackResponse.ok) return []; - data = await fallbackResponse.json() as any[]; - } catch { - return []; - } - } - - if (!Array.isArray(data)) return []; - - return data.slice(0, pageSize).map((raw: any): GithubRepo => ({ - fullName: `${raw.author}/${raw.name}`, - description: raw.description || '', - language: raw.language || '', - stars: raw.stars || 0, - starsToday: raw.currentPeriodStars || 0, - forks: raw.forks || 0, - url: raw.url || `https://github.com/${raw.author}/${raw.name}`, - })); + return []; } // ---------- Handler ----------