fix(seeds): improve resilience and fix dead APIs across seed scripts (#1644)

* fix(seeds): improve resilience and fix dead APIs across seed scripts - Fix wrong domain in seed-service-statuses (worldmonitor.app to api.worldmonitor.app) - Fix Kalshi API domain migration (trading-api.kalshi.com to api.elections.kalshi.com) - Replace dead trending APIs (gitterapp.com, herokuapp.com) with OSSInsight + GitHub Search - Fix case-sensitive HTML detection in seed-usni-fleet (lowercase doctype not matched) - Add Promise.allSettled rejection logging across 8 seed scripts - Wrap fetch loops in try-catch (seed-supply-chain-trade, seed-economy) so a single network error no longer kills the entire function - Update list-trending-repos.ts RPC handler to match seed changes * fix(seeds): correct OSSInsight response parsing and period-aware GitHub Search fallback - OSSInsight returns {data: {rows: [...]}} not {data: [...]}, fix both seed and handler - GitHub Search fallback now respects period parameter (daily=1d, weekly=7d, monthly=30d) * fix(seeds): correct OSSInsight period values (past_week/past_month, not past_7_days/past_28_days)
2026-04-25 17:14:57 +02:00 · 2026-03-15 13:31:41 +04:00
parent 249c088639
commit 65543d71d5
12 changed files with 187 additions and 104 deletions
--- a/scripts/seed-aviation.mjs
+++ b/scripts/seed-aviation.mjs
@@ -249,6 +249,9 @@ async function fetchAll() {
  const opsData = ops.status === 'fulfilled' ? ops.value : null;
  const newsData = news.status === 'fulfilled' ? news.value : null;

+  if (ops.status === 'rejected') console.warn(`  AirportOps failed: ${ops.reason?.message || ops.reason}`);
+  if (news.status === 'rejected') console.warn(`  AviationNews failed: ${news.reason?.message || news.reason}`);
+
  if (!opsData && !newsData) throw new Error('All aviation fetches failed');

  // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write)
--- a/scripts/seed-conflict-intel.mjs
+++ b/scripts/seed-conflict-intel.mjs
@@ -260,6 +260,11 @@ async function fetchAll() {
  const pi = pizzint.status === 'fulfilled' ? pizzint.value : null;
  const gd = gdelt.status === 'fulfilled' ? gdelt.value : null;

+  if (acled.status === 'rejected') console.warn(`  ACLED failed: ${acled.reason?.message || acled.reason}`);
+  if (hapi.status === 'rejected') console.warn(`  HAPI failed: ${hapi.reason?.message || hapi.reason}`);
+  if (pizzint.status === 'rejected') console.warn(`  PizzINT failed: ${pizzint.reason?.message || pizzint.reason}`);
+  if (gdelt.status === 'rejected') console.warn(`  GDELT failed: ${gdelt.reason?.message || gdelt.reason}`);
+
  if (!ac && !pi) throw new Error('All conflict/intel fetches failed');

  // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write)
--- a/scripts/seed-economy.mjs
+++ b/scripts/seed-economy.mjs
@@ -112,24 +112,28 @@ async function fetchEnergyCapacity() {

  const series = [];
  for (const source of CAPACITY_SOURCES) {
-    let yearTotals;
-    if (source.code === 'COL') {
-      yearTotals = await fetchCapacityForSource('COL', apiKey, startYear);
-      if (yearTotals.size === 0) {
-        const merged = new Map();
-        for (const sub of COAL_SUBTYPES) {
-          const subMap = await fetchCapacityForSource(sub, apiKey, startYear);
-          for (const [year, mw] of subMap) merged.set(year, (merged.get(year) ?? 0) + mw);
+    try {
+      let yearTotals;
+      if (source.code === 'COL') {
+        yearTotals = await fetchCapacityForSource('COL', apiKey, startYear);
+        if (yearTotals.size === 0) {
+          const merged = new Map();
+          for (const sub of COAL_SUBTYPES) {
+            const subMap = await fetchCapacityForSource(sub, apiKey, startYear);
+            for (const [year, mw] of subMap) merged.set(year, (merged.get(year) ?? 0) + mw);
+          }
+          yearTotals = merged;
        }
-        yearTotals = merged;
+      } else {
+        yearTotals = await fetchCapacityForSource(source.code, apiKey, startYear);
      }
-    } else {
-      yearTotals = await fetchCapacityForSource(source.code, apiKey, startYear);
+      const data = Array.from(yearTotals.entries())
+        .sort(([a], [b]) => a - b)
+        .map(([year, mw]) => ({ year, capacityMw: mw }));
+      series.push({ energySource: source.code, name: source.name, data });
+    } catch (e) {
+      console.warn(`  EIA ${source.code}: ${e.message}`);
    }
-    const data = Array.from(yearTotals.entries())
-      .sort(([a], [b]) => a - b)
-      .map(([year, mw]) => ({ year, capacityMw: mw }));
-    series.push({ energySource: source.code, name: source.name, data });
  }
  console.log(`  Energy capacity: ${series.length} sources`);
  return { series };
@@ -371,6 +375,11 @@ async function fetchAll() {
  const fr = fredResults.status === 'fulfilled' ? fredResults.value : null;
  const ms = macroSignals.status === 'fulfilled' ? macroSignals.value : null;

+  if (energyPrices.status === 'rejected') console.warn(`  EnergyPrices failed: ${energyPrices.reason?.message || energyPrices.reason}`);
+  if (energyCapacity.status === 'rejected') console.warn(`  EnergyCapacity failed: ${energyCapacity.reason?.message || energyCapacity.reason}`);
+  if (fredResults.status === 'rejected') console.warn(`  FRED failed: ${fredResults.reason?.message || fredResults.reason}`);
+  if (macroSignals.status === 'rejected') console.warn(`  MacroSignals failed: ${macroSignals.reason?.message || macroSignals.reason}`);
+
  if (!ep && !fr && !ms) throw new Error('All economic fetches failed');

  // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write)
--- a/scripts/seed-infra.mjs
+++ b/scripts/seed-infra.mjs
@@ -55,6 +55,8 @@ async function main() {
    warmPing('Cable Health', '/api/infrastructure/v1/get-cable-health'),
  ]);

+  for (const r of results) { if (r.status === 'rejected') console.warn(`  Warm-ping failed: ${r.reason?.message || r.reason}`); }
+
  const ok = results.filter(r => r.status === 'fulfilled' && r.value).length;
  const total = results.length;
  const duration = Date.now() - start;
--- a/scripts/seed-military-maritime-news.mjs
+++ b/scripts/seed-military-maritime-news.mjs
@@ -58,6 +58,8 @@ async function main() {
    warmPing('Nav Warnings', '/api/maritime/v1/list-navigational-warnings'),
  ]);

+  for (const r of results) { if (r.status === 'rejected') console.warn(`  Warm-ping failed: ${r.reason?.message || r.reason}`); }
+
  const ok = results.filter(r => r.status === 'fulfilled' && r.value).length;
  const total = results.length;
  const duration = Date.now() - start;
--- a/scripts/seed-prediction-markets.mjs
+++ b/scripts/seed-prediction-markets.mjs
@@ -13,7 +13,7 @@ const CANONICAL_KEY = 'prediction:markets-bootstrap:v1';
 const CACHE_TTL = 1800; // 30 min — matches client poll interval

 const GAMMA_BASE = 'https://gamma-api.polymarket.com';
-const KALSHI_BASE = 'https://trading-api.kalshi.com/trade-api/v2';
+const KALSHI_BASE = 'https://api.elections.kalshi.com/trade-api/v2';
 const FETCH_TIMEOUT = 10_000;
 const TAG_DELAY_MS = 300;

--- a/scripts/seed-research.mjs
+++ b/scripts/seed-research.mjs
@@ -227,38 +227,61 @@ async function fetchTechEvents() {

 // ─── Trending Repos ───

+const OSSINSIGHT_LANG_MAP = { python: 'Python', javascript: 'JavaScript', typescript: 'TypeScript' };
+
+async function fetchTrendingFromOSSInsight(lang) {
+  const ossLang = OSSINSIGHT_LANG_MAP[lang] || lang;
+  const resp = await fetch(
+    `https://api.ossinsight.io/v1/trends/repos/?language=${ossLang}&period=past_24_hours`,
+    {
+      headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
+      signal: AbortSignal.timeout(10_000),
+    },
+  );
+  if (!resp.ok) return null;
+  const json = await resp.json();
+  const rows = json?.data?.rows;
+  if (!Array.isArray(rows)) return null;
+  return rows.slice(0, 50).map(r => ({
+    fullName: r.repo_name || '', description: r.description || '',
+    language: r.primary_language || lang, stars: r.stars || 0,
+    starsToday: 0, forks: r.forks || 0,
+    url: r.repo_name ? `https://github.com/${r.repo_name}` : '',
+  }));
+}
+
+async function fetchTrendingFromGitHubSearch(lang) {
+  const since = new Date(Date.now() - 7 * 86400_000).toISOString().slice(0, 10);
+  const resp = await fetch(
+    `https://api.github.com/search/repositories?q=language:${lang}+created:>${since}&sort=stars&order=desc&per_page=50`,
+    {
+      headers: { Accept: 'application/vnd.github+json', 'User-Agent': CHROME_UA },
+      signal: AbortSignal.timeout(10_000),
+    },
+  );
+  if (!resp.ok) return null;
+  const data = await resp.json();
+  if (!Array.isArray(data?.items)) return null;
+  return data.items.map(r => ({
+    fullName: r.full_name, description: r.description || '',
+    language: r.language || '', stars: r.stargazers_count || 0,
+    starsToday: 0, forks: r.forks_count || 0,
+    url: r.html_url,
+  }));
+}
+
 async function fetchTrendingRepos() {
  const languages = ['python', 'javascript', 'typescript'];
  const results = {};

  for (const lang of languages) {
    try {
-      let data;
-      const primaryUrl = `https://api.gitterapp.com/repositories?language=${lang}&since=daily`;
-      const resp = await fetch(primaryUrl, {
-        headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
-        signal: AbortSignal.timeout(10_000),
-      });
-      if (resp.ok) {
-        data = await resp.json();
-      } else {
-        const fallback = await fetch(`https://gh-trending-api.herokuapp.com/repositories/${lang}?since=daily`, {
-          headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
-          signal: AbortSignal.timeout(10_000),
-        });
-        if (fallback.ok) data = await fallback.json();
-      }
-
-      if (!Array.isArray(data)) { console.warn(`  Trending ${lang}: not an array`); continue; }
-      const repos = data.slice(0, 50).map(r => ({
-        fullName: `${r.author}/${r.name}`, description: r.description || '',
-        language: r.language || '', stars: r.stars || 0,
-        starsToday: r.currentPeriodStars || 0, forks: r.forks || 0,
-        url: r.url || `https://github.com/${r.author}/${r.name}`,
-      }));
+      let repos = await fetchTrendingFromOSSInsight(lang);
+      if (!repos) repos = await fetchTrendingFromGitHubSearch(lang);
+      if (!repos || repos.length === 0) { console.warn(`  Trending ${lang}: no data from any source`); continue; }

      const cacheKey = `research:trending:v1:${lang}:daily:50`;
-      if (repos.length > 0) results[cacheKey] = { repos, pagination: undefined };
+      results[cacheKey] = { repos, pagination: undefined };
      console.log(`  Trending ${lang}: ${repos.length} repos`);
      await sleep(500);
    } catch (e) {
@@ -287,6 +310,11 @@ async function fetchAll() {
    trending: trending.status === 'fulfilled' ? trending.value : null,
  };

+  if (arxiv.status === 'rejected') console.warn(`  arXiv failed: ${arxiv.reason?.message || arxiv.reason}`);
+  if (hn.status === 'rejected') console.warn(`  HN failed: ${hn.reason?.message || hn.reason}`);
+  if (techEvents.status === 'rejected') console.warn(`  TechEvents failed: ${techEvents.reason?.message || techEvents.reason}`);
+  if (trending.status === 'rejected') console.warn(`  Trending failed: ${trending.reason?.message || trending.reason}`);
+
  if (!allData.arxiv && !allData.hn && !allData.trending) throw new Error('All research fetches failed');

  // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write)
--- a/scripts/seed-security-advisories.mjs
+++ b/scripts/seed-security-advisories.mjs
@@ -179,8 +179,10 @@ function buildByCountryMap(advisories) {
 async function fetchAll() {
  const results = await Promise.allSettled(ADVISORY_FEEDS.map(fetchFeed));
  const all = [];
-  for (const r of results) {
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
    if (r.status === 'fulfilled') all.push(...r.value);
+    else console.warn(`  Feed ${ADVISORY_FEEDS[i]?.name || i} failed: ${r.reason?.message || r.reason}`);
  }

  const seen = new Set();
--- a/scripts/seed-service-statuses.mjs
+++ b/scripts/seed-service-statuses.mjs
@@ -12,7 +12,7 @@ import { loadEnvFile, CHROME_UA, getRedisCredentials, logSeedResult, extendExist

 loadEnvFile(import.meta.url);

-const RPC_URL = 'https://worldmonitor.app/api/infrastructure/v1/list-service-statuses';
+const RPC_URL = 'https://api.worldmonitor.app/api/infrastructure/v1/list-service-statuses';
 const CANONICAL_KEY = 'infra:service-statuses:v1';

 async function warmPing() {
--- a/scripts/seed-supply-chain-trade.mjs
+++ b/scripts/seed-supply-chain-trade.mjs
@@ -47,28 +47,32 @@ async function fetchShippingRates() {

  const indices = [];
  for (const cfg of SHIPPING_SERIES) {
-    const params = new URLSearchParams({
-      series_id: cfg.seriesId, api_key: apiKey, file_type: 'json',
-      frequency: cfg.frequency, sort_order: 'desc', limit: '24',
-    });
-    const resp = await fetch(`https://api.stlouisfed.org/fred/series/observations?${params}`, {
-      headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
-      signal: AbortSignal.timeout(10_000),
-    });
-    if (!resp.ok) { console.warn(`  FRED ${cfg.seriesId}: HTTP ${resp.status}`); continue; }
-    const data = await resp.json();
-    const observations = (data.observations || [])
-      .map(o => { const v = parseFloat(o.value); return isNaN(v) || o.value === '.' ? null : { date: o.date, value: v }; })
-      .filter(Boolean).reverse();
-    if (observations.length === 0) continue;
-    const current = observations[observations.length - 1].value;
-    const previous = observations.length > 1 ? observations[observations.length - 2].value : current;
-    const changePct = previous !== 0 ? ((current - previous) / previous) * 100 : 0;
-    indices.push({
-      indexId: cfg.seriesId, name: cfg.name, currentValue: current, previousValue: previous,
-      changePct, unit: cfg.unit, history: observations, spikeAlert: detectSpike(observations),
-    });
-    await sleep(200);
+    try {
+      const params = new URLSearchParams({
+        series_id: cfg.seriesId, api_key: apiKey, file_type: 'json',
+        frequency: cfg.frequency, sort_order: 'desc', limit: '24',
+      });
+      const resp = await fetch(`https://api.stlouisfed.org/fred/series/observations?${params}`, {
+        headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
+        signal: AbortSignal.timeout(10_000),
+      });
+      if (!resp.ok) { console.warn(`  FRED ${cfg.seriesId}: HTTP ${resp.status}`); continue; }
+      const data = await resp.json();
+      const observations = (data.observations || [])
+        .map(o => { const v = parseFloat(o.value); return isNaN(v) || o.value === '.' ? null : { date: o.date, value: v }; })
+        .filter(Boolean).reverse();
+      if (observations.length === 0) continue;
+      const current = observations[observations.length - 1].value;
+      const previous = observations.length > 1 ? observations[observations.length - 2].value : current;
+      const changePct = previous !== 0 ? ((current - previous) / previous) * 100 : 0;
+      indices.push({
+        indexId: cfg.seriesId, name: cfg.name, currentValue: current, previousValue: previous,
+        changePct, unit: cfg.unit, history: observations, spikeAlert: detectSpike(observations),
+      });
+      await sleep(200);
+    } catch (e) {
+      console.warn(`  FRED ${cfg.seriesId}: ${e.message}`);
+    }
  }
  console.log(`  Shipping rates: ${indices.length} indices`);
  return { indices, fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
@@ -307,6 +311,12 @@ async function fetchAll() {
  const fl = flows.status === 'fulfilled' ? flows.value : null;
  const ta = tariffs.status === 'fulfilled' ? tariffs.value : null;

+  if (shipping.status === 'rejected') console.warn(`  Shipping failed: ${shipping.reason?.message || shipping.reason}`);
+  if (barriers.status === 'rejected') console.warn(`  Barriers failed: ${barriers.reason?.message || barriers.reason}`);
+  if (restrictions.status === 'rejected') console.warn(`  Restrictions failed: ${restrictions.reason?.message || restrictions.reason}`);
+  if (flows.status === 'rejected') console.warn(`  Flows failed: ${flows.reason?.message || flows.reason}`);
+  if (tariffs.status === 'rejected') console.warn(`  Tariffs failed: ${tariffs.reason?.message || tariffs.reason}`);
+
  if (!sh && !ba && !re) throw new Error('All supply-chain/trade fetches failed');

  // Write secondary keys BEFORE returning (runSeed calls process.exit after primary write)
--- a/scripts/seed-usni-fleet.mjs
+++ b/scripts/seed-usni-fleet.mjs
@@ -47,7 +47,8 @@ function fetchDirect(url) {
      stream.on('data', (c) => chunks.push(c));
      stream.on('end', () => {
        const body = Buffer.concat(chunks).toString();
-        if (body.trimStart().startsWith('<!DOCTYPE') || body.trimStart().startsWith('<html')) {
+        const trimmed = body.trimStart().toLowerCase();
+        if (trimmed.startsWith('<!doctype') || trimmed.startsWith('<html')) {
          reject(new Error('Cloudflare block (HTML response)'));
          return;
        }
@@ -88,7 +89,8 @@ function fetchViaHttpProxy(url, proxyAuth) {
        stream.on('data', (c) => chunks.push(c));
        stream.on('end', () => {
          const body = Buffer.concat(chunks).toString();
-          if (body.trimStart().startsWith('<!DOCTYPE') || body.trimStart().startsWith('<html')) {
+          const trimmed = body.trimStart().toLowerCase();
+          if (trimmed.startsWith('<!doctype') || trimmed.startsWith('<html')) {
            reject(new Error('Cloudflare block via proxy (HTML response)'));
            return;
          }
--- a/server/worldmonitor/research/v1/list-trending-repos.ts
+++ b/server/worldmonitor/research/v1/list-trending-repos.ts
@@ -1,8 +1,8 @@
 /**
 * RPC: listTrendingRepos
 *
- * Fetches trending GitHub repos from gitterapp JSON API with
- * herokuapp fallback. Returns empty array on any failure.
+ * Fetches trending GitHub repos from OSSInsight API (PingCAP-backed)
+ * with GitHub Search API fallback. Returns empty array on any failure.
 */

 import type {
@@ -18,52 +18,72 @@ import { cachedFetchJson } from '../../../_shared/redis';
 const REDIS_CACHE_KEY = 'research:trending:v1';
 const REDIS_CACHE_TTL = 3600; // 1 hr — daily trending data

+const OSSINSIGHT_LANG: Record<string, string> = {
+  python: 'Python', javascript: 'JavaScript', typescript: 'TypeScript',
+  go: 'Go', rust: 'Rust', java: 'Java', 'c++': 'C++', c: 'C',
+};
+
+const OSSINSIGHT_PERIOD: Record<string, string> = {
+  daily: 'past_24_hours', weekly: 'past_week', monthly: 'past_month',
+};
+
 // ---------- Fetch ----------

+async function fetchFromOSSInsight(language: string, period: string, pageSize: number): Promise<GithubRepo[] | null> {
+  const ossLang = OSSINSIGHT_LANG[language] || language;
+  const ossPeriod = OSSINSIGHT_PERIOD[period] || 'past_24_hours';
+  const resp = await fetch(
+    `https://api.ossinsight.io/v1/trends/repos/?language=${ossLang}&period=${ossPeriod}`,
+    { headers: { Accept: 'application/json', 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(10_000) },
+  );
+  if (!resp.ok) return null;
+  const json = await resp.json() as any;
+  const rows = json?.data?.rows;
+  if (!Array.isArray(rows)) return null;
+  return rows.slice(0, pageSize).map((r: any): GithubRepo => ({
+    fullName: r.repo_name || '', description: r.description || '',
+    language: r.primary_language || language, stars: r.stars || 0,
+    starsToday: 0, forks: r.forks || 0,
+    url: r.repo_name ? `https://github.com/${r.repo_name}` : '',
+  }));
+}
+
+const GH_SEARCH_DAYS: Record<string, number> = { daily: 1, weekly: 7, monthly: 30 };
+
+async function fetchFromGitHubSearch(language: string, period: string, pageSize: number): Promise<GithubRepo[] | null> {
+  const days = GH_SEARCH_DAYS[period] || 7;
+  const since = new Date(Date.now() - days * 86400_000).toISOString().slice(0, 10);
+  const resp = await fetch(
+    `https://api.github.com/search/repositories?q=language:${language}+created:>${since}&sort=stars&order=desc&per_page=${pageSize}`,
+    { headers: { Accept: 'application/vnd.github+json', 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(10_000) },
+  );
+  if (!resp.ok) return null;
+  const data = await resp.json() as any;
+  if (!Array.isArray(data?.items)) return null;
+  return data.items.map((r: any): GithubRepo => ({
+    fullName: r.full_name, description: r.description || '',
+    language: r.language || '', stars: r.stargazers_count || 0,
+    starsToday: 0, forks: r.forks_count || 0,
+    url: r.html_url,
+  }));
+}
+
 async function fetchTrendingRepos(req: ListTrendingReposRequest): Promise<GithubRepo[]> {
  const language = req.language || 'python';
  const period = req.period || 'daily';
  const pageSize = clampInt(req.pageSize, 50, 1, 100);

-  // Primary API
-  const primaryUrl = `https://api.gitterapp.com/repositories?language=${language}&since=${period}`;
-  let data: any[];
+  try {
+    const repos = await fetchFromOSSInsight(language, period, pageSize);
+    if (repos && repos.length > 0) return repos;
+  } catch { /* fall through */ }

  try {
-    const response = await fetch(primaryUrl, {
-      headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
-      signal: AbortSignal.timeout(10000),
-    });
+    const repos = await fetchFromGitHubSearch(language, period, pageSize);
+    if (repos && repos.length > 0) return repos;
+  } catch { /* fall through */ }

-    if (!response.ok) throw new Error('Primary API failed');
-    data = await response.json() as any[];
-  } catch {
-    // Fallback API
-    try {
-      const fallbackUrl = `https://gh-trending-api.herokuapp.com/repositories/${language}?since=${period}`;
-      const fallbackResponse = await fetch(fallbackUrl, {
-        headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
-        signal: AbortSignal.timeout(10000),
-      });
-
-      if (!fallbackResponse.ok) return [];
-      data = await fallbackResponse.json() as any[];
-    } catch {
-      return [];
-    }
-  }
-
-  if (!Array.isArray(data)) return [];
-
-  return data.slice(0, pageSize).map((raw: any): GithubRepo => ({
-    fullName: `${raw.author}/${raw.name}`,
-    description: raw.description || '',
-    language: raw.language || '',
-    stars: raw.stars || 0,
-    starsToday: raw.currentPeriodStars || 0,
-    forks: raw.forks || 0,
-    url: raw.url || `https://github.com/${raw.author}/${raw.name}`,
-  }));
+  return [];
 }

 // ---------- Handler ----------