mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
467608c2d7
Mechanical fixes across 13 files: - isNaN() → Number.isNaN() (all values already numeric from parseFloat/parseInt) - let → const where never reassigned - Math.pow() → ** operator - Unnecessary continue in for loop - Useless string escape in test description - Missing parseInt radix parameter - Remove unused private class member (write-only counter) - Prefix unused function parameter with _ Config: suppress noImportantStyles (CSS !important is intentional) and useLiteralKeys (bracket notation used for computed/dynamic keys) in biome.json. Remaining 49 warnings are all noExcessiveCognitiveComplexity (already configured as warn, safe to address incrementally).
574 lines
26 KiB
JavaScript
Executable File
574 lines
26 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
import { loadEnvFile, CHROME_UA, runSeed, writeExtraKeyWithMeta, sleep, verifySeedKey } from './_seed-utils.mjs';
|
|
|
|
loadEnvFile(import.meta.url);
|
|
|
|
// ─── Keys (must match handler cache keys exactly) ───
|
|
const KEYS = {
|
|
shipping: 'supply_chain:shipping:v2',
|
|
barriers: 'trade:barriers:v1:tariff-gap:50',
|
|
restrictions: 'trade:restrictions:v1:tariff-overview:50',
|
|
customsRevenue: 'trade:customs-revenue:v1',
|
|
};
|
|
|
|
const SHIPPING_TTL = 3600;
|
|
const TRADE_TTL = 21600;
|
|
|
|
const MAJOR_REPORTERS = ['840', '156', '276', '392', '826', '356', '076', '643', '410', '036', '124', '484', '250', '380', '528'];
|
|
|
|
const WTO_MEMBER_CODES = {
|
|
'840': 'United States', '156': 'China', '276': 'Germany', '392': 'Japan',
|
|
'826': 'United Kingdom', '250': 'France', '356': 'India', '643': 'Russia',
|
|
'076': 'Brazil', '410': 'South Korea', '036': 'Australia', '124': 'Canada',
|
|
'484': 'Mexico', '380': 'Italy', '528': 'Netherlands', '000': 'World',
|
|
};
|
|
|
|
// ─── Shipping Rates (FRED) ───
|
|
|
|
const SHIPPING_SERIES = [
|
|
{ seriesId: 'PCU483111483111', name: 'Deep Sea Freight Producer Price Index', unit: 'index', frequency: 'm' },
|
|
{ seriesId: 'TSIFRGHT', name: 'Freight Transportation Services Index', unit: 'index', frequency: 'm' },
|
|
];
|
|
|
|
function detectSpike(history) {
|
|
if (!history || history.length < 3) return false;
|
|
const values = history.map(h => typeof h === 'number' ? h : h.value).filter(v => Number.isFinite(v));
|
|
if (values.length < 3) return false;
|
|
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
|
const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
|
|
const stdDev = Math.sqrt(variance);
|
|
if (stdDev === 0) return false;
|
|
return values[values.length - 1] > mean + 2 * stdDev;
|
|
}
|
|
|
|
async function fetchShippingRates() {
|
|
const apiKey = process.env.FRED_API_KEY;
|
|
if (!apiKey) throw new Error('Missing FRED_API_KEY');
|
|
|
|
const indices = [];
|
|
for (const cfg of SHIPPING_SERIES) {
|
|
try {
|
|
const params = new URLSearchParams({
|
|
series_id: cfg.seriesId, api_key: apiKey, file_type: 'json',
|
|
frequency: cfg.frequency, sort_order: 'desc', limit: '24',
|
|
});
|
|
const resp = await fetch(`https://api.stlouisfed.org/fred/series/observations?${params}`, {
|
|
headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
|
|
signal: AbortSignal.timeout(10_000),
|
|
});
|
|
if (!resp.ok) { console.warn(` FRED ${cfg.seriesId}: HTTP ${resp.status}`); continue; }
|
|
const data = await resp.json();
|
|
const observations = (data.observations || [])
|
|
.map(o => { const v = parseFloat(o.value); return Number.isNaN(v) || o.value === '.' ? null : { date: o.date, value: v }; })
|
|
.filter(Boolean).reverse();
|
|
if (observations.length === 0) continue;
|
|
const current = observations[observations.length - 1].value;
|
|
const previous = observations.length > 1 ? observations[observations.length - 2].value : current;
|
|
const changePct = previous !== 0 ? ((current - previous) / previous) * 100 : 0;
|
|
indices.push({
|
|
indexId: cfg.seriesId, name: cfg.name, currentValue: current, previousValue: previous,
|
|
changePct, unit: cfg.unit, history: observations, spikeAlert: detectSpike(observations),
|
|
});
|
|
await sleep(200);
|
|
} catch (e) {
|
|
console.warn(` FRED ${cfg.seriesId}: ${e.message}`);
|
|
}
|
|
}
|
|
console.log(` Shipping rates: ${indices.length} indices`);
|
|
return { indices, fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
|
|
}
|
|
|
|
// ─── Container Indices (Shanghai Shipping Exchange) ───
|
|
|
|
async function fetchSSEIndex(indexName, indexId, dataItemType, displayName, unit) {
|
|
try {
|
|
const resp = await fetch(`https://en.sse.net.cn/currentIndex?indexName=${indexName}`, {
|
|
headers: { 'User-Agent': CHROME_UA, Accept: 'application/json' },
|
|
signal: AbortSignal.timeout(10_000),
|
|
});
|
|
if (!resp.ok) { console.warn(` SSE ${indexName}: HTTP ${resp.status}`); return []; }
|
|
const json = await resp.json();
|
|
const lines = json?.data?.lineDataList;
|
|
if (!Array.isArray(lines)) { console.warn(` SSE ${indexName}: no lineDataList`); return []; }
|
|
const composite = lines.find(l => l.dataItemTypeName === dataItemType);
|
|
if (!composite) { console.warn(` SSE ${indexName}: ${dataItemType} not found`); return []; }
|
|
const currentValue = composite.currentContent;
|
|
const previousValue = composite.lastContent;
|
|
if (typeof currentValue !== 'number') return [];
|
|
const changePct = typeof composite.percentage === 'number' ? composite.percentage
|
|
: (previousValue > 0 ? ((currentValue - previousValue) / previousValue) * 100 : 0);
|
|
const observationDate = json.data?.currentDate || new Date().toISOString().slice(0, 10);
|
|
return [{
|
|
indexId, name: displayName, currentValue, previousValue: previousValue ?? currentValue,
|
|
changePct, unit, history: [], spikeAlert: false, _observationDate: observationDate,
|
|
}];
|
|
} catch (e) {
|
|
console.warn(` SSE ${indexName}: ${e.message}`);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async function fetchSCFI() {
|
|
return fetchSSEIndex('scfi', 'SCFI', 'SCFI_T', 'SCFI - Shanghai Container Freight', 'index');
|
|
}
|
|
|
|
async function fetchCCFI() {
|
|
return fetchSSEIndex('ccfi', 'CCFI', 'CCFI_T', 'CCFI - China Container Freight', 'index');
|
|
}
|
|
|
|
// ─── Baltic Dry Index (HandyBulk scrape) ───
|
|
|
|
const BDI_INDEX_MAP = [
|
|
{ label: 'Dry', id: 'BDI', name: 'BDI - Baltic Dry Index' },
|
|
{ label: 'Capesize', id: 'BCI', name: 'BCI - Baltic Capesize Index' },
|
|
{ label: 'Panamax', id: 'BPI', name: 'BPI - Baltic Panamax Index' },
|
|
{ label: 'Supramax', id: 'BSI', name: 'BSI - Baltic Supramax Index' },
|
|
{ label: 'Handysize', id: 'BHSI', name: 'BHSI - Baltic Handysize Index' },
|
|
];
|
|
|
|
async function fetchBDI() {
|
|
try {
|
|
const resp = await fetch('https://www.handybulk.com/baltic-dry-index/', {
|
|
headers: { 'User-Agent': CHROME_UA, 'Accept-Encoding': 'gzip, deflate' },
|
|
signal: AbortSignal.timeout(10_000),
|
|
redirect: 'manual',
|
|
});
|
|
if (!resp.ok && resp.status !== 301 && resp.status !== 302) {
|
|
console.warn(` BDI: HTTP ${resp.status}`);
|
|
return [];
|
|
}
|
|
const contentLength = parseInt(resp.headers.get('content-length') || '0', 10);
|
|
if (contentLength > 1_000_000) { console.warn(' BDI: response too large'); return []; }
|
|
const html = await resp.text();
|
|
if (html.length > 1_000_000) { console.warn(' BDI: body too large'); return []; }
|
|
|
|
// Parse article date from heading (e.g., "13-March-2026" or "13-Mar-2026")
|
|
const dateMatch = html.match(/(\d{1,2})-(\w+)-(\d{4})/);
|
|
let articleDate = new Date().toISOString().slice(0, 10);
|
|
if (dateMatch) {
|
|
const parsed = new Date(`${dateMatch[2]} ${dateMatch[1]}, ${dateMatch[3]}`);
|
|
if (!Number.isNaN(parsed.getTime())) articleDate = parsed.toISOString().slice(0, 10);
|
|
}
|
|
|
|
const indices = [];
|
|
for (const cfg of BDI_INDEX_MAP) {
|
|
const patterns = [
|
|
new RegExp(`Baltic ${cfg.label} Index \\(${cfg.id}\\)[^.]*?(?:reach|to|at)\\s+([\\d,]+)\\s*points`, 'i'),
|
|
new RegExp(`${cfg.id}[^.]*?(?:reach|to|at)\\s+([\\d,]+)\\s*points`, 'i'),
|
|
new RegExp(`Baltic ${cfg.label} Index \\(${cfg.id}\\)[^.]*?([\\d,]+)\\s*points`, 'i'),
|
|
];
|
|
let currentValue = null;
|
|
for (const re of patterns) {
|
|
const m = html.match(re);
|
|
if (m) { currentValue = parseFloat(m[1].replace(/,/g, '')); break; }
|
|
}
|
|
if (currentValue == null || !Number.isFinite(currentValue)) continue;
|
|
|
|
let changePct = 0;
|
|
let previousValue = currentValue;
|
|
const deltaRe = new RegExp(`${cfg.id}\\)?[^.]*?(increased|decreased|gained|lost|dropped|rose)\\s+by\\s+([\\d,]+)\\s+points`, 'i');
|
|
const deltaMatch = html.match(deltaRe);
|
|
if (deltaMatch) {
|
|
const delta = parseFloat(deltaMatch[2].replace(/,/g, ''));
|
|
const isNeg = /decreased|lost|dropped/i.test(deltaMatch[1]);
|
|
const signedDelta = isNeg ? -delta : delta;
|
|
previousValue = currentValue - signedDelta;
|
|
changePct = previousValue !== 0 ? (signedDelta / previousValue) * 100 : 0;
|
|
}
|
|
|
|
indices.push({
|
|
indexId: cfg.id, name: cfg.name, currentValue, previousValue,
|
|
changePct, unit: 'index', history: [], spikeAlert: false, _observationDate: articleDate,
|
|
});
|
|
}
|
|
console.log(` BDI: ${indices.length} indices parsed`);
|
|
return indices;
|
|
} catch (e) {
|
|
console.warn(` BDI: ${e.message}`);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
// ─── History accumulation (inline in canonical payload) ───
|
|
|
|
function accumulateHistory(newIndices, previousPayload) {
|
|
if (!previousPayload?.indices?.length) {
|
|
for (const idx of newIndices) delete idx._observationDate;
|
|
return newIndices;
|
|
}
|
|
const prevMap = new Map();
|
|
for (const idx of previousPayload.indices) {
|
|
if (idx.indexId) prevMap.set(idx.indexId, idx);
|
|
}
|
|
const fallbackDate = new Date().toISOString().slice(0, 10);
|
|
for (const idx of newIndices) {
|
|
const prev = prevMap.get(idx.indexId);
|
|
const existingHistory = prev?.history ?? [];
|
|
if (idx.history?.length > 0) { delete idx._observationDate; continue; }
|
|
const obsDate = idx._observationDate || fallbackDate;
|
|
const last = existingHistory[existingHistory.length - 1];
|
|
const newHistory = [...existingHistory];
|
|
if (!last || last.date !== obsDate) {
|
|
newHistory.push({ date: obsDate, value: idx.currentValue });
|
|
}
|
|
idx.history = newHistory.slice(-24);
|
|
delete idx._observationDate;
|
|
}
|
|
return newIndices;
|
|
}
|
|
|
|
// ─── WTO helpers ───
|
|
|
|
async function wtoFetch(path, params) {
|
|
const apiKey = process.env.WTO_API_KEY;
|
|
if (!apiKey) { console.warn('[WTO] WTO_API_KEY not set'); return null; }
|
|
const url = new URL(`https://api.wto.org/timeseries/v1${path}`);
|
|
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
|
|
const resp = await fetch(url.toString(), {
|
|
headers: { 'Ocp-Apim-Subscription-Key': apiKey, 'User-Agent': CHROME_UA },
|
|
signal: AbortSignal.timeout(15_000),
|
|
});
|
|
if (resp.status === 204) return { Dataset: [] };
|
|
if (!resp.ok) { console.warn(`[WTO] HTTP ${resp.status} for ${path}`); return null; }
|
|
return resp.json();
|
|
}
|
|
|
|
// ─── Trade Flows (WTO) — pre-seed major reporters vs World + key bilateral pairs ───
|
|
|
|
const BILATERAL_PAIRS = [
|
|
['840', '156'], // US ↔ China
|
|
['840', '276'], // US ↔ Germany
|
|
['840', '392'], // US ↔ Japan
|
|
['840', '124'], // US ↔ Canada
|
|
['840', '484'], // US ↔ Mexico
|
|
['156', '840'], // China ↔ US
|
|
['156', '276'], // China ↔ Germany
|
|
['826', '156'], // UK ↔ China
|
|
['000', '156'], // World ↔ China
|
|
['000', '840'], // World ↔ US
|
|
];
|
|
|
|
function parseFlowRows(data, indicator) {
|
|
const dataset = Array.isArray(data) ? data : data?.Dataset ?? data?.dataset ?? [];
|
|
return dataset.map(row => {
|
|
const year = parseInt(row.Year ?? row.year ?? '', 10);
|
|
const value = parseFloat(row.Value ?? row.value ?? '');
|
|
return !Number.isNaN(year) && !Number.isNaN(value) ? { year, indicator, value } : null;
|
|
}).filter(Boolean);
|
|
}
|
|
|
|
function buildFlowRecords(rows, reporterCode, partnerCode) {
|
|
const byYear = new Map();
|
|
for (const row of rows) {
|
|
if (!byYear.has(row.year)) byYear.set(row.year, { exports: 0, imports: 0 });
|
|
const e = byYear.get(row.year);
|
|
if (row.indicator === 'ITS_MTV_AX') e.exports = row.value; else e.imports = row.value;
|
|
}
|
|
const sortedYears = [...byYear.keys()].sort((a, b) => a - b);
|
|
return sortedYears.map((year, i) => {
|
|
const cur = byYear.get(year);
|
|
const prev = i > 0 ? byYear.get(sortedYears[i - 1]) : null;
|
|
return {
|
|
reportingCountry: WTO_MEMBER_CODES[reporterCode] ?? reporterCode,
|
|
partnerCountry: partnerCode === '000' ? 'World' : (WTO_MEMBER_CODES[partnerCode] ?? partnerCode),
|
|
year, exportValueUsd: cur.exports, importValueUsd: cur.imports,
|
|
yoyExportChange: prev?.exports > 0 ? Math.round(((cur.exports - prev.exports) / prev.exports) * 10000) / 100 : 0,
|
|
yoyImportChange: prev?.imports > 0 ? Math.round(((cur.imports - prev.imports) / prev.imports) * 10000) / 100 : 0,
|
|
productSector: 'Total merchandise',
|
|
};
|
|
});
|
|
}
|
|
|
|
async function fetchFlowPair(reporter, partner, years, flows) {
|
|
const currentYear = new Date().getFullYear();
|
|
const startYear = currentYear - years;
|
|
const base = { r: reporter, p: partner, ps: `${startYear}-${currentYear}`, pc: 'TO', fmt: 'json', mode: 'full', max: '500' };
|
|
const [exportsResult, importsResult] = await Promise.allSettled([
|
|
wtoFetch('/data', { ...base, i: 'ITS_MTV_AX' }),
|
|
wtoFetch('/data', { ...base, i: 'ITS_MTV_AM' }),
|
|
]);
|
|
const exportsData = exportsResult.status === 'fulfilled' ? exportsResult.value : null;
|
|
const importsData = importsResult.status === 'fulfilled' ? importsResult.value : null;
|
|
const rows = [...(exportsData ? parseFlowRows(exportsData, 'ITS_MTV_AX') : []), ...(importsData ? parseFlowRows(importsData, 'ITS_MTV_AM') : [])];
|
|
const records = buildFlowRecords(rows, reporter, partner);
|
|
const cacheKey = `trade:flows:v1:${reporter}:${partner}:${years}`;
|
|
if (records.length > 0) {
|
|
flows[cacheKey] = { flows: records, fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
|
|
}
|
|
}
|
|
|
|
async function fetchTradeFlows() {
|
|
const flows = {};
|
|
const years = 10;
|
|
|
|
for (const reporter of MAJOR_REPORTERS) {
|
|
await fetchFlowPair(reporter, '000', years, flows);
|
|
await sleep(500);
|
|
}
|
|
|
|
for (const [reporter, partner] of BILATERAL_PAIRS) {
|
|
await fetchFlowPair(reporter, partner, years, flows);
|
|
await sleep(500);
|
|
}
|
|
|
|
console.log(` Trade flows: ${Object.keys(flows).length} pairs (${MAJOR_REPORTERS.length} world + ${BILATERAL_PAIRS.length} bilateral)`);
|
|
return flows;
|
|
}
|
|
|
|
// ─── Trade Barriers (WTO) ───
|
|
|
|
async function fetchTradeBarriers() {
|
|
const currentYear = new Date().getFullYear();
|
|
const reporters = MAJOR_REPORTERS.join(',');
|
|
|
|
const [agriResult, nonAgriResult] = await Promise.allSettled([
|
|
wtoFetch('/data', { i: 'TP_A_0160', r: reporters, ps: `${currentYear - 3}-${currentYear}`, fmt: 'json', mode: 'full', max: '500' }),
|
|
wtoFetch('/data', { i: 'TP_A_0430', r: reporters, ps: `${currentYear - 3}-${currentYear}`, fmt: 'json', mode: 'full', max: '500' }),
|
|
]);
|
|
const agriData = agriResult.status === 'fulfilled' ? agriResult.value : null;
|
|
const nonAgriData = nonAgriResult.status === 'fulfilled' ? nonAgriResult.value : null;
|
|
if (!agriData && !nonAgriData) return null;
|
|
|
|
const parseRows = (data) => {
|
|
const dataset = Array.isArray(data) ? data : data?.Dataset ?? data?.dataset ?? [];
|
|
return dataset.map(row => {
|
|
const year = parseInt(row.Year ?? row.year ?? '0', 10);
|
|
const value = parseFloat(row.Value ?? row.value ?? '');
|
|
const cc = String(row.ReportingEconomyCode ?? '');
|
|
return !Number.isNaN(year) && !Number.isNaN(value) && cc ? { country: WTO_MEMBER_CODES[cc] ?? '', countryCode: cc, year, value } : null;
|
|
}).filter(Boolean);
|
|
};
|
|
|
|
const latestByCountry = (rows) => {
|
|
const m = new Map();
|
|
for (const r of rows) { const e = m.get(r.countryCode); if (!e || r.year > e.year) m.set(r.countryCode, r); }
|
|
return m;
|
|
};
|
|
|
|
const latestAgri = latestByCountry(agriData ? parseRows(agriData) : []);
|
|
const latestNonAgri = latestByCountry(nonAgriData ? parseRows(nonAgriData) : []);
|
|
const allCodes = new Set([...latestAgri.keys(), ...latestNonAgri.keys()]);
|
|
|
|
const barriers = [];
|
|
for (const code of allCodes) {
|
|
const agri = latestAgri.get(code);
|
|
const nonAgri = latestNonAgri.get(code);
|
|
if (!agri && !nonAgri) continue;
|
|
const agriRate = agri?.value ?? 0;
|
|
const nonAgriRate = nonAgri?.value ?? 0;
|
|
const gap = agriRate - nonAgriRate;
|
|
const country = agri?.country ?? nonAgri?.country ?? code;
|
|
const year = String(agri?.year ?? nonAgri?.year ?? '');
|
|
barriers.push({
|
|
id: `${code}-tariff-gap-${year}`, notifyingCountry: country,
|
|
title: `Agricultural tariff: ${agriRate.toFixed(1)}% vs Non-agricultural: ${nonAgriRate.toFixed(1)}% (gap: ${gap > 0 ? '+' : ''}${gap.toFixed(1)}pp)`,
|
|
measureType: gap > 10 ? 'High agricultural protection' : gap > 5 ? 'Moderate agricultural protection' : 'Low tariff gap',
|
|
productDescription: 'Agricultural vs Non-agricultural products',
|
|
objective: gap > 0 ? 'Agricultural sector protection' : 'Uniform tariff structure',
|
|
status: gap > 10 ? 'high' : gap > 5 ? 'moderate' : 'low',
|
|
dateDistributed: year, sourceUrl: 'https://stats.wto.org',
|
|
});
|
|
}
|
|
barriers.sort((a, b) => {
|
|
const gapA = parseFloat(a.title.match(/gap: ([+-]?\d+\.?\d*)/)?.[1] ?? '0');
|
|
const gapB = parseFloat(b.title.match(/gap: ([+-]?\d+\.?\d*)/)?.[1] ?? '0');
|
|
return gapB - gapA;
|
|
});
|
|
console.log(` Trade barriers: ${barriers.length} countries`);
|
|
return { barriers: barriers.slice(0, 50), fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
|
|
}
|
|
|
|
// ─── Trade Restrictions (WTO) ───
|
|
|
|
async function fetchTradeRestrictions() {
|
|
const currentYear = new Date().getFullYear();
|
|
const data = await wtoFetch('/data', {
|
|
i: 'TP_A_0010', r: MAJOR_REPORTERS.join(','),
|
|
ps: `${currentYear - 3}-${currentYear}`, fmt: 'json', mode: 'full', max: '500',
|
|
});
|
|
if (!data) return null;
|
|
|
|
const dataset = Array.isArray(data) ? data : data?.Dataset ?? data?.dataset ?? [];
|
|
const latestByCountry = new Map();
|
|
for (const row of dataset) {
|
|
const code = String(row.ReportingEconomyCode ?? '');
|
|
const year = parseInt(row.Year ?? row.year ?? '0', 10);
|
|
const existing = latestByCountry.get(code);
|
|
if (!existing || year > parseInt(existing.Year ?? existing.year ?? '0', 10)) latestByCountry.set(code, row);
|
|
}
|
|
|
|
const restrictions = [...latestByCountry.values()].map(row => {
|
|
const value = parseFloat(row.Value ?? row.value ?? '');
|
|
if (Number.isNaN(value)) return null;
|
|
const cc = String(row.ReportingEconomyCode ?? '');
|
|
const year = String(row.Year ?? row.year ?? '');
|
|
return {
|
|
id: `${cc}-${year}-${row.IndicatorCode ?? ''}`,
|
|
reportingCountry: WTO_MEMBER_CODES[cc] ?? String(row.ReportingEconomy ?? ''),
|
|
affectedCountry: 'All trading partners', productSector: 'All products',
|
|
measureType: 'MFN Applied Tariff', description: `Average tariff rate: ${value.toFixed(1)}%`,
|
|
status: value > 10 ? 'high' : value > 5 ? 'moderate' : 'low',
|
|
notifiedAt: year, sourceUrl: 'https://stats.wto.org',
|
|
};
|
|
}).filter(Boolean).sort((a, b) => {
|
|
const rateA = parseFloat(a.description.match(/[\d.]+/)?.[0] ?? '0');
|
|
const rateB = parseFloat(b.description.match(/[\d.]+/)?.[0] ?? '0');
|
|
return rateB - rateA;
|
|
}).slice(0, 50);
|
|
|
|
console.log(` Trade restrictions: ${restrictions.length} countries`);
|
|
return { restrictions, fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
|
|
}
|
|
|
|
// ─── Tariff Trends (WTO) — pre-seed major reporters ───
|
|
|
|
async function fetchTariffTrends() {
|
|
const currentYear = new Date().getFullYear();
|
|
const trends = {};
|
|
|
|
for (const reporter of MAJOR_REPORTERS) {
|
|
const years = 10;
|
|
const data = await wtoFetch('/data', {
|
|
i: 'TP_A_0010', r: reporter,
|
|
ps: `${currentYear - years}-${currentYear}`, fmt: 'json', mode: 'full', max: '500',
|
|
});
|
|
if (!data) { await sleep(500); continue; }
|
|
const dataset = Array.isArray(data) ? data : data?.Dataset ?? data?.dataset ?? [];
|
|
const datapoints = dataset.map(row => {
|
|
const year = parseInt(row.Year ?? row.year ?? '', 10);
|
|
const tariffRate = parseFloat(row.Value ?? row.value ?? '');
|
|
if (Number.isNaN(year) || Number.isNaN(tariffRate)) return null;
|
|
return {
|
|
reportingCountry: WTO_MEMBER_CODES[reporter] ?? reporter,
|
|
partnerCountry: 'World', productSector: 'All products',
|
|
year, tariffRate: Math.round(tariffRate * 100) / 100,
|
|
boundRate: 0, indicatorCode: 'TP_A_0010',
|
|
};
|
|
}).filter(Boolean).sort((a, b) => a.year - b.year);
|
|
|
|
if (datapoints.length > 0) {
|
|
const cacheKey = `trade:tariffs:v1:${reporter}:all:${years}`;
|
|
trends[cacheKey] = { datapoints, fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
|
|
}
|
|
await sleep(500);
|
|
}
|
|
console.log(` Tariff trends: ${Object.keys(trends).length} countries`);
|
|
return trends;
|
|
}
|
|
|
|
// ─── US Treasury Customs Revenue ───
|
|
|
|
const TREASURY_MTS_URL = 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v1/accounting/mts/mts_table_9';
|
|
|
|
async function fetchCustomsRevenue() {
|
|
const threeYearsAgo = `${new Date().getFullYear() - 3}-01-01`;
|
|
const fields = 'record_date,current_month_rcpt_outly_amt,current_fytd_rcpt_outly_amt,record_fiscal_year,record_calendar_year,record_calendar_month';
|
|
const url = `${TREASURY_MTS_URL}?fields=${fields}&filter=classification_desc:eq:Customs%20Duties,record_date:gte:${threeYearsAgo}&sort=-record_date&page[size]=50`;
|
|
|
|
const resp = await fetch(url, {
|
|
headers: { 'User-Agent': CHROME_UA, Accept: 'application/json' },
|
|
signal: AbortSignal.timeout(15_000),
|
|
});
|
|
if (!resp.ok) throw new Error(`Treasury MTS HTTP ${resp.status}`);
|
|
const json = await resp.json();
|
|
const rows = json.data;
|
|
if (!Array.isArray(rows) || rows.length === 0) throw new Error('Treasury MTS returned no data');
|
|
if (rows.length > 100) throw new Error(`Treasury MTS returned unexpected row count: ${rows.length}`);
|
|
|
|
const months = rows
|
|
.map(r => {
|
|
const monthly = parseFloat(r.current_month_rcpt_outly_amt);
|
|
const fytd = parseFloat(r.current_fytd_rcpt_outly_amt);
|
|
if (!Number.isFinite(monthly) || !Number.isFinite(fytd)) return null;
|
|
return {
|
|
recordDate: r.record_date,
|
|
fiscalYear: parseInt(r.record_fiscal_year, 10) || 0,
|
|
calendarYear: parseInt(r.record_calendar_year, 10) || 0,
|
|
calendarMonth: parseInt(r.record_calendar_month, 10) || 0,
|
|
monthlyAmountBillions: Math.round((monthly / 1e9) * 100) / 100,
|
|
fytdAmountBillions: Math.round((fytd / 1e9) * 100) / 100,
|
|
};
|
|
})
|
|
.filter(Boolean)
|
|
.reverse();
|
|
|
|
console.log(` Treasury customs revenue: ${months.length} months (${months[0]?.recordDate} to ${months[months.length - 1]?.recordDate})`);
|
|
return { months, fetchedAt: new Date().toISOString(), upstreamUnavailable: false };
|
|
}
|
|
|
|
// ─── Main ───
|
|
|
|
async function fetchAll() {
|
|
const [shipping, scfi, ccfi, bdi, barriers, restrictions, flows, tariffs, customs] = await Promise.allSettled([
|
|
fetchShippingRates(),
|
|
fetchSCFI(),
|
|
fetchCCFI(),
|
|
fetchBDI(),
|
|
fetchTradeBarriers(),
|
|
fetchTradeRestrictions(),
|
|
fetchTradeFlows(),
|
|
fetchTariffTrends(),
|
|
fetchCustomsRevenue(),
|
|
]);
|
|
|
|
const sh = shipping.status === 'fulfilled' ? shipping.value : null;
|
|
const scfiResult = scfi.status === 'fulfilled' ? scfi.value : [];
|
|
const ccfiResult = ccfi.status === 'fulfilled' ? ccfi.value : [];
|
|
const bdiResult = bdi.status === 'fulfilled' ? bdi.value : [];
|
|
const ba = barriers.status === 'fulfilled' ? barriers.value : null;
|
|
const re = restrictions.status === 'fulfilled' ? restrictions.value : null;
|
|
const fl = flows.status === 'fulfilled' ? flows.value : null;
|
|
const ta = tariffs.status === 'fulfilled' ? tariffs.value : null;
|
|
const cu = customs.status === 'fulfilled' ? customs.value : null;
|
|
|
|
if (shipping.status === 'rejected') console.warn(` Shipping failed: ${shipping.reason?.message || shipping.reason}`);
|
|
if (scfi.status === 'rejected') console.warn(` SCFI failed: ${scfi.reason?.message || scfi.reason}`);
|
|
if (ccfi.status === 'rejected') console.warn(` CCFI failed: ${ccfi.reason?.message || ccfi.reason}`);
|
|
if (bdi.status === 'rejected') console.warn(` BDI failed: ${bdi.reason?.message || bdi.reason}`);
|
|
if (barriers.status === 'rejected') console.warn(` Barriers failed: ${barriers.reason?.message || barriers.reason}`);
|
|
if (restrictions.status === 'rejected') console.warn(` Restrictions failed: ${restrictions.reason?.message || restrictions.reason}`);
|
|
if (flows.status === 'rejected') console.warn(` Flows failed: ${flows.reason?.message || flows.reason}`);
|
|
if (tariffs.status === 'rejected') console.warn(` Tariffs failed: ${tariffs.reason?.message || tariffs.reason}`);
|
|
if (customs.status === 'rejected') console.warn(` Treasury customs failed: ${customs.reason?.message || customs.reason}`);
|
|
|
|
const allIndices = [
|
|
...(sh?.indices || []),
|
|
...scfiResult,
|
|
...ccfiResult,
|
|
...bdiResult,
|
|
];
|
|
|
|
if (allIndices.length === 0 && !ba && !re) throw new Error('All supply-chain/trade fetches failed');
|
|
|
|
// History accumulation: read previous payload, merge history
|
|
let previousPayload = null;
|
|
try { previousPayload = await verifySeedKey(KEYS.shipping); } catch { /* ignore */ }
|
|
const mergedIndices = accumulateHistory(allIndices, previousPayload);
|
|
console.log(` Merged shipping indices: ${mergedIndices.length} (FRED: ${sh?.indices?.length ?? 0}, SCFI: ${scfiResult.length}, CCFI: ${ccfiResult.length}, BDI: ${bdiResult.length})`);
|
|
|
|
// Write secondary keys BEFORE returning (runSeed calls process.exit after primary write)
|
|
if (ba) await writeExtraKeyWithMeta(KEYS.barriers, ba, TRADE_TTL, ba.barriers?.length ?? 0);
|
|
if (re) await writeExtraKeyWithMeta(KEYS.restrictions, re, TRADE_TTL, re.restrictions?.length ?? 0);
|
|
if (fl) { for (const [key, data] of Object.entries(fl)) await writeExtraKeyWithMeta(key, data, TRADE_TTL, data.flows?.length ?? 0); }
|
|
if (ta) { for (const [key, data] of Object.entries(ta)) await writeExtraKeyWithMeta(key, data, TRADE_TTL, data.datapoints?.length ?? 0); }
|
|
if (cu) await writeExtraKeyWithMeta(KEYS.customsRevenue, cu, TRADE_TTL, cu.months?.length ?? 0);
|
|
|
|
return mergedIndices.length > 0
|
|
? { indices: mergedIndices, fetchedAt: new Date().toISOString(), upstreamUnavailable: false }
|
|
: { indices: [], fetchedAt: new Date().toISOString(), upstreamUnavailable: true };
|
|
}
|
|
|
|
function validate(data) {
|
|
return data?.indices?.length > 0;
|
|
}
|
|
|
|
runSeed('supply_chain', 'shipping', KEYS.shipping, fetchAll, {
|
|
validateFn: validate,
|
|
ttlSeconds: SHIPPING_TTL,
|
|
sourceVersion: 'fred-wto-sse-bdi',
|
|
}).catch((err) => {
|
|
const _cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + _cause);
|
|
process.exit(1);
|
|
});
|