mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
eacf991812
* fix(trade): fetch Budget Lab tariff from GitHub embed + new pattern The Budget Lab page is now a Next.js SPA (no static HTML content). The report is embedded from GitHub: Budget-Lab-Yale/tariff-impact- tracker/website/html/tariff_impacts_report_drupal.html Changes: - Fetch from GitHub raw URL first (stable), fall back to original - Added "stood at X%" regex pattern which matches the current rate (11.1% post-SCOTUS) instead of the older "reaching X%" (10.6%) * fix(trade): replace Budget Lab HTML scraper with FRED API Budget Lab page is a Next.js SPA, scraping kept breaking. Now computes US effective tariff rate from official FRED data: - B235RC1Q027SBEA (customs duties, quarterly SAAR) - IEAMGSN (goods imports, quarterly SAAR) - Rate = customs / imports × 100 Uses existing fredFetchJson() infrastructure. No scraping, no fragile regex, no SPA bypass needed. Pure API. * fix(trade): build full FRED API URLs for tariff rate series fredFetchJson() expects a full URL, not a series ID. Built proper FRED API URLs with series_id, api_key, and params. Also extract .observations from FRED response structure. * fix(trade): use matching FRED import series for tariff rate IEAMGSN (Millions, NSA) was incompatible with B235RC1Q027SBEA (Billions, SAAR), producing ~0.03% instead of ~11%. Switched to A255RC1Q027SBEA (Imports of goods, Billions, Quarterly, SAAR) which matches the customs duties series exactly. Verified: 364.3B / 3217.4B × 100 = 11.3% (Q4 2025). * test(trade): update tariff test for FRED-based effective rate * fix(trade): pass _proxyAuth to FRED tariff rate calls
82 lines
2.7 KiB
JavaScript
82 lines
2.7 KiB
JavaScript
/**
|
|
* Pure parse helpers for trade-data seed scripts.
|
|
* Extracted so test files can import directly without new Function() hacks.
|
|
*/
|
|
|
|
export const BUDGET_LAB_TARIFFS_URL = 'https://budgetlab.yale.edu/research/tracking-economic-effects-tariffs';
|
|
|
|
const MONTH_MAP = {
|
|
january: '01', february: '02', march: '03', april: '04',
|
|
may: '05', june: '06', july: '07', august: '08',
|
|
september: '09', october: '10', november: '11', december: '12',
|
|
};
|
|
|
|
export function htmlToPlainText(html) {
|
|
return String(html ?? '')
|
|
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
|
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
|
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
.replace(/<\/?[A-Za-z][A-Za-z0-9:-]*(?:\s[^<>]*?)?>/g, ' ')
|
|
.replace(/ /gi, ' ')
|
|
.replace(/&/gi, '&')
|
|
.replace(/"/gi, '"')
|
|
.replace(/'/gi, '\'')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
/**
|
|
* Convert a human-readable date string like "March 2, 2026" to ISO "2026-03-02".
|
|
* Falls back to '' on failure.
|
|
*/
|
|
export function toIsoDate(value) {
|
|
const text = String(value ?? '').trim();
|
|
if (!text) return '';
|
|
if (/^\d{4}-\d{2}-\d{2}/.test(text)) return text.slice(0, 10);
|
|
const m = text.match(/^([A-Za-z]+)\s+(\d{1,2}),?\s+(\d{4})/);
|
|
if (m) {
|
|
const mm = MONTH_MAP[m[1].toLowerCase()];
|
|
if (mm) return `${m[3]}-${mm}-${m[2].padStart(2, '0')}`;
|
|
}
|
|
return '';
|
|
}
|
|
|
|
/**
|
|
* Parse the Yale Budget Lab tariff-tracking page and extract effective tariff rate.
|
|
*
|
|
* Tries three patterns in priority order:
|
|
* 1. "effective tariff rate reaching X% in [month year]"
|
|
* 2. "average effective [U.S.] tariff rate ... to X% ... in/by [month year]"
|
|
* 3. Same as 2 but no period capture
|
|
*
|
|
* Returns null when no recognisable rate is found.
|
|
*/
|
|
export function parseBudgetLabEffectiveTariffHtml(html) {
|
|
const text = htmlToPlainText(html);
|
|
if (!text) return null;
|
|
|
|
const updatedAt = toIsoDate(text.match(/\bUpdated:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})/i)?.[1] ?? '');
|
|
const patterns = [
|
|
/effective tariff rate (?:stood at|was|is)\s+(\d+(?:\.\d+)?)%/i,
|
|
/effective tariff rate reaching\s+(\d+(?:\.\d+)?)%\s+in\s+([A-Za-z]+\s+\d{4})/i,
|
|
/average effective (?:u\.s\.\s*)?tariff rate[^.]{0,180}?\bto\s+(\d+(?:\.\d+)?)%[^.]{0,180}?\b(?:in|by)\s+([A-Za-z]+\s+\d{4})/i,
|
|
/average effective (?:u\.s\.\s*)?tariff rate[^.]{0,180}?\bto\s+(\d+(?:\.\d+)?)%/i,
|
|
];
|
|
|
|
for (const pattern of patterns) {
|
|
const match = text.match(pattern);
|
|
if (!match) continue;
|
|
const tariffRate = parseFloat(match[1]);
|
|
if (!Number.isFinite(tariffRate)) continue;
|
|
return {
|
|
sourceName: 'Yale Budget Lab',
|
|
sourceUrl: BUDGET_LAB_TARIFFS_URL,
|
|
observationPeriod: match[2] ?? '',
|
|
updatedAt,
|
|
tariffRate: Math.round(tariffRate * 100) / 100,
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|