From 45c98284da5b32013f48c1b71bb5b0f5ac028b95 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Tue, 14 Apr 2026 09:05:50 +0400 Subject: [PATCH] fix(trade): correct UN Comtrade reporter codes for India and Taiwan (#3081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(trade): correct UN Comtrade reporter codes for India and Taiwan seed-trade-flows was fetching India (356) and Taiwan (158) using UN M49 codes. UN Comtrade registers India as 699 and Taiwan as 490 ("Other Asia, nes"), so every fetch silently returned count:0 — 10 of 30 reporter×commodity pairs yielded zero records per run. Live probe confirms 699→500 India rows, 490→159 Taiwan rows. - Update reporter codes in seed-trade-flows.mjs and its consumer list-comtrade-flows.ts. - Update ISO2_TO_COMTRADE in _comtrade-reporters.ts and seed-energy-spine.mjs so energy-shock and sector-dependency RPCs resolve the correct Comtrade keys for IN/TW. - Add IN/TW overrides to seed-comtrade-bilateral-hs4 and seed-recovery-import-hhi (they iterate shared/un-to-iso2.json which must remain pure M49 for other callers). - Fix partner-dedupe bug in seed-trade-flows: the preview endpoint returns partner-level rows; keying by (flowCode, year) without summing kept only the last partner seen, so tradeValueUsd was a random counterparty's value, not the World aggregate. Sum across partners and label as World. - Add a 70% coverage floor on reporter×commodity pairs so an entire reporter silently flatlining now throws in Phase 1 (TTL extend, no seed-meta refresh) rather than publishing a partial snapshot. - Sync energy-shock test fixture. * fix(trade): apply Comtrade IN/TW overrides to runtime consumers too Follow-up to PR review: the seed-side fix was incomplete because two request-time consumers still mapped iso2 → M49 (356/158) when hitting Comtrade or reading the (now-rekeyed) seeded cache. - server/worldmonitor/supply-chain/v1/_bilateral-hs4-lazy.ts: apply the IN=699 / TW=490 override when deriving ISO2_TO_UN, so the lazy bilateral-hs4 fetch path used by get-route-impact and get-country-chokepoint-index stops silently returning count:0 for India and Taiwan when the seeded cache is cold. - src/utils/country-codes.ts: add iso2ToComtradeReporterCode helper with the override baked in. Keep iso2ToUnCode as pure M49 (used elsewhere for legitimate M49 semantics). - src/app/country-intel.ts: switch the listComtradeFlows call on the country brief page to the new helper so IN/TW resolve to the same reporter codes the seeder now writes under. --- scripts/seed-comtrade-bilateral-hs4.mjs | 2 ++ scripts/seed-energy-spine.mjs | 4 +-- scripts/seed-recovery-import-hhi.mjs | 7 ++++ scripts/seed-trade-flows.mjs | 33 ++++++++++++++----- .../intelligence/v1/_comtrade-reporters.ts | 4 +-- .../supply-chain/v1/_bilateral-hs4-lazy.ts | 7 ++++ .../trade/v1/list-comtrade-flows.ts | 2 +- src/app/country-intel.ts | 4 +-- src/utils/country-codes.ts | 9 +++++ tests/energy-shock-seed.test.mts | 2 +- 10 files changed, 58 insertions(+), 16 deletions(-) diff --git a/scripts/seed-comtrade-bilateral-hs4.mjs b/scripts/seed-comtrade-bilateral-hs4.mjs index 7a2ff0a3c..3c29442b6 100644 --- a/scripts/seed-comtrade-bilateral-hs4.mjs +++ b/scripts/seed-comtrade-bilateral-hs4.mjs @@ -85,6 +85,8 @@ const COMTRADE_REPORTER_OVERRIDES = { FR: '251', // UN M49 standard is 250, but Comtrade registers France as reporter 251 IT: '381', // UN M49 standard is 380, but Comtrade registers Italy as reporter 381 US: '842', // UN M49 standard is 840, but Comtrade registers the US as reporter 842 + IN: '699', // UN M49 standard is 356, Comtrade registers India as reporter 699 + TW: '490', // M49 has no entry; Comtrade reports Taiwan as 490 "Other Asia, nes" }; /** diff --git a/scripts/seed-energy-spine.mjs b/scripts/seed-energy-spine.mjs index 880002ccd..b0d28fd47 100644 --- a/scripts/seed-energy-spine.mjs +++ b/scripts/seed-energy-spine.mjs @@ -30,8 +30,8 @@ const ISO2_TO_COMTRADE = { CN: '156', RU: '643', IR: '364', - IN: '356', - TW: '158', + IN: '699', + TW: '490', }; // Chokepoints supported by the shock model for comtrade-mapped countries. diff --git a/scripts/seed-recovery-import-hhi.mjs b/scripts/seed-recovery-import-hhi.mjs index 65b1bf22f..7a67ad970 100644 --- a/scripts/seed-recovery-import-hhi.mjs +++ b/scripts/seed-recovery-import-hhi.mjs @@ -39,9 +39,16 @@ if (COMTRADE_KEYS.length === 0) { const COMTRADE_URL = 'https://comtradeapi.un.org/data/v1/get/C/A/HS'; const PER_KEY_DELAY_MS = 600; +// UN M49 codes mostly match UN Comtrade reporterCodes, except for India (699, +// not 356) and Taiwan (490 "Other Asia, nes", not 158). Using M49 codes for +// these silently returns count:0 from the Comtrade API. +const COMTRADE_REPORTER_OVERRIDES = { IN: '699', TW: '490' }; const ISO2_TO_UN = Object.fromEntries( Object.entries(UN_TO_ISO2).map(([un, iso2]) => [iso2, un]), ); +for (const [iso2, code] of Object.entries(COMTRADE_REPORTER_OVERRIDES)) { + ISO2_TO_UN[iso2] = code; +} const ALL_REPORTERS = Object.values(UN_TO_ISO2).filter(c => c.length === 2); diff --git a/scripts/seed-trade-flows.mjs b/scripts/seed-trade-flows.mjs index 6b6b0c410..8e4299223 100644 --- a/scripts/seed-trade-flows.mjs +++ b/scripts/seed-trade-flows.mjs @@ -12,6 +12,10 @@ const KEY_PREFIX = 'comtrade:flows'; const COMTRADE_BASE = 'https://comtradeapi.un.org/public/v1'; const INTER_REQUEST_DELAY_MS = 3_000; const ANOMALY_THRESHOLD = 0.30; // 30% YoY change +// Require at least this fraction of (reporter × commodity) pairs to return +// non-empty flows. Guards against an entire reporter silently flatlining +// (e.g., wrong reporterCode → HTTP 200 with count:0 for every commodity). +const MIN_COVERAGE_RATIO = 0.70; // Strategic reporters: US, China, Russia, Iran, India, Taiwan const REPORTERS = [ @@ -19,8 +23,8 @@ const REPORTERS = [ { code: '156', name: 'China' }, { code: '643', name: 'Russia' }, { code: '364', name: 'Iran' }, - { code: '356', name: 'India' }, - { code: '158', name: 'Taiwan' }, + { code: '699', name: 'India' }, + { code: '490', name: 'Taiwan' }, ]; // Strategic HS commodity codes @@ -50,9 +54,10 @@ async function fetchFlows(reporter, commodity) { const records = data?.data ?? []; if (!Array.isArray(records)) return []; - // Group by (flowCode, year) to keep exports and imports separate. - // Using just year as key caused the second flow direction (M after X) to - // silently overwrite the first, producing wrong YoY values. + // The preview endpoint returns partner-level rows (one per counterparty). + // Aggregate to World totals per (flowCode, year) by summing, so YoY is + // computed against full-year totals. Keying on (flowCode, year) without + // summing would silently drop every partner except the last one seen. const byFlowYear = new Map(); // key: `${flowCode}:${year}` for (const r of records) { const year = Number(r.period ?? r.refYear ?? r.refMonth?.slice(0, 4) ?? 0); @@ -60,10 +65,14 @@ async function fetchFlows(reporter, commodity) { const flowCode = String(r.flowCode ?? r.rgDesc ?? 'X'); const val = Number(r.primaryValue ?? r.cifvalue ?? r.fobvalue ?? 0); const wt = Number(r.netWgt ?? 0); - const partnerCode = String(r.partnerCode ?? r.partner2Code ?? '000').padStart(3, '0'); - const partnerName = String(r.partnerDesc ?? r.partner2Desc ?? 'World'); const mapKey = `${flowCode}:${year}`; - byFlowYear.set(mapKey, { year, flowCode, val, wt, partnerCode, partnerName }); + const prev = byFlowYear.get(mapKey); + if (prev) { + prev.val += val; + prev.wt += wt; + } else { + byFlowYear.set(mapKey, { year, flowCode, val, wt, partnerCode: '000', partnerName: 'World' }); + } } // Derive the set of (flowCode, year) pairs sorted for YoY lookup. @@ -121,6 +130,14 @@ async function fetchAllFlows() { } } + const total = REPORTERS.length * COMMODITIES.length; + const populated = Object.values(perKeyFlows).filter((v) => (v.flows?.length ?? 0) > 0).length; + const coverage = populated / total; + console.log(` Coverage: ${populated}/${total} (${(coverage * 100).toFixed(0)}%) reporter×commodity pairs populated`); + if (coverage < MIN_COVERAGE_RATIO) { + throw new Error(`coverage ${populated}/${total} below floor ${MIN_COVERAGE_RATIO}; refusing to publish partial snapshot`); + } + return { flows: allFlows, perKeyFlows, fetchedAt: new Date().toISOString() }; } diff --git a/server/worldmonitor/intelligence/v1/_comtrade-reporters.ts b/server/worldmonitor/intelligence/v1/_comtrade-reporters.ts index 26b5301b0..ab364c2b0 100644 --- a/server/worldmonitor/intelligence/v1/_comtrade-reporters.ts +++ b/server/worldmonitor/intelligence/v1/_comtrade-reporters.ts @@ -75,7 +75,7 @@ export const ISO2_TO_COMTRADE: Record = { ID: '360', IE: '372', IL: '376', - IN: '356', + IN: '699', IQ: '368', IR: '364', IS: '352', @@ -174,7 +174,7 @@ export const ISO2_TO_COMTRADE: Record = { TR: '792', TT: '780', TV: '798', - TW: '158', + TW: '490', TZ: '834', UA: '804', UG: '800', diff --git a/server/worldmonitor/supply-chain/v1/_bilateral-hs4-lazy.ts b/server/worldmonitor/supply-chain/v1/_bilateral-hs4-lazy.ts index 33681a9e5..1671763fa 100644 --- a/server/worldmonitor/supply-chain/v1/_bilateral-hs4-lazy.ts +++ b/server/worldmonitor/supply-chain/v1/_bilateral-hs4-lazy.ts @@ -43,9 +43,16 @@ const HS4_LABELS: Record = { '8704': 'Commercial Vehicles', '8708': 'Auto Parts', }; +// UN M49 mostly matches UN Comtrade reporterCodes, except India (699, not 356) +// and Taiwan (490 "Other Asia, nes", not 158). Using M49 codes silently yields +// count:0 from the Comtrade API for these two countries. +const COMTRADE_REPORTER_OVERRIDES: Record = { IN: '699', TW: '490' }; const ISO2_TO_UN: Record = Object.fromEntries( Object.entries(UN_TO_ISO2 as Record).map(([un, iso]) => [iso, un]), ); +for (const [iso2, code] of Object.entries(COMTRADE_REPORTER_OVERRIDES)) { + ISO2_TO_UN[iso2] = code; +} let fetchInFlight = false; diff --git a/server/worldmonitor/trade/v1/list-comtrade-flows.ts b/server/worldmonitor/trade/v1/list-comtrade-flows.ts index dd7909edd..f4b0c809e 100644 --- a/server/worldmonitor/trade/v1/list-comtrade-flows.ts +++ b/server/worldmonitor/trade/v1/list-comtrade-flows.ts @@ -10,7 +10,7 @@ import { isCallerPremium } from '../../../_shared/premium-check'; const KEY_PREFIX = 'comtrade:flows'; // Strategic reporters and commodities mirrored from the seed script. -const REPORTERS = ['842', '156', '643', '364', '356', '158']; +const REPORTERS = ['842', '156', '643', '364', '699', '490']; const CMD_CODES = ['2709', '2711', '7108', '8542', '9301']; function isValidCode(c: string): boolean { diff --git a/src/app/country-intel.ts b/src/app/country-intel.ts index fddc03a02..75d1fc66e 100644 --- a/src/app/country-intel.ts +++ b/src/app/country-intel.ts @@ -44,7 +44,7 @@ import type { StrategicPosturePanel } from '@/components/StrategicPosturePanel'; import type { NewsItem } from '@/types'; import { getNearbyInfrastructure } from '@/services/related-assets'; import { toFlagEmoji } from '@/utils/country-flag'; -import { iso2ToIso3, iso2ToUnCode } from '@/utils/country-codes'; +import { iso2ToIso3, iso2ToComtradeReporterCode } from '@/utils/country-codes'; import { buildDependencyGraph } from '@/services/infrastructure-cascade'; import { getActiveFrameworkForPanel, subscribeFrameworkChange } from '@/services/analysis-framework-store'; import { fetchMultiSectorExposure, fetchCountryProducts, fetchMultiSectorCostShock } from '@/services/supply-chain'; @@ -610,7 +610,7 @@ export class CountryIntelManager implements AppModule { if (this.ctx.countryBriefPage?.getCode() === code) this.ctx.countryBriefPage.updateSanctionsPressure?.(null); }); - const unCode = iso2ToUnCode(code); + const unCode = iso2ToComtradeReporterCode(code); if (unCode) { tradeClient.listComtradeFlows({ reporterCode: unCode, cmdCode: '', anomaliesOnly: false }).then(resp => { if (this.ctx.countryBriefPage?.getCode() !== code) return; diff --git a/src/utils/country-codes.ts b/src/utils/country-codes.ts index 26aa5aef8..6e001e0d7 100644 --- a/src/utils/country-codes.ts +++ b/src/utils/country-codes.ts @@ -71,3 +71,12 @@ export function iso2ToIso3(iso2: string): string | null { export function iso2ToUnCode(iso2: string): string | null { return ISO2_TO_UN[iso2.toUpperCase()] ?? null; } + +// UN M49 codes mostly match UN Comtrade reporterCodes except India (699, not +// 356) and Taiwan (490 "Other Asia, nes", not 158). Use this helper wherever +// an iso2 feeds into a Comtrade reporterCode lookup or seeded comtrade:* key. +const ISO2_TO_COMTRADE_OVERRIDES: Record = { IN: '699', TW: '490' }; +export function iso2ToComtradeReporterCode(iso2: string): string | null { + const upper = iso2.toUpperCase(); + return ISO2_TO_COMTRADE_OVERRIDES[upper] ?? ISO2_TO_UN[upper] ?? null; +} diff --git a/tests/energy-shock-seed.test.mts b/tests/energy-shock-seed.test.mts index 47f84582a..bc536841a 100644 --- a/tests/energy-shock-seed.test.mts +++ b/tests/energy-shock-seed.test.mts @@ -79,7 +79,7 @@ describe('energy shock scenario computation', () => { it('returns hasData=false when country has no Comtrade data (no numeric code mapping)', () => { const ISO2_TO_COMTRADE: Record = { - US: '842', CN: '156', RU: '643', IR: '364', IN: '356', TW: '158', + US: '842', CN: '156', RU: '643', IR: '364', IN: '699', TW: '490', }; const unsupportedCountries = ['DE', 'FR', 'JP', 'KR', 'BR', 'SA']; for (const code of unsupportedCountries) {