mirror of
https://github.com/koala73/worldmonitor.git
synced 2026-04-25 17:14:57 +02:00
feat(gold): SPDR GLD physical holdings flows (PR B) (#3037)
* feat(gold): SPDR GLD physical holdings flows (PR B) * fix(gold): strip UTF-8 BOM from SPDR CSV header (greptile P2 #3037)
This commit is contained in:
@@ -170,6 +170,7 @@ const STANDALONE_KEYS = {
|
||||
recoveryImportHhi: 'resilience:recovery:import-hhi:v1',
|
||||
recoveryFuelStocks: 'resilience:recovery:fuel-stocks:v1',
|
||||
goldExtended: 'market:gold-extended:v1',
|
||||
goldEtfFlows: 'market:gold-etf-flows:v1',
|
||||
};
|
||||
|
||||
const SEED_META = {
|
||||
@@ -197,6 +198,7 @@ const SEED_META = {
|
||||
marketQuotes: { key: 'seed-meta:market:stocks', maxStaleMin: 30 },
|
||||
commodityQuotes: { key: 'seed-meta:market:commodities', maxStaleMin: 30 },
|
||||
goldExtended: { key: 'seed-meta:market:gold-extended', maxStaleMin: 30 },
|
||||
goldEtfFlows: { key: 'seed-meta:market:gold-etf-flows', maxStaleMin: 2880 }, // SPDR publishes daily; 2× = 48h tolerance
|
||||
// RPC/warm-ping keys — seed-meta written by relay loops or handlers
|
||||
// serviceStatuses: moved to ON_DEMAND — RPC-populated, no dedicated seed, goes stale when no users visit
|
||||
cableHealth: { key: 'seed-meta:cable-health', maxStaleMin: 90 }, // ais-relay warm-ping runs every 30min; 90min = 3× interval catches missed pings without false positives
|
||||
|
||||
@@ -31,6 +31,7 @@ const SEED_DOMAINS = {
|
||||
'market:stocks': { key: 'seed-meta:market:stocks', intervalMin: 15 },
|
||||
'market:commodities': { key: 'seed-meta:market:commodities', intervalMin: 15 },
|
||||
'market:gold-extended': { key: 'seed-meta:market:gold-extended', intervalMin: 15 },
|
||||
'market:gold-etf-flows': { key: 'seed-meta:market:gold-etf-flows', intervalMin: 1440 },
|
||||
'market:sectors': { key: 'seed-meta:market:sectors', intervalMin: 15 },
|
||||
'aviation:faa': { key: 'seed-meta:aviation:faa', intervalMin: 45 },
|
||||
'news:insights': { key: 'seed-meta:news:insights', intervalMin: 15 },
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1949,6 +1949,8 @@ components:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/GoldDriver'
|
||||
etfFlows:
|
||||
$ref: '#/components/schemas/GoldEtfFlows'
|
||||
GoldCrossCurrencyPrice:
|
||||
type: object
|
||||
properties:
|
||||
@@ -2046,3 +2048,40 @@ components:
|
||||
correlation30d:
|
||||
type: number
|
||||
format: double
|
||||
GoldEtfFlows:
|
||||
type: object
|
||||
properties:
|
||||
asOfDate:
|
||||
type: string
|
||||
tonnes:
|
||||
type: number
|
||||
format: double
|
||||
aumUsd:
|
||||
type: number
|
||||
format: double
|
||||
nav:
|
||||
type: number
|
||||
format: double
|
||||
changeW1Tonnes:
|
||||
type: number
|
||||
format: double
|
||||
changeM1Tonnes:
|
||||
type: number
|
||||
format: double
|
||||
changeY1Tonnes:
|
||||
type: number
|
||||
format: double
|
||||
changeW1Pct:
|
||||
type: number
|
||||
format: double
|
||||
changeM1Pct:
|
||||
type: number
|
||||
format: double
|
||||
changeY1Pct:
|
||||
type: number
|
||||
format: double
|
||||
sparkline90d:
|
||||
type: array
|
||||
items:
|
||||
type: number
|
||||
format: double
|
||||
|
||||
@@ -53,6 +53,20 @@ message GoldDriver {
|
||||
double correlation_30d = 5;
|
||||
}
|
||||
|
||||
message GoldEtfFlows {
|
||||
string as_of_date = 1;
|
||||
double tonnes = 2;
|
||||
double aum_usd = 3;
|
||||
double nav = 4;
|
||||
double change_w1_tonnes = 5;
|
||||
double change_m1_tonnes = 6;
|
||||
double change_y1_tonnes = 7;
|
||||
double change_w1_pct = 8;
|
||||
double change_m1_pct = 9;
|
||||
double change_y1_pct = 10;
|
||||
repeated double sparkline_90d = 11;
|
||||
}
|
||||
|
||||
message GetGoldIntelligenceRequest {}
|
||||
|
||||
message GetGoldIntelligenceResponse {
|
||||
@@ -72,4 +86,5 @@ message GetGoldIntelligenceResponse {
|
||||
GoldReturns returns = 14;
|
||||
GoldRange52w range_52w = 15;
|
||||
repeated GoldDriver drivers = 16;
|
||||
GoldEtfFlows etf_flows = 17;
|
||||
}
|
||||
|
||||
@@ -7,4 +7,6 @@ await runBundle('market-backup', [
|
||||
{ label: 'ETF-Flows', script: 'seed-etf-flows.mjs', seedMetaKey: 'market:etf-flows', intervalMs: 15 * MIN, timeoutMs: 120_000 },
|
||||
{ label: 'Gulf-Quotes', script: 'seed-gulf-quotes.mjs', seedMetaKey: 'market:gulf-quotes', intervalMs: 10 * MIN, timeoutMs: 120_000 },
|
||||
{ label: 'Token-Panels', script: 'seed-token-panels.mjs', seedMetaKey: 'market:token-panels', intervalMs: 30 * MIN, timeoutMs: 120_000 },
|
||||
// SPDR GLD publishes holdings once daily (~16:30 ET). 2h cadence = retries on Cloudflare blocks + catches late publish.
|
||||
{ label: 'Gold-ETF-Flows', script: 'seed-gold-etf-flows.mjs', seedMetaKey: 'market:gold-etf-flows', intervalMs: 120 * MIN, timeoutMs: 60_000 },
|
||||
]);
|
||||
|
||||
151
scripts/seed-gold-etf-flows.mjs
Normal file
151
scripts/seed-gold-etf-flows.mjs
Normal file
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { loadEnvFile, CHROME_UA, runSeed } from './_seed-utils.mjs';
|
||||
loadEnvFile(import.meta.url);
|
||||
|
||||
const GLD_KEY = 'market:gold-etf-flows:v1';
|
||||
const GLD_TTL = 86400;
|
||||
const GLD_URL = 'https://www.spdrgoldshares.com/assets/dynamic/GLD/GLD_US_archive_EN.csv';
|
||||
|
||||
// SPDR publishes a daily CSV of GLD holdings. Columns observed (header order):
|
||||
// Date, Gold (oz), Total Net Assets, NAV, Shares Outstanding
|
||||
// Some archive versions use tonnes directly. We parse defensively — either Gold
|
||||
// column is converted to tonnes on the way out (1 tonne = 32,150.7 troy oz).
|
||||
const TROY_OZ_PER_TONNE = 32_150.7;
|
||||
|
||||
function parseCsv(text) {
|
||||
const lines = text.split(/\r?\n/).filter(l => l.trim().length > 0);
|
||||
if (lines.length < 2) return { header: [], rows: [] };
|
||||
const splitLine = (l) => {
|
||||
// Handles simple CSV with optional double-quoted cells containing commas.
|
||||
const out = [];
|
||||
let cur = '';
|
||||
let inQuote = false;
|
||||
for (let i = 0; i < l.length; i++) {
|
||||
const ch = l[i];
|
||||
if (ch === '"') { inQuote = !inQuote; continue; }
|
||||
if (ch === ',' && !inQuote) { out.push(cur.trim()); cur = ''; continue; }
|
||||
cur += ch;
|
||||
}
|
||||
out.push(cur.trim());
|
||||
return out;
|
||||
};
|
||||
// Strip UTF-8 BOM from first header cell — SPDR's CSV has been observed
|
||||
// both with and without one; without this, findCol('date') silently returns
|
||||
// -1 and the outer 30-row guard throws a misleading "format may have changed".
|
||||
const header = splitLine(lines[0]).map(h => h.trim().toLowerCase().replace(/^\ufeff/, ''));
|
||||
const rows = lines.slice(1).map(splitLine);
|
||||
return { header, rows };
|
||||
}
|
||||
|
||||
function toNum(s) {
|
||||
if (!s) return NaN;
|
||||
const n = parseFloat(String(s).replace(/[,$"]/g, ''));
|
||||
return Number.isFinite(n) ? n : NaN;
|
||||
}
|
||||
|
||||
function parseIsoDate(s) {
|
||||
// SPDR typically writes "DD-MMM-YY" (e.g. 10-Apr-26) or "M/D/YYYY". Normalize.
|
||||
if (!s) return '';
|
||||
const raw = String(s).trim();
|
||||
if (/^\d{4}-\d{2}-\d{2}/.test(raw)) return raw.slice(0, 10);
|
||||
const m1 = raw.match(/^(\d{1,2})[-/](\w{3})[-/](\d{2,4})$/);
|
||||
if (m1) {
|
||||
const [, d, mon, y] = m1;
|
||||
const months = { jan: 1, feb: 2, mar: 3, apr: 4, may: 5, jun: 6, jul: 7, aug: 8, sep: 9, oct: 10, nov: 11, dec: 12 };
|
||||
const mm = months[mon.toLowerCase()];
|
||||
if (!mm) return '';
|
||||
const yyyy = y.length === 2 ? (parseInt(y, 10) >= 50 ? `19${y}` : `20${y}`) : y;
|
||||
return `${yyyy}-${String(mm).padStart(2, '0')}-${String(parseInt(d, 10)).padStart(2, '0')}`;
|
||||
}
|
||||
const m2 = raw.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
|
||||
if (m2) {
|
||||
const [, mm, dd, yyyy] = m2;
|
||||
return `${yyyy}-${mm.padStart(2, '0')}-${dd.padStart(2, '0')}`;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
export function parseGldArchive(csvText) {
|
||||
const { header, rows } = parseCsv(csvText);
|
||||
if (!header.length || !rows.length) return [];
|
||||
|
||||
const findCol = (...candidates) => {
|
||||
for (const c of candidates) {
|
||||
const idx = header.findIndex(h => h === c || h.startsWith(c));
|
||||
if (idx !== -1) return idx;
|
||||
}
|
||||
return -1;
|
||||
};
|
||||
const idxDate = findCol('date');
|
||||
const idxOz = findCol('gold troy oz', 'gold (oz)', 'gold oz', 'ounces');
|
||||
const idxTonnes = findCol('gold (tonnes)', 'gold tonnes', 'tonnes', 'metric tonnes');
|
||||
const idxAum = findCol('total net assets', 'net assets', 'aum');
|
||||
const idxNav = findCol('nav', 'price per share', 'share price');
|
||||
if (idxDate === -1 || (idxOz === -1 && idxTonnes === -1)) return [];
|
||||
|
||||
const out = [];
|
||||
for (const r of rows) {
|
||||
const date = parseIsoDate(r[idxDate]);
|
||||
if (!date) continue;
|
||||
let tonnes = NaN;
|
||||
if (idxTonnes !== -1) tonnes = toNum(r[idxTonnes]);
|
||||
if (!Number.isFinite(tonnes) && idxOz !== -1) {
|
||||
const oz = toNum(r[idxOz]);
|
||||
if (Number.isFinite(oz) && oz > 0) tonnes = oz / TROY_OZ_PER_TONNE;
|
||||
}
|
||||
if (!Number.isFinite(tonnes) || tonnes <= 0) continue;
|
||||
const aum = idxAum !== -1 ? toNum(r[idxAum]) : NaN;
|
||||
const nav = idxNav !== -1 ? toNum(r[idxNav]) : NaN;
|
||||
out.push({ date, tonnes, aum: Number.isFinite(aum) ? aum : 0, nav: Number.isFinite(nav) ? nav : 0 });
|
||||
}
|
||||
// Sort ascending by date so index arithmetic for deltas is obvious.
|
||||
out.sort((a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0));
|
||||
return out;
|
||||
}
|
||||
|
||||
export function computeFlows(history) {
|
||||
if (!history.length) return null;
|
||||
const latest = history[history.length - 1];
|
||||
const byAgo = (days) => history[Math.max(0, history.length - 1 - days)];
|
||||
const w1 = byAgo(5);
|
||||
const m1 = byAgo(21);
|
||||
const y1 = byAgo(252);
|
||||
const pct = (from, to) => from > 0 ? ((to - from) / from) * 100 : 0;
|
||||
const spark = history.slice(-90).map(p => p.tonnes);
|
||||
return {
|
||||
asOfDate: latest.date,
|
||||
tonnes: +latest.tonnes.toFixed(2),
|
||||
aumUsd: +latest.aum.toFixed(0),
|
||||
nav: +latest.nav.toFixed(2),
|
||||
changeW1Tonnes: +(latest.tonnes - w1.tonnes).toFixed(2),
|
||||
changeM1Tonnes: +(latest.tonnes - m1.tonnes).toFixed(2),
|
||||
changeY1Tonnes: +(latest.tonnes - y1.tonnes).toFixed(2),
|
||||
changeW1Pct: +pct(w1.tonnes, latest.tonnes).toFixed(2),
|
||||
changeM1Pct: +pct(m1.tonnes, latest.tonnes).toFixed(2),
|
||||
changeY1Pct: +pct(y1.tonnes, latest.tonnes).toFixed(2),
|
||||
sparkline90d: spark,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchGldFlows() {
|
||||
const resp = await fetch(GLD_URL, {
|
||||
headers: { 'User-Agent': CHROME_UA, Accept: 'text/csv,text/plain,*/*' },
|
||||
signal: AbortSignal.timeout(20_000),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`SPDR GLD archive HTTP ${resp.status}`);
|
||||
const text = await resp.text();
|
||||
const history = parseGldArchive(text);
|
||||
if (history.length < 30) throw new Error(`Parsed only ${history.length} rows — SPDR format may have changed`);
|
||||
const flows = computeFlows(history);
|
||||
if (!flows) throw new Error('flows computation returned null');
|
||||
return { updatedAt: new Date().toISOString(), ...flows };
|
||||
}
|
||||
|
||||
if (process.argv[1]?.endsWith('seed-gold-etf-flows.mjs')) {
|
||||
runSeed('market', 'gold-etf-flows', GLD_KEY, fetchGldFlows, {
|
||||
ttlSeconds: GLD_TTL,
|
||||
validateFn: data => Number.isFinite(data?.tonnes) && data.tonnes > 0,
|
||||
recordCount: () => 1,
|
||||
}).catch(err => { console.error('FATAL:', err.message || err); process.exit(1); });
|
||||
}
|
||||
@@ -9,12 +9,14 @@ import type {
|
||||
GoldReturns,
|
||||
GoldRange52w,
|
||||
GoldDriver,
|
||||
GoldEtfFlows,
|
||||
} from '../../../../src/generated/server/worldmonitor/market/v1/service_server';
|
||||
import { getCachedJson } from '../../../_shared/redis';
|
||||
|
||||
const COMMODITY_KEY = 'market:commodities-bootstrap:v1';
|
||||
const COT_KEY = 'market:cot:v1';
|
||||
const GOLD_EXTENDED_KEY = 'market:gold-extended:v1';
|
||||
const GOLD_ETF_FLOWS_KEY = 'market:gold-etf-flows:v1';
|
||||
|
||||
interface RawQuote {
|
||||
symbol: string;
|
||||
@@ -73,6 +75,21 @@ interface GoldExtendedPayload {
|
||||
drivers?: GoldExtendedDriver[];
|
||||
}
|
||||
|
||||
interface GoldEtfFlowsPayload {
|
||||
updatedAt: string;
|
||||
asOfDate: string;
|
||||
tonnes: number;
|
||||
aumUsd: number;
|
||||
nav: number;
|
||||
changeW1Tonnes: number;
|
||||
changeM1Tonnes: number;
|
||||
changeY1Tonnes: number;
|
||||
changeW1Pct: number;
|
||||
changeM1Pct: number;
|
||||
changeY1Pct: number;
|
||||
sparkline90d: number[];
|
||||
}
|
||||
|
||||
const XAU_FX = [
|
||||
{ symbol: 'EURUSD=X', label: 'EUR', flag: '\u{1F1EA}\u{1F1FA}', multiply: false },
|
||||
{ symbol: 'GBPUSD=X', label: 'GBP', flag: '\u{1F1EC}\u{1F1E7}', multiply: false },
|
||||
@@ -154,10 +171,11 @@ export async function getGoldIntelligence(
|
||||
_req: GetGoldIntelligenceRequest,
|
||||
): Promise<GetGoldIntelligenceResponse> {
|
||||
try {
|
||||
const [rawPayload, rawCot, rawExtended] = await Promise.all([
|
||||
const [rawPayload, rawCot, rawExtended, rawEtfFlows] = await Promise.all([
|
||||
getCachedJson(COMMODITY_KEY, true) as Promise<{ quotes?: RawQuote[] } | null>,
|
||||
getCachedJson(COT_KEY, true) as Promise<{ instruments?: RawCotInstrument[]; reportDate?: string } | null>,
|
||||
getCachedJson(GOLD_EXTENDED_KEY, true) as Promise<GoldExtendedPayload | null>,
|
||||
getCachedJson(GOLD_ETF_FLOWS_KEY, true) as Promise<GoldEtfFlowsPayload | null>,
|
||||
]);
|
||||
|
||||
const rawQuotes = rawPayload?.quotes;
|
||||
@@ -208,6 +226,22 @@ export async function getGoldIntelligence(
|
||||
correlation30d: d.correlation30d,
|
||||
}));
|
||||
|
||||
const etfFlows: GoldEtfFlows | undefined = rawEtfFlows && Number.isFinite(rawEtfFlows.tonnes) && rawEtfFlows.tonnes > 0
|
||||
? {
|
||||
asOfDate: rawEtfFlows.asOfDate,
|
||||
tonnes: rawEtfFlows.tonnes,
|
||||
aumUsd: rawEtfFlows.aumUsd,
|
||||
nav: rawEtfFlows.nav,
|
||||
changeW1Tonnes: rawEtfFlows.changeW1Tonnes,
|
||||
changeM1Tonnes: rawEtfFlows.changeM1Tonnes,
|
||||
changeY1Tonnes: rawEtfFlows.changeY1Tonnes,
|
||||
changeW1Pct: rawEtfFlows.changeW1Pct,
|
||||
changeM1Pct: rawEtfFlows.changeM1Pct,
|
||||
changeY1Pct: rawEtfFlows.changeY1Pct,
|
||||
sparkline90d: rawEtfFlows.sparkline90d ?? [],
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
goldPrice,
|
||||
goldChangePct: gold?.change ?? 0,
|
||||
@@ -223,6 +257,7 @@ export async function getGoldIntelligence(
|
||||
returns,
|
||||
range52w,
|
||||
drivers,
|
||||
etfFlows,
|
||||
// updatedAt reflects the *enrichment* layer's freshness. If the extended
|
||||
// key is missing we deliberately emit empty so the panel renders "Updated —"
|
||||
// rather than a misleading "just now" stamp while session/returns/drivers
|
||||
|
||||
@@ -17,6 +17,19 @@ interface SessionRange { dayHigh: number; dayLow: number; prevClose: number }
|
||||
interface Returns { w1: number; m1: number; ytd: number; y1: number }
|
||||
interface Range52w { hi: number; lo: number; positionPct: number }
|
||||
interface Driver { symbol: string; label: string; value: number; changePct: number; correlation30d: number }
|
||||
interface EtfFlows {
|
||||
asOfDate: string;
|
||||
tonnes: number;
|
||||
aumUsd: number;
|
||||
nav: number;
|
||||
changeW1Tonnes: number;
|
||||
changeM1Tonnes: number;
|
||||
changeY1Tonnes: number;
|
||||
changeW1Pct: number;
|
||||
changeM1Pct: number;
|
||||
changeY1Pct: number;
|
||||
sparkline90d: number[];
|
||||
}
|
||||
|
||||
interface GoldIntelligenceData {
|
||||
goldPrice: number;
|
||||
@@ -33,6 +46,7 @@ interface GoldIntelligenceData {
|
||||
returns?: Returns;
|
||||
range52w?: Range52w;
|
||||
drivers: Driver[];
|
||||
etfFlows?: EtfFlows;
|
||||
updatedAt: string;
|
||||
unavailable?: boolean;
|
||||
}
|
||||
@@ -285,6 +299,42 @@ export class GoldIntelligencePanel extends Panel {
|
||||
</div>`;
|
||||
}
|
||||
|
||||
private renderEtfFlows(d: GoldIntelligenceData): string {
|
||||
const f = d.etfFlows;
|
||||
if (!f || !Number.isFinite(f.tonnes) || f.tonnes <= 0) return '';
|
||||
|
||||
const chip = (label: string, deltaT: number, deltaPct: number) => {
|
||||
const color = deltaT >= 0 ? '#2ecc71' : '#e74c3c';
|
||||
const tSign = deltaT >= 0 ? '+' : '';
|
||||
const pSign = deltaPct >= 0 ? '+' : '';
|
||||
return `<div style="flex:1;text-align:center;padding:4px;background:rgba(255,255,255,0.03);border-radius:4px">
|
||||
<div style="font-size:9px;color:var(--text-dim)">${escapeHtml(label)}</div>
|
||||
<div style="font-size:11px;font-weight:600;color:${color}">${tSign}${deltaT.toFixed(1)}t</div>
|
||||
<div style="font-size:9px;color:${color}">${pSign}${deltaPct.toFixed(2)}%</div>
|
||||
</div>`;
|
||||
};
|
||||
|
||||
const aumStr = f.aumUsd >= 1e9 ? `$${(f.aumUsd / 1e9).toFixed(1)}B` : f.aumUsd > 0 ? `$${(f.aumUsd / 1e6).toFixed(0)}M` : '--';
|
||||
const spark = f.sparkline90d.length > 1 ? miniSparkline(f.sparkline90d, f.changeM1Pct, 80, 20) : '';
|
||||
|
||||
return `<div class="energy-tape-section" style="margin-top:10px">
|
||||
<div class="energy-section-title">Physical Flows (GLD)</div>
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;gap:8px;margin-bottom:4px">
|
||||
<div>
|
||||
<span style="font-size:14px;font-weight:700">${escapeHtml(f.tonnes.toFixed(1))} <span style="font-size:10px;color:var(--text-dim);font-weight:500">tonnes</span></span>
|
||||
<span style="font-size:10px;color:var(--text-dim);margin-left:6px">AUM ${escapeHtml(aumStr)}${f.nav > 0 ? ` • NAV $${f.nav.toFixed(2)}` : ''}</span>
|
||||
</div>
|
||||
${spark}
|
||||
</div>
|
||||
<div style="display:flex;gap:4px;margin-top:4px">
|
||||
${chip('1W', f.changeW1Tonnes, f.changeW1Pct)}
|
||||
${chip('1M', f.changeM1Tonnes, f.changeM1Pct)}
|
||||
${chip('1Y', f.changeY1Tonnes, f.changeY1Pct)}
|
||||
</div>
|
||||
<div style="font-size:9px;color:var(--text-dim);margin-top:4px;text-align:right">SPDR GLD • as of ${escapeHtml(f.asOfDate)}</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
private renderDrivers(d: GoldIntelligenceData): string {
|
||||
if (!d.drivers?.length) return '';
|
||||
const rows = d.drivers.map(dr => {
|
||||
@@ -312,6 +362,7 @@ export class GoldIntelligencePanel extends Panel {
|
||||
this.renderMetals(d),
|
||||
this.renderFx(d),
|
||||
this.renderPositioning(d),
|
||||
this.renderEtfFlows(d),
|
||||
this.renderDrivers(d),
|
||||
].join('');
|
||||
this.setContent(`<div style="padding:10px 14px">${html}</div>`);
|
||||
|
||||
@@ -523,6 +523,7 @@ export interface GetGoldIntelligenceResponse {
|
||||
returns?: GoldReturns;
|
||||
range52w?: GoldRange52w;
|
||||
drivers: GoldDriver[];
|
||||
etfFlows?: GoldEtfFlows;
|
||||
}
|
||||
|
||||
export interface GoldCrossCurrencyPrice {
|
||||
@@ -574,6 +575,20 @@ export interface GoldDriver {
|
||||
correlation30d: number;
|
||||
}
|
||||
|
||||
export interface GoldEtfFlows {
|
||||
asOfDate: string;
|
||||
tonnes: number;
|
||||
aumUsd: number;
|
||||
nav: number;
|
||||
changeW1Tonnes: number;
|
||||
changeM1Tonnes: number;
|
||||
changeY1Tonnes: number;
|
||||
changeW1Pct: number;
|
||||
changeM1Pct: number;
|
||||
changeY1Pct: number;
|
||||
sparkline90d: number[];
|
||||
}
|
||||
|
||||
export interface FieldViolation {
|
||||
field: string;
|
||||
description: string;
|
||||
|
||||
@@ -523,6 +523,7 @@ export interface GetGoldIntelligenceResponse {
|
||||
returns?: GoldReturns;
|
||||
range52w?: GoldRange52w;
|
||||
drivers: GoldDriver[];
|
||||
etfFlows?: GoldEtfFlows;
|
||||
}
|
||||
|
||||
export interface GoldCrossCurrencyPrice {
|
||||
@@ -574,6 +575,20 @@ export interface GoldDriver {
|
||||
correlation30d: number;
|
||||
}
|
||||
|
||||
export interface GoldEtfFlows {
|
||||
asOfDate: string;
|
||||
tonnes: number;
|
||||
aumUsd: number;
|
||||
nav: number;
|
||||
changeW1Tonnes: number;
|
||||
changeM1Tonnes: number;
|
||||
changeY1Tonnes: number;
|
||||
changeW1Pct: number;
|
||||
changeM1Pct: number;
|
||||
changeY1Pct: number;
|
||||
sparkline90d: number[];
|
||||
}
|
||||
|
||||
export interface FieldViolation {
|
||||
field: string;
|
||||
description: string;
|
||||
|
||||
122
tests/gold-etf-flows-seed.test.mjs
Normal file
122
tests/gold-etf-flows-seed.test.mjs
Normal file
@@ -0,0 +1,122 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { parseGldArchive, computeFlows } from '../scripts/seed-gold-etf-flows.mjs';
|
||||
|
||||
describe('seed-gold-etf-flows: parseGldArchive', () => {
|
||||
it('parses tonnes column directly when present', () => {
|
||||
const csv = `Date,Gold (Tonnes),Total Net Assets,NAV
|
||||
10-Apr-26,905.20,90500000000,78.50
|
||||
09-Apr-26,904.10,90000000000,78.20`;
|
||||
const rows = parseGldArchive(csv);
|
||||
assert.equal(rows.length, 2);
|
||||
// Sorted ascending
|
||||
assert.equal(rows[0].date, '2026-04-09');
|
||||
assert.equal(rows[1].date, '2026-04-10');
|
||||
assert.equal(rows[1].tonnes, 905.20);
|
||||
assert.equal(rows[1].aum, 90500000000);
|
||||
});
|
||||
|
||||
it('falls back to troy oz → tonnes conversion', () => {
|
||||
const csv = `Date,Gold Troy Oz,Total Net Assets,NAV
|
||||
10-Apr-26,29097063.5,90500000000,78.50`;
|
||||
const rows = parseGldArchive(csv);
|
||||
assert.equal(rows.length, 1);
|
||||
// 29,097,063.5 / 32,150.7 ≈ 905.02
|
||||
assert.ok(Math.abs(rows[0].tonnes - 905.02) < 0.1, `got ${rows[0].tonnes}`);
|
||||
});
|
||||
|
||||
it('handles M/D/YYYY date format', () => {
|
||||
const csv = `Date,Gold (Tonnes)
|
||||
4/10/2026,905.20`;
|
||||
const rows = parseGldArchive(csv);
|
||||
assert.equal(rows[0]?.date, '2026-04-10');
|
||||
});
|
||||
|
||||
it('skips rows with zero or negative tonnage', () => {
|
||||
const csv = `Date,Gold (Tonnes)
|
||||
10-Apr-26,905.20
|
||||
09-Apr-26,0
|
||||
08-Apr-26,-5`;
|
||||
const rows = parseGldArchive(csv);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].tonnes, 905.20);
|
||||
});
|
||||
|
||||
it('strips UTF-8 BOM from the first header cell', () => {
|
||||
// Regression guard (PR #3037 review): SPDR has been observed serving the
|
||||
// CSV with a leading UTF-8 BOM. Without stripping, findCol('date') would
|
||||
// return -1 and parseGldArchive silently returns [].
|
||||
const csv = `\uFEFFDate,Gold (Tonnes)
|
||||
10-Apr-26,905.20`;
|
||||
const rows = parseGldArchive(csv);
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].tonnes, 905.20);
|
||||
});
|
||||
|
||||
it('returns empty on malformed CSV', () => {
|
||||
assert.deepEqual(parseGldArchive(''), []);
|
||||
assert.deepEqual(parseGldArchive('junk\ndata'), []);
|
||||
});
|
||||
|
||||
it('strips commas and dollar signs from numeric cells', () => {
|
||||
const csv = `Date,Gold (Tonnes),Total Net Assets
|
||||
10-Apr-26,"905.20","$90,500,000,000"`;
|
||||
const rows = parseGldArchive(csv);
|
||||
assert.equal(rows[0].aum, 90500000000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('seed-gold-etf-flows: computeFlows', () => {
|
||||
// Build a 260-day synthetic history (~1 trading year + slack)
|
||||
const buildHistory = (tonnesFn) => {
|
||||
const out = [];
|
||||
const start = new Date('2025-04-15T00:00:00Z');
|
||||
for (let i = 0; i < 260; i++) {
|
||||
const d = new Date(start.getTime() + i * 86400000);
|
||||
out.push({ date: d.toISOString().slice(0, 10), tonnes: tonnesFn(i), aum: 0, nav: 0 });
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
it('returns null on empty history', () => {
|
||||
assert.equal(computeFlows([]), null);
|
||||
});
|
||||
|
||||
it('computes 1W / 1M / 1Y tonnage deltas correctly', () => {
|
||||
// Linear +1 tonne/day
|
||||
const history = buildHistory(i => 800 + i);
|
||||
const flows = computeFlows(history);
|
||||
// latest = 800 + 259 = 1059; 5d ago = 1054 → +5 tonnes; 21d ago = 1038 → +21; 252d ago = 807 → +252
|
||||
assert.equal(flows.tonnes, 1059);
|
||||
assert.equal(flows.changeW1Tonnes, 5);
|
||||
assert.equal(flows.changeM1Tonnes, 21);
|
||||
assert.equal(flows.changeY1Tonnes, 252);
|
||||
});
|
||||
|
||||
it('sparkline is last 90 days of tonnage', () => {
|
||||
const history = buildHistory(i => 800 + i);
|
||||
const flows = computeFlows(history);
|
||||
assert.equal(flows.sparkline90d.length, 90);
|
||||
assert.equal(flows.sparkline90d[0], 800 + 170); // 260 - 90 = index 170
|
||||
assert.equal(flows.sparkline90d[89], 1059);
|
||||
});
|
||||
|
||||
it('handles short histories (<252 days) without crashing', () => {
|
||||
const history = buildHistory(i => 800 + i).slice(0, 10);
|
||||
const flows = computeFlows(history);
|
||||
assert.ok(flows !== null);
|
||||
// With <5 days of prior data, changeW1 uses oldest row as baseline
|
||||
assert.ok(Number.isFinite(flows.changeW1Tonnes));
|
||||
assert.ok(Number.isFinite(flows.changeY1Tonnes));
|
||||
});
|
||||
|
||||
it('percent deltas are zero when baseline is zero', () => {
|
||||
const history = [
|
||||
{ date: '2026-04-09', tonnes: 0, aum: 0, nav: 0 },
|
||||
{ date: '2026-04-10', tonnes: 900, aum: 0, nav: 0 },
|
||||
];
|
||||
const flows = computeFlows(history);
|
||||
assert.equal(flows.changeW1Pct, 0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user