diff --git a/api/bootstrap.js b/api/bootstrap.js index f2d8b17fa..d2f6fb315 100644 --- a/api/bootstrap.js +++ b/api/bootstrap.js @@ -56,6 +56,10 @@ const BOOTSTRAP_CACHE_KEYS = { securityAdvisories: 'intelligence:advisories-bootstrap:v1', customsRevenue: 'trade:customs-revenue:v1', sanctionsPressure: 'sanctions:pressure:v1', + consumerPricesOverview: 'consumer-prices:overview:ae', + consumerPricesCategories: 'consumer-prices:categories:ae:30d', + consumerPricesMovers: 'consumer-prices:movers:ae:30d', + consumerPricesSpread: 'consumer-prices:retailer-spread:ae:essentials-ae', groceryBasket: 'economic:grocery-basket:v1', bigmac: 'economic:bigmac:v1', nationalDebt: 'economic:national-debt:v1', @@ -73,6 +77,7 @@ const SLOW_KEYS = new Set([ 'securityAdvisories', 'customsRevenue', 'sanctionsPressure', + 'consumerPricesOverview', 'consumerPricesCategories', 'consumerPricesMovers', 'consumerPricesSpread', 'groceryBasket', 'bigmac', 'nationalDebt', diff --git a/api/consumer-prices/v1/[rpc].ts b/api/consumer-prices/v1/[rpc].ts new file mode 100644 index 000000000..9ef4173b2 --- /dev/null +++ b/api/consumer-prices/v1/[rpc].ts @@ -0,0 +1,9 @@ +export const config = { runtime: 'edge' }; + +import { createDomainGateway, serverOptions } from '../../../server/gateway'; +import { createConsumerPricesServiceRoutes } from '../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; +import { consumerPricesHandler } from '../../../server/worldmonitor/consumer-prices/v1/handler'; + +export default createDomainGateway( + createConsumerPricesServiceRoutes(consumerPricesHandler, serverOptions), +); diff --git a/api/health.js b/api/health.js index 049901f0a..7501f89ae 100644 --- a/api/health.js +++ b/api/health.js @@ -37,6 +37,11 @@ const BOOTSTRAP_KEYS = { customsRevenue: 'trade:customs-revenue:v1', sanctionsPressure: 'sanctions:pressure:v1', radiationWatch: 'radiation:observations:v1', + consumerPricesOverview: 'consumer-prices:overview:ae', + consumerPricesCategories: 'consumer-prices:categories:ae:30d', + consumerPricesMovers: 'consumer-prices:movers:ae:30d', + consumerPricesSpread: 'consumer-prices:retailer-spread:ae:essentials-ae', + consumerPricesFreshness: 'consumer-prices:freshness:ae', groceryBasket: 'economic:grocery-basket:v1', bigmac: 'economic:bigmac:v1', nationalDebt: 'economic:national-debt:v1', @@ -143,6 +148,11 @@ const SEED_META = { thermalEscalation: { key: 'seed-meta:thermal:escalation', maxStaleMin: 240 }, nationalDebt: { key: 'seed-meta:economic:national-debt', maxStaleMin: 10080 }, // 7 days — monthly seed tariffTrendsUs: { key: 'seed-meta:trade:tariffs:v1:840:all:10', maxStaleMin: 900 }, + consumerPricesOverview: { key: 'seed-meta:consumer-prices:overview:ae', maxStaleMin: 240 }, + consumerPricesCategories: { key: 'seed-meta:consumer-prices:categories:ae', maxStaleMin: 240 }, + consumerPricesMovers: { key: 'seed-meta:consumer-prices:movers:ae:30d', maxStaleMin: 240 }, + consumerPricesSpread: { key: 'seed-meta:consumer-prices:spread:ae', maxStaleMin: 720 }, + consumerPricesFreshness: { key: 'seed-meta:consumer-prices:freshness:ae', maxStaleMin: 30 }, }; // Standalone keys that are populated on-demand by RPC handlers (not seeds). diff --git a/consumer-prices-core/.env.example b/consumer-prices-core/.env.example new file mode 100644 index 000000000..64388cc71 --- /dev/null +++ b/consumer-prices-core/.env.example @@ -0,0 +1,33 @@ +# ─── Database ──────────────────────────────────────────────────────────────── +DATABASE_URL=postgresql://user:password@localhost:5432/consumer_prices + +# ─── Redis ──────────────────────────────────────────────────────────────────── +REDIS_URL=redis://localhost:6379 + +# ─── Acquisition Providers ──────────────────────────────────────────────────── +# Firecrawl — https://firecrawl.dev +FIRECRAWL_API_KEY= + +# Exa — https://exa.ai +EXA_API_KEY= + +# Parallel P0 +P0_API_KEY= +P0_BASE_URL=https://api.parallelai.dev/v1 + +# ─── Object Storage (optional, for raw artifact retention) ─────────────────── +ARTIFACTS_BUCKET_URL= +ARTIFACTS_BUCKET_KEY= +ARTIFACTS_BUCKET_SECRET= + +# ─── API Server ─────────────────────────────────────────────────────────────── +PORT=3400 +HOST=0.0.0.0 +LOG_LEVEL=info + +# ─── Security ───────────────────────────────────────────────────────────────── +# Shared secret between WorldMonitor seed job and this service +WORLDMONITOR_SNAPSHOT_API_KEY= + +# Allow CORS from specific origin (default: *) +CORS_ORIGIN=* diff --git a/consumer-prices-core/Dockerfile b/consumer-prices-core/Dockerfile new file mode 100644 index 000000000..31c56900a --- /dev/null +++ b/consumer-prices-core/Dockerfile @@ -0,0 +1,41 @@ +FROM node:20-slim AS base +WORKDIR /app + +# Install Playwright dependencies +RUN apt-get update && apt-get install -y \ + chromium \ + fonts-liberation \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libgdk-pixbuf2.0-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libx11-6 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxrandr2 \ + libxrender1 \ + --no-install-recommends && rm -rf /var/lib/apt/lists/* + +ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium + +# Install dependencies +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +# Build +COPY tsconfig.json ./ +COPY src ./src +COPY configs ./configs +RUN npm run build + +# Runtime +ENV NODE_ENV=production +EXPOSE 3400 +CMD ["node", "dist/api/server.js"] diff --git a/consumer-prices-core/configs/baskets/essentials_ae.yaml b/consumer-prices-core/configs/baskets/essentials_ae.yaml new file mode 100644 index 000000000..3ed839ad8 --- /dev/null +++ b/consumer-prices-core/configs/baskets/essentials_ae.yaml @@ -0,0 +1,119 @@ +basket: + slug: essentials-ae + name: Essentials Basket UAE + marketCode: ae + methodology: fixed + baseDate: "2025-01-01" + description: > + Core household essentials tracked weekly across UAE retailers. + Weighted to reflect a typical household of 4 in the UAE. + Does not represent official CPI. Tracks consumer price pressure only. + + items: + - id: eggs_12 + category: eggs + canonicalName: Eggs Fresh 12 Pack + weight: 0.12 + baseUnit: ct + substitutionGroup: eggs + minBaseQty: 10 + maxBaseQty: 15 + + - id: milk_1l + category: dairy + canonicalName: Full Fat Fresh Milk 1L + weight: 0.10 + baseUnit: ml + substitutionGroup: milk_full_fat + minBaseQty: 900 + maxBaseQty: 1100 + + - id: bread_white + category: bread + canonicalName: White Sliced Bread 600g + weight: 0.08 + baseUnit: g + substitutionGroup: bread_white + minBaseQty: 500 + maxBaseQty: 700 + + - id: rice_basmati_1kg + category: rice + canonicalName: Basmati Rice 1kg + weight: 0.10 + baseUnit: g + substitutionGroup: rice_basmati + minBaseQty: 900 + maxBaseQty: 1100 + + - id: cooking_oil_sunflower_1l + category: cooking_oil + canonicalName: Sunflower Oil 1L + weight: 0.08 + baseUnit: ml + substitutionGroup: cooking_oil_sunflower + minBaseQty: 900 + maxBaseQty: 1100 + + - id: chicken_whole_1kg + category: chicken + canonicalName: Whole Chicken Fresh 1kg + weight: 0.12 + baseUnit: g + substitutionGroup: chicken_whole + minBaseQty: 800 + maxBaseQty: 1200 + + - id: tomatoes_1kg + category: tomatoes + canonicalName: Tomatoes Fresh 1kg + weight: 0.08 + baseUnit: g + substitutionGroup: tomatoes + minBaseQty: 800 + maxBaseQty: 1200 + + - id: onions_1kg + category: onions + canonicalName: Onions 1kg + weight: 0.06 + baseUnit: g + substitutionGroup: onions + minBaseQty: 800 + maxBaseQty: 1200 + + - id: water_1_5l + category: water + canonicalName: Drinking Water 1.5L + weight: 0.08 + baseUnit: ml + substitutionGroup: water_still + minBaseQty: 1400 + maxBaseQty: 1600 + + - id: sugar_1kg + category: sugar + canonicalName: White Sugar 1kg + weight: 0.06 + baseUnit: g + substitutionGroup: sugar_white + minBaseQty: 900 + maxBaseQty: 1100 + + - id: cheese_processed_200g + category: dairy + canonicalName: Processed Cheese Slices 200g + weight: 0.06 + baseUnit: g + substitutionGroup: cheese_processed + minBaseQty: 150 + maxBaseQty: 250 + + - id: yogurt_500g + category: dairy + canonicalName: Plain Yogurt 500g + weight: 0.06 + baseUnit: g + substitutionGroup: yogurt_plain + minBaseQty: 450 + maxBaseQty: 550 diff --git a/consumer-prices-core/configs/brands/aliases.json b/consumer-prices-core/configs/brands/aliases.json new file mode 100644 index 000000000..347619fc2 --- /dev/null +++ b/consumer-prices-core/configs/brands/aliases.json @@ -0,0 +1,28 @@ +{ + "aliases": { + "Almarai": ["almarai", "al marai", "الأمراء"], + "Barakat": ["barakat", "بركات"], + "Nada": ["nada", "ندى"], + "Lactalis": ["lactel", "lactalis"], + "President": ["président", "president"], + "Lurpak": ["lurpak", "لورباك"], + "Baladna": ["baladna"], + "KDD": ["kdd", "Kuwait Dairy Company"], + "Saudia": ["saudia dairy", "saudia"], + "Al Ain": ["al ain", "العين", "al-ain"], + "Masafi": ["masafi", "مسافي"], + "Evian": ["evian"], + "Volvic": ["volvic"], + "Nestle": ["nestle", "nestlé", "نستلة"], + "Kelloggs": ["kellogg's", "kelloggs", "kellog"], + "Uncle Ben's": ["uncle ben's", "uncle bens"], + "Tilda": ["tilda"], + "Daawat": ["daawat", "dawat"], + "India Gate": ["india gate"], + "Carrefour": ["carrefour", "كارفور"], + "Lulu": ["lulu", "lulu hypermarket"], + "Nadec": ["nadec"], + "Nabil": ["nabil"], + "Americana": ["americana"] + } +} diff --git a/consumer-prices-core/configs/retailers/carrefour_ae.yaml b/consumer-prices-core/configs/retailers/carrefour_ae.yaml new file mode 100644 index 000000000..226f108f7 --- /dev/null +++ b/consumer-prices-core/configs/retailers/carrefour_ae.yaml @@ -0,0 +1,21 @@ +retailer: + slug: carrefour_ae + name: Carrefour UAE + marketCode: ae + currencyCode: AED + adapter: exa-search + baseUrl: https://www.carrefouruae.com + enabled: true + + acquisition: + provider: exa + + rateLimit: + requestsPerMinute: 20 + maxConcurrency: 2 + delayBetweenRequestsMs: 3000 + + discovery: + mode: search + maxPages: 20 + seeds: [] diff --git a/consumer-prices-core/configs/retailers/lulu_ae.yaml b/consumer-prices-core/configs/retailers/lulu_ae.yaml new file mode 100644 index 000000000..41fc1dd08 --- /dev/null +++ b/consumer-prices-core/configs/retailers/lulu_ae.yaml @@ -0,0 +1,21 @@ +retailer: + slug: lulu_ae + name: Lulu Hypermarket UAE + marketCode: ae + currencyCode: AED + adapter: exa-search + baseUrl: https://www.luluhypermarket.com + enabled: true + + acquisition: + provider: exa + + rateLimit: + requestsPerMinute: 15 + maxConcurrency: 1 + delayBetweenRequestsMs: 4000 + + discovery: + mode: search + maxPages: 20 + seeds: [] diff --git a/consumer-prices-core/configs/retailers/noon_grocery_ae.yaml b/consumer-prices-core/configs/retailers/noon_grocery_ae.yaml new file mode 100644 index 000000000..f11bfc6b7 --- /dev/null +++ b/consumer-prices-core/configs/retailers/noon_grocery_ae.yaml @@ -0,0 +1,21 @@ +retailer: + slug: noon_grocery_ae + name: Noon Grocery UAE + marketCode: ae + currencyCode: AED + adapter: exa-search + baseUrl: https://www.noon.com + enabled: true + + acquisition: + provider: exa + + rateLimit: + requestsPerMinute: 10 + maxConcurrency: 1 + delayBetweenRequestsMs: 6000 + + discovery: + mode: search + maxPages: 20 + seeds: [] diff --git a/consumer-prices-core/migrations/001_initial.sql b/consumer-prices-core/migrations/001_initial.sql new file mode 100644 index 000000000..a902e8a22 --- /dev/null +++ b/consumer-prices-core/migrations/001_initial.sql @@ -0,0 +1,195 @@ +-- Consumer Prices Core: Initial Schema +-- Run: psql $DATABASE_URL < migrations/001_initial.sql + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- ─── Retailers ──────────────────────────────────────────────────────────────── + +CREATE TABLE retailers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + slug VARCHAR(64) NOT NULL UNIQUE, + name VARCHAR(128) NOT NULL, + market_code CHAR(2) NOT NULL, + country_code CHAR(2) NOT NULL, + currency_code CHAR(3) NOT NULL, + adapter_key VARCHAR(32) NOT NULL DEFAULT 'generic', + base_url TEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE retailer_targets ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + retailer_id UUID NOT NULL REFERENCES retailers(id) ON DELETE CASCADE, + target_type VARCHAR(32) NOT NULL CHECK (target_type IN ('category_url','product_url','search_query')), + target_ref TEXT NOT NULL, + category_slug VARCHAR(64) NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + last_scraped_at TIMESTAMPTZ +); + +-- ─── Products ───────────────────────────────────────────────────────────────── + +CREATE TABLE canonical_products ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + canonical_name VARCHAR(256) NOT NULL, + brand_norm VARCHAR(128), + category VARCHAR(64) NOT NULL, + variant_norm VARCHAR(128), + size_value NUMERIC(12,4), + size_unit VARCHAR(16), + base_quantity NUMERIC(12,4), + base_unit VARCHAR(16), + active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (canonical_name, brand_norm, category, variant_norm, size_value, size_unit) +); + +CREATE TABLE retailer_products ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + retailer_id UUID NOT NULL REFERENCES retailers(id) ON DELETE CASCADE, + retailer_sku VARCHAR(128), + canonical_product_id UUID REFERENCES canonical_products(id), + source_url TEXT NOT NULL, + raw_title TEXT NOT NULL, + raw_brand TEXT, + raw_size_text TEXT, + image_url TEXT, + category_text TEXT, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + active BOOLEAN NOT NULL DEFAULT TRUE, + UNIQUE (retailer_id, source_url) +); + +CREATE INDEX idx_retailer_products_retailer ON retailer_products(retailer_id); +CREATE INDEX idx_retailer_products_canonical ON retailer_products(canonical_product_id) WHERE canonical_product_id IS NOT NULL; + +-- ─── Observations ───────────────────────────────────────────────────────────── + +CREATE TABLE scrape_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + retailer_id UUID NOT NULL REFERENCES retailers(id), + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + finished_at TIMESTAMPTZ, + status VARCHAR(16) NOT NULL DEFAULT 'running' + CHECK (status IN ('running','completed','failed','partial')), + trigger_type VARCHAR(16) NOT NULL DEFAULT 'scheduled' + CHECK (trigger_type IN ('scheduled','manual')), + pages_attempted INT NOT NULL DEFAULT 0, + pages_succeeded INT NOT NULL DEFAULT 0, + errors_count INT NOT NULL DEFAULT 0, + config_version VARCHAR(32) NOT NULL DEFAULT '1' +); + +CREATE TABLE price_observations ( + id BIGSERIAL PRIMARY KEY, + retailer_product_id UUID NOT NULL REFERENCES retailer_products(id), + scrape_run_id UUID NOT NULL REFERENCES scrape_runs(id), + observed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + price NUMERIC(12,2) NOT NULL, + list_price NUMERIC(12,2), + promo_price NUMERIC(12,2), + currency_code CHAR(3) NOT NULL, + unit_price NUMERIC(12,4), + unit_basis_qty NUMERIC(12,4), + unit_basis_unit VARCHAR(16), + in_stock BOOLEAN NOT NULL DEFAULT TRUE, + promo_text TEXT, + raw_payload_json JSONB NOT NULL DEFAULT '{}', + raw_hash VARCHAR(64) NOT NULL +); + +CREATE INDEX idx_price_obs_product_time ON price_observations(retailer_product_id, observed_at DESC); +CREATE INDEX idx_price_obs_run ON price_observations(scrape_run_id); + +-- ─── Matching ───────────────────────────────────────────────────────────────── + +CREATE TABLE product_matches ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + retailer_product_id UUID NOT NULL REFERENCES retailer_products(id), + canonical_product_id UUID NOT NULL REFERENCES canonical_products(id), + basket_item_id UUID, + match_score NUMERIC(5,2) NOT NULL, + match_status VARCHAR(16) NOT NULL DEFAULT 'review' + CHECK (match_status IN ('auto','review','approved','rejected')), + evidence_json JSONB NOT NULL DEFAULT '{}', + reviewed_by VARCHAR(64), + reviewed_at TIMESTAMPTZ, + UNIQUE (retailer_product_id, canonical_product_id) +); + +-- ─── Baskets ────────────────────────────────────────────────────────────────── + +CREATE TABLE baskets ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + slug VARCHAR(64) NOT NULL UNIQUE, + name VARCHAR(128) NOT NULL, + market_code CHAR(2) NOT NULL, + methodology VARCHAR(16) NOT NULL CHECK (methodology IN ('fixed','value')), + base_date DATE NOT NULL, + description TEXT, + active BOOLEAN NOT NULL DEFAULT TRUE +); + +CREATE TABLE basket_items ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + basket_id UUID NOT NULL REFERENCES baskets(id) ON DELETE CASCADE, + category VARCHAR(64) NOT NULL, + canonical_product_id UUID REFERENCES canonical_products(id), + substitution_group VARCHAR(64), + weight NUMERIC(5,4) NOT NULL, + qualification_rules_json JSONB, + active BOOLEAN NOT NULL DEFAULT TRUE +); + +ALTER TABLE product_matches ADD CONSTRAINT fk_pm_basket_item + FOREIGN KEY (basket_item_id) REFERENCES basket_items(id); + +-- ─── Analytics ──────────────────────────────────────────────────────────────── + +CREATE TABLE computed_indices ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + basket_id UUID NOT NULL REFERENCES baskets(id), + retailer_id UUID REFERENCES retailers(id), + category VARCHAR(64), + metric_date DATE NOT NULL, + metric_key VARCHAR(64) NOT NULL, + metric_value NUMERIC(14,4) NOT NULL, + methodology_version VARCHAR(16) NOT NULL DEFAULT '1', + UNIQUE (basket_id, retailer_id, category, metric_date, metric_key) +); + +CREATE INDEX idx_computed_indices_basket_date ON computed_indices(basket_id, metric_date DESC); + +-- ─── Operational ────────────────────────────────────────────────────────────── + +CREATE TABLE source_artifacts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + scrape_run_id UUID NOT NULL REFERENCES scrape_runs(id), + retailer_product_id UUID REFERENCES retailer_products(id), + artifact_type VARCHAR(16) NOT NULL CHECK (artifact_type IN ('html','screenshot','parsed_json')), + storage_key TEXT NOT NULL, + content_type VARCHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE data_source_health ( + retailer_id UUID PRIMARY KEY REFERENCES retailers(id), + last_successful_run_at TIMESTAMPTZ, + last_run_status VARCHAR(16), + parse_success_rate NUMERIC(5,2), + avg_freshness_minutes NUMERIC(8,2), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ─── Updated-at trigger ─────────────────────────────────────────────────────── + +CREATE OR REPLACE FUNCTION set_updated_at() +RETURNS TRIGGER LANGUAGE plpgsql AS $$ +BEGIN NEW.updated_at = NOW(); RETURN NEW; END; +$$; + +CREATE TRIGGER retailers_updated_at BEFORE UPDATE ON retailers + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/consumer-prices-core/package.json b/consumer-prices-core/package.json new file mode 100644 index 000000000..9559ba516 --- /dev/null +++ b/consumer-prices-core/package.json @@ -0,0 +1,40 @@ +{ + "name": "consumer-prices-core", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "start": "node dist/api/server.js", + "dev": "tsx watch src/api/server.ts", + "jobs:scrape": "tsx src/jobs/scrape.ts", + "jobs:aggregate": "tsx src/jobs/aggregate.ts", + "jobs:publish": "tsx src/jobs/publish.ts", + "migrate": "tsx src/db/migrate.ts", + "validate": "tsx src/cli/validate.ts", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@fastify/cors": "^9.0.1", + "dotenv": "^16.4.5", + "exa-js": "^1.7.0", + "fastify": "^4.28.1", + "js-yaml": "^4.1.0", + "jsdom": "^25.0.1", + "pg": "^8.13.1", + "pino": "^9.4.0", + "playwright": "^1.47.2", + "redis": "^4.7.0", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/js-yaml": "^4.0.9", + "@types/jsdom": "^21.1.7", + "@types/node": "^22.7.5", + "@types/pg": "^8.11.10", + "tsx": "^4.19.1", + "typescript": "^5.6.3", + "vitest": "^2.1.2" + } +} diff --git a/consumer-prices-core/src/acquisition/exa.ts b/consumer-prices-core/src/acquisition/exa.ts new file mode 100644 index 000000000..a072ff239 --- /dev/null +++ b/consumer-prices-core/src/acquisition/exa.ts @@ -0,0 +1,85 @@ +import Exa from 'exa-js'; +import type { AcquisitionProvider, ExtractResult, ExtractSchema, FetchOptions, FetchResult, SearchOptions, SearchResult } from './types.js'; + +export class ExaProvider implements AcquisitionProvider { + readonly name = 'exa' as const; + + private client: Exa; + + constructor(apiKey: string) { + this.client = new Exa(apiKey); + } + + async fetch(url: string, _opts: FetchOptions = {}): Promise { + const result = await this.client.getContents([url], { + text: { maxCharacters: 100_000 }, + highlights: { numSentences: 5, highlightsPerUrl: 3 }, + }); + + const item = result.results[0]; + if (!item) throw new Error(`Exa returned no content for ${url}`); + + return { + url, + html: item.text ?? '', + markdown: item.text ?? '', + statusCode: 200, + provider: this.name, + fetchedAt: new Date(), + metadata: { highlights: item.highlights }, + }; + } + + async search(query: string, opts: SearchOptions = {}): Promise { + const result = await this.client.search(query, { + numResults: opts.numResults ?? 10, + type: opts.type ?? 'neural', + includeDomains: opts.includeDomains, + startPublishedDate: opts.startPublishedDate, + useAutoprompt: true, + }); + + return result.results.map((r) => ({ + url: r.url, + title: r.title ?? '', + text: r.text, + highlights: r.highlights, + score: r.score, + publishedDate: r.publishedDate, + })); + } + + async extract>( + url: string, + schema: ExtractSchema, + _opts: FetchOptions = {}, + ): Promise> { + const prompt = `Extract the following fields from this product page: ${Object.entries(schema.fields) + .map(([k, v]) => `${k} (${v.type}): ${v.description}`) + .join(', ')}`; + + const result = await this.client.getContents([url], { + text: { maxCharacters: 50_000 }, + summary: { query: prompt }, + }); + + const item = result.results[0]; + if (!item) throw new Error(`Exa returned no content for ${url}`); + + return { + url, + data: (item as unknown as { summary?: T }).summary ?? ({} as T), + provider: this.name, + fetchedAt: new Date(), + }; + } + + async validate(): Promise { + try { + await this.client.search('test', { numResults: 1 }); + return true; + } catch { + return false; + } + } +} diff --git a/consumer-prices-core/src/acquisition/firecrawl.ts b/consumer-prices-core/src/acquisition/firecrawl.ts new file mode 100644 index 000000000..28be0c2d3 --- /dev/null +++ b/consumer-prices-core/src/acquisition/firecrawl.ts @@ -0,0 +1,142 @@ +import type { AcquisitionProvider, ExtractResult, ExtractSchema, FetchOptions, FetchResult, SearchOptions, SearchResult } from './types.js'; + +interface FirecrawlScrapeResponse { + success: boolean; + data?: { + html?: string; + markdown?: string; + metadata?: Record; + }; + error?: string; +} + +interface FirecrawlSearchResponse { + success: boolean; + data?: Array<{ url: string; title: string; description?: string; markdown?: string }>; +} + +interface FirecrawlExtractResponse { + success: boolean; + data?: Record; +} + +export class FirecrawlProvider implements AcquisitionProvider { + readonly name = 'firecrawl' as const; + + private readonly baseUrl = 'https://api.firecrawl.dev/v1'; + + constructor(private readonly apiKey: string) {} + + private headers() { + return { + Authorization: `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', + }; + } + + async fetch(url: string, opts: FetchOptions = {}): Promise { + const resp = await fetch(`${this.baseUrl}/scrape`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ + url, + formats: ['html', 'markdown'], + waitFor: opts.waitForSelector ? 2000 : 0, + timeout: opts.timeout ?? 30_000, + headers: opts.headers, + }), + signal: AbortSignal.timeout((opts.timeout ?? 30_000) + 5_000), + }); + + if (!resp.ok) throw new Error(`Firecrawl scrape failed: HTTP ${resp.status}`); + + const data = (await resp.json()) as FirecrawlScrapeResponse; + if (!data.success || !data.data) { + throw new Error(`Firecrawl error: ${data.error ?? 'unknown'}`); + } + + return { + url, + html: data.data.html ?? '', + markdown: data.data.markdown ?? '', + statusCode: 200, + provider: this.name, + fetchedAt: new Date(), + metadata: data.data.metadata, + }; + } + + async search(query: string, opts: SearchOptions = {}): Promise { + const resp = await fetch(`${this.baseUrl}/search`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ + query, + limit: opts.numResults ?? 10, + includeDomains: opts.includeDomains, + scrapeOptions: { formats: ['markdown'] }, + }), + }); + + if (!resp.ok) throw new Error(`Firecrawl search failed: HTTP ${resp.status}`); + + const data = (await resp.json()) as FirecrawlSearchResponse; + return (data.data ?? []).map((r) => ({ + url: r.url, + title: r.title, + text: r.description ?? r.markdown, + })); + } + + async extract>( + url: string, + schema: ExtractSchema, + opts: FetchOptions = {}, + ): Promise> { + const jsonSchema: Record = { + type: 'object', + properties: Object.fromEntries( + Object.entries(schema.fields).map(([k, v]) => [ + k, + { type: v.type, description: v.description }, + ]), + ), + }; + + const resp = await fetch(`${this.baseUrl}/scrape`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ + url, + formats: ['extract'], + extract: { schema: jsonSchema }, + timeout: opts.timeout ?? 30_000, + }), + }); + + if (!resp.ok) throw new Error(`Firecrawl extract failed: HTTP ${resp.status}`); + + const data = (await resp.json()) as FirecrawlExtractResponse; + + return { + url, + data: (data.data ?? {}) as T, + provider: this.name, + fetchedAt: new Date(), + }; + } + + async validate(): Promise { + try { + const resp = await fetch(`${this.baseUrl}/scrape`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ url: 'https://example.com', formats: ['markdown'] }), + signal: AbortSignal.timeout(10_000), + }); + return resp.ok; + } catch { + return false; + } + } +} diff --git a/consumer-prices-core/src/acquisition/p0.ts b/consumer-prices-core/src/acquisition/p0.ts new file mode 100644 index 000000000..ed71c5399 --- /dev/null +++ b/consumer-prices-core/src/acquisition/p0.ts @@ -0,0 +1,103 @@ +/** + * Parallel P0 acquisition provider. + * P0 is a high-throughput scraping API that handles JS rendering, + * anti-bot, and proxy rotation. Compatible with its REST API. + */ +import type { AcquisitionProvider, FetchOptions, FetchResult, SearchOptions, SearchResult } from './types.js'; + +interface P0ScrapeResponse { + success: boolean; + html?: string; + markdown?: string; + statusCode?: number; + error?: string; +} + +interface P0SearchResponse { + results?: Array<{ url: string; title: string; snippet?: string }>; +} + +export class P0Provider implements AcquisitionProvider { + readonly name = 'p0' as const; + + private readonly baseUrl: string; + + constructor( + private readonly apiKey: string, + baseUrl = 'https://api.parallelai.dev/v1', + ) { + this.baseUrl = baseUrl; + } + + private headers() { + return { + 'x-api-key': this.apiKey, + 'Content-Type': 'application/json', + }; + } + + async fetch(url: string, opts: FetchOptions = {}): Promise { + const resp = await fetch(`${this.baseUrl}/scrape`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ + url, + render_js: true, + wait_for: opts.waitForSelector, + timeout: Math.floor((opts.timeout ?? 30_000) / 1_000), + output_format: 'html', + premium_proxy: true, + }), + signal: AbortSignal.timeout((opts.timeout ?? 30_000) + 10_000), + }); + + if (!resp.ok) throw new Error(`P0 scrape failed: HTTP ${resp.status}`); + + const data = (await resp.json()) as P0ScrapeResponse; + if (!data.success && !data.html) { + throw new Error(`P0 error: ${data.error ?? 'no content'}`); + } + + return { + url, + html: data.html ?? '', + markdown: data.markdown, + statusCode: data.statusCode ?? 200, + provider: this.name, + fetchedAt: new Date(), + }; + } + + async search(query: string, opts: SearchOptions = {}): Promise { + const resp = await fetch(`${this.baseUrl}/search`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ + query, + num_results: opts.numResults ?? 10, + include_domains: opts.includeDomains, + }), + }); + + if (!resp.ok) throw new Error(`P0 search failed: HTTP ${resp.status}`); + + const data = (await resp.json()) as P0SearchResponse; + return (data.results ?? []).map((r) => ({ + url: r.url, + title: r.title, + text: r.snippet, + })); + } + + async validate(): Promise { + try { + const resp = await fetch(`${this.baseUrl}/health`, { + headers: this.headers(), + signal: AbortSignal.timeout(5_000), + }); + return resp.ok; + } catch { + return false; + } + } +} diff --git a/consumer-prices-core/src/acquisition/playwright.ts b/consumer-prices-core/src/acquisition/playwright.ts new file mode 100644 index 000000000..2af34f201 --- /dev/null +++ b/consumer-prices-core/src/acquisition/playwright.ts @@ -0,0 +1,73 @@ +import { chromium, type Browser, type BrowserContext } from 'playwright'; +import type { AcquisitionProvider, FetchOptions, FetchResult, SearchOptions, SearchResult } from './types.js'; + +const DEFAULT_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'; + +export class PlaywrightProvider implements AcquisitionProvider { + readonly name = 'playwright' as const; + + private browser: Browser | null = null; + private context: BrowserContext | null = null; + + private async getContext(): Promise { + if (!this.browser) { + this.browser = await chromium.launch({ headless: true }); + } + if (!this.context) { + this.context = await this.browser.newContext({ + userAgent: DEFAULT_UA, + locale: 'en-US', + viewport: { width: 1280, height: 900 }, + extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' }, + }); + } + return this.context; + } + + async fetch(url: string, opts: FetchOptions = {}): Promise { + const ctx = await this.getContext(); + const page = await ctx.newPage(); + + const timeout = opts.timeout ?? 30_000; + + try { + if (opts.headers) { + await page.setExtraHTTPHeaders(opts.headers); + } + + const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout }); + + if (opts.waitForSelector) { + await page.waitForSelector(opts.waitForSelector, { timeout: 10_000 }).catch(() => {}); + } + + const html = await page.content(); + const statusCode = response?.status() ?? 200; + + return { url, html, statusCode, provider: this.name, fetchedAt: new Date() }; + } finally { + await page.close(); + } + } + + async search(_query: string, _opts?: SearchOptions): Promise { + throw new Error('PlaywrightProvider does not support search mode. Use Exa instead.'); + } + + async validate(): Promise { + try { + await this.getContext(); + return true; + } catch { + return false; + } + } + + async teardown(): Promise { + await this.context?.close(); + await this.browser?.close(); + this.context = null; + this.browser = null; + } +} diff --git a/consumer-prices-core/src/acquisition/registry.ts b/consumer-prices-core/src/acquisition/registry.ts new file mode 100644 index 000000000..40b89de9d --- /dev/null +++ b/consumer-prices-core/src/acquisition/registry.ts @@ -0,0 +1,59 @@ +import { ExaProvider } from './exa.js'; +import { FirecrawlProvider } from './firecrawl.js'; +import { P0Provider } from './p0.js'; +import { PlaywrightProvider } from './playwright.js'; +import type { AcquisitionConfig, AcquisitionProvider, AcquisitionProviderName, FetchOptions, FetchResult } from './types.js'; + +const _providers = new Map(); + +export function initProviders(env: Record) { + _providers.set('playwright', new PlaywrightProvider()); + + if (env.EXA_API_KEY) { + _providers.set('exa', new ExaProvider(env.EXA_API_KEY)); + } + if (env.FIRECRAWL_API_KEY) { + _providers.set('firecrawl', new FirecrawlProvider(env.FIRECRAWL_API_KEY)); + } + if (env.P0_API_KEY) { + _providers.set('p0', new P0Provider(env.P0_API_KEY, env.P0_BASE_URL)); + } +} + +export function getProvider(name: AcquisitionProviderName): AcquisitionProvider { + const p = _providers.get(name); + if (!p) throw new Error(`Acquisition provider '${name}' is not configured. Set the required API key env var.`); + return p; +} + +export async function teardownAll(): Promise { + for (const p of _providers.values()) { + await p.teardown?.(); + } + _providers.clear(); +} + +/** + * Fetch a URL using the provider chain defined in config. + * Tries primary provider first; on failure, tries fallback. + */ +export async function fetchWithFallback( + url: string, + config: AcquisitionConfig, + opts?: FetchOptions, +): Promise { + const primary = getProvider(config.provider); + const mergedOpts = { ...config.options, ...opts }; + + try { + return await primary.fetch(url, mergedOpts); + } catch (err) { + if (!config.fallback) throw err; + + const msg = err instanceof Error ? err.message : String(err); + console.warn(`[acquisition] ${config.provider} failed for ${url}: ${msg}. Falling back to ${config.fallback}.`); + + const fallback = getProvider(config.fallback); + return fallback.fetch(url, mergedOpts); + } +} diff --git a/consumer-prices-core/src/acquisition/types.ts b/consumer-prices-core/src/acquisition/types.ts new file mode 100644 index 000000000..8c15c9734 --- /dev/null +++ b/consumer-prices-core/src/acquisition/types.ts @@ -0,0 +1,78 @@ +export type AcquisitionProviderName = 'playwright' | 'exa' | 'firecrawl' | 'p0'; + +export interface FetchOptions { + waitForSelector?: string; + timeout?: number; + headers?: Record; + retries?: number; + userAgent?: string; +} + +export interface SearchOptions { + numResults?: number; + includeDomains?: string[]; + startPublishedDate?: string; + type?: 'keyword' | 'neural'; +} + +export interface ExtractSchema { + fields: Record; +} + +export interface FetchResult { + url: string; + html: string; + markdown?: string; + statusCode: number; + provider: AcquisitionProviderName; + fetchedAt: Date; + metadata?: Record; +} + +export interface SearchResult { + url: string; + title: string; + text?: string; + highlights?: string[]; + score?: number; + publishedDate?: string; +} + +export interface ExtractResult> { + url: string; + data: T; + provider: AcquisitionProviderName; + fetchedAt: Date; +} + +export interface AcquisitionProvider { + readonly name: AcquisitionProviderName; + + /** Fetch a URL, returning HTML content. */ + fetch(url: string, opts?: FetchOptions): Promise; + + /** Search for pages matching a query (Exa primary, others may not support). */ + search?(query: string, opts?: SearchOptions): Promise; + + /** Extract structured data from a URL using a schema hint. */ + extract?>(url: string, schema: ExtractSchema, opts?: FetchOptions): Promise>; + + /** Validate provider is configured and reachable. */ + validate(): Promise; + + /** Clean up resources (close browser, etc.) */ + teardown?(): Promise; +} + +export interface AcquisitionConfig { + /** Primary acquisition provider. */ + provider: AcquisitionProviderName; + /** Fallback provider if primary fails. */ + fallback?: AcquisitionProviderName; + /** Provider-specific options. */ + options?: FetchOptions; + /** Use search mode instead of direct URL fetch (Exa only). */ + searchMode?: boolean; + /** Query template for search mode: use {category}, {product}, {market} tokens. */ + searchQueryTemplate?: string; +} diff --git a/consumer-prices-core/src/adapters/exa-search.ts b/consumer-prices-core/src/adapters/exa-search.ts new file mode 100644 index 000000000..8ee437a34 --- /dev/null +++ b/consumer-prices-core/src/adapters/exa-search.ts @@ -0,0 +1,218 @@ +/** + * ExaSearchAdapter — acquires prices via Exa AI neural search + summary extraction. + * Ported from scripts/seed-grocery-basket.mjs (PR #1904). + * + * Instead of fetching category pages and parsing CSS selectors, this adapter: + * 1. Discovers targets from the basket YAML config (one target per basket item) + * 2. Calls Exa with contents.summary to get AI-extracted price text from retailer pages + * 3. Uses regex to extract the price from the summary + * + * Basket → product match is written automatically (match_status: 'auto') + * because the search is item-specific — no ambiguity in what was searched. + */ +import { loadAllBasketConfigs } from '../config/loader.js'; +import type { AdapterContext, FetchResult, ParsedProduct, RetailerAdapter, Target } from './types.js'; +import type { RetailerConfig } from '../config/types.js'; + +const CHROME_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; + +const CCY = + 'USD|GBP|EUR|JPY|CNY|INR|AUD|CAD|BRL|MXN|ZAR|TRY|NGN|KRW|SGD|PKR|AED|SAR|QAR|KWD|BHD|OMR|EGP|JOD|LBP|KES|ARS|IDR|PHP'; + +const SYMBOL_MAP: Record = { + '£': 'GBP', + '€': 'EUR', + '¥': 'JPY', + '₩': 'KRW', + '₹': 'INR', + '₦': 'NGN', + 'R$': 'BRL', +}; + +const CURRENCY_MIN: Record = { + NGN: 50, + IDR: 500, + ARS: 50, + KRW: 1000, + ZAR: 2, + PKR: 20, + LBP: 1000, +}; + +const PRICE_PATTERNS = [ + new RegExp(`(\\d+(?:\\.\\d{1,3})?)\\s*(${CCY})`, 'i'), + new RegExp(`(${CCY})\\s*(\\d+(?:\\.\\d{1,3})?)`, 'i'), +]; + +function matchPrice(text: string, expectedCurrency: string): number | null { + for (const re of PRICE_PATTERNS) { + const match = text.match(re); + if (match) { + const [price, currency] = /^\d/.test(match[1]) + ? [parseFloat(match[1]), match[2].toUpperCase()] + : [parseFloat(match[2]), match[1].toUpperCase()]; + if (currency !== expectedCurrency) continue; + const minPrice = CURRENCY_MIN[currency] ?? 0; + if (price > minPrice && price < 100_000) return price; + } + } + for (const [sym, iso] of Object.entries(SYMBOL_MAP)) { + if (iso !== expectedCurrency) continue; + const re = new RegExp(`${sym.replace('$', '\\$')}\\s*(\\d+(?:[.,]\\d{1,3})?)`, 'i'); + const m = text.match(re); + if (m) { + const price = parseFloat(m[1].replace(',', '.')); + const minPrice = CURRENCY_MIN[iso] ?? 0; + if (price > minPrice && price < 100_000) return price; + } + } + return null; +} + +interface ExaResult { + url?: string; + title?: string; + summary?: string; +} + +interface SearchPayload { + exaResults: ExaResult[]; + basketSlug: string; + itemCategory: string; + canonicalName: string; +} + +export class ExaSearchAdapter implements RetailerAdapter { + readonly key = 'exa-search'; + + constructor(private readonly apiKey: string) {} + + async discoverTargets(ctx: AdapterContext): Promise { + const baskets = loadAllBasketConfigs().filter((b) => b.marketCode === ctx.config.marketCode); + const domain = new URL(ctx.config.baseUrl).hostname; + const targets: Target[] = []; + + for (const basket of baskets) { + for (const item of basket.items) { + targets.push({ + id: item.id, + url: ctx.config.baseUrl, + category: item.category, + metadata: { + canonicalName: item.canonicalName, + domain, + basketSlug: basket.slug, + currency: ctx.config.currencyCode, + }, + }); + } + } + + return targets; + } + + async fetchTarget(ctx: AdapterContext, target: Target): Promise { + if (!this.apiKey) throw new Error('EXA_API_KEY is required for exa-search adapter'); + + const { canonicalName, domain, currency, basketSlug } = target.metadata as { + canonicalName: string; + domain: string; + currency: string; + basketSlug: string; + }; + + const body = { + query: `${canonicalName} ${currency} retail price`, + numResults: 5, + type: 'auto', + includeDomains: [domain], + contents: { + summary: { + query: `What is the retail price of this product? State amount and ISO currency code (e.g. ${currency} 12.50).`, + }, + }, + }; + + const resp = await fetch('https://api.exa.ai/search', { + method: 'POST', + headers: { + 'x-api-key': this.apiKey, + 'Content-Type': 'application/json', + 'User-Agent': CHROME_UA, + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(15_000), + }); + + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + throw new Error(`Exa search failed HTTP ${resp.status}: ${text.slice(0, 120)}`); + } + + const data = (await resp.json()) as { results?: ExaResult[] }; + const payload: SearchPayload = { + exaResults: data.results ?? [], + basketSlug, + itemCategory: target.category, + canonicalName, + }; + + return { + url: target.url, + html: JSON.stringify(payload), + statusCode: 200, + fetchedAt: new Date(), + }; + } + + async parseListing(ctx: AdapterContext, result: FetchResult): Promise { + const payload = JSON.parse(result.html) as SearchPayload; + const currency = ctx.config.currencyCode; + + for (const r of payload.exaResults) { + const price = + matchPrice(r.summary ?? '', currency) ?? + matchPrice(r.title ?? '', currency); + + if (price !== null) { + return [ + { + sourceUrl: r.url ?? ctx.config.baseUrl, + rawTitle: r.title ?? payload.canonicalName, + rawBrand: null, + rawSizeText: null, + imageUrl: null, + categoryText: payload.itemCategory, + retailerSku: null, + price, + listPrice: null, + promoPrice: null, + promoText: null, + inStock: true, + rawPayload: { + exaUrl: r.url, + summary: r.summary, + basketSlug: payload.basketSlug, + itemCategory: payload.itemCategory, + canonicalName: payload.canonicalName, + }, + }, + ]; + } + } + + return []; + } + + async parseProduct(_ctx: AdapterContext, _result: FetchResult): Promise { + throw new Error('ExaSearchAdapter does not support single-product parsing'); + } + + async validateConfig(config: RetailerConfig): Promise { + const errors: string[] = []; + if (!this.apiKey) errors.push('EXA_API_KEY env var is required for adapter: exa-search'); + if (!config.baseUrl) errors.push('baseUrl is required'); + return errors; + } +} diff --git a/consumer-prices-core/src/adapters/generic.ts b/consumer-prices-core/src/adapters/generic.ts new file mode 100644 index 000000000..86264a192 --- /dev/null +++ b/consumer-prices-core/src/adapters/generic.ts @@ -0,0 +1,165 @@ +/** + * Generic config-driven adapter. + * Uses CSS selectors from the retailer YAML to extract products. + * Works with any acquisition provider (Playwright, Firecrawl, Exa, P0). + */ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore — jsdom types provided via @types/jsdom +import { JSDOM } from 'jsdom'; +import { fetchWithFallback } from '../acquisition/registry.js'; +import type { AdapterContext, FetchResult, ParsedProduct, RetailerAdapter, Target } from './types.js'; +import type { RetailerConfig } from '../config/types.js'; + +function parsePrice(text: string | null | undefined, config: RetailerConfig): number | null { + if (!text) return null; + + const fmt = config.extraction?.priceFormat; + let clean = text; + + if (fmt?.currencySymbols) { + for (const sym of fmt.currencySymbols) { + clean = clean.replace(sym, ''); + } + } + + const dec = fmt?.decimalSeparator ?? '.'; + const thou = fmt?.thousandsSeparator ?? ','; + + clean = clean.replace(new RegExp(`\\${thou}`, 'g'), '').replace(dec, '.').replace(/[^\d.]/g, '').trim(); + + const val = parseFloat(clean); + return isNaN(val) ? null : val; +} + +function selectText(doc: Document, selector: string): string | null { + if (!selector) return null; + + if (selector.includes('::attr(')) { + const [sel, attr] = selector.replace(')', '').split('::attr('); + const el = doc.querySelector(sel.trim()); + return el?.getAttribute(attr.trim()) ?? null; + } + + return doc.querySelector(selector)?.textContent?.trim() ?? null; +} + +export class GenericPlaywrightAdapter implements RetailerAdapter { + readonly key = 'generic'; + + async discoverTargets(ctx: AdapterContext): Promise { + return ctx.config.discovery.seeds.map((s) => ({ + id: s.id, + url: s.url.startsWith('http') ? s.url : `${ctx.config.baseUrl}${s.url}`, + category: s.category ?? s.id, + })); + } + + async fetchTarget(ctx: AdapterContext, target: Target): Promise { + const result = await fetchWithFallback(target.url, ctx.config.acquisition, ctx.config.rateLimit ? { + timeout: 30_000, + } : undefined); + + return { + url: result.url, + html: result.html, + markdown: result.markdown, + statusCode: result.statusCode, + fetchedAt: result.fetchedAt, + }; + } + + async parseListing(ctx: AdapterContext, result: FetchResult): Promise { + const selectors = ctx.config.extraction?.productCard; + if (!selectors) return []; + + const dom = new JSDOM(result.html); + const doc = dom.window.document; + const cards = doc.querySelectorAll(selectors.container); + + const products: ParsedProduct[] = []; + + for (const card of cards) { + try { + const rawTitle = selectText(card as unknown as Document, selectors.title) ?? ''; + if (!rawTitle) continue; + + const priceText = selectText(card as unknown as Document, selectors.price); + const price = parsePrice(priceText, ctx.config); + if (!price) continue; + + const listPriceText = selectors.listPrice + ? selectText(card as unknown as Document, selectors.listPrice) + : null; + const listPrice = parsePrice(listPriceText, ctx.config); + + const relUrl = selectText(card as unknown as Document, selectors.url) ?? ''; + const sourceUrl = relUrl.startsWith('http') ? relUrl : `${ctx.config.baseUrl}${relUrl}`; + + products.push({ + sourceUrl, + rawTitle, + rawBrand: selectors.brand ? selectText(card as unknown as Document, selectors.brand) : null, + rawSizeText: selectors.sizeText + ? selectText(card as unknown as Document, selectors.sizeText) + : null, + imageUrl: selectors.imageUrl + ? selectText(card as unknown as Document, selectors.imageUrl) + : null, + categoryText: null, + retailerSku: selectors.sku ? selectText(card as unknown as Document, selectors.sku) : null, + price, + listPrice, + promoPrice: price < (listPrice ?? price) ? price : null, + promoText: null, + inStock: true, + rawPayload: { title: rawTitle, price: priceText, url: relUrl }, + }); + } catch (err) { + ctx.logger.warn(`[generic] parse error on card: ${err}`); + } + } + + return products; + } + + async parseProduct(ctx: AdapterContext, result: FetchResult): Promise { + const selectors = ctx.config.extraction?.productPage; + const dom = new JSDOM(result.html); + const doc = dom.window.document; + + const rawTitle = selectors?.title ? (selectText(doc, selectors.title) ?? '') : ''; + const priceText = selectors?.price ? selectText(doc, selectors.price) : null; + const price = parsePrice(priceText, ctx.config) ?? 0; + + const jsonld = selectors?.jsonld ? doc.querySelector(selectors.jsonld)?.textContent : null; + let jsonldData: Record = {}; + if (jsonld) { + try { jsonldData = JSON.parse(jsonld) as Record; } catch {} + } + + return { + sourceUrl: result.url, + rawTitle: rawTitle || (jsonldData.name as string) || '', + rawBrand: (jsonldData.brand as { name?: string })?.name ?? null, + rawSizeText: null, + imageUrl: (jsonldData.image as string) ?? null, + categoryText: selectors?.categoryPath ? selectText(doc, selectors.categoryPath) : null, + retailerSku: selectors?.sku ? selectText(doc, selectors.sku) : null, + price, + listPrice: null, + promoPrice: null, + promoText: null, + inStock: true, + rawPayload: { title: rawTitle, price: priceText, jsonld: jsonldData }, + }; + } + + async validateConfig(config: RetailerConfig): Promise { + const errors: string[] = []; + if (!config.baseUrl) errors.push('baseUrl is required'); + if (!config.discovery.seeds?.length) errors.push('at least one discovery seed is required'); + if (!config.extraction?.productCard?.container) errors.push('extraction.productCard.container is required'); + return errors; + } +} + diff --git a/consumer-prices-core/src/adapters/types.ts b/consumer-prices-core/src/adapters/types.ts new file mode 100644 index 000000000..314060154 --- /dev/null +++ b/consumer-prices-core/src/adapters/types.ts @@ -0,0 +1,48 @@ +import type { RetailerConfig } from '../config/types.js'; + +export interface ParsedProduct { + sourceUrl: string; + rawTitle: string; + rawBrand: string | null; + rawSizeText: string | null; + imageUrl: string | null; + categoryText: string | null; + retailerSku: string | null; + price: number; + listPrice: number | null; + promoPrice: number | null; + promoText: string | null; + inStock: boolean; + rawPayload: Record; +} + +export interface AdapterContext { + config: RetailerConfig; + runId: string; + logger: { info: (msg: string, ...args: unknown[]) => void; warn: (msg: string, ...args: unknown[]) => void; error: (msg: string, ...args: unknown[]) => void }; +} + +export interface Target { + id: string; + url: string; + category: string; + metadata?: Record; +} + +export interface FetchResult { + url: string; + html: string; + markdown?: string; + statusCode: number; + fetchedAt: Date; +} + +export interface RetailerAdapter { + readonly key: string; + + discoverTargets(ctx: AdapterContext): Promise; + fetchTarget(ctx: AdapterContext, target: Target): Promise; + parseListing(ctx: AdapterContext, result: FetchResult): Promise; + parseProduct(ctx: AdapterContext, result: FetchResult): Promise; + validateConfig(config: RetailerConfig): Promise; +} diff --git a/consumer-prices-core/src/api/routes/health.ts b/consumer-prices-core/src/api/routes/health.ts new file mode 100644 index 000000000..706595e91 --- /dev/null +++ b/consumer-prices-core/src/api/routes/health.ts @@ -0,0 +1,22 @@ +import type { FastifyInstance } from 'fastify'; +import { getPool } from '../../db/client.js'; + +export async function healthRoutes(fastify: FastifyInstance) { + fastify.get('/', async (_request, reply) => { + const checks: Record = {}; + + try { + await getPool().query('SELECT 1'); + checks.postgres = 'ok'; + } catch { + checks.postgres = 'fail'; + } + + const healthy = Object.values(checks).every((v) => v === 'ok'); + return reply.status(healthy ? 200 : 503).send({ + status: healthy ? 'ok' : 'degraded', + checks, + timestamp: new Date().toISOString(), + }); + }); +} diff --git a/consumer-prices-core/src/api/routes/worldmonitor.ts b/consumer-prices-core/src/api/routes/worldmonitor.ts new file mode 100644 index 000000000..dfae24816 --- /dev/null +++ b/consumer-prices-core/src/api/routes/worldmonitor.ts @@ -0,0 +1,84 @@ +import type { FastifyInstance } from 'fastify'; +import { + buildBasketSeriesSnapshot, + buildCategoriesSnapshot, + buildFreshnessSnapshot, + buildMoversSnapshot, + buildOverviewSnapshot, + buildRetailerSpreadSnapshot, +} from '../../snapshots/worldmonitor.js'; + +export async function worldmonitorRoutes(fastify: FastifyInstance) { + fastify.get('/overview', async (request, reply) => { + const { market = 'ae' } = request.query as { market?: string }; + try { + const data = await buildOverviewSnapshot(market); + return reply.send(data); + } catch (err) { + fastify.log.error(err); + return reply.status(500).send({ error: 'failed to build overview snapshot' }); + } + }); + + fastify.get('/movers', async (request, reply) => { + const { market = 'ae', days = '30' } = request.query as { market?: string; days?: string }; + try { + const data = await buildMoversSnapshot(market, parseInt(days, 10)); + return reply.send(data); + } catch (err) { + fastify.log.error(err); + return reply.status(500).send({ error: 'failed to build movers snapshot' }); + } + }); + + fastify.get('/retailer-spread', async (request, reply) => { + const { market = 'ae', basket = 'essentials-ae' } = request.query as { + market?: string; + basket?: string; + }; + try { + const data = await buildRetailerSpreadSnapshot(market, basket); + return reply.send(data); + } catch (err) { + fastify.log.error(err); + return reply.status(500).send({ error: 'failed to build retailer spread snapshot' }); + } + }); + + fastify.get('/freshness', async (request, reply) => { + const { market = 'ae' } = request.query as { market?: string }; + try { + const data = await buildFreshnessSnapshot(market); + return reply.send(data); + } catch (err) { + fastify.log.error(err); + return reply.status(500).send({ error: 'failed to build freshness snapshot' }); + } + }); + + fastify.get('/categories', async (request, reply) => { + const { market = 'ae', range = '30d' } = request.query as { market?: string; range?: string }; + try { + const data = await buildCategoriesSnapshot(market, range); + return reply.send(data); + } catch (err) { + fastify.log.error(err); + return reply.status(500).send({ error: 'failed to build categories snapshot' }); + } + }); + + fastify.get('/basket-series', async (request, reply) => { + const { market = 'ae', basket = 'essentials-ae', range = '30d' } = request.query as { + market?: string; + basket?: string; + range?: string; + }; + try { + const data = await buildBasketSeriesSnapshot(market, basket, range); + return reply.send(data); + } catch (err) { + fastify.log.error(err); + return reply.status(500).send({ error: 'failed to build basket series snapshot' }); + } + }); +} diff --git a/consumer-prices-core/src/api/server.ts b/consumer-prices-core/src/api/server.ts new file mode 100644 index 000000000..728c098b1 --- /dev/null +++ b/consumer-prices-core/src/api/server.ts @@ -0,0 +1,39 @@ +import 'dotenv/config'; +import Fastify from 'fastify'; +import cors from '@fastify/cors'; +import { worldmonitorRoutes } from './routes/worldmonitor.js'; +import { healthRoutes } from './routes/health.js'; + +const server = Fastify({ logger: { level: process.env.LOG_LEVEL ?? 'info' } }); + +await server.register(cors, { + origin: process.env.CORS_ORIGIN ?? '*', + methods: ['GET'], +}); + +const API_KEY = process.env.WORLDMONITOR_SNAPSHOT_API_KEY; + +server.addHook('onRequest', async (request, reply) => { + if (request.url === '/health') return; + + if (API_KEY) { + const provided = request.headers['x-api-key']; + if (provided !== API_KEY) { + await reply.status(401).send({ error: 'unauthorized' }); + } + } +}); + +await server.register(worldmonitorRoutes, { prefix: '/wm/consumer-prices/v1' }); +await server.register(healthRoutes, { prefix: '/health' }); + +const port = parseInt(process.env.PORT ?? '3400', 10); +const host = process.env.HOST ?? '0.0.0.0'; + +try { + await server.listen({ port, host }); + console.log(`consumer-prices-core listening on ${host}:${port}`); +} catch (err) { + server.log.error(err); + process.exit(1); +} diff --git a/consumer-prices-core/src/config/loader.ts b/consumer-prices-core/src/config/loader.ts new file mode 100644 index 000000000..1d84b6472 --- /dev/null +++ b/consumer-prices-core/src/config/loader.ts @@ -0,0 +1,40 @@ +import { readFileSync, readdirSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import yaml from 'js-yaml'; +import { BasketConfigSchema, RetailerConfigSchema } from './types.js'; +import type { BasketConfig, RetailerConfig } from './types.js'; + +const CONFIG_DIR = join(dirname(fileURLToPath(import.meta.url)), '../../configs'); + +export function loadRetailerConfig(slug: string): RetailerConfig { + const filePath = join(CONFIG_DIR, 'retailers', `${slug}.yaml`); + const raw = readFileSync(filePath, 'utf8'); + const parsed = RetailerConfigSchema.parse(yaml.load(raw)); + return parsed.retailer; +} + +export function loadAllRetailerConfigs(): RetailerConfig[] { + const dir = join(CONFIG_DIR, 'retailers'); + const files = readdirSync(dir).filter((f) => f.endsWith('.yaml')); + return files.map((f) => { + const raw = readFileSync(join(dir, f), 'utf8'); + return RetailerConfigSchema.parse(yaml.load(raw)).retailer; + }); +} + +export function loadBasketConfig(slug: string): BasketConfig { + const filePath = join(CONFIG_DIR, 'baskets', `${slug}.yaml`); + const raw = readFileSync(filePath, 'utf8'); + const parsed = BasketConfigSchema.parse(yaml.load(raw)); + return parsed.basket; +} + +export function loadAllBasketConfigs(): BasketConfig[] { + const dir = join(CONFIG_DIR, 'baskets'); + const files = readdirSync(dir).filter((f) => f.endsWith('.yaml')); + return files.map((f) => { + const raw = readFileSync(join(dir, f), 'utf8'); + return BasketConfigSchema.parse(yaml.load(raw)).basket; + }); +} diff --git a/consumer-prices-core/src/config/types.ts b/consumer-prices-core/src/config/types.ts new file mode 100644 index 000000000..c5683feb1 --- /dev/null +++ b/consumer-prices-core/src/config/types.ts @@ -0,0 +1,110 @@ +import { z } from 'zod'; + +export const AcquisitionConfigSchema = z.object({ + provider: z.enum(['playwright', 'exa', 'firecrawl', 'p0']), + fallback: z.enum(['playwright', 'exa', 'firecrawl', 'p0']).optional(), + options: z + .object({ + waitForSelector: z.string().optional(), + timeout: z.number().optional(), + retries: z.number().optional(), + }) + .optional(), + searchMode: z.boolean().optional(), + searchQueryTemplate: z.string().optional(), +}); + +export const RateLimitSchema = z.object({ + requestsPerMinute: z.number().default(30), + maxConcurrency: z.number().default(2), + delayBetweenRequestsMs: z.number().default(2_000), +}); + +export const ProductCardSelectorsSchema = z.object({ + container: z.string(), + title: z.string(), + price: z.string(), + listPrice: z.string().optional(), + url: z.string(), + imageUrl: z.string().optional(), + sizeText: z.string().optional(), + inStock: z.string().optional(), + sku: z.string().optional(), + brand: z.string().optional(), +}); + +export const ProductPageSelectorsSchema = z.object({ + title: z.string(), + sku: z.string().optional(), + categoryPath: z.string().optional(), + jsonld: z.string().optional(), + price: z.string().optional(), + brand: z.string().optional(), + sizeText: z.string().optional(), +}); + +export const DiscoverySeedSchema = z.object({ + id: z.string(), + url: z.string(), + category: z.string().optional(), +}); + +export const RetailerConfigSchema = z.object({ + retailer: z.object({ + slug: z.string(), + name: z.string(), + marketCode: z.string().length(2), + currencyCode: z.string().length(3), + adapter: z.enum(['generic', 'exa-search', 'custom']).default('generic'), + baseUrl: z.string().url(), + rateLimit: RateLimitSchema.optional(), + acquisition: AcquisitionConfigSchema, + discovery: z.object({ + mode: z.enum(['category_urls', 'sitemap', 'search']).default('category_urls'), + seeds: z.array(DiscoverySeedSchema), + paginationSelector: z.string().optional(), + maxPages: z.number().default(20), + }), + extraction: z.object({ + productCard: ProductCardSelectorsSchema.optional(), + productPage: ProductPageSelectorsSchema.optional(), + priceFormat: z + .object({ + decimalSeparator: z.string().default('.'), + thousandsSeparator: z.string().default(','), + currencySymbols: z.array(z.string()).default([]), + }) + .optional(), + }), + enabled: z.boolean().default(true), + }), +}); + +export type RetailerConfig = z.infer['retailer']; + +export const BasketItemSchema = z.object({ + id: z.string(), + category: z.string(), + canonicalName: z.string(), + weight: z.number().min(0).max(1), + baseUnit: z.string(), + substitutionGroup: z.string().optional(), + minBaseQty: z.number().optional(), + maxBaseQty: z.number().optional(), + qualificationRules: z.record(z.unknown()).optional(), +}); + +export const BasketConfigSchema = z.object({ + basket: z.object({ + slug: z.string(), + name: z.string(), + marketCode: z.string().length(2), + methodology: z.enum(['fixed', 'value']), + baseDate: z.string(), + description: z.string().optional(), + items: z.array(BasketItemSchema), + }), +}); + +export type BasketConfig = z.infer['basket']; +export type BasketItem = z.infer; diff --git a/consumer-prices-core/src/db/client.ts b/consumer-prices-core/src/db/client.ts new file mode 100644 index 000000000..0602e6d7b --- /dev/null +++ b/consumer-prices-core/src/db/client.ts @@ -0,0 +1,38 @@ +import pg from 'pg'; + +const { Pool } = pg; + +let _pool: pg.Pool | null = null; + +export function getPool(): pg.Pool { + if (!_pool) { + const databaseUrl = process.env.DATABASE_URL; + if (!databaseUrl) throw new Error('DATABASE_URL is not set'); + + _pool = new Pool({ + connectionString: databaseUrl, + max: 10, + idleTimeoutMillis: 30_000, + connectionTimeoutMillis: 5_000, + ssl: databaseUrl.includes('localhost') ? false : true, + }); + + _pool.on('error', (err) => { + console.error('[db] pool error:', err.message); + }); + } + return _pool; +} + +export async function query>( + sql: string, + params?: unknown[], +): Promise> { + const pool = getPool(); + return pool.query(sql, params); +} + +export async function closePool(): Promise { + await _pool?.end(); + _pool = null; +} diff --git a/consumer-prices-core/src/db/migrate.ts b/consumer-prices-core/src/db/migrate.ts new file mode 100644 index 000000000..33f24f97d --- /dev/null +++ b/consumer-prices-core/src/db/migrate.ts @@ -0,0 +1,60 @@ +/** + * Simple forward-only migration runner. + * Run: tsx src/db/migrate.ts + */ +import { readFileSync, readdirSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import 'dotenv/config'; +import { getPool } from './client.js'; + +const MIGRATIONS_DIR = join(dirname(fileURLToPath(import.meta.url)), '../../migrations'); + +async function run() { + const pool = getPool(); + + await pool.query(` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(64) PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `); + + const applied = await pool.query<{ version: string }>(`SELECT version FROM schema_migrations ORDER BY version`); + const appliedSet = new Set(applied.rows.map((r) => r.version)); + + const files = readdirSync(MIGRATIONS_DIR) + .filter((f) => f.endsWith('.sql')) + .sort(); + + for (const file of files) { + const version = file.replace('.sql', ''); + if (appliedSet.has(version)) { + console.log(` [skip] ${file}`); + continue; + } + + const sql = readFileSync(join(MIGRATIONS_DIR, file), 'utf8'); + console.log(` [run] ${file}`); + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + await client.query(sql); + await client.query(`INSERT INTO schema_migrations (version) VALUES ($1)`, [version]); + await client.query('COMMIT'); + console.log(` [done] ${file}`); + } catch (err) { + await client.query('ROLLBACK'); + console.error(` [fail] ${file}:`, err); + process.exit(1); + } finally { + client.release(); + } + } + + await pool.end(); + console.log('Migrations complete.'); +} + +run().catch(console.error); diff --git a/consumer-prices-core/src/db/models.ts b/consumer-prices-core/src/db/models.ts new file mode 100644 index 000000000..5398dc66b --- /dev/null +++ b/consumer-prices-core/src/db/models.ts @@ -0,0 +1,116 @@ +export interface Retailer { + id: string; + slug: string; + name: string; + marketCode: string; + countryCode: string; + currencyCode: string; + adapterKey: string; + baseUrl: string; + active: boolean; + createdAt: Date; + updatedAt: Date; +} + +export interface RetailerTarget { + id: string; + retailerId: string; + targetType: 'category_url' | 'product_url' | 'search_query'; + targetRef: string; + categorySlug: string; + enabled: boolean; + lastScrapedAt: Date | null; +} + +export interface CanonicalProduct { + id: string; + canonicalName: string; + brandNorm: string | null; + category: string; + variantNorm: string | null; + sizeValue: number | null; + sizeUnit: string | null; + baseQuantity: number | null; + baseUnit: string | null; + active: boolean; + createdAt: Date; +} + +export interface RetailerProduct { + id: string; + retailerId: string; + retailerSku: string | null; + canonicalProductId: string | null; + sourceUrl: string; + rawTitle: string; + rawBrand: string | null; + rawSizeText: string | null; + imageUrl: string | null; + categoryText: string | null; + firstSeenAt: Date; + lastSeenAt: Date; + active: boolean; +} + +export interface PriceObservation { + id: string; + retailerProductId: string; + scrapeRunId: string; + observedAt: Date; + price: number; + listPrice: number | null; + promoPrice: number | null; + currencyCode: string; + unitPrice: number | null; + unitBasisQty: number | null; + unitBasisUnit: string | null; + inStock: boolean; + promoText: string | null; + rawPayloadJson: Record; + rawHash: string; +} + +export interface ScrapeRun { + id: string; + retailerId: string; + startedAt: Date; + finishedAt: Date | null; + status: 'running' | 'completed' | 'failed' | 'partial'; + triggerType: 'scheduled' | 'manual'; + pagesAttempted: number; + pagesSucceeded: number; + errorsCount: number; + configVersion: string; +} + +export interface ProductMatch { + id: string; + retailerProductId: string; + canonicalProductId: string; + basketItemId: string | null; + matchScore: number; + matchStatus: 'auto' | 'review' | 'approved' | 'rejected'; + evidenceJson: Record; + reviewedBy: string | null; + reviewedAt: Date | null; +} + +export interface ComputedIndex { + id: string; + basketId: string; + retailerId: string | null; + category: string | null; + metricDate: Date; + metricKey: string; + metricValue: number; + methodologyVersion: string; +} + +export interface DataSourceHealth { + retailerId: string; + lastSuccessfulRunAt: Date | null; + lastRunStatus: string | null; + parseSuccessRate: number | null; + avgFreshnessMinutes: number | null; + updatedAt: Date; +} diff --git a/consumer-prices-core/src/db/queries/matches.ts b/consumer-prices-core/src/db/queries/matches.ts new file mode 100644 index 000000000..ca47597c5 --- /dev/null +++ b/consumer-prices-core/src/db/queries/matches.ts @@ -0,0 +1,38 @@ +import { query } from '../client.js'; + +export async function upsertProductMatch(input: { + retailerProductId: string; + canonicalProductId: string; + basketItemId: string; + matchScore: number; + matchStatus: 'auto' | 'approved'; +}): Promise { + await query( + `INSERT INTO product_matches + (retailer_product_id, canonical_product_id, basket_item_id, match_score, match_status, evidence_json) + VALUES ($1,$2,$3,$4,$5,'{}') + ON CONFLICT (retailer_product_id, canonical_product_id) + DO UPDATE SET + basket_item_id = EXCLUDED.basket_item_id, + match_score = EXCLUDED.match_score, + match_status = EXCLUDED.match_status`, + [ + input.retailerProductId, + input.canonicalProductId, + input.basketItemId, + input.matchScore, + input.matchStatus, + ], + ); +} + +export async function getBasketItemId(basketSlug: string, category: string): Promise { + const result = await query<{ id: string }>( + `SELECT bi.id FROM basket_items bi + JOIN baskets b ON b.id = bi.basket_id + WHERE b.slug = $1 AND bi.category = $2 AND bi.active = true + LIMIT 1`, + [basketSlug, category], + ); + return result.rows[0]?.id ?? null; +} diff --git a/consumer-prices-core/src/db/queries/observations.ts b/consumer-prices-core/src/db/queries/observations.ts new file mode 100644 index 000000000..5c6e00512 --- /dev/null +++ b/consumer-prices-core/src/db/queries/observations.ts @@ -0,0 +1,91 @@ +import { createHash } from 'node:crypto'; +import { query } from '../client.js'; +import type { PriceObservation } from '../models.js'; + +export interface InsertObservationInput { + retailerProductId: string; + scrapeRunId: string; + price: number; + listPrice?: number | null; + promoPrice?: number | null; + currencyCode: string; + unitPrice?: number | null; + unitBasisQty?: number | null; + unitBasisUnit?: string | null; + inStock?: boolean; + promoText?: string | null; + rawPayloadJson: Record; +} + +export function hashPayload(payload: Record): string { + return createHash('sha256').update(JSON.stringify(payload)).digest('hex').slice(0, 64); +} + +export async function insertObservation(input: InsertObservationInput): Promise { + const rawHash = hashPayload(input.rawPayloadJson); + + const existing = await query<{ id: string }>( + `SELECT id FROM price_observations WHERE retailer_product_id = $1 AND raw_hash = $2 ORDER BY observed_at DESC LIMIT 1`, + [input.retailerProductId, rawHash], + ); + if (existing.rows.length > 0) return existing.rows[0].id; + + const result = await query<{ id: string }>( + `INSERT INTO price_observations + (retailer_product_id, scrape_run_id, observed_at, price, list_price, promo_price, + currency_code, unit_price, unit_basis_qty, unit_basis_unit, in_stock, promo_text, + raw_payload_json, raw_hash) + VALUES ($1,$2,NOW(),$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) + RETURNING id`, + [ + input.retailerProductId, + input.scrapeRunId, + input.price, + input.listPrice ?? null, + input.promoPrice ?? null, + input.currencyCode, + input.unitPrice ?? null, + input.unitBasisQty ?? null, + input.unitBasisUnit ?? null, + input.inStock ?? true, + input.promoText ?? null, + JSON.stringify(input.rawPayloadJson), + rawHash, + ], + ); + return result.rows[0].id; +} + +export async function getLatestObservations( + retailerProductIds: string[], +): Promise { + if (retailerProductIds.length === 0) return []; + + const result = await query( + `SELECT DISTINCT ON (retailer_product_id) * + FROM price_observations + WHERE retailer_product_id = ANY($1) AND in_stock = true + ORDER BY retailer_product_id, observed_at DESC`, + [retailerProductIds], + ); + return result.rows; +} + +export async function getPriceHistory( + retailerProductId: string, + daysBack: number, +): Promise> { + const result = await query<{ date: Date; price: number; unit_price: number | null }>( + `SELECT date_trunc('day', observed_at) AS date, + AVG(price)::numeric(12,2) AS price, + AVG(unit_price)::numeric(12,4) AS unit_price + FROM price_observations + WHERE retailer_product_id = $1 + AND observed_at > NOW() - ($2 || ' days')::INTERVAL + AND in_stock = true + GROUP BY 1 + ORDER BY 1`, + [retailerProductId, daysBack], + ); + return result.rows.map((r) => ({ date: r.date, price: r.price, unitPrice: r.unit_price })); +} diff --git a/consumer-prices-core/src/db/queries/products.ts b/consumer-prices-core/src/db/queries/products.ts new file mode 100644 index 000000000..bd2f826a4 --- /dev/null +++ b/consumer-prices-core/src/db/queries/products.ts @@ -0,0 +1,88 @@ +import { query } from '../client.js'; +import type { CanonicalProduct, RetailerProduct } from '../models.js'; + +export async function upsertRetailerProduct(input: { + retailerId: string; + retailerSku: string | null; + sourceUrl: string; + rawTitle: string; + rawBrand?: string | null; + rawSizeText?: string | null; + imageUrl?: string | null; + categoryText?: string | null; +}): Promise { + const result = await query<{ id: string }>( + `INSERT INTO retailer_products + (retailer_id, retailer_sku, source_url, raw_title, raw_brand, raw_size_text, + image_url, category_text, first_seen_at, last_seen_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NOW(),NOW()) + ON CONFLICT (retailer_id, source_url) DO UPDATE + SET raw_title = EXCLUDED.raw_title, + raw_brand = EXCLUDED.raw_brand, + raw_size_text = EXCLUDED.raw_size_text, + image_url = EXCLUDED.image_url, + category_text = EXCLUDED.category_text, + last_seen_at = NOW() + RETURNING id`, + [ + input.retailerId, + input.retailerSku ?? null, + input.sourceUrl, + input.rawTitle, + input.rawBrand ?? null, + input.rawSizeText ?? null, + input.imageUrl ?? null, + input.categoryText ?? null, + ], + ); + return result.rows[0].id; +} + +export async function getRetailerProductsByRetailer(retailerId: string): Promise { + const result = await query( + `SELECT * FROM retailer_products WHERE retailer_id = $1 AND active = true`, + [retailerId], + ); + return result.rows; +} + +export async function getCanonicalProducts(marketCode?: string): Promise { + const result = await query( + `SELECT * FROM canonical_products WHERE active = true ORDER BY canonical_name`, + [], + ); + return result.rows; +} + +export async function upsertCanonicalProduct(input: { + canonicalName: string; + brandNorm?: string | null; + category: string; + variantNorm?: string | null; + sizeValue?: number | null; + sizeUnit?: string | null; + baseQuantity?: number | null; + baseUnit?: string | null; +}): Promise { + const result = await query<{ id: string }>( + `INSERT INTO canonical_products + (canonical_name, brand_norm, category, variant_norm, size_value, size_unit, + base_quantity, base_unit) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) + ON CONFLICT (canonical_name, brand_norm, category, variant_norm, size_value, size_unit) + DO UPDATE SET base_quantity = EXCLUDED.base_quantity, base_unit = EXCLUDED.base_unit + RETURNING id`, + [ + input.canonicalName, + input.brandNorm ?? null, + input.category, + input.variantNorm ?? null, + input.sizeValue ?? null, + input.sizeUnit ?? null, + input.baseQuantity ?? null, + input.baseUnit ?? null, + ], + ); + return result.rows[0].id; +} + diff --git a/consumer-prices-core/src/jobs/aggregate.ts b/consumer-prices-core/src/jobs/aggregate.ts new file mode 100644 index 000000000..49f266b3e --- /dev/null +++ b/consumer-prices-core/src/jobs/aggregate.ts @@ -0,0 +1,208 @@ +/** + * Aggregate job: computes basket indices from latest price observations. + * Produces Fixed Basket Index and Value Basket Index per methodology. + */ +import { query } from '../db/client.js'; +import { loadAllBasketConfigs } from '../config/loader.js'; + +const logger = { + info: (msg: string, ...args: unknown[]) => console.log(`[aggregate] ${msg}`, ...args), + warn: (msg: string, ...args: unknown[]) => console.warn(`[aggregate] ${msg}`, ...args), +}; + +interface BasketRow { + basketItemId: string; + category: string; + weight: number; + retailerProductId: string; + retailerSlug: string; + price: number; + unitPrice: number | null; + currencyCode: string; + observedAt: Date; +} + + +async function getBasketRows(basketSlug: string, marketCode: string): Promise { + const result = await query<{ + basket_item_id: string; + category: string; + weight: string; + retailer_product_id: string; + retailer_slug: string; + price: string; + unit_price: string | null; + currency_code: string; + observed_at: Date; + }>( + `SELECT bi.id AS basket_item_id, + bi.category, + bi.weight, + rp.id AS retailer_product_id, + r.slug AS retailer_slug, + po.price, + po.unit_price, + po.currency_code, + po.observed_at + FROM baskets b + JOIN basket_items bi ON bi.basket_id = b.id AND bi.active = true + JOIN product_matches pm ON pm.basket_item_id = bi.id AND pm.match_status IN ('auto','approved') + JOIN retailer_products rp ON rp.id = pm.retailer_product_id AND rp.active = true + JOIN retailers r ON r.id = rp.retailer_id AND r.market_code = $2 AND r.active = true + JOIN LATERAL ( + SELECT price, unit_price, currency_code, observed_at + FROM price_observations + WHERE retailer_product_id = rp.id AND in_stock = true + ORDER BY observed_at DESC LIMIT 1 + ) po ON true + WHERE b.slug = $1`, + [basketSlug, marketCode], + ); + + return result.rows.map((r) => ({ + basketItemId: r.basket_item_id, + category: r.category, + weight: parseFloat(r.weight), + retailerProductId: r.retailer_product_id, + retailerSlug: r.retailer_slug, + price: parseFloat(r.price), + unitPrice: r.unit_price ? parseFloat(r.unit_price) : null, + currencyCode: r.currency_code, + observedAt: r.observed_at, + })); +} + +async function getBaselinePrices(basketItemIds: string[], baseDate: string): Promise> { + const result = await query<{ basket_item_id: string; price: string }>( + `SELECT pm.basket_item_id, AVG(po.price)::numeric(12,2) AS price + FROM price_observations po + JOIN product_matches pm ON pm.retailer_product_id = po.retailer_product_id + WHERE pm.basket_item_id = ANY($1) + AND po.in_stock = true + AND DATE_TRUNC('day', po.observed_at) = $2::date + GROUP BY pm.basket_item_id`, + [basketItemIds, baseDate], + ); + const map = new Map(); + for (const row of result.rows) { + map.set(row.basket_item_id, parseFloat(row.price)); + } + return map; +} + +function computeFixedIndex(rows: BasketRow[], baselines: Map): number { + let weightedSum = 0; + let totalWeight = 0; + + const byItem = new Map(); + for (const r of rows) { + if (!byItem.has(r.basketItemId)) byItem.set(r.basketItemId, []); + byItem.get(r.basketItemId)!.push(r); + } + + for (const [itemId, itemRows] of byItem) { + const base = baselines.get(itemId); + if (!base) continue; + + const avgPrice = itemRows.reduce((s, r) => s + r.price, 0) / itemRows.length; + const weight = itemRows[0].weight; + + weightedSum += weight * (avgPrice / base); + totalWeight += weight; + } + + if (totalWeight === 0) return 100; + return 100 * (weightedSum / totalWeight); +} + +function computeValueIndex(rows: BasketRow[], baselines: Map): number { + // Value index: same as fixed index but using the cheapest available price + // per basket item (floor price across retailers), not the average. + const byItem = new Map(); + for (const r of rows) { + if (!byItem.has(r.basketItemId)) byItem.set(r.basketItemId, []); + byItem.get(r.basketItemId)!.push(r); + } + + let weightedSum = 0; + let totalWeight = 0; + + for (const [itemId, itemRows] of byItem) { + const base = baselines.get(itemId); + if (!base) continue; + + const floorPrice = itemRows.reduce((min, r) => Math.min(min, r.price), Infinity); + const weight = itemRows[0].weight; + + weightedSum += weight * (floorPrice / base); + totalWeight += weight; + } + + if (totalWeight === 0) return 100; + return 100 * (weightedSum / totalWeight); +} + +async function writeComputedIndex( + basketId: string, + retailerId: string | null, + category: string | null, + metricKey: string, + metricValue: number, +) { + await query( + `INSERT INTO computed_indices (basket_id, retailer_id, category, metric_date, metric_key, metric_value, methodology_version) + VALUES ($1,$2,$3,NOW()::date,$4,$5,'1') + ON CONFLICT (basket_id, retailer_id, category, metric_date, metric_key) + DO UPDATE SET metric_value = EXCLUDED.metric_value, methodology_version = EXCLUDED.methodology_version`, + [basketId, retailerId, category, metricKey, metricValue], + ); +} + +export async function aggregateBasket(basketSlug: string, marketCode: string) { + const configs = loadAllBasketConfigs(); + const basketConfig = configs.find((b) => b.slug === basketSlug && b.marketCode === marketCode); + if (!basketConfig) { + logger.warn(`Basket ${basketSlug}:${marketCode} not found in config`); + return; + } + + const basketResult = await query<{ id: string }>(`SELECT id FROM baskets WHERE slug = $1`, [basketSlug]); + if (!basketResult.rows.length) { + logger.warn(`Basket ${basketSlug} not found in DB — run seed first`); + return; + } + const basketId = basketResult.rows[0].id; + + const rows = await getBasketRows(basketSlug, marketCode); + if (rows.length === 0) { + logger.warn(`No matched products for ${basketSlug}:${marketCode}`); + return; + } + + const uniqueItemIds = [...new Set(rows.map((r) => r.basketItemId))]; + const baselines = await getBaselinePrices(uniqueItemIds, basketConfig.baseDate); + + const essentialsIndex = computeFixedIndex(rows, baselines); + const valueIndex = computeValueIndex(rows, baselines); + + const coverageCount = new Set(rows.map((r) => r.basketItemId)).size; + const totalItems = basketConfig.items.length; + const coveragePct = (coverageCount / totalItems) * 100; + + await writeComputedIndex(basketId, null, null, 'essentials_index', essentialsIndex); + await writeComputedIndex(basketId, null, null, 'value_index', valueIndex); + await writeComputedIndex(basketId, null, null, 'coverage_pct', coveragePct); + + logger.info(`${basketSlug}:${marketCode} essentials=${essentialsIndex.toFixed(2)} value=${valueIndex.toFixed(2)} coverage=${coveragePct.toFixed(1)}%`); +} + +export async function aggregateAll() { + const configs = loadAllBasketConfigs(); + for (const c of configs) { + await aggregateBasket(c.slug, c.marketCode); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + aggregateAll().catch(console.error); +} diff --git a/consumer-prices-core/src/jobs/publish.ts b/consumer-prices-core/src/jobs/publish.ts new file mode 100644 index 000000000..c357a7acf --- /dev/null +++ b/consumer-prices-core/src/jobs/publish.ts @@ -0,0 +1,133 @@ +/** + * Publish job: builds compact WorldMonitor snapshot payloads and writes to Redis. + * This is the handoff point between consumer-prices-core and WorldMonitor. + */ +import { createClient } from 'redis'; +import { + buildBasketSeriesSnapshot, + buildCategoriesSnapshot, + buildFreshnessSnapshot, + buildMoversSnapshot, + buildOverviewSnapshot, + buildRetailerSpreadSnapshot, +} from '../snapshots/worldmonitor.js'; +import { loadAllBasketConfigs, loadAllRetailerConfigs } from '../config/loader.js'; + +const logger = { + info: (msg: string, ...args: unknown[]) => console.log(`[publish] ${msg}`, ...args), + warn: (msg: string, ...args: unknown[]) => console.warn(`[publish] ${msg}`, ...args), + error: (msg: string, ...args: unknown[]) => console.error(`[publish] ${msg}`, ...args), +}; + +function makeKey(parts: string[]): string { + return parts.join(':'); +} + +function recordCount(data: unknown): number { + if (!data || typeof data !== 'object') return 1; + const d = data as Record; + const arr = d.retailers ?? d.risers ?? d.essentialsSeries ?? d.categories; + return Array.isArray(arr) ? arr.length : 1; +} + +async function writeSnapshot( + redis: ReturnType, + key: string, + data: unknown, + ttlSeconds: number, +) { + const json = JSON.stringify(data); + await redis.setEx(key, ttlSeconds, json); + await redis.setEx( + makeKey(['seed-meta', key]), + ttlSeconds * 2, + JSON.stringify({ fetchedAt: Date.now(), recordCount: recordCount(data) }), + ); + logger.info(` wrote ${key} (${json.length} bytes, ttl=${ttlSeconds}s)`); +} + +export async function publishAll() { + const redisUrl = process.env.REDIS_URL; + if (!redisUrl) throw new Error('REDIS_URL is not set'); + + const redis = createClient({ url: redisUrl }); + await redis.connect(); + + try { + const retailers = loadAllRetailerConfigs().filter((r) => r.enabled); + const markets = [...new Set(retailers.map((r) => r.marketCode))]; + const baskets = loadAllBasketConfigs(); + + for (const marketCode of markets) { + logger.info(`Publishing snapshots for market: ${marketCode}`); + + try { + const overview = await buildOverviewSnapshot(marketCode); + await writeSnapshot(redis, makeKey(['consumer-prices', 'overview', marketCode]), overview, 1800); + } catch (err) { + logger.error(`overview:${marketCode} failed: ${err}`); + } + + for (const days of [7, 30]) { + try { + const movers = await buildMoversSnapshot(marketCode, days); + await writeSnapshot(redis, makeKey(['consumer-prices', 'movers', marketCode, `${days}d`]), movers, 1800); + } catch (err) { + logger.error(`movers:${marketCode}:${days}d failed: ${err}`); + } + } + + try { + const freshness = await buildFreshnessSnapshot(marketCode); + await writeSnapshot(redis, makeKey(['consumer-prices', 'freshness', marketCode]), freshness, 600); + } catch (err) { + logger.error(`freshness:${marketCode} failed: ${err}`); + } + + for (const range of ['7d', '30d', '90d']) { + try { + const categories = await buildCategoriesSnapshot(marketCode, range); + await writeSnapshot(redis, makeKey(['consumer-prices', 'categories', marketCode, range]), categories, 1800); + } catch (err) { + logger.error(`categories:${marketCode}:${range} failed: ${err}`); + } + } + + for (const basket of baskets.filter((b) => b.marketCode === marketCode)) { + try { + const spread = await buildRetailerSpreadSnapshot(marketCode, basket.slug); + await writeSnapshot( + redis, + makeKey(['consumer-prices', 'retailer-spread', marketCode, basket.slug]), + spread, + 1800, + ); + } catch (err) { + logger.error(`spread:${marketCode}:${basket.slug} failed: ${err}`); + } + + for (const range of ['7d', '30d', '90d']) { + try { + const series = await buildBasketSeriesSnapshot(marketCode, basket.slug, range); + await writeSnapshot( + redis, + makeKey(['consumer-prices', 'basket-series', marketCode, basket.slug, range]), + series, + 3600, + ); + } catch (err) { + logger.error(`basket-series:${marketCode}:${basket.slug}:${range} failed: ${err}`); + } + } + } + } + + logger.info('Publish complete'); + } finally { + await redis.disconnect(); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + publishAll().catch(console.error); +} diff --git a/consumer-prices-core/src/jobs/scrape.ts b/consumer-prices-core/src/jobs/scrape.ts new file mode 100644 index 000000000..73d95f63e --- /dev/null +++ b/consumer-prices-core/src/jobs/scrape.ts @@ -0,0 +1,187 @@ +/** + * Scrape job: discovers targets and writes price observations to Postgres. + * Respects per-retailer rate limits and acquisition provider config. + */ +import { query } from '../db/client.js'; +import { insertObservation } from '../db/queries/observations.js'; +import { upsertRetailerProduct } from '../db/queries/products.js'; +import { parseSize, unitPrice as calcUnitPrice } from '../normalizers/size.js'; +import { loadAllRetailerConfigs, loadRetailerConfig } from '../config/loader.js'; +import { initProviders, teardownAll } from '../acquisition/registry.js'; +import { GenericPlaywrightAdapter } from '../adapters/generic.js'; +import { ExaSearchAdapter } from '../adapters/exa-search.js'; +import type { AdapterContext } from '../adapters/types.js'; +import { upsertCanonicalProduct } from '../db/queries/products.js'; +import { getBasketItemId, upsertProductMatch } from '../db/queries/matches.js'; + +const logger = { + info: (msg: string, ...args: unknown[]) => console.log(`[scrape] ${msg}`, ...args), + warn: (msg: string, ...args: unknown[]) => console.warn(`[scrape] ${msg}`, ...args), + error: (msg: string, ...args: unknown[]) => console.error(`[scrape] ${msg}`, ...args), +}; + +async function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +async function getOrCreateRetailer(slug: string, config: ReturnType) { + const existing = await query<{ id: string }>(`SELECT id FROM retailers WHERE slug = $1`, [slug]); + if (existing.rows.length > 0) return existing.rows[0].id; + + const result = await query<{ id: string }>( + `INSERT INTO retailers (slug, name, market_code, country_code, currency_code, adapter_key, base_url) + VALUES ($1,$2,$3,$3,$4,$5,$6) RETURNING id`, + [slug, config.name, config.marketCode, config.currencyCode, config.adapter, config.baseUrl], + ); + return result.rows[0].id; +} + +async function createScrapeRun(retailerId: string): Promise { + const result = await query<{ id: string }>( + `INSERT INTO scrape_runs (retailer_id, started_at, status, trigger_type, pages_attempted, pages_succeeded, errors_count, config_version) + VALUES ($1, NOW(), 'running', 'scheduled', 0, 0, 0, '1') RETURNING id`, + [retailerId], + ); + return result.rows[0].id; +} + +async function updateScrapeRun( + runId: string, + status: string, + pagesAttempted: number, + pagesSucceeded: number, + errorsCount: number, +) { + await query( + `UPDATE scrape_runs SET status=$2, finished_at=NOW(), pages_attempted=$3, pages_succeeded=$4, errors_count=$5 WHERE id=$1`, + [runId, status, pagesAttempted, pagesSucceeded, errorsCount], + ); +} + +export async function scrapeRetailer(slug: string) { + initProviders(process.env as Record); + + const config = loadRetailerConfig(slug); + if (!config.enabled) { + logger.info(`${slug} is disabled, skipping`); + return; + } + + const retailerId = await getOrCreateRetailer(slug, config); + const runId = await createScrapeRun(retailerId); + + logger.info(`Run ${runId} started for ${slug}`); + + const adapter = + config.adapter === 'exa-search' + ? new ExaSearchAdapter((process.env.EXA_API_KEYS || process.env.EXA_API_KEY || '').split(/[\n,]+/)[0].trim()) + : new GenericPlaywrightAdapter(); + const ctx: AdapterContext = { config, runId, logger }; + + const targets = await adapter.discoverTargets(ctx); + logger.info(`Discovered ${targets.length} targets`); + + let pagesAttempted = 0; + let pagesSucceeded = 0; + let errorsCount = 0; + + const delay = config.rateLimit?.delayBetweenRequestsMs ?? 2_000; + + for (const target of targets) { + pagesAttempted++; + try { + const fetchResult = await adapter.fetchTarget(ctx, target); + const products = await adapter.parseListing(ctx, fetchResult); + + logger.info(` [${target.id}] parsed ${products.length} products`); + + for (const product of products) { + const productId = await upsertRetailerProduct({ + retailerId, + retailerSku: product.retailerSku, + sourceUrl: product.sourceUrl, + rawTitle: product.rawTitle, + rawBrand: product.rawBrand, + rawSizeText: product.rawSizeText, + imageUrl: product.imageUrl, + categoryText: product.categoryText ?? target.category, + }); + + const parsed = parseSize(product.rawSizeText); + const up = parsed ? calcUnitPrice(product.price, parsed) : null; + + await insertObservation({ + retailerProductId: productId, + scrapeRunId: runId, + price: product.price, + listPrice: product.listPrice, + promoPrice: product.promoPrice, + currencyCode: config.currencyCode, + unitPrice: up, + unitBasisQty: parsed?.baseQuantity ?? null, + unitBasisUnit: parsed?.baseUnit ?? null, + inStock: product.inStock, + promoText: product.promoText, + rawPayloadJson: product.rawPayload, + }); + + // For exa-search adapter: auto-create product → basket match since we + // searched for a specific basket item (no ambiguity in what was scraped). + if ( + config.adapter === 'exa-search' && + product.rawPayload.basketSlug && + product.rawPayload.itemCategory + ) { + try { + const canonicalId = await upsertCanonicalProduct({ + canonicalName: (product.rawPayload.canonicalName as string) || product.rawTitle, + category: product.categoryText ?? target.category, + }); + const basketItemId = await getBasketItemId( + product.rawPayload.basketSlug as string, + product.rawPayload.itemCategory as string, + ); + if (basketItemId) { + await upsertProductMatch({ + retailerProductId: productId, + canonicalProductId: canonicalId, + basketItemId, + matchScore: 1.0, + matchStatus: 'auto', + }); + } + } catch (matchErr) { + logger.warn(` [${target.id}] product match failed: ${matchErr}`); + } + } + } + + pagesSucceeded++; + } catch (err) { + errorsCount++; + logger.error(` [${target.id}] failed: ${err}`); + } + + if (pagesAttempted < targets.length) await sleep(delay); + } + + const status = errorsCount === 0 ? 'completed' : pagesSucceeded > 0 ? 'partial' : 'failed'; + await updateScrapeRun(runId, status, pagesAttempted, pagesSucceeded, errorsCount); + logger.info(`Run ${runId} finished: ${status} (${pagesSucceeded}/${pagesAttempted} pages)`); + await teardownAll(); +} + +export async function scrapeAll() { + initProviders(process.env as Record); + const configs = loadAllRetailerConfigs().filter((c) => c.enabled); + logger.info(`Scraping ${configs.length} retailers`); + for (const c of configs) { + await scrapeRetailer(c.slug); + } +} + +if (process.argv[2]) { + scrapeRetailer(process.argv[2]).catch(console.error); +} else { + scrapeAll().catch(console.error); +} diff --git a/consumer-prices-core/src/matchers/canonical.ts b/consumer-prices-core/src/matchers/canonical.ts new file mode 100644 index 000000000..67adc2435 --- /dev/null +++ b/consumer-prices-core/src/matchers/canonical.ts @@ -0,0 +1,88 @@ +import { normalizeBrand } from '../normalizers/brand.js'; +import { parseSize } from '../normalizers/size.js'; +import { tokenOverlap } from '../normalizers/title.js'; +import type { CanonicalProduct } from '../db/models.js'; + +export interface RawProduct { + rawTitle: string; + rawBrand?: string | null; + rawSizeText?: string | null; + categoryText?: string | null; +} + +export interface MatchResult { + canonicalProductId: string; + score: number; + status: 'auto' | 'review' | 'reject'; + evidence: { + brandExact: boolean; + categoryExact: boolean; + titleOverlap: number; + sizeExact: boolean; + sizeClose: boolean; + packCountMatch: boolean; + unitPriceRatioOk: boolean; + }; +} + +export function scoreMatch(raw: RawProduct, canonical: CanonicalProduct): MatchResult { + let score = 0; + + const rawBrandNorm = normalizeBrand(raw.rawBrand)?.toLowerCase(); + const canonBrandNorm = canonical.brandNorm?.toLowerCase(); + const brandExact = !!(rawBrandNorm && canonBrandNorm && rawBrandNorm === canonBrandNorm); + if (brandExact) score += 30; + + const rawCategory = (raw.categoryText ?? '').toLowerCase(); + const canonCategory = canonical.category.toLowerCase(); + const categoryExact = rawCategory.includes(canonCategory) || canonCategory.includes(rawCategory); + if (categoryExact) score += 20; + + const overlap = tokenOverlap(raw.rawTitle, canonical.canonicalName); + score += Math.round(overlap * 15); + + const rawParsed = parseSize(raw.rawSizeText); + const canonHasSize = canonical.sizeValue !== null; + + let sizeExact = false; + let sizeClose = false; + let packCountMatch = false; + + if (rawParsed && canonHasSize && canonical.baseUnit === rawParsed.baseUnit) { + const ratio = rawParsed.baseQuantity / (canonical.baseQuantity ?? rawParsed.baseQuantity); + sizeExact = Math.abs(ratio - 1) < 0.01; + sizeClose = Math.abs(ratio - 1) < 0.05; + packCountMatch = rawParsed.packCount === 1; + + if (sizeExact) score += 20; + else if (sizeClose) score += 10; + if (packCountMatch) score += 10; + } + + const status: MatchResult['status'] = score >= 85 ? 'auto' : score >= 70 ? 'review' : 'reject'; + + return { + canonicalProductId: canonical.id, + score, + status, + evidence: { + brandExact, + categoryExact, + titleOverlap: overlap, + sizeExact, + sizeClose, + packCountMatch, + unitPriceRatioOk: false, + }, + }; +} + +export function bestMatch(raw: RawProduct, candidates: CanonicalProduct[]): MatchResult | null { + if (candidates.length === 0) return null; + + const scored = candidates.map((c) => scoreMatch(raw, c)); + scored.sort((a, b) => b.score - a.score); + + const best = scored[0]; + return best.status === 'reject' ? null : best; +} diff --git a/consumer-prices-core/src/normalizers/brand.ts b/consumer-prices-core/src/normalizers/brand.ts new file mode 100644 index 000000000..60d334390 --- /dev/null +++ b/consumer-prices-core/src/normalizers/brand.ts @@ -0,0 +1,46 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +interface BrandAliases { + aliases: Record; +} + +let _aliases: Map | null = null; + +function loadAliases(): Map { + if (_aliases) return _aliases; + + const filePath = join(dirname(fileURLToPath(import.meta.url)), '../../../configs/brands/aliases.json'); + const map = new Map(); + + if (existsSync(filePath)) { + const data = JSON.parse(readFileSync(filePath, 'utf8')) as BrandAliases; + for (const [canonical, variants] of Object.entries(data.aliases)) { + for (const v of variants) { + map.set(v.toLowerCase(), canonical); + } + } + } + + _aliases = map; + return map; +} + +export function normalizeBrand(raw: string | null | undefined): string | null { + if (!raw) return null; + + const cleaned = raw + .trim() + .toLowerCase() + .replace(/[^\w\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + + const aliases = loadAliases(); + return aliases.get(cleaned) ?? titleCase(cleaned); +} + +function titleCase(s: string): string { + return s.replace(/\b\w/g, (c) => c.toUpperCase()); +} diff --git a/consumer-prices-core/src/normalizers/size.ts b/consumer-prices-core/src/normalizers/size.ts new file mode 100644 index 000000000..6f462acd4 --- /dev/null +++ b/consumer-prices-core/src/normalizers/size.ts @@ -0,0 +1,84 @@ +/** + * Parses and normalizes product size strings into base units. + * Handles patterns like: 2x200g, 6x1L, 500ml, 24 rolls, 3 ct, 1kg, 12 pods + */ + +export interface ParsedSize { + packCount: number; + sizeValue: number; + sizeUnit: string; + baseQuantity: number; + baseUnit: string; + rawText: string; +} + +const UNIT_MAP: Record = { + kg: { base: 'g', factor: 1000 }, + g: { base: 'g', factor: 1 }, + mg: { base: 'g', factor: 0.001 }, + l: { base: 'ml', factor: 1000 }, + lt: { base: 'ml', factor: 1000 }, + ltr: { base: 'ml', factor: 1000 }, + litre: { base: 'ml', factor: 1000 }, + liter: { base: 'ml', factor: 1000 }, + ml: { base: 'ml', factor: 1 }, + cl: { base: 'ml', factor: 10 }, + oz: { base: 'g', factor: 28.3495 }, + lb: { base: 'g', factor: 453.592 }, + ct: { base: 'ct', factor: 1 }, + pc: { base: 'ct', factor: 1 }, + pcs: { base: 'ct', factor: 1 }, + piece: { base: 'ct', factor: 1 }, + pieces: { base: 'ct', factor: 1 }, + roll: { base: 'ct', factor: 1 }, + rolls: { base: 'ct', factor: 1 }, + pod: { base: 'ct', factor: 1 }, + pods: { base: 'ct', factor: 1 }, + sheet: { base: 'ct', factor: 1 }, + sheets: { base: 'ct', factor: 1 }, + sachet: { base: 'ct', factor: 1 }, + sachets: { base: 'ct', factor: 1 }, +}; + +const PACK_PATTERN = /^(\d+)\s*[x×]\s*(.+)$/i; +const SIZE_PATTERN = /(\d+(?:\.\d+)?)\s*([a-z]+)/i; + +export function parseSize(raw: string | null | undefined): ParsedSize | null { + if (!raw) return null; + + const text = raw.trim().toLowerCase(); + + let packCount = 1; + let sizeStr = text; + + const packMatch = PACK_PATTERN.exec(text); + if (packMatch) { + packCount = parseInt(packMatch[1], 10); + sizeStr = packMatch[2].trim(); + } + + const sizeMatch = SIZE_PATTERN.exec(sizeStr); + if (!sizeMatch) return null; + + const sizeValue = parseFloat(sizeMatch[1]); + const rawUnit = sizeMatch[2].toLowerCase().replace(/\.$/, ''); + const unitDef = UNIT_MAP[rawUnit]; + + if (!unitDef) return null; + + const baseQuantity = packCount * sizeValue * unitDef.factor; + + return { + packCount, + sizeValue, + sizeUnit: rawUnit, + baseQuantity, + baseUnit: unitDef.base, + rawText: raw, + }; +} + +export function unitPrice(price: number, size: ParsedSize): number { + if (size.baseQuantity === 0) return price; + return price / size.baseQuantity; +} diff --git a/consumer-prices-core/src/normalizers/title.ts b/consumer-prices-core/src/normalizers/title.ts new file mode 100644 index 000000000..081b641d5 --- /dev/null +++ b/consumer-prices-core/src/normalizers/title.ts @@ -0,0 +1,37 @@ +const PROMO_TOKENS = new Set([ + 'fresh', 'save', 'sale', 'deal', 'offer', 'limited', 'new', 'best', 'value', + 'buy', 'get', 'free', 'bonus', 'extra', 'special', 'exclusive', 'online only', + 'website exclusive', 'price drop', 'clearance', 'now', 'only', +]); + +const STOP_WORDS = new Set(['a', 'an', 'the', 'with', 'and', 'or', 'in', 'of', 'for', 'to', 'by']); + +export function cleanTitle(raw: string): string { + return raw + .trim() + .toLowerCase() + .replace(/\s+/g, ' ') + .replace(/[^\w\s\-&]/g, ' ') + .split(' ') + .filter((t) => t.length > 1 && !PROMO_TOKENS.has(t)) + .join(' ') + .trim(); +} + +export function titleTokens(title: string): string[] { + return cleanTitle(title) + .split(' ') + .filter((t) => t.length > 2 && !STOP_WORDS.has(t)); +} + +export function tokenOverlap(a: string, b: string): number { + const ta = new Set(titleTokens(a)); + const tb = new Set(titleTokens(b)); + if (ta.size === 0 || tb.size === 0) return 0; + + let shared = 0; + for (const t of ta) { + if (tb.has(t)) shared++; + } + return shared / Math.min(ta.size, tb.size); +} diff --git a/consumer-prices-core/src/snapshots/worldmonitor.ts b/consumer-prices-core/src/snapshots/worldmonitor.ts new file mode 100644 index 000000000..1a6801e62 --- /dev/null +++ b/consumer-prices-core/src/snapshots/worldmonitor.ts @@ -0,0 +1,555 @@ +/** + * Builds compact WorldMonitor-ready snapshot payloads from computed indices. + * All types are shaped to match the proto-generated TypeScript interfaces so + * snapshots can be written to Redis and read directly by WorldMonitor handlers. + */ +import { query } from '../db/client.js'; + +// --------------------------------------------------------------------------- +// Snapshot interfaces — mirror proto-generated response types exactly +// (asOf is int64 → string per protobuf JSON mapping) +// --------------------------------------------------------------------------- + +export interface WMCategorySnapshot { + slug: string; + name: string; + wowPct: number; + momPct: number; + currentIndex: number; + sparkline: number[]; + coveragePct: number; + itemCount: number; +} + +export interface WMOverviewSnapshot { + marketCode: string; + asOf: string; + currencyCode: string; + essentialsIndex: number; + valueBasketIndex: number; + wowPct: number; + momPct: number; + retailerSpreadPct: number; + coveragePct: number; + freshnessLagMin: number; + topCategories: WMCategorySnapshot[]; + upstreamUnavailable: false; +} + +export interface WMPriceMover { + productId: string; + title: string; + category: string; + retailerSlug: string; + changePct: number; + currentPrice: number; + currencyCode: string; +} + +export interface WMMoversSnapshot { + marketCode: string; + asOf: string; + range: string; + risers: WMPriceMover[]; + fallers: WMPriceMover[]; + upstreamUnavailable: false; +} + +export interface WMRetailerSpread { + slug: string; + name: string; + basketTotal: number; + deltaVsCheapest: number; + deltaVsCheapestPct: number; + itemCount: number; + freshnessMin: number; + currencyCode: string; +} + +export interface WMRetailerSpreadSnapshot { + marketCode: string; + asOf: string; + basketSlug: string; + currencyCode: string; + retailers: WMRetailerSpread[]; + spreadPct: number; + upstreamUnavailable: false; +} + +export interface WMRetailerFreshness { + slug: string; + name: string; + lastRunAt: string; + status: string; + parseSuccessRate: number; + freshnessMin: number; +} + +export interface WMFreshnessSnapshot { + marketCode: string; + asOf: string; + retailers: WMRetailerFreshness[]; + overallFreshnessMin: number; + stalledCount: number; + upstreamUnavailable: false; +} + +export interface WMBasketPoint { + date: string; + index: number; +} + +export interface WMBasketSeriesSnapshot { + marketCode: string; + basketSlug: string; + asOf: string; + currencyCode: string; + range: string; + essentialsSeries: WMBasketPoint[]; + valueSeries: WMBasketPoint[]; + upstreamUnavailable: false; +} + +// --------------------------------------------------------------------------- +// Private helpers +// --------------------------------------------------------------------------- + +async function buildTopCategories(basketId: string): Promise { + const result = await query<{ + category: string; + current_index: number | null; + prev_week_index: number | null; + coverage_pct: number | null; + }>( + `WITH today AS ( + SELECT category, metric_key, metric_value::float AS metric_value + FROM computed_indices + WHERE basket_id = $1 AND category IS NOT NULL AND retailer_id IS NULL AND metric_date = CURRENT_DATE + ), + last_week AS ( + SELECT category, metric_key, metric_value::float AS metric_value + FROM computed_indices + WHERE basket_id = $1 AND category IS NOT NULL AND retailer_id IS NULL + AND metric_date = ( + SELECT MAX(metric_date) FROM computed_indices + WHERE basket_id = $1 AND category IS NOT NULL + AND metric_date < CURRENT_DATE - INTERVAL '6 days' + ) + ) + SELECT + t.category, + MAX(CASE WHEN t.metric_key = 'essentials_index' THEN t.metric_value END) AS current_index, + MAX(CASE WHEN lw.metric_key = 'essentials_index' THEN lw.metric_value END) AS prev_week_index, + MAX(CASE WHEN t.metric_key = 'coverage_pct' THEN t.metric_value END) AS coverage_pct + FROM (SELECT DISTINCT category FROM today) cats + JOIN today t ON t.category = cats.category + LEFT JOIN last_week lw ON lw.category = cats.category AND lw.metric_key = t.metric_key + GROUP BY cats.category + HAVING MAX(CASE WHEN t.metric_key = 'essentials_index' THEN 1 ELSE 0 END) = 1 + ORDER BY ABS(COALESCE(MAX(CASE WHEN t.metric_key = 'essentials_index' THEN t.metric_value END), 100) - 100) DESC + LIMIT 8`, + [basketId], + ); + + return result.rows.map((r) => { + const cur = r.current_index ?? 100; + const prev = r.prev_week_index; + const wowPct = prev && prev > 0 ? Math.round(((cur - prev) / prev) * 100 * 10) / 10 : 0; + const slug = r.category + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + return { + slug, + name: r.category.charAt(0).toUpperCase() + r.category.slice(1), + wowPct, + momPct: 0, // TODO: requires 30-day baseline per category + currentIndex: Math.round(cur * 10) / 10, + sparkline: [], // TODO: requires per-category date series query + coveragePct: Math.round((r.coverage_pct ?? 0) * 10) / 10, + itemCount: 0, // TODO: requires basket_items count query per category + }; + }); +} + +// --------------------------------------------------------------------------- +// Public builders +// --------------------------------------------------------------------------- + +export async function buildOverviewSnapshot(marketCode: string): Promise { + const now = Date.now(); + + // Resolve basket id for category queries + const basketIdResult = await query<{ id: string }>( + `SELECT b.id FROM baskets b WHERE b.market_code = $1 LIMIT 1`, + [marketCode], + ); + const basketId = basketIdResult.rows[0]?.id ?? null; + + const [indexResult, prevWeekResult, prevMonthResult, spreadResult, currencyResult, freshnessResult] = + await Promise.all([ + query<{ metric_key: string; metric_value: string }>( + `SELECT ci.metric_key, ci.metric_value + FROM computed_indices ci + JOIN baskets b ON b.id = ci.basket_id + WHERE b.market_code = $1 + AND ci.retailer_id IS NULL AND ci.category IS NULL + AND ci.metric_date = ( + SELECT MAX(metric_date) FROM computed_indices ci2 JOIN baskets b2 ON b2.id = ci2.basket_id + WHERE b2.market_code = $1 AND ci2.retailer_id IS NULL + )`, + [marketCode], + ), + query<{ metric_key: string; metric_value: string }>( + `SELECT ci.metric_key, ci.metric_value + FROM computed_indices ci + JOIN baskets b ON b.id = ci.basket_id + WHERE b.market_code = $1 + AND ci.retailer_id IS NULL AND ci.category IS NULL + AND ci.metric_date = ( + SELECT MAX(metric_date) FROM computed_indices ci2 JOIN baskets b2 ON b2.id = ci2.basket_id + WHERE b2.market_code = $1 AND ci2.retailer_id IS NULL + AND ci2.metric_date < CURRENT_DATE - INTERVAL '6 days' + )`, + [marketCode], + ), + query<{ metric_key: string; metric_value: string }>( + `SELECT ci.metric_key, ci.metric_value + FROM computed_indices ci + JOIN baskets b ON b.id = ci.basket_id + WHERE b.market_code = $1 + AND ci.retailer_id IS NULL AND ci.category IS NULL + AND ci.metric_date = ( + SELECT MAX(metric_date) FROM computed_indices ci2 JOIN baskets b2 ON b2.id = ci2.basket_id + WHERE b2.market_code = $1 AND ci2.retailer_id IS NULL + AND ci2.metric_date < CURRENT_DATE - INTERVAL '29 days' + )`, + [marketCode], + ), + query<{ spread_pct: string }>( + `SELECT metric_value AS spread_pct FROM computed_indices ci + JOIN baskets b ON b.id = ci.basket_id + WHERE b.market_code = $1 AND ci.metric_key = 'retailer_spread_pct' + ORDER BY ci.metric_date DESC LIMIT 1`, + [marketCode], + ), + query<{ currency_code: string }>( + `SELECT currency_code FROM retailers WHERE market_code = $1 AND active = true LIMIT 1`, + [marketCode], + ), + query<{ avg_lag_min: string }>( + `SELECT AVG(EXTRACT(EPOCH FROM (NOW() - last_successful_run_at)) / 60)::int AS avg_lag_min + FROM data_source_health dsh + JOIN retailers r ON r.id = dsh.retailer_id + WHERE r.market_code = $1`, + [marketCode], + ), + ]); + + const metrics: Record = {}; + for (const row of indexResult.rows) metrics[row.metric_key] = parseFloat(row.metric_value); + + const prevWeek: Record = {}; + for (const row of prevWeekResult.rows) prevWeek[row.metric_key] = parseFloat(row.metric_value); + + const prevMonth: Record = {}; + for (const row of prevMonthResult.rows) prevMonth[row.metric_key] = parseFloat(row.metric_value); + + const ess = metrics.essentials_index ?? 100; + const val = metrics.value_index ?? 100; + const prevEss = prevWeek.essentials_index; + const prevMonthEss = prevMonth.essentials_index; + const wowPct = prevEss ? Math.round(((ess - prevEss) / prevEss) * 100 * 10) / 10 : 0; + const momPct = prevMonthEss ? Math.round(((ess - prevMonthEss) / prevMonthEss) * 100 * 10) / 10 : 0; + + const topCategories = basketId ? await buildTopCategories(basketId) : []; + + return { + marketCode, + asOf: String(now), + currencyCode: currencyResult.rows[0]?.currency_code ?? 'USD', + essentialsIndex: Math.round(ess * 10) / 10, + valueBasketIndex: Math.round(val * 10) / 10, + wowPct, + momPct, + retailerSpreadPct: spreadResult.rows[0]?.spread_pct + ? Math.round(parseFloat(spreadResult.rows[0].spread_pct) * 10) / 10 + : 0, + coveragePct: Math.round((metrics.coverage_pct ?? 0) * 10) / 10, + freshnessLagMin: freshnessResult.rows[0]?.avg_lag_min + ? parseInt(freshnessResult.rows[0].avg_lag_min, 10) + : 0, + topCategories, + upstreamUnavailable: false, + }; +} + +export async function buildMoversSnapshot( + marketCode: string, + rangeDays: number, +): Promise { + const now = Date.now(); + const range = `${rangeDays}d`; + + const result = await query<{ + product_id: string; + raw_title: string; + category_text: string; + retailer_slug: string; + current_price: string; + currency_code: string; + change_pct: string; + }>( + `WITH latest AS ( + SELECT DISTINCT ON (rp.id) rp.id, rp.raw_title, rp.category_text, r.slug AS retailer_slug, + po.price, r.currency_code + FROM retailer_products rp + JOIN retailers r ON r.id = rp.retailer_id AND r.market_code = $1 AND r.active = true + JOIN price_observations po ON po.retailer_product_id = rp.id AND po.in_stock = true + ORDER BY rp.id, po.observed_at DESC + ), + past AS ( + SELECT DISTINCT ON (rp.id) rp.id, po.price AS past_price + FROM retailer_products rp + JOIN retailers r ON r.id = rp.retailer_id AND r.market_code = $1 + JOIN price_observations po ON po.retailer_product_id = rp.id + AND po.observed_at BETWEEN NOW() - ($2 || ' days')::INTERVAL - INTERVAL '1 day' + AND NOW() - ($2 || ' days')::INTERVAL + ORDER BY rp.id, po.observed_at DESC + ) + SELECT l.id AS product_id, l.raw_title, l.category_text, l.retailer_slug, + l.price AS current_price, l.currency_code, + ROUND(((l.price - p.past_price) / p.past_price * 100)::numeric, 2) AS change_pct + FROM latest l + JOIN past p ON p.id = l.id + WHERE p.past_price > 0 + ORDER BY ABS((l.price - p.past_price) / p.past_price) DESC + LIMIT 30`, + [marketCode, rangeDays], + ); + + const all = result.rows.map((r) => ({ + productId: r.product_id, + title: r.raw_title, + category: r.category_text ?? 'other', + retailerSlug: r.retailer_slug, + currentPrice: parseFloat(r.current_price), + currencyCode: r.currency_code, + changePct: parseFloat(r.change_pct), + })); + + return { + marketCode, + asOf: String(now), + range, + risers: all.filter((r) => r.changePct > 0).slice(0, 10), + fallers: all.filter((r) => r.changePct < 0).slice(0, 10), + upstreamUnavailable: false, + }; +} + +export async function buildRetailerSpreadSnapshot( + marketCode: string, + basketSlug: string, +): Promise { + const now = Date.now(); + + const result = await query<{ + retailer_slug: string; + retailer_name: string; + basket_total: string; + item_count: string; + currency_code: string; + freshness_min: string | null; + }>( + `SELECT r.slug AS retailer_slug, r.name AS retailer_name, r.currency_code, + SUM(po.price) AS basket_total, COUNT(*) AS item_count, + EXTRACT(EPOCH FROM (NOW() - MAX(po.observed_at))) / 60 AS freshness_min + FROM baskets b + JOIN basket_items bi ON bi.basket_id = b.id AND bi.active = true + JOIN product_matches pm ON pm.basket_item_id = bi.id AND pm.match_status IN ('auto','approved') + JOIN retailer_products rp ON rp.id = pm.retailer_product_id AND rp.active = true + JOIN retailers r ON r.id = rp.retailer_id AND r.market_code = $2 AND r.active = true + JOIN LATERAL ( + SELECT price, observed_at + FROM price_observations + WHERE retailer_product_id = rp.id AND in_stock = true + ORDER BY observed_at DESC LIMIT 1 + ) po ON true + WHERE b.slug = $1 + GROUP BY r.slug, r.name, r.currency_code + ORDER BY basket_total ASC`, + [basketSlug, marketCode], + ); + + const retailers: WMRetailerSpread[] = result.rows.map((r) => ({ + slug: r.retailer_slug, + name: r.retailer_name, + basketTotal: parseFloat(r.basket_total), + deltaVsCheapest: 0, + deltaVsCheapestPct: 0, + itemCount: parseInt(r.item_count, 10), + freshnessMin: r.freshness_min ? parseInt(r.freshness_min, 10) : 0, + currencyCode: r.currency_code, + })); + + if (retailers.length > 0) { + const cheapest = retailers[0].basketTotal; + for (const r of retailers) { + r.deltaVsCheapest = Math.round((r.basketTotal - cheapest) * 100) / 100; + r.deltaVsCheapestPct = + cheapest > 0 ? Math.round(((r.basketTotal - cheapest) / cheapest) * 100 * 10) / 10 : 0; + } + } + + const spreadPct = + retailers.length >= 2 + ? Math.round( + ((retailers[retailers.length - 1].basketTotal - retailers[0].basketTotal) / + retailers[0].basketTotal) * + 100 * + 10, + ) / 10 + : 0; + + return { + marketCode, + asOf: String(now), + basketSlug, + currencyCode: result.rows[0]?.currency_code ?? 'USD', + retailers, + spreadPct, + upstreamUnavailable: false, + }; +} + +export async function buildFreshnessSnapshot(marketCode: string): Promise { + const now = Date.now(); + + const result = await query<{ + slug: string; + name: string; + last_run_at: Date | null; + last_run_status: string | null; + parse_success_rate: string | null; + freshness_min: string | null; + }>( + `SELECT r.slug, r.name, + dsh.last_successful_run_at AS last_run_at, + dsh.last_run_status, + dsh.parse_success_rate, + EXTRACT(EPOCH FROM (NOW() - dsh.last_successful_run_at)) / 60 AS freshness_min + FROM retailers r + LEFT JOIN data_source_health dsh ON dsh.retailer_id = r.id + WHERE r.market_code = $1 AND r.active = true`, + [marketCode], + ); + + const retailers: WMRetailerFreshness[] = result.rows.map((r) => ({ + slug: r.slug, + name: r.name, + lastRunAt: r.last_run_at ? r.last_run_at.toISOString() : '', + status: r.last_run_status ?? 'unknown', + parseSuccessRate: r.parse_success_rate ? parseFloat(r.parse_success_rate) : 0, + freshnessMin: r.freshness_min ? parseInt(r.freshness_min, 10) : 0, + })); + + const freshnessValues = retailers.map((r) => r.freshnessMin).filter((v) => v > 0); + const overallFreshnessMin = + freshnessValues.length > 0 + ? Math.round(freshnessValues.reduce((a, b) => a + b, 0) / freshnessValues.length) + : 0; + + const stalledCount = retailers.filter((r) => r.freshnessMin === 0 || r.freshnessMin > 240).length; + + return { + marketCode, + asOf: String(now), + retailers, + overallFreshnessMin, + stalledCount, + upstreamUnavailable: false, + }; +} + +export async function buildBasketSeriesSnapshot( + marketCode: string, + basketSlug: string, + range: string, +): Promise { + const now = Date.now(); + const days = parseInt(range.replace('d', ''), 10) || 30; + + const [essResult, valResult, currencyResult] = await Promise.all([ + query<{ metric_date: Date; metric_value: string }>( + `SELECT ci.metric_date, ci.metric_value + FROM computed_indices ci + JOIN baskets b ON b.id = ci.basket_id + WHERE b.slug = $1 AND b.market_code = $2 + AND ci.metric_key = 'essentials_index' + AND ci.retailer_id IS NULL AND ci.category IS NULL + AND ci.metric_date >= CURRENT_DATE - ($3 || ' days')::INTERVAL + ORDER BY ci.metric_date ASC`, + [basketSlug, marketCode, days], + ), + query<{ metric_date: Date; metric_value: string }>( + `SELECT ci.metric_date, ci.metric_value + FROM computed_indices ci + JOIN baskets b ON b.id = ci.basket_id + WHERE b.slug = $1 AND b.market_code = $2 + AND ci.metric_key = 'value_index' + AND ci.retailer_id IS NULL AND ci.category IS NULL + AND ci.metric_date >= CURRENT_DATE - ($3 || ' days')::INTERVAL + ORDER BY ci.metric_date ASC`, + [basketSlug, marketCode, days], + ), + query<{ currency_code: string }>( + `SELECT currency_code FROM retailers WHERE market_code = $1 AND active = true LIMIT 1`, + [marketCode], + ), + ]); + + return { + marketCode, + basketSlug, + asOf: String(now), + currencyCode: currencyResult.rows[0]?.currency_code ?? 'USD', + range, + essentialsSeries: essResult.rows.map((r) => ({ + date: r.metric_date.toISOString().slice(0, 10), + index: Math.round(parseFloat(r.metric_value) * 10) / 10, + })), + valueSeries: valResult.rows.map((r) => ({ + date: r.metric_date.toISOString().slice(0, 10), + index: Math.round(parseFloat(r.metric_value) * 10) / 10, + })), + upstreamUnavailable: false, + }; +} + +export interface WMCategoriesSnapshot { + marketCode: string; + asOf: string; + range: string; + categories: WMCategorySnapshot[]; + upstreamUnavailable: false; +} + +export async function buildCategoriesSnapshot(marketCode: string, range: string): Promise { + const now = Date.now(); + + const basketIdResult = await query<{ id: string }>( + `SELECT b.id FROM baskets b WHERE b.market_code = $1 LIMIT 1`, + [marketCode], + ); + const basketId = basketIdResult.rows[0]?.id ?? null; + const categories = basketId ? await buildTopCategories(basketId) : []; + + return { + marketCode, + asOf: String(now), + range, + categories, + upstreamUnavailable: false, + }; +} diff --git a/consumer-prices-core/tests/unit/matcher.test.ts b/consumer-prices-core/tests/unit/matcher.test.ts new file mode 100644 index 000000000..224af034f --- /dev/null +++ b/consumer-prices-core/tests/unit/matcher.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from 'vitest'; +import { scoreMatch, bestMatch } from '../../src/matchers/canonical.js'; +import type { CanonicalProduct } from '../../src/db/models.js'; + +const baseCanonical: CanonicalProduct = { + id: 'c1', + canonicalName: 'Basmati Rice 1kg', + brandNorm: 'Tilda', + category: 'rice', + variantNorm: null, + sizeValue: 1000, + sizeUnit: 'g', + baseQuantity: 1000, + baseUnit: 'g', + active: true, + createdAt: new Date(), +}; + +describe('scoreMatch', () => { + it('gives high score for exact brand+category+size match', () => { + const result = scoreMatch( + { rawTitle: 'Tilda Basmati Rice 1kg', rawBrand: 'Tilda', rawSizeText: '1kg', categoryText: 'rice' }, + baseCanonical, + ); + expect(result.score).toBeGreaterThanOrEqual(85); + expect(result.status).toBe('auto'); + }); + + it('gives review score for partial match (no brand)', () => { + const result = scoreMatch( + { rawTitle: 'Basmati Rice 1kg', rawBrand: null, rawSizeText: '1kg', categoryText: 'rice' }, + baseCanonical, + ); + expect(result.score).toBeGreaterThanOrEqual(50); + }); + + it('rejects clearly wrong product', () => { + const result = scoreMatch( + { rawTitle: 'Sunflower Oil 2L', rawBrand: 'Generic', rawSizeText: '2L', categoryText: 'oil' }, + baseCanonical, + ); + expect(result.status).toBe('reject'); + }); +}); + +describe('bestMatch', () => { + it('returns null when no candidates', () => { + expect(bestMatch({ rawTitle: 'Eggs 12 Pack' }, [])).toBeNull(); + }); + + it('returns best scoring non-reject match', () => { + const result = bestMatch( + { rawTitle: 'Tilda Basmati Rice 1kg', rawBrand: 'Tilda', rawSizeText: '1kg', categoryText: 'rice' }, + [baseCanonical], + ); + expect(result?.canonicalProductId).toBe('c1'); + expect(result?.score).toBeGreaterThanOrEqual(85); + }); +}); diff --git a/consumer-prices-core/tests/unit/size.test.ts b/consumer-prices-core/tests/unit/size.test.ts new file mode 100644 index 000000000..3ae719e16 --- /dev/null +++ b/consumer-prices-core/tests/unit/size.test.ts @@ -0,0 +1,60 @@ +import { describe, it, expect } from 'vitest'; +import { parseSize, unitPrice } from '../../src/normalizers/size.js'; + +describe('parseSize', () => { + it('parses simple gram weights', () => { + const r = parseSize('500g'); + expect(r?.baseQuantity).toBe(500); + expect(r?.baseUnit).toBe('g'); + expect(r?.packCount).toBe(1); + }); + + it('parses kilograms and converts to grams', () => { + const r = parseSize('1kg'); + expect(r?.baseQuantity).toBe(1000); + expect(r?.baseUnit).toBe('g'); + }); + + it('parses multi-pack patterns (2x200g)', () => { + const r = parseSize('2x200g'); + expect(r?.packCount).toBe(2); + expect(r?.sizeValue).toBe(200); + expect(r?.baseQuantity).toBe(400); + }); + + it('parses multi-pack with × symbol', () => { + const r = parseSize('6×1L'); + expect(r?.packCount).toBe(6); + expect(r?.baseQuantity).toBe(6000); + expect(r?.baseUnit).toBe('ml'); + }); + + it('parses litre variants', () => { + expect(parseSize('1L')?.baseQuantity).toBe(1000); + expect(parseSize('1.5l')?.baseQuantity).toBe(1500); + expect(parseSize('500ml')?.baseQuantity).toBe(500); + }); + + it('parses count units', () => { + const r = parseSize('12 rolls'); + expect(r?.baseQuantity).toBe(12); + expect(r?.baseUnit).toBe('ct'); + }); + + it('parses piece counts', () => { + const r = parseSize('24 pcs'); + expect(r?.baseQuantity).toBe(24); + }); + + it('returns null for unparseable text', () => { + expect(parseSize('large')).toBeNull(); + expect(parseSize(null)).toBeNull(); + expect(parseSize('')).toBeNull(); + }); + + it('computes unit price correctly', () => { + const size = parseSize('1kg')!; + const up = unitPrice(10, size); + expect(up).toBeCloseTo(0.01); // 10 AED per 1000g = 0.01 per g + }); +}); diff --git a/consumer-prices-core/tests/unit/title.test.ts b/consumer-prices-core/tests/unit/title.test.ts new file mode 100644 index 000000000..48dcbf577 --- /dev/null +++ b/consumer-prices-core/tests/unit/title.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect } from 'vitest'; +import { cleanTitle, tokenOverlap } from '../../src/normalizers/title.js'; + +describe('cleanTitle', () => { + it('strips promo tokens', () => { + const r = cleanTitle('Fresh Organic Eggs - SAVE NOW!'); + expect(r).not.toContain('fresh'); + expect(r).not.toContain('save'); + expect(r).toContain('organic'); + expect(r).toContain('eggs'); + }); + + it('lowercases and normalizes whitespace', () => { + const r = cleanTitle(' Basmati Rice '); + expect(r).toBe('basmati rice'); + }); +}); + +describe('tokenOverlap', () => { + it('returns 1 for identical titles', () => { + expect(tokenOverlap('Basmati Rice 1kg', 'Basmati Rice 1kg')).toBe(1); + }); + + it('returns 0 for completely different titles', () => { + expect(tokenOverlap('Eggs 12 Pack', 'Sunflower Oil 1L')).toBe(0); + }); + + it('returns partial overlap for partial matches', () => { + const r = tokenOverlap('Basmati Rice 1kg Pack', 'Basmati Rice Premium'); + expect(r).toBeGreaterThan(0.5); + }); +}); diff --git a/consumer-prices-core/tsconfig.json b/consumer-prices-core/tsconfig.json new file mode 100644 index 000000000..55c7353e5 --- /dev/null +++ b/consumer-prices-core/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "tests"] +} diff --git a/docs/api/ConsumerPricesService.openapi.json b/docs/api/ConsumerPricesService.openapi.json new file mode 100644 index 000000000..7d26fb8af --- /dev/null +++ b/docs/api/ConsumerPricesService.openapi.json @@ -0,0 +1 @@ +{"components":{"schemas":{"BasketPoint":{"description":"BasketPoint is a single data point in a basket index time series.","properties":{"date":{"description":"date is the ISO 8601 date string (YYYY-MM-DD).","type":"string"},"index":{"description":"index is the basket index value (base = 100).","format":"double","type":"number"}},"type":"object"},"CategorySnapshot":{"description":"CategorySnapshot holds price index data for a single product category.","properties":{"coveragePct":{"description":"coverage_pct is the percentage of basket items observed for this category.","format":"double","type":"number"},"currentIndex":{"description":"current_index is the current price index value (base = 100).","format":"double","type":"number"},"itemCount":{"description":"item_count is the number of observed products in this category.","format":"int32","type":"integer"},"momPct":{"description":"mom_pct is the month-over-month percentage change.","format":"double","type":"number"},"name":{"description":"name is the human-readable category label.","type":"string"},"slug":{"description":"slug is the machine-readable category identifier (e.g. \"eggs\", \"rice\").","type":"string"},"sparkline":{"items":{"description":"sparkline is an ordered sequence of index values for the selected range.","format":"double","type":"number"},"type":"array"},"wowPct":{"description":"wow_pct is the week-over-week percentage change.","format":"double","type":"number"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GetConsumerPriceBasketSeriesRequest":{"description":"GetConsumerPriceBasketSeriesRequest parameters for time series data.","properties":{"basketSlug":{"description":"basket_slug selects the basket (e.g. \"essentials-ae\").","type":"string"},"marketCode":{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","type":"string"},"range":{"description":"range is one of \"7d\", \"30d\", \"90d\", \"180d\".","type":"string"}},"type":"object"},"GetConsumerPriceBasketSeriesResponse":{"description":"GetConsumerPriceBasketSeriesResponse contains the basket index time series.","properties":{"asOf":{"description":"as_of is the Unix millisecond timestamp of the snapshot.","format":"int64","type":"string"},"basketSlug":{"description":"basket_slug echoes the requested basket.","type":"string"},"currencyCode":{"description":"currency_code is the ISO 4217 currency code.","type":"string"},"essentialsSeries":{"items":{"$ref":"#/components/schemas/BasketPoint"},"type":"array"},"marketCode":{"description":"market_code echoes the requested market.","type":"string"},"range":{"description":"range echoes the requested range.","type":"string"},"upstreamUnavailable":{"description":"upstream_unavailable is true when the companion service could not be reached.","type":"boolean"},"valueSeries":{"items":{"$ref":"#/components/schemas/BasketPoint"},"type":"array"}},"type":"object"},"GetConsumerPriceFreshnessRequest":{"description":"GetConsumerPriceFreshnessRequest parameters for the freshness RPC.","properties":{"marketCode":{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","type":"string"}},"type":"object"},"GetConsumerPriceFreshnessResponse":{"description":"GetConsumerPriceFreshnessResponse describes feed health for all retailers.","properties":{"asOf":{"description":"as_of is the Unix millisecond timestamp of the snapshot.","format":"int64","type":"string"},"marketCode":{"description":"market_code echoes the requested market.","type":"string"},"overallFreshnessMin":{"description":"overall_freshness_min is the average freshness lag across all retailers.","format":"int32","type":"integer"},"retailers":{"items":{"$ref":"#/components/schemas/RetailerFreshnessInfo"},"type":"array"},"stalledCount":{"description":"stalled_count is the number of retailers with no recent successful scrape.","format":"int32","type":"integer"},"upstreamUnavailable":{"description":"upstream_unavailable is true when the companion service could not be reached.","type":"boolean"}},"type":"object"},"GetConsumerPriceOverviewRequest":{"description":"GetConsumerPriceOverviewRequest parameters for the overview RPC.","properties":{"basketSlug":{"description":"basket_slug selects which basket to use (e.g. \"essentials-ae\").","type":"string"},"marketCode":{"description":"market_code is the ISO 3166-1 alpha-2 market identifier (e.g. \"ae\").","type":"string"}},"type":"object"},"GetConsumerPriceOverviewResponse":{"description":"GetConsumerPriceOverviewResponse contains headline basket and coverage metrics.","properties":{"asOf":{"description":"as_of is the Unix millisecond timestamp of the snapshot.","format":"int64","type":"string"},"coveragePct":{"description":"coverage_pct is the fraction of basket items with current observations.","format":"double","type":"number"},"currencyCode":{"description":"currency_code is the ISO 4217 currency for price values.","type":"string"},"essentialsIndex":{"description":"essentials_index is the fixed basket index value (base = 100).","format":"double","type":"number"},"freshnessLagMin":{"description":"freshness_lag_min is the average minutes since last observation across all retailers.","format":"int32","type":"integer"},"marketCode":{"description":"market_code echoes the requested market.","type":"string"},"momPct":{"description":"mom_pct is the month-over-month percentage change in the essentials index.","format":"double","type":"number"},"retailerSpreadPct":{"description":"retailer_spread_pct is the basket cost spread between cheapest and most expensive retailer.","format":"double","type":"number"},"topCategories":{"items":{"$ref":"#/components/schemas/CategorySnapshot"},"type":"array"},"upstreamUnavailable":{"description":"upstream_unavailable is true when the companion service could not be reached.","type":"boolean"},"valueBasketIndex":{"description":"value_basket_index is the value basket index value (base = 100).","format":"double","type":"number"},"wowPct":{"description":"wow_pct is the week-over-week percentage change in the essentials index.","format":"double","type":"number"}},"type":"object"},"ListConsumerPriceCategoriesRequest":{"description":"ListConsumerPriceCategoriesRequest parameters for category listing.","properties":{"basketSlug":{"description":"basket_slug selects the basket scope.","type":"string"},"marketCode":{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","type":"string"},"range":{"description":"range is one of \"7d\", \"30d\", \"90d\", \"180d\".","type":"string"}},"type":"object"},"ListConsumerPriceCategoriesResponse":{"description":"ListConsumerPriceCategoriesResponse holds category-level price snapshots.","properties":{"asOf":{"description":"as_of is the Unix millisecond timestamp of the snapshot.","format":"int64","type":"string"},"categories":{"items":{"$ref":"#/components/schemas/CategorySnapshot"},"type":"array"},"marketCode":{"description":"market_code echoes the requested market.","type":"string"},"range":{"description":"range echoes the requested range.","type":"string"},"upstreamUnavailable":{"description":"upstream_unavailable is true when the companion service could not be reached.","type":"boolean"}},"type":"object"},"ListConsumerPriceMoversRequest":{"description":"ListConsumerPriceMoversRequest parameters for the movers RPC.","properties":{"categorySlug":{"description":"category_slug filters to a single category when set.","type":"string"},"limit":{"description":"limit caps the number of risers and fallers returned (default 10).","format":"int32","type":"integer"},"marketCode":{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","type":"string"},"range":{"description":"range is one of \"7d\", \"30d\", \"90d\".","type":"string"}},"type":"object"},"ListConsumerPriceMoversResponse":{"description":"ListConsumerPriceMoversResponse holds the top price movers.","properties":{"asOf":{"description":"as_of is the Unix millisecond timestamp of the snapshot.","format":"int64","type":"string"},"fallers":{"items":{"$ref":"#/components/schemas/PriceMover"},"type":"array"},"marketCode":{"description":"market_code echoes the requested market.","type":"string"},"range":{"description":"range echoes the requested range.","type":"string"},"risers":{"items":{"$ref":"#/components/schemas/PriceMover"},"type":"array"},"upstreamUnavailable":{"description":"upstream_unavailable is true when the companion service could not be reached.","type":"boolean"}},"type":"object"},"ListRetailerPriceSpreadsRequest":{"description":"ListRetailerPriceSpreadsRequest parameters for the retailer spread RPC.","properties":{"basketSlug":{"description":"basket_slug selects which basket to compare across retailers.","type":"string"},"marketCode":{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","type":"string"}},"type":"object"},"ListRetailerPriceSpreadsResponse":{"description":"ListRetailerPriceSpreadsResponse holds cheapest-basket rankings.","properties":{"asOf":{"description":"as_of is the Unix millisecond timestamp of the snapshot.","format":"int64","type":"string"},"basketSlug":{"description":"basket_slug echoes the requested basket.","type":"string"},"currencyCode":{"description":"currency_code is the ISO 4217 currency code.","type":"string"},"marketCode":{"description":"market_code echoes the requested market.","type":"string"},"retailers":{"items":{"$ref":"#/components/schemas/RetailerSpread"},"type":"array"},"spreadPct":{"description":"spread_pct is the percentage difference between cheapest and most expensive retailer.","format":"double","type":"number"},"upstreamUnavailable":{"description":"upstream_unavailable is true when the companion service could not be reached.","type":"boolean"}},"type":"object"},"PriceMover":{"description":"PriceMover describes a product with a notable upward or downward price move.","properties":{"category":{"description":"category is the product category slug.","type":"string"},"changePct":{"description":"change_pct is the signed percentage change over the selected window.","format":"double","type":"number"},"currencyCode":{"description":"currency_code is the ISO 4217 currency code.","type":"string"},"currentPrice":{"description":"current_price is the latest observed price.","format":"double","type":"number"},"productId":{"description":"product_id is the retailer product identifier.","type":"string"},"retailerSlug":{"description":"retailer_slug identifies the retailer where this move was observed.","type":"string"},"title":{"description":"title is the normalized product title.","type":"string"}},"type":"object"},"RetailerFreshnessInfo":{"description":"RetailerFreshnessInfo describes the operational health of one retailer feed.","properties":{"freshnessMin":{"description":"freshness_min is minutes since last successful observation.","format":"int32","type":"integer"},"lastRunAt":{"description":"last_run_at is the Unix millisecond timestamp of the last successful scrape.","format":"int64","type":"string"},"name":{"description":"name is the retailer display name.","type":"string"},"parseSuccessRate":{"description":"parse_success_rate is the fraction of pages parsed successfully (0–1).","format":"double","type":"number"},"slug":{"description":"slug is the retailer identifier.","type":"string"},"status":{"description":"status is one of \"ok\", \"stale\", \"failed\", \"unknown\".","type":"string"}},"type":"object"},"RetailerSpread":{"description":"RetailerSpread holds the basket cost breakdown for one retailer.","properties":{"basketTotal":{"description":"basket_total is the sum of matched basket item prices at this retailer.","format":"double","type":"number"},"currencyCode":{"description":"currency_code is the ISO 4217 currency code.","type":"string"},"deltaVsCheapest":{"description":"delta_vs_cheapest is the absolute price difference vs the cheapest retailer.","format":"double","type":"number"},"deltaVsCheapestPct":{"description":"delta_vs_cheapest_pct is the percentage difference vs the cheapest retailer.","format":"double","type":"number"},"freshnessMin":{"description":"freshness_min is minutes since the last successful scrape for this retailer.","format":"int32","type":"integer"},"itemCount":{"description":"item_count is the number of matched basket items observed.","format":"int32","type":"integer"},"name":{"description":"name is the retailer display name.","type":"string"},"slug":{"description":"slug is the retailer identifier.","type":"string"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ConsumerPricesService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/consumer-prices/v1/get-consumer-price-basket-series":{"get":{"description":"GetConsumerPriceBasketSeries retrieves the basket index time series.","operationId":"GetConsumerPriceBasketSeries","parameters":[{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","in":"query","name":"market_code","required":false,"schema":{"type":"string"}},{"description":"basket_slug selects the basket (e.g. \"essentials-ae\").","in":"query","name":"basket_slug","required":false,"schema":{"type":"string"}},{"description":"range is one of \"7d\", \"30d\", \"90d\", \"180d\".","in":"query","name":"range","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetConsumerPriceBasketSeriesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetConsumerPriceBasketSeries","tags":["ConsumerPricesService"]}},"/api/consumer-prices/v1/get-consumer-price-freshness":{"get":{"description":"GetConsumerPriceFreshness retrieves feed freshness and coverage health per retailer.","operationId":"GetConsumerPriceFreshness","parameters":[{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","in":"query","name":"market_code","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetConsumerPriceFreshnessResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetConsumerPriceFreshness","tags":["ConsumerPricesService"]}},"/api/consumer-prices/v1/get-consumer-price-overview":{"get":{"description":"GetConsumerPriceOverview retrieves headline basket indices and coverage metrics.","operationId":"GetConsumerPriceOverview","parameters":[{"description":"market_code is the ISO 3166-1 alpha-2 market identifier (e.g. \"ae\").","in":"query","name":"market_code","required":false,"schema":{"type":"string"}},{"description":"basket_slug selects which basket to use (e.g. \"essentials-ae\").","in":"query","name":"basket_slug","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetConsumerPriceOverviewResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetConsumerPriceOverview","tags":["ConsumerPricesService"]}},"/api/consumer-prices/v1/list-consumer-price-categories":{"get":{"description":"ListConsumerPriceCategories retrieves category summaries with sparklines.","operationId":"ListConsumerPriceCategories","parameters":[{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","in":"query","name":"market_code","required":false,"schema":{"type":"string"}},{"description":"basket_slug selects the basket scope.","in":"query","name":"basket_slug","required":false,"schema":{"type":"string"}},{"description":"range is one of \"7d\", \"30d\", \"90d\", \"180d\".","in":"query","name":"range","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListConsumerPriceCategoriesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListConsumerPriceCategories","tags":["ConsumerPricesService"]}},"/api/consumer-prices/v1/list-consumer-price-movers":{"get":{"description":"ListConsumerPriceMovers retrieves the largest upward and downward item price moves.","operationId":"ListConsumerPriceMovers","parameters":[{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","in":"query","name":"market_code","required":false,"schema":{"type":"string"}},{"description":"range is one of \"7d\", \"30d\", \"90d\".","in":"query","name":"range","required":false,"schema":{"type":"string"}},{"description":"limit caps the number of risers and fallers returned (default 10).","in":"query","name":"limit","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"category_slug filters to a single category when set.","in":"query","name":"category_slug","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListConsumerPriceMoversResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListConsumerPriceMovers","tags":["ConsumerPricesService"]}},"/api/consumer-prices/v1/list-retailer-price-spreads":{"get":{"description":"ListRetailerPriceSpreads retrieves cheapest-basket comparisons across retailers.","operationId":"ListRetailerPriceSpreads","parameters":[{"description":"market_code is the ISO 3166-1 alpha-2 market identifier.","in":"query","name":"market_code","required":false,"schema":{"type":"string"}},{"description":"basket_slug selects which basket to compare across retailers.","in":"query","name":"basket_slug","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListRetailerPriceSpreadsResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListRetailerPriceSpreads","tags":["ConsumerPricesService"]}}}} \ No newline at end of file diff --git a/docs/api/ConsumerPricesService.openapi.yaml b/docs/api/ConsumerPricesService.openapi.yaml new file mode 100644 index 000000000..eed49c581 --- /dev/null +++ b/docs/api/ConsumerPricesService.openapi.yaml @@ -0,0 +1,673 @@ +openapi: 3.1.0 +info: + title: ConsumerPricesService API + version: 1.0.0 +paths: + /api/consumer-prices/v1/get-consumer-price-overview: + get: + tags: + - ConsumerPricesService + summary: GetConsumerPriceOverview + description: GetConsumerPriceOverview retrieves headline basket indices and coverage metrics. + operationId: GetConsumerPriceOverview + parameters: + - name: market_code + in: query + description: market_code is the ISO 3166-1 alpha-2 market identifier (e.g. "ae"). + required: false + schema: + type: string + - name: basket_slug + in: query + description: basket_slug selects which basket to use (e.g. "essentials-ae"). + required: false + schema: + type: string + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/GetConsumerPriceOverviewResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /api/consumer-prices/v1/get-consumer-price-basket-series: + get: + tags: + - ConsumerPricesService + summary: GetConsumerPriceBasketSeries + description: GetConsumerPriceBasketSeries retrieves the basket index time series. + operationId: GetConsumerPriceBasketSeries + parameters: + - name: market_code + in: query + description: market_code is the ISO 3166-1 alpha-2 market identifier. + required: false + schema: + type: string + - name: basket_slug + in: query + description: basket_slug selects the basket (e.g. "essentials-ae"). + required: false + schema: + type: string + - name: range + in: query + description: range is one of "7d", "30d", "90d", "180d". + required: false + schema: + type: string + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/GetConsumerPriceBasketSeriesResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /api/consumer-prices/v1/list-consumer-price-categories: + get: + tags: + - ConsumerPricesService + summary: ListConsumerPriceCategories + description: ListConsumerPriceCategories retrieves category summaries with sparklines. + operationId: ListConsumerPriceCategories + parameters: + - name: market_code + in: query + description: market_code is the ISO 3166-1 alpha-2 market identifier. + required: false + schema: + type: string + - name: basket_slug + in: query + description: basket_slug selects the basket scope. + required: false + schema: + type: string + - name: range + in: query + description: range is one of "7d", "30d", "90d", "180d". + required: false + schema: + type: string + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ListConsumerPriceCategoriesResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /api/consumer-prices/v1/list-consumer-price-movers: + get: + tags: + - ConsumerPricesService + summary: ListConsumerPriceMovers + description: ListConsumerPriceMovers retrieves the largest upward and downward item price moves. + operationId: ListConsumerPriceMovers + parameters: + - name: market_code + in: query + description: market_code is the ISO 3166-1 alpha-2 market identifier. + required: false + schema: + type: string + - name: range + in: query + description: range is one of "7d", "30d", "90d". + required: false + schema: + type: string + - name: limit + in: query + description: limit caps the number of risers and fallers returned (default 10). + required: false + schema: + type: integer + format: int32 + - name: category_slug + in: query + description: category_slug filters to a single category when set. + required: false + schema: + type: string + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ListConsumerPriceMoversResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /api/consumer-prices/v1/list-retailer-price-spreads: + get: + tags: + - ConsumerPricesService + summary: ListRetailerPriceSpreads + description: ListRetailerPriceSpreads retrieves cheapest-basket comparisons across retailers. + operationId: ListRetailerPriceSpreads + parameters: + - name: market_code + in: query + description: market_code is the ISO 3166-1 alpha-2 market identifier. + required: false + schema: + type: string + - name: basket_slug + in: query + description: basket_slug selects which basket to compare across retailers. + required: false + schema: + type: string + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ListRetailerPriceSpreadsResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /api/consumer-prices/v1/get-consumer-price-freshness: + get: + tags: + - ConsumerPricesService + summary: GetConsumerPriceFreshness + description: GetConsumerPriceFreshness retrieves feed freshness and coverage health per retailer. + operationId: GetConsumerPriceFreshness + parameters: + - name: market_code + in: query + description: market_code is the ISO 3166-1 alpha-2 market identifier. + required: false + schema: + type: string + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/GetConsumerPriceFreshnessResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' +components: + schemas: + Error: + type: object + properties: + message: + type: string + description: Error message (e.g., 'user not found', 'database connection failed') + description: Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize. + FieldViolation: + type: object + properties: + field: + type: string + description: The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key') + description: + type: string + description: Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing') + required: + - field + - description + description: FieldViolation describes a single validation error for a specific field. + ValidationError: + type: object + properties: + violations: + type: array + items: + $ref: '#/components/schemas/FieldViolation' + description: List of validation violations + required: + - violations + description: ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong. + GetConsumerPriceOverviewRequest: + type: object + properties: + marketCode: + type: string + description: market_code is the ISO 3166-1 alpha-2 market identifier (e.g. "ae"). + basketSlug: + type: string + description: basket_slug selects which basket to use (e.g. "essentials-ae"). + description: GetConsumerPriceOverviewRequest parameters for the overview RPC. + GetConsumerPriceOverviewResponse: + type: object + properties: + marketCode: + type: string + description: market_code echoes the requested market. + asOf: + type: string + format: int64 + description: as_of is the Unix millisecond timestamp of the snapshot. + currencyCode: + type: string + description: currency_code is the ISO 4217 currency for price values. + essentialsIndex: + type: number + format: double + description: essentials_index is the fixed basket index value (base = 100). + valueBasketIndex: + type: number + format: double + description: value_basket_index is the value basket index value (base = 100). + wowPct: + type: number + format: double + description: wow_pct is the week-over-week percentage change in the essentials index. + momPct: + type: number + format: double + description: mom_pct is the month-over-month percentage change in the essentials index. + retailerSpreadPct: + type: number + format: double + description: retailer_spread_pct is the basket cost spread between cheapest and most expensive retailer. + coveragePct: + type: number + format: double + description: coverage_pct is the fraction of basket items with current observations. + freshnessLagMin: + type: integer + format: int32 + description: freshness_lag_min is the average minutes since last observation across all retailers. + topCategories: + type: array + items: + $ref: '#/components/schemas/CategorySnapshot' + upstreamUnavailable: + type: boolean + description: upstream_unavailable is true when the companion service could not be reached. + description: GetConsumerPriceOverviewResponse contains headline basket and coverage metrics. + CategorySnapshot: + type: object + properties: + slug: + type: string + description: slug is the machine-readable category identifier (e.g. "eggs", "rice"). + name: + type: string + description: name is the human-readable category label. + wowPct: + type: number + format: double + description: wow_pct is the week-over-week percentage change. + momPct: + type: number + format: double + description: mom_pct is the month-over-month percentage change. + currentIndex: + type: number + format: double + description: current_index is the current price index value (base = 100). + sparkline: + type: array + items: + type: number + format: double + description: sparkline is an ordered sequence of index values for the selected range. + coveragePct: + type: number + format: double + description: coverage_pct is the percentage of basket items observed for this category. + itemCount: + type: integer + format: int32 + description: item_count is the number of observed products in this category. + description: CategorySnapshot holds price index data for a single product category. + GetConsumerPriceBasketSeriesRequest: + type: object + properties: + marketCode: + type: string + description: market_code is the ISO 3166-1 alpha-2 market identifier. + basketSlug: + type: string + description: basket_slug selects the basket (e.g. "essentials-ae"). + range: + type: string + description: range is one of "7d", "30d", "90d", "180d". + description: GetConsumerPriceBasketSeriesRequest parameters for time series data. + GetConsumerPriceBasketSeriesResponse: + type: object + properties: + marketCode: + type: string + description: market_code echoes the requested market. + basketSlug: + type: string + description: basket_slug echoes the requested basket. + asOf: + type: string + format: int64 + description: as_of is the Unix millisecond timestamp of the snapshot. + currencyCode: + type: string + description: currency_code is the ISO 4217 currency code. + range: + type: string + description: range echoes the requested range. + essentialsSeries: + type: array + items: + $ref: '#/components/schemas/BasketPoint' + valueSeries: + type: array + items: + $ref: '#/components/schemas/BasketPoint' + upstreamUnavailable: + type: boolean + description: upstream_unavailable is true when the companion service could not be reached. + description: GetConsumerPriceBasketSeriesResponse contains the basket index time series. + BasketPoint: + type: object + properties: + date: + type: string + description: date is the ISO 8601 date string (YYYY-MM-DD). + index: + type: number + format: double + description: index is the basket index value (base = 100). + description: BasketPoint is a single data point in a basket index time series. + ListConsumerPriceCategoriesRequest: + type: object + properties: + marketCode: + type: string + description: market_code is the ISO 3166-1 alpha-2 market identifier. + basketSlug: + type: string + description: basket_slug selects the basket scope. + range: + type: string + description: range is one of "7d", "30d", "90d", "180d". + description: ListConsumerPriceCategoriesRequest parameters for category listing. + ListConsumerPriceCategoriesResponse: + type: object + properties: + marketCode: + type: string + description: market_code echoes the requested market. + asOf: + type: string + format: int64 + description: as_of is the Unix millisecond timestamp of the snapshot. + range: + type: string + description: range echoes the requested range. + categories: + type: array + items: + $ref: '#/components/schemas/CategorySnapshot' + upstreamUnavailable: + type: boolean + description: upstream_unavailable is true when the companion service could not be reached. + description: ListConsumerPriceCategoriesResponse holds category-level price snapshots. + ListConsumerPriceMoversRequest: + type: object + properties: + marketCode: + type: string + description: market_code is the ISO 3166-1 alpha-2 market identifier. + range: + type: string + description: range is one of "7d", "30d", "90d". + limit: + type: integer + format: int32 + description: limit caps the number of risers and fallers returned (default 10). + categorySlug: + type: string + description: category_slug filters to a single category when set. + description: ListConsumerPriceMoversRequest parameters for the movers RPC. + ListConsumerPriceMoversResponse: + type: object + properties: + marketCode: + type: string + description: market_code echoes the requested market. + asOf: + type: string + format: int64 + description: as_of is the Unix millisecond timestamp of the snapshot. + range: + type: string + description: range echoes the requested range. + risers: + type: array + items: + $ref: '#/components/schemas/PriceMover' + fallers: + type: array + items: + $ref: '#/components/schemas/PriceMover' + upstreamUnavailable: + type: boolean + description: upstream_unavailable is true when the companion service could not be reached. + description: ListConsumerPriceMoversResponse holds the top price movers. + PriceMover: + type: object + properties: + productId: + type: string + description: product_id is the retailer product identifier. + title: + type: string + description: title is the normalized product title. + category: + type: string + description: category is the product category slug. + retailerSlug: + type: string + description: retailer_slug identifies the retailer where this move was observed. + changePct: + type: number + format: double + description: change_pct is the signed percentage change over the selected window. + currentPrice: + type: number + format: double + description: current_price is the latest observed price. + currencyCode: + type: string + description: currency_code is the ISO 4217 currency code. + description: PriceMover describes a product with a notable upward or downward price move. + ListRetailerPriceSpreadsRequest: + type: object + properties: + marketCode: + type: string + description: market_code is the ISO 3166-1 alpha-2 market identifier. + basketSlug: + type: string + description: basket_slug selects which basket to compare across retailers. + description: ListRetailerPriceSpreadsRequest parameters for the retailer spread RPC. + ListRetailerPriceSpreadsResponse: + type: object + properties: + marketCode: + type: string + description: market_code echoes the requested market. + asOf: + type: string + format: int64 + description: as_of is the Unix millisecond timestamp of the snapshot. + basketSlug: + type: string + description: basket_slug echoes the requested basket. + currencyCode: + type: string + description: currency_code is the ISO 4217 currency code. + retailers: + type: array + items: + $ref: '#/components/schemas/RetailerSpread' + spreadPct: + type: number + format: double + description: spread_pct is the percentage difference between cheapest and most expensive retailer. + upstreamUnavailable: + type: boolean + description: upstream_unavailable is true when the companion service could not be reached. + description: ListRetailerPriceSpreadsResponse holds cheapest-basket rankings. + RetailerSpread: + type: object + properties: + slug: + type: string + description: slug is the retailer identifier. + name: + type: string + description: name is the retailer display name. + basketTotal: + type: number + format: double + description: basket_total is the sum of matched basket item prices at this retailer. + deltaVsCheapest: + type: number + format: double + description: delta_vs_cheapest is the absolute price difference vs the cheapest retailer. + deltaVsCheapestPct: + type: number + format: double + description: delta_vs_cheapest_pct is the percentage difference vs the cheapest retailer. + itemCount: + type: integer + format: int32 + description: item_count is the number of matched basket items observed. + freshnessMin: + type: integer + format: int32 + description: freshness_min is minutes since the last successful scrape for this retailer. + currencyCode: + type: string + description: currency_code is the ISO 4217 currency code. + description: RetailerSpread holds the basket cost breakdown for one retailer. + GetConsumerPriceFreshnessRequest: + type: object + properties: + marketCode: + type: string + description: market_code is the ISO 3166-1 alpha-2 market identifier. + description: GetConsumerPriceFreshnessRequest parameters for the freshness RPC. + GetConsumerPriceFreshnessResponse: + type: object + properties: + marketCode: + type: string + description: market_code echoes the requested market. + asOf: + type: string + format: int64 + description: as_of is the Unix millisecond timestamp of the snapshot. + retailers: + type: array + items: + $ref: '#/components/schemas/RetailerFreshnessInfo' + overallFreshnessMin: + type: integer + format: int32 + description: overall_freshness_min is the average freshness lag across all retailers. + stalledCount: + type: integer + format: int32 + description: stalled_count is the number of retailers with no recent successful scrape. + upstreamUnavailable: + type: boolean + description: upstream_unavailable is true when the companion service could not be reached. + description: GetConsumerPriceFreshnessResponse describes feed health for all retailers. + RetailerFreshnessInfo: + type: object + properties: + slug: + type: string + description: slug is the retailer identifier. + name: + type: string + description: name is the retailer display name. + lastRunAt: + type: string + format: int64 + description: last_run_at is the Unix millisecond timestamp of the last successful scrape. + status: + type: string + description: status is one of "ok", "stale", "failed", "unknown". + parseSuccessRate: + type: number + format: double + description: parse_success_rate is the fraction of pages parsed successfully (0–1). + freshnessMin: + type: integer + format: int32 + description: freshness_min is minutes since last successful observation. + description: RetailerFreshnessInfo describes the operational health of one retailer feed. diff --git a/proto/worldmonitor/consumer_prices/v1/consumer_prices_data.proto b/proto/worldmonitor/consumer_prices/v1/consumer_prices_data.proto new file mode 100644 index 000000000..ad2ee5587 --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/consumer_prices_data.proto @@ -0,0 +1,85 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +// CategorySnapshot holds price index data for a single product category. +message CategorySnapshot { + // slug is the machine-readable category identifier (e.g. "eggs", "rice"). + string slug = 1; + // name is the human-readable category label. + string name = 2; + // wow_pct is the week-over-week percentage change. + double wow_pct = 3; + // mom_pct is the month-over-month percentage change. + double mom_pct = 4; + // current_index is the current price index value (base = 100). + double current_index = 5; + // sparkline is an ordered sequence of index values for the selected range. + repeated double sparkline = 6; + // coverage_pct is the percentage of basket items observed for this category. + double coverage_pct = 7; + // item_count is the number of observed products in this category. + int32 item_count = 8; +} + +// PriceMover describes a product with a notable upward or downward price move. +message PriceMover { + // product_id is the retailer product identifier. + string product_id = 1; + // title is the normalized product title. + string title = 2; + // category is the product category slug. + string category = 3; + // retailer_slug identifies the retailer where this move was observed. + string retailer_slug = 4; + // change_pct is the signed percentage change over the selected window. + double change_pct = 5; + // current_price is the latest observed price. + double current_price = 6; + // currency_code is the ISO 4217 currency code. + string currency_code = 7; +} + +// RetailerSpread holds the basket cost breakdown for one retailer. +message RetailerSpread { + // slug is the retailer identifier. + string slug = 1; + // name is the retailer display name. + string name = 2; + // basket_total is the sum of matched basket item prices at this retailer. + double basket_total = 3; + // delta_vs_cheapest is the absolute price difference vs the cheapest retailer. + double delta_vs_cheapest = 4; + // delta_vs_cheapest_pct is the percentage difference vs the cheapest retailer. + double delta_vs_cheapest_pct = 5; + // item_count is the number of matched basket items observed. + int32 item_count = 6; + // freshness_min is minutes since the last successful scrape for this retailer. + int32 freshness_min = 7; + // currency_code is the ISO 4217 currency code. + string currency_code = 8; +} + +// BasketPoint is a single data point in a basket index time series. +message BasketPoint { + // date is the ISO 8601 date string (YYYY-MM-DD). + string date = 1; + // index is the basket index value (base = 100). + double index = 2; +} + +// RetailerFreshnessInfo describes the operational health of one retailer feed. +message RetailerFreshnessInfo { + // slug is the retailer identifier. + string slug = 1; + // name is the retailer display name. + string name = 2; + // last_run_at is the Unix millisecond timestamp of the last successful scrape. + int64 last_run_at = 3; + // status is one of "ok", "stale", "failed", "unknown". + string status = 4; + // parse_success_rate is the fraction of pages parsed successfully (0–1). + double parse_success_rate = 5; + // freshness_min is minutes since last successful observation. + int32 freshness_min = 6; +} diff --git a/proto/worldmonitor/consumer_prices/v1/get_consumer_price_basket_series.proto b/proto/worldmonitor/consumer_prices/v1/get_consumer_price_basket_series.proto new file mode 100644 index 000000000..6a5d24215 --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/get_consumer_price_basket_series.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/consumer_prices_data.proto"; + +// GetConsumerPriceBasketSeriesRequest parameters for time series data. +message GetConsumerPriceBasketSeriesRequest { + // market_code is the ISO 3166-1 alpha-2 market identifier. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // basket_slug selects the basket (e.g. "essentials-ae"). + string basket_slug = 2 [(sebuf.http.query) = { name: "basket_slug" }]; + // range is one of "7d", "30d", "90d", "180d". + string range = 3 [(sebuf.http.query) = { name: "range" }]; +} + +// GetConsumerPriceBasketSeriesResponse contains the basket index time series. +message GetConsumerPriceBasketSeriesResponse { + // market_code echoes the requested market. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // basket_slug echoes the requested basket. + string basket_slug = 2 [(sebuf.http.query) = { name: "basket_slug" }]; + // as_of is the Unix millisecond timestamp of the snapshot. + int64 as_of = 3; + // currency_code is the ISO 4217 currency code. + string currency_code = 4; + // range echoes the requested range. + string range = 5; + // essentials_series is the fixed basket index series. + repeated BasketPoint essentials_series = 6; + // value_series is the value basket index series. + repeated BasketPoint value_series = 7; + // upstream_unavailable is true when the companion service could not be reached. + bool upstream_unavailable = 8; +} diff --git a/proto/worldmonitor/consumer_prices/v1/get_consumer_price_freshness.proto b/proto/worldmonitor/consumer_prices/v1/get_consumer_price_freshness.proto new file mode 100644 index 000000000..eed135fd4 --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/get_consumer_price_freshness.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/consumer_prices_data.proto"; + +// GetConsumerPriceFreshnessRequest parameters for the freshness RPC. +message GetConsumerPriceFreshnessRequest { + // market_code is the ISO 3166-1 alpha-2 market identifier. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; +} + +// GetConsumerPriceFreshnessResponse describes feed health for all retailers. +message GetConsumerPriceFreshnessResponse { + // market_code echoes the requested market. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // as_of is the Unix millisecond timestamp of the snapshot. + int64 as_of = 2; + // retailers holds per-retailer freshness data. + repeated RetailerFreshnessInfo retailers = 3; + // overall_freshness_min is the average freshness lag across all retailers. + int32 overall_freshness_min = 4; + // stalled_count is the number of retailers with no recent successful scrape. + int32 stalled_count = 5; + // upstream_unavailable is true when the companion service could not be reached. + bool upstream_unavailable = 6; +} diff --git a/proto/worldmonitor/consumer_prices/v1/get_consumer_price_overview.proto b/proto/worldmonitor/consumer_prices/v1/get_consumer_price_overview.proto new file mode 100644 index 000000000..1b22c885b --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/get_consumer_price_overview.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/consumer_prices_data.proto"; + +// GetConsumerPriceOverviewRequest parameters for the overview RPC. +message GetConsumerPriceOverviewRequest { + // market_code is the ISO 3166-1 alpha-2 market identifier (e.g. "ae"). + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // basket_slug selects which basket to use (e.g. "essentials-ae"). + string basket_slug = 2 [(sebuf.http.query) = { name: "basket_slug" }]; +} + +// GetConsumerPriceOverviewResponse contains headline basket and coverage metrics. +message GetConsumerPriceOverviewResponse { + // market_code echoes the requested market. + string market_code = 1; + // as_of is the Unix millisecond timestamp of the snapshot. + int64 as_of = 2; + // currency_code is the ISO 4217 currency for price values. + string currency_code = 3; + // essentials_index is the fixed basket index value (base = 100). + double essentials_index = 4; + // value_basket_index is the value basket index value (base = 100). + double value_basket_index = 5; + // wow_pct is the week-over-week percentage change in the essentials index. + double wow_pct = 6; + // mom_pct is the month-over-month percentage change in the essentials index. + double mom_pct = 7; + // retailer_spread_pct is the basket cost spread between cheapest and most expensive retailer. + double retailer_spread_pct = 8; + // coverage_pct is the fraction of basket items with current observations. + double coverage_pct = 9; + // freshness_lag_min is the average minutes since last observation across all retailers. + int32 freshness_lag_min = 10; + // top_categories holds the top category movers. + repeated CategorySnapshot top_categories = 11; + // upstream_unavailable is true when the companion service could not be reached. + bool upstream_unavailable = 12; +} diff --git a/proto/worldmonitor/consumer_prices/v1/list_consumer_price_categories.proto b/proto/worldmonitor/consumer_prices/v1/list_consumer_price_categories.proto new file mode 100644 index 000000000..de5360dbc --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/list_consumer_price_categories.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/consumer_prices_data.proto"; + +// ListConsumerPriceCategoriesRequest parameters for category listing. +message ListConsumerPriceCategoriesRequest { + // market_code is the ISO 3166-1 alpha-2 market identifier. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // basket_slug selects the basket scope. + string basket_slug = 2 [(sebuf.http.query) = { name: "basket_slug" }]; + // range is one of "7d", "30d", "90d", "180d". + string range = 3 [(sebuf.http.query) = { name: "range" }]; +} + +// ListConsumerPriceCategoriesResponse holds category-level price snapshots. +message ListConsumerPriceCategoriesResponse { + // market_code echoes the requested market. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // as_of is the Unix millisecond timestamp of the snapshot. + int64 as_of = 2; + // range echoes the requested range. + string range = 3 [(sebuf.http.query) = { name: "range" }]; + // categories holds all category snapshots, sorted by absolute mom_pct descending. + repeated CategorySnapshot categories = 4; + // upstream_unavailable is true when the companion service could not be reached. + bool upstream_unavailable = 5; +} diff --git a/proto/worldmonitor/consumer_prices/v1/list_consumer_price_movers.proto b/proto/worldmonitor/consumer_prices/v1/list_consumer_price_movers.proto new file mode 100644 index 000000000..bae24f1d7 --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/list_consumer_price_movers.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/consumer_prices_data.proto"; + +// ListConsumerPriceMoversRequest parameters for the movers RPC. +message ListConsumerPriceMoversRequest { + // market_code is the ISO 3166-1 alpha-2 market identifier. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // range is one of "7d", "30d", "90d". + string range = 2 [(sebuf.http.query) = { name: "range" }]; + // limit caps the number of risers and fallers returned (default 10). + int32 limit = 3 [(sebuf.http.query) = { name: "limit" }]; + // category_slug filters to a single category when set. + string category_slug = 4 [(sebuf.http.query) = { name: "category_slug" }]; +} + +// ListConsumerPriceMoversResponse holds the top price movers. +message ListConsumerPriceMoversResponse { + // market_code echoes the requested market. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // as_of is the Unix millisecond timestamp of the snapshot. + int64 as_of = 2; + // range echoes the requested range. + string range = 3 [(sebuf.http.query) = { name: "range" }]; + // risers holds the products with the largest upward price moves. + repeated PriceMover risers = 4; + // fallers holds the products with the largest downward price moves. + repeated PriceMover fallers = 5; + // upstream_unavailable is true when the companion service could not be reached. + bool upstream_unavailable = 6; +} diff --git a/proto/worldmonitor/consumer_prices/v1/list_retailer_price_spreads.proto b/proto/worldmonitor/consumer_prices/v1/list_retailer_price_spreads.proto new file mode 100644 index 000000000..b0b7c3800 --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/list_retailer_price_spreads.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/consumer_prices_data.proto"; + +// ListRetailerPriceSpreadsRequest parameters for the retailer spread RPC. +message ListRetailerPriceSpreadsRequest { + // market_code is the ISO 3166-1 alpha-2 market identifier. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // basket_slug selects which basket to compare across retailers. + string basket_slug = 2 [(sebuf.http.query) = { name: "basket_slug" }]; +} + +// ListRetailerPriceSpreadsResponse holds cheapest-basket rankings. +message ListRetailerPriceSpreadsResponse { + // market_code echoes the requested market. + string market_code = 1 [(sebuf.http.query) = { name: "market_code" }]; + // as_of is the Unix millisecond timestamp of the snapshot. + int64 as_of = 2; + // basket_slug echoes the requested basket. + string basket_slug = 3; + // currency_code is the ISO 4217 currency code. + string currency_code = 4; + // retailers holds retailer basket totals, sorted cheapest first. + repeated RetailerSpread retailers = 5; + // spread_pct is the percentage difference between cheapest and most expensive retailer. + double spread_pct = 6; + // upstream_unavailable is true when the companion service could not be reached. + bool upstream_unavailable = 7; +} diff --git a/proto/worldmonitor/consumer_prices/v1/service.proto b/proto/worldmonitor/consumer_prices/v1/service.proto new file mode 100644 index 000000000..9e1110b7e --- /dev/null +++ b/proto/worldmonitor/consumer_prices/v1/service.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package worldmonitor.consumer_prices.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/consumer_prices/v1/get_consumer_price_overview.proto"; +import "worldmonitor/consumer_prices/v1/get_consumer_price_basket_series.proto"; +import "worldmonitor/consumer_prices/v1/list_consumer_price_categories.proto"; +import "worldmonitor/consumer_prices/v1/list_consumer_price_movers.proto"; +import "worldmonitor/consumer_prices/v1/list_retailer_price_spreads.proto"; +import "worldmonitor/consumer_prices/v1/get_consumer_price_freshness.proto"; + +// ConsumerPricesService provides APIs for consumer price intelligence. +// Data is sourced from the consumer-prices-core companion service via Redis snapshots. +service ConsumerPricesService { + option (sebuf.http.service_config) = {base_path: "/api/consumer-prices/v1"}; + + // GetConsumerPriceOverview retrieves headline basket indices and coverage metrics. + rpc GetConsumerPriceOverview(GetConsumerPriceOverviewRequest) returns (GetConsumerPriceOverviewResponse) { + option (sebuf.http.config) = {path: "/get-consumer-price-overview", method: HTTP_METHOD_GET}; + } + + // GetConsumerPriceBasketSeries retrieves the basket index time series. + rpc GetConsumerPriceBasketSeries(GetConsumerPriceBasketSeriesRequest) returns (GetConsumerPriceBasketSeriesResponse) { + option (sebuf.http.config) = {path: "/get-consumer-price-basket-series", method: HTTP_METHOD_GET}; + } + + // ListConsumerPriceCategories retrieves category summaries with sparklines. + rpc ListConsumerPriceCategories(ListConsumerPriceCategoriesRequest) returns (ListConsumerPriceCategoriesResponse) { + option (sebuf.http.config) = {path: "/list-consumer-price-categories", method: HTTP_METHOD_GET}; + } + + // ListConsumerPriceMovers retrieves the largest upward and downward item price moves. + rpc ListConsumerPriceMovers(ListConsumerPriceMoversRequest) returns (ListConsumerPriceMoversResponse) { + option (sebuf.http.config) = {path: "/list-consumer-price-movers", method: HTTP_METHOD_GET}; + } + + // ListRetailerPriceSpreads retrieves cheapest-basket comparisons across retailers. + rpc ListRetailerPriceSpreads(ListRetailerPriceSpreadsRequest) returns (ListRetailerPriceSpreadsResponse) { + option (sebuf.http.config) = {path: "/list-retailer-price-spreads", method: HTTP_METHOD_GET}; + } + + // GetConsumerPriceFreshness retrieves feed freshness and coverage health per retailer. + rpc GetConsumerPriceFreshness(GetConsumerPriceFreshnessRequest) returns (GetConsumerPriceFreshnessResponse) { + option (sebuf.http.config) = {path: "/get-consumer-price-freshness", method: HTTP_METHOD_GET}; + } +} diff --git a/scripts/seed-consumer-prices.mjs b/scripts/seed-consumer-prices.mjs new file mode 100644 index 000000000..6694a376a --- /dev/null +++ b/scripts/seed-consumer-prices.mjs @@ -0,0 +1,201 @@ +#!/usr/bin/env node + +/** + * Seed script: fetches compact snapshot payloads from consumer-prices-core + * and writes them to Upstash Redis for WorldMonitor bootstrap hydration. + * + * Run manually: node scripts/seed-consumer-prices.mjs + * Deployed as: Railway cron service (same pattern as ais-relay loops) + * + * Memory: runSeed() calls process.exit(0) — use extraKeys for all keys. + */ + +import { loadEnvFile, CHROME_UA, writeExtraKeyWithMeta } from './_seed-utils.mjs'; + +loadEnvFile(import.meta.url); + +const BASE_URL = process.env.CONSUMER_PRICES_CORE_BASE_URL; +const API_KEY = process.env.CONSUMER_PRICES_CORE_API_KEY; +const MARKET = process.env.CONSUMER_PRICES_DEFAULT_MARKET || 'ae'; +const BASKET = 'essentials-ae'; + +if (!BASE_URL) { + console.warn('[consumer-prices] CONSUMER_PRICES_CORE_BASE_URL not set — writing empty placeholders'); +} + +async function fetchSnapshot(path) { + if (!BASE_URL) return null; + const url = `${BASE_URL.replace(/\/$/, '')}${path}`; + try { + const resp = await fetch(url, { + headers: { + 'User-Agent': CHROME_UA, + ...(API_KEY ? { 'x-api-key': API_KEY } : {}), + }, + signal: AbortSignal.timeout(20_000), + }); + if (!resp.ok) { + console.warn(` [consumer-prices] ${path} HTTP ${resp.status}`); + return null; + } + return resp.json(); + } catch (err) { + console.warn(` [consumer-prices] ${path} error: ${err.message}`); + return null; + } +} + +function emptyOverview(market) { + return { + marketCode: market, + asOf: String(Date.now()), + currencyCode: 'AED', + essentialsIndex: 0, + valueBasketIndex: 0, + wowPct: 0, + momPct: 0, + retailerSpreadPct: 0, + coveragePct: 0, + freshnessLagMin: 0, + topCategories: [], + upstreamUnavailable: true, + }; +} + +function emptyMovers(market, range) { + return { marketCode: market, asOf: String(Date.now()), range, risers: [], fallers: [], upstreamUnavailable: true }; +} + +function emptySpread(market, basket) { + return { marketCode: market, asOf: String(Date.now()), basketSlug: basket, currencyCode: 'AED', retailers: [], spreadPct: 0, upstreamUnavailable: true }; +} + +function emptyFreshness(market) { + return { marketCode: market, asOf: String(Date.now()), retailers: [], overallFreshnessMin: 0, stalledCount: 0, upstreamUnavailable: true }; +} + +function emptyBasketSeries(market, basket, range) { + return { marketCode: market, basketSlug: basket, asOf: String(Date.now()), currencyCode: 'AED', range, essentialsSeries: [], valueSeries: [], upstreamUnavailable: true }; +} + +function emptyCategories(market, range) { + return { marketCode: market, asOf: String(Date.now()), range, categories: [], upstreamUnavailable: true }; +} + +async function run() { + console.log(`[consumer-prices] seeding market=${MARKET} basket=${BASKET}`); + + const TTL_OVERVIEW = 1800; // 30 min + const TTL_MOVERS = 1800; // 30 min + const TTL_SPREAD = 3600; // 60 min + const TTL_FRESHNESS = 600; // 10 min + const TTL_SERIES = 3600; // 60 min + const TTL_CATEGORIES = 1800; // 30 min + + // Fetch all snapshots in parallel + const [overview, movers30d, movers7d, spread, freshness, series30d, series7d, series90d, + categories30d, categories7d, categories90d] = await Promise.all([ + fetchSnapshot(`/wm/consumer-prices/v1/overview?market=${MARKET}`), + fetchSnapshot(`/wm/consumer-prices/v1/movers?market=${MARKET}&days=30`), + fetchSnapshot(`/wm/consumer-prices/v1/movers?market=${MARKET}&days=7`), + fetchSnapshot(`/wm/consumer-prices/v1/retailer-spread?market=${MARKET}&basket=${BASKET}`), + fetchSnapshot(`/wm/consumer-prices/v1/freshness?market=${MARKET}`), + fetchSnapshot(`/wm/consumer-prices/v1/basket-series?market=${MARKET}&basket=${BASKET}&range=30d`), + fetchSnapshot(`/wm/consumer-prices/v1/basket-series?market=${MARKET}&basket=${BASKET}&range=7d`), + fetchSnapshot(`/wm/consumer-prices/v1/basket-series?market=${MARKET}&basket=${BASKET}&range=90d`), + fetchSnapshot(`/wm/consumer-prices/v1/categories?market=${MARKET}&range=30d`), + fetchSnapshot(`/wm/consumer-prices/v1/categories?market=${MARKET}&range=7d`), + fetchSnapshot(`/wm/consumer-prices/v1/categories?market=${MARKET}&range=90d`), + ]); + + const writes = [ + { + key: `consumer-prices:overview:${MARKET}`, + data: overview ?? emptyOverview(MARKET), + ttl: TTL_OVERVIEW, + metaKey: `seed-meta:consumer-prices:overview:${MARKET}`, + }, + { + key: `consumer-prices:movers:${MARKET}:30d`, + data: movers30d ?? emptyMovers(MARKET, '30d'), + ttl: TTL_MOVERS, + metaKey: `seed-meta:consumer-prices:movers:${MARKET}:30d`, + }, + { + key: `consumer-prices:movers:${MARKET}:7d`, + data: movers7d ?? emptyMovers(MARKET, '7d'), + ttl: TTL_MOVERS, + metaKey: `seed-meta:consumer-prices:movers:${MARKET}:7d`, + }, + { + key: `consumer-prices:retailer-spread:${MARKET}:${BASKET}`, + data: spread ?? emptySpread(MARKET, BASKET), + ttl: TTL_SPREAD, + metaKey: `seed-meta:consumer-prices:spread:${MARKET}`, + }, + { + key: `consumer-prices:freshness:${MARKET}`, + data: freshness ?? emptyFreshness(MARKET), + ttl: TTL_FRESHNESS, + metaKey: `seed-meta:consumer-prices:freshness:${MARKET}`, + }, + { + key: `consumer-prices:basket-series:${MARKET}:${BASKET}:30d`, + data: series30d ?? emptyBasketSeries(MARKET, BASKET, '30d'), + ttl: TTL_SERIES, + metaKey: `seed-meta:consumer-prices:basket-series:${MARKET}:${BASKET}:30d`, + }, + { + key: `consumer-prices:basket-series:${MARKET}:${BASKET}:7d`, + data: series7d ?? emptyBasketSeries(MARKET, BASKET, '7d'), + ttl: TTL_SERIES, + metaKey: `seed-meta:consumer-prices:basket-series:${MARKET}:${BASKET}:7d`, + }, + { + key: `consumer-prices:basket-series:${MARKET}:${BASKET}:90d`, + data: series90d ?? emptyBasketSeries(MARKET, BASKET, '90d'), + ttl: TTL_SERIES, + metaKey: `seed-meta:consumer-prices:basket-series:${MARKET}:${BASKET}:90d`, + }, + { + key: `consumer-prices:categories:${MARKET}:30d`, + data: categories30d ?? emptyCategories(MARKET, '30d'), + ttl: TTL_CATEGORIES, + metaKey: `seed-meta:consumer-prices:categories:${MARKET}:30d`, + }, + { + key: `consumer-prices:categories:${MARKET}:7d`, + data: categories7d ?? emptyCategories(MARKET, '7d'), + ttl: TTL_CATEGORIES, + metaKey: `seed-meta:consumer-prices:categories:${MARKET}:7d`, + }, + { + key: `consumer-prices:categories:${MARKET}:90d`, + data: categories90d ?? emptyCategories(MARKET, '90d'), + ttl: TTL_CATEGORIES, + metaKey: `seed-meta:consumer-prices:categories:${MARKET}:90d`, + }, + ]; + + let failed = 0; + for (const { key, data, ttl, metaKey } of writes) { + try { + const recordCount = Array.isArray(data.retailers ?? data.categories ?? data.risers) + ? (data.retailers ?? data.categories ?? data.risers ?? []).length + : 1; + await writeExtraKeyWithMeta(key, data, ttl, recordCount, metaKey); + console.log(` [consumer-prices] wrote ${key} (${recordCount} records)`); + } catch (err) { + console.error(` [consumer-prices] failed ${key}: ${err.message}`); + failed++; + } + } + + console.log(`[consumer-prices] done. ${writes.length - failed}/${writes.length} keys written.`); + process.exit(failed > 0 ? 1 : 0); +} + +run().catch((err) => { + console.error('[consumer-prices] seed failed:', err); + process.exit(1); +}); diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index e8a234a14..01d4783cb 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -77,6 +77,8 @@ export const BOOTSTRAP_TIERS: Record = { securityAdvisories: 'slow', forecasts: 'fast', customsRevenue: 'slow', + consumerPricesOverview: 'slow', consumerPricesCategories: 'slow', + consumerPricesMovers: 'slow', consumerPricesSpread: 'slow', groceryBasket: 'slow', bigmac: 'slow', cryptoSectors: 'slow', diff --git a/server/gateway.ts b/server/gateway.ts index ae74e79cb..7488eaf9e 100644 --- a/server/gateway.ts +++ b/server/gateway.ts @@ -153,6 +153,13 @@ const RPC_CACHE_TIER: Record = { '/api/infrastructure/v1/list-temporal-anomalies': 'medium', '/api/webcam/v1/get-webcam-image': 'no-store', '/api/webcam/v1/list-webcams': 'no-store', + + '/api/consumer-prices/v1/get-consumer-price-overview': 'static', + '/api/consumer-prices/v1/get-consumer-price-basket-series': 'slow', + '/api/consumer-prices/v1/list-consumer-price-categories': 'static', + '/api/consumer-prices/v1/list-consumer-price-movers': 'static', + '/api/consumer-prices/v1/list-retailer-price-spreads': 'static', + '/api/consumer-prices/v1/get-consumer-price-freshness': 'slow', }; const PREMIUM_RPC_PATHS = new Set([ diff --git a/server/worldmonitor/consumer-prices/v1/get-consumer-price-basket-series.ts b/server/worldmonitor/consumer-prices/v1/get-consumer-price-basket-series.ts new file mode 100644 index 000000000..cb15906e6 --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/get-consumer-price-basket-series.ts @@ -0,0 +1,41 @@ +import type { + GetConsumerPriceBasketSeriesRequest, + GetConsumerPriceBasketSeriesResponse, +} from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const DEFAULT_MARKET = 'ae'; +const DEFAULT_BASKET = 'essentials-ae'; +const DEFAULT_RANGE = '30d'; + +const VALID_RANGES = new Set(['7d', '30d', '90d', '180d']); + +export async function getConsumerPriceBasketSeries( + _ctx: unknown, + req: GetConsumerPriceBasketSeriesRequest, +): Promise { + const market = req.marketCode || DEFAULT_MARKET; + const basket = req.basketSlug || DEFAULT_BASKET; + const range = VALID_RANGES.has(req.range ?? '') ? req.range! : DEFAULT_RANGE; + + const key = `consumer-prices:basket-series:${market}:${basket}:${range}`; + + const EMPTY: GetConsumerPriceBasketSeriesResponse = { + marketCode: market, + basketSlug: basket, + asOf: '0', + currencyCode: 'AED', + range, + essentialsSeries: [], + valueSeries: [], + upstreamUnavailable: true, + }; + + try { + const result = await getCachedJson(key, true) as GetConsumerPriceBasketSeriesResponse | null; + return result ?? EMPTY; + } catch { + return EMPTY; + } +} diff --git a/server/worldmonitor/consumer-prices/v1/get-consumer-price-freshness.ts b/server/worldmonitor/consumer-prices/v1/get-consumer-price-freshness.ts new file mode 100644 index 000000000..196a343ec --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/get-consumer-price-freshness.ts @@ -0,0 +1,32 @@ +import type { + GetConsumerPriceFreshnessRequest, + GetConsumerPriceFreshnessResponse, +} from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const DEFAULT_MARKET = 'ae'; + +export async function getConsumerPriceFreshness( + _ctx: unknown, + req: GetConsumerPriceFreshnessRequest, +): Promise { + const market = req.marketCode || DEFAULT_MARKET; + const key = `consumer-prices:freshness:${market}`; + + const EMPTY: GetConsumerPriceFreshnessResponse = { + marketCode: market, + asOf: '0', + retailers: [], + overallFreshnessMin: 0, + stalledCount: 0, + upstreamUnavailable: true, + }; + + try { + const result = await getCachedJson(key, true) as GetConsumerPriceFreshnessResponse | null; + return result ?? EMPTY; + } catch { + return EMPTY; + } +} diff --git a/server/worldmonitor/consumer-prices/v1/get-consumer-price-overview.ts b/server/worldmonitor/consumer-prices/v1/get-consumer-price-overview.ts new file mode 100644 index 000000000..ad815d639 --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/get-consumer-price-overview.ts @@ -0,0 +1,38 @@ +import type { + GetConsumerPriceOverviewRequest, + GetConsumerPriceOverviewResponse, +} from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const DEFAULT_MARKET = 'ae'; + +const EMPTY: GetConsumerPriceOverviewResponse = { + marketCode: DEFAULT_MARKET, + asOf: '0', + currencyCode: 'AED', + essentialsIndex: 0, + valueBasketIndex: 0, + wowPct: 0, + momPct: 0, + retailerSpreadPct: 0, + coveragePct: 0, + freshnessLagMin: 0, + topCategories: [], + upstreamUnavailable: true, +}; + +export async function getConsumerPriceOverview( + _ctx: unknown, + req: GetConsumerPriceOverviewRequest, +): Promise { + const market = req.marketCode || DEFAULT_MARKET; + const key = `consumer-prices:overview:${market}`; + + try { + const result = await getCachedJson(key, true) as GetConsumerPriceOverviewResponse | null; + return result ?? { ...EMPTY, marketCode: market }; + } catch { + return { ...EMPTY, marketCode: market }; + } +} diff --git a/server/worldmonitor/consumer-prices/v1/handler.ts b/server/worldmonitor/consumer-prices/v1/handler.ts new file mode 100644 index 000000000..5c867130e --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/handler.ts @@ -0,0 +1,17 @@ +import type { ConsumerPricesServiceHandler } from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getConsumerPriceOverview } from './get-consumer-price-overview'; +import { getConsumerPriceBasketSeries } from './get-consumer-price-basket-series'; +import { listConsumerPriceCategories } from './list-consumer-price-categories'; +import { listConsumerPriceMovers } from './list-consumer-price-movers'; +import { listRetailerPriceSpreads } from './list-retailer-price-spreads'; +import { getConsumerPriceFreshness } from './get-consumer-price-freshness'; + +export const consumerPricesHandler: ConsumerPricesServiceHandler = { + getConsumerPriceOverview, + getConsumerPriceBasketSeries, + listConsumerPriceCategories, + listConsumerPriceMovers, + listRetailerPriceSpreads, + getConsumerPriceFreshness, +}; diff --git a/server/worldmonitor/consumer-prices/v1/list-consumer-price-categories.ts b/server/worldmonitor/consumer-prices/v1/list-consumer-price-categories.ts new file mode 100644 index 000000000..163555dea --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/list-consumer-price-categories.ts @@ -0,0 +1,34 @@ +import type { + ListConsumerPriceCategoriesRequest, + ListConsumerPriceCategoriesResponse, +} from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const DEFAULT_MARKET = 'ae'; +const DEFAULT_RANGE = '30d'; +const VALID_RANGES = new Set(['7d', '30d', '90d', '180d']); + +export async function listConsumerPriceCategories( + _ctx: unknown, + req: ListConsumerPriceCategoriesRequest, +): Promise { + const market = req.marketCode || DEFAULT_MARKET; + const range = VALID_RANGES.has(req.range ?? '') ? req.range! : DEFAULT_RANGE; + const key = `consumer-prices:categories:${market}:${range}`; + + const EMPTY: ListConsumerPriceCategoriesResponse = { + marketCode: market, + asOf: '0', + range, + categories: [], + upstreamUnavailable: true, + }; + + try { + const result = await getCachedJson(key, true) as ListConsumerPriceCategoriesResponse | null; + return result ?? EMPTY; + } catch { + return EMPTY; + } +} diff --git a/server/worldmonitor/consumer-prices/v1/list-consumer-price-movers.ts b/server/worldmonitor/consumer-prices/v1/list-consumer-price-movers.ts new file mode 100644 index 000000000..0e773eb5c --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/list-consumer-price-movers.ts @@ -0,0 +1,43 @@ +import type { + ListConsumerPriceMoversRequest, + ListConsumerPriceMoversResponse, +} from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const DEFAULT_MARKET = 'ae'; +const DEFAULT_RANGE = '30d'; +const VALID_RANGES = new Set(['7d', '30d', '90d']); + +export async function listConsumerPriceMovers( + _ctx: unknown, + req: ListConsumerPriceMoversRequest, +): Promise { + const market = req.marketCode || DEFAULT_MARKET; + const range = VALID_RANGES.has(req.range ?? '') ? req.range! : DEFAULT_RANGE; + const key = `consumer-prices:movers:${market}:${range}`; + + const EMPTY: ListConsumerPriceMoversResponse = { + marketCode: market, + asOf: '0', + range, + risers: [], + fallers: [], + upstreamUnavailable: true, + }; + + try { + const cached = await getCachedJson(key, true) as ListConsumerPriceMoversResponse | null; + if (!cached) return EMPTY; + + const limit = req.limit ?? 10; + const filterCategory = req.categorySlug; + + const filter = (movers: typeof cached.risers) => + (filterCategory ? movers.filter((m) => m.category === filterCategory) : movers).slice(0, limit); + + return { ...cached, risers: filter(cached.risers), fallers: filter(cached.fallers) }; + } catch { + return EMPTY; + } +} diff --git a/server/worldmonitor/consumer-prices/v1/list-retailer-price-spreads.ts b/server/worldmonitor/consumer-prices/v1/list-retailer-price-spreads.ts new file mode 100644 index 000000000..9e43e6ba8 --- /dev/null +++ b/server/worldmonitor/consumer-prices/v1/list-retailer-price-spreads.ts @@ -0,0 +1,35 @@ +import type { + ListRetailerPriceSpreadsRequest, + ListRetailerPriceSpreadsResponse, +} from '../../../../src/generated/server/worldmonitor/consumer_prices/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const DEFAULT_MARKET = 'ae'; +const DEFAULT_BASKET = 'essentials-ae'; + +export async function listRetailerPriceSpreads( + _ctx: unknown, + req: ListRetailerPriceSpreadsRequest, +): Promise { + const market = req.marketCode || DEFAULT_MARKET; + const basket = req.basketSlug || DEFAULT_BASKET; + const key = `consumer-prices:retailer-spread:${market}:${basket}`; + + const EMPTY: ListRetailerPriceSpreadsResponse = { + marketCode: market, + asOf: '0', + basketSlug: basket, + currencyCode: 'AED', + retailers: [], + spreadPct: 0, + upstreamUnavailable: true, + }; + + try { + const result = await getCachedJson(key, true) as ListRetailerPriceSpreadsResponse | null; + return result ?? EMPTY; + } catch { + return EMPTY; + } +} diff --git a/src/app/panel-layout.ts b/src/app/panel-layout.ts index 38f6e1cf8..ff0ce86c8 100644 --- a/src/app/panel-layout.ts +++ b/src/app/panel-layout.ts @@ -18,6 +18,7 @@ import { PredictionPanel, MonitorPanel, EconomicPanel, + ConsumerPricesPanel, EnergyComplexPanel, GdeltIntelPanel, LiveNewsPanel, @@ -581,6 +582,7 @@ export class PanelLayoutManager implements AppModule { this.createNewsPanel('ipo', 'panels.ipo'); this.createNewsPanel('thinktanks', 'panels.thinktanks'); this.createPanel('economic', () => new EconomicPanel()); + this.createPanel('consumer-prices', () => new ConsumerPricesPanel()); this.createPanel('trade-policy', () => new TradePolicyPanel()); this.createPanel('sanctions-pressure', () => new SanctionsPressurePanel()); diff --git a/src/components/ConsumerPricesPanel.ts b/src/components/ConsumerPricesPanel.ts new file mode 100644 index 000000000..a93ed91ca --- /dev/null +++ b/src/components/ConsumerPricesPanel.ts @@ -0,0 +1,412 @@ +import { Panel } from './Panel'; +import { t } from '@/services/i18n'; +import { escapeHtml } from '@/utils/sanitize'; +import { sparkline } from '@/utils/sparkline'; +import { + fetchConsumerPriceOverview, + fetchConsumerPriceCategories, + fetchConsumerPriceMovers, + fetchRetailerPriceSpreads, + fetchConsumerPriceFreshness, + DEFAULT_MARKET, + DEFAULT_BASKET, + type GetConsumerPriceOverviewResponse, + type ListConsumerPriceCategoriesResponse, + type ListConsumerPriceMoversResponse, + type ListRetailerPriceSpreadsResponse, + type GetConsumerPriceFreshnessResponse, + type CategorySnapshot, + type PriceMover, + type RetailerSpread, +} from '@/services/consumer-prices'; + +type TabId = 'overview' | 'categories' | 'movers' | 'spread' | 'health'; + +const SETTINGS_KEY = 'wm-consumer-prices-v1'; +const CHANGE_EVENT = 'wm-consumer-prices-settings-changed'; + +interface PanelSettings { + market: string; + basket: string; + range: '7d' | '30d' | '90d'; + tab: TabId; + categoryFilter: string | null; +} + +const DEFAULT_SETTINGS: PanelSettings = { + market: DEFAULT_MARKET, + basket: DEFAULT_BASKET, + range: '30d', + tab: 'overview', + categoryFilter: null, +}; + +function loadSettings(): PanelSettings { + try { + const raw = localStorage.getItem(SETTINGS_KEY); + if (raw) return { ...DEFAULT_SETTINGS, ...JSON.parse(raw) }; + } catch {} + return { ...DEFAULT_SETTINGS }; +} + +function saveSettings(s: PanelSettings): void { + try { + localStorage.setItem(SETTINGS_KEY, JSON.stringify(s)); + window.dispatchEvent(new CustomEvent(CHANGE_EVENT, { detail: s })); + } catch {} +} + +function pctBadge(val: number | null | undefined, invertColor = false): string { + if (val == null || val === 0) return '—'; + const cls = invertColor + ? val > 0 ? 'cp-badge--red' : 'cp-badge--green' + : val > 0 ? 'cp-badge--green' : 'cp-badge--red'; + const sign = val > 0 ? '+' : ''; + return `${sign}${val.toFixed(1)}%`; +} + +function pricePressureBadge(wowPct: number): string { + if (Math.abs(wowPct) < 0.5) return 'Stable'; + if (wowPct >= 2) return 'Rising'; + if (wowPct > 0.5) return 'Mild Rise'; + return 'Easing'; +} + +function freshnessLabel(min: number | null): string { + if (min == null || min === 0) return 'Unknown'; + if (min < 60) return `${min}m ago`; + if (min < 1440) return `${Math.round(min / 60)}h ago`; + return `${Math.round(min / 1440)}d ago`; +} + +function freshnessClass(min: number | null): string { + if (min == null) return 'cp-fresh--unknown'; + if (min <= 60) return 'cp-fresh--ok'; + if (min <= 240) return 'cp-fresh--warn'; + return 'cp-fresh--stale'; +} + +export class ConsumerPricesPanel extends Panel { + private overview: GetConsumerPriceOverviewResponse | null = null; + private categories: ListConsumerPriceCategoriesResponse | null = null; + private movers: ListConsumerPriceMoversResponse | null = null; + private spread: ListRetailerPriceSpreadsResponse | null = null; + private freshness: GetConsumerPriceFreshnessResponse | null = null; + private settings: PanelSettings = loadSettings(); + private loading = false; // tracks in-flight fetch to avoid duplicates + + constructor() { + super({ + id: 'consumer-prices', + title: t('panels.consumerPrices'), + defaultRowSpan: 2, + infoTooltip: t('components.consumerPrices.infoTooltip'), + }); + + this.content.addEventListener('click', (e) => this.handleClick(e)); + } + + private handleClick(e: Event): void { + const target = e.target as HTMLElement; + + const tab = target.closest('.panel-tab') as HTMLElement | null; + if (tab?.dataset.tab) { + this.settings.tab = tab.dataset.tab as TabId; + saveSettings(this.settings); + this.render(); + return; + } + + const catRow = target.closest('[data-category]') as HTMLElement | null; + if (catRow?.dataset.category) { + this.settings.categoryFilter = catRow.dataset.category; + this.settings.tab = 'movers'; + saveSettings(this.settings); + this.render(); + return; + } + + const rangeBtn = target.closest('[data-range]') as HTMLElement | null; + if (rangeBtn?.dataset.range) { + this.settings.range = rangeBtn.dataset.range as PanelSettings['range']; + saveSettings(this.settings); + this.loadData(); + return; + } + + const clearFilter = target.closest('[data-clear-filter]'); + if (clearFilter) { + this.settings.categoryFilter = null; + saveSettings(this.settings); + this.render(); + } + } + + public async loadData(): Promise { + if (this.loading) return; + this.loading = true; + this.showLoading(); + + const { market, basket, range } = this.settings; + + const [overview, categories, movers, spread, freshness] = await Promise.all([ + fetchConsumerPriceOverview(market, basket), + fetchConsumerPriceCategories(market, basket, range), + fetchConsumerPriceMovers(market, range), + fetchRetailerPriceSpreads(market, basket), + fetchConsumerPriceFreshness(market), + ]); + + this.overview = overview; + this.categories = categories; + this.movers = movers; + this.spread = spread; + this.freshness = freshness; + this.loading = false; + this.render(); + } + + private render(): void { + const { tab, range, categoryFilter } = this.settings; + + const tabs: Array<{ id: TabId; label: string }> = [ + { id: 'overview', label: t('components.consumerPrices.tabs.overview') }, + { id: 'categories', label: t('components.consumerPrices.tabs.categories') }, + { id: 'movers', label: t('components.consumerPrices.tabs.movers') }, + { id: 'spread', label: t('components.consumerPrices.tabs.spread') }, + { id: 'health', label: t('components.consumerPrices.tabs.health') }, + ]; + + const tabsHtml = ` +
+ ${tabs.map((t_) => ` + + `).join('')} +
+ `; + + const rangeHtml = ` +
+ ${(['7d', '30d', '90d'] as const).map((r) => ` + + `).join('')} +
+ `; + + let bodyHtml = ''; + const noData = this.overview?.upstreamUnavailable; + + switch (tab) { + case 'overview': + bodyHtml = this.renderOverview(); + break; + case 'categories': + bodyHtml = rangeHtml + this.renderCategories(); + break; + case 'movers': + bodyHtml = rangeHtml + (categoryFilter + ? `
Filtered: ${escapeHtml(categoryFilter)}
` + : '') + this.renderMovers(); + break; + case 'spread': + bodyHtml = this.renderSpread(); + break; + case 'health': + bodyHtml = this.renderHealth(); + break; + } + + this.setContent(` +
+ ${tabsHtml} + ${noData && tab === 'overview' ? '
Data collection starting — check back soon
' : ''} +
${bodyHtml}
+
+ `); + } + + private renderOverview(): string { + const d = this.overview; + if (!d || !d.asOf || d.asOf === '0') return this.renderEmptyState('No price data available yet'); + + return ` +
+
+
Essentials Basket
+
${d.essentialsIndex > 0 ? d.essentialsIndex.toFixed(1) : '—'}
+
Index (base 100)
+
+
+
Value Basket
+
${d.valueBasketIndex > 0 ? d.valueBasketIndex.toFixed(1) : '—'}
+
Index (base 100)
+
+
+
Week-over-Week
+
${pctBadge(d.wowPct, true)}
+
${pricePressureBadge(d.wowPct)}
+
+
+
Month-over-Month
+
${pctBadge(d.momPct, true)}
+
+
+
Retailer Spread
+
${d.retailerSpreadPct > 0 ? `${d.retailerSpreadPct.toFixed(1)}%` : '—'}
+
Cheapest vs most exp.
+
+
+
Coverage
+
${d.coveragePct > 0 ? `${d.coveragePct.toFixed(0)}%` : '—'}
+
+ ${freshnessLabel(d.freshnessLagMin)} +
+
+
+ ${d.topCategories?.length ? ` +
Top Category Movers
+
+ ${d.topCategories.slice(0, 5).map((c) => this.renderCategoryMini(c)).join('')} +
+ ` : ''} + `; + } + + private renderCategoryMini(c: CategorySnapshot): string { + const spark = c.sparkline?.length ? sparkline(c.sparkline, 'var(--accent)', 40, 16) : ''; + return ` +
+ ${escapeHtml(c.name)} + ${spark} + ${pctBadge(c.momPct, true)} +
+ `; + } + + private renderCategories(): string { + const cats = this.categories?.categories; + if (!cats?.length) return this.renderEmptyState('No category data yet'); + + return ` + + + + + + + + + + + + ${cats.map((c) => ` + + + + + + + + `).join('')} + +
CategoryWoWMoMTrendCoverage
${escapeHtml(c.name)}${pctBadge(c.wowPct, true)}${pctBadge(c.momPct, true)}${c.sparkline?.length ? sparkline(c.sparkline, 'var(--accent)', 48, 18) : '—'}${c.coveragePct > 0 ? `${c.coveragePct.toFixed(0)}%` : '—'}
+ `; + } + + private renderMovers(): string { + const d = this.movers; + if (!d) return this.renderEmptyState('No price movement data yet'); + + const { categoryFilter } = this.settings; + const filterFn = (m: PriceMover) => !categoryFilter || m.category === categoryFilter; + + const risers = (d.risers ?? []).filter(filterFn).slice(0, 8); + const fallers = (d.fallers ?? []).filter(filterFn).slice(0, 8); + + if (!risers.length && !fallers.length) return this.renderEmptyState('No movers for this selection'); + + return ` +
+
+
Rising
+ ${risers.map((m) => this.renderMoverRow(m, 'up')).join('') || '
None
'} +
+
+
Falling
+ ${fallers.map((m) => this.renderMoverRow(m, 'down')).join('') || '
None
'} +
+
+ `; + } + + private renderMoverRow(m: PriceMover, dir: 'up' | 'down'): string { + const sign = m.changePct > 0 ? '+' : ''; + return ` +
+
${escapeHtml(m.title)}
+
+ ${escapeHtml(m.category)} + ${escapeHtml(m.retailerSlug)} +
+
${sign}${m.changePct.toFixed(1)}%
+
+ `; + } + + private renderSpread(): string { + const d = this.spread; + if (!d?.retailers?.length) return this.renderEmptyState('Retailer comparison starts once data is collected'); + + return ` +
+ Spread: ${d.spreadPct.toFixed(1)}% + ${escapeHtml(d.basketSlug)} · ${escapeHtml(d.currencyCode)} +
+
+ ${d.retailers.map((r, i) => this.renderSpreadRow(r, i, d.currencyCode)).join('')} +
+ `; + } + + private renderSpreadRow(r: RetailerSpread, rank: number, currency: string): string { + const isChepeast = rank === 0; + return ` +
+
#${rank + 1}
+
${escapeHtml(r.name)}
+
${currency} ${r.basketTotal.toFixed(2)}
+
${isChepeast ? 'Cheapest' : pctBadge(r.deltaVsCheapestPct, true)}
+
${r.itemCount} items
+
${freshnessLabel(r.freshnessMin)}
+
+ `; + } + + private renderHealth(): string { + const d = this.freshness; + if (!d?.retailers?.length) return this.renderEmptyState('Health data not yet available'); + + return ` +
+ Overall freshness: ${freshnessLabel(d.overallFreshnessMin)} + ${d.stalledCount > 0 ? `${d.stalledCount} stalled` : ''} +
+
+ ${d.retailers.map((r) => ` +
+ ${escapeHtml(r.name)} + ${r.status} + ${r.parseSuccessRate > 0 ? `${(r.parseSuccessRate * 100).toFixed(0)}% parse` : '—'} + ${freshnessLabel(r.freshnessMin)} +
+ `).join('')} +
+ `; + } + + private renderEmptyState(msg: string): string { + return `
${escapeHtml(msg)}
`; + } +} diff --git a/src/components/index.ts b/src/components/index.ts index 1f017f998..27e1117aa 100644 --- a/src/components/index.ts +++ b/src/components/index.ts @@ -65,4 +65,5 @@ export * from './MilitaryCorrelationPanel'; export * from './EscalationCorrelationPanel'; export * from './EconomicCorrelationPanel'; export * from './DisasterCorrelationPanel'; +export * from './ConsumerPricesPanel'; export { NationalDebtPanel } from './NationalDebtPanel'; diff --git a/src/config/panels.ts b/src/config/panels.ts index 2c1ab67e2..9f1b19510 100644 --- a/src/config/panels.ts +++ b/src/config/panels.ts @@ -56,6 +56,7 @@ const FULL_PANELS: Record = { 'satellite-fires': { name: 'Fires', enabled: true, priority: 2 }, 'macro-signals': { name: 'Market Regime', enabled: true, priority: 2 }, 'gulf-economies': { name: 'Gulf Economies', enabled: false, priority: 2 }, + 'consumer-prices': { name: 'Consumer Prices', enabled: false, priority: 2 }, 'grocery-basket': { name: 'Grocery Index', enabled: false, priority: 2 }, 'bigmac': { name: 'Big Mac Index', enabled: false, priority: 2 }, 'etf-flows': { name: 'BTC ETF Tracker', enabled: true, priority: 2 }, @@ -410,6 +411,7 @@ const FINANCE_PANELS: Record = { 'gcc-investments': { name: 'GCC Investments', enabled: true, priority: 2 }, gccNews: { name: 'GCC Business News', enabled: true, priority: 2 }, 'gulf-economies': { name: 'Gulf Economies', enabled: true, priority: 1 }, + 'consumer-prices': { name: 'Consumer Prices', enabled: true, priority: 1 }, polymarket: { name: 'Predictions', enabled: true, priority: 2 }, 'airline-intel': { name: 'Airline Intelligence', enabled: true, priority: 2 }, 'world-clock': { name: 'World Clock', enabled: true, priority: 2 }, @@ -702,6 +704,7 @@ const COMMODITY_PANELS: Record = { economic: { name: 'Macro Stress', enabled: true, priority: 1 }, 'gulf-economies': { name: 'Gulf & OPEC Economies', enabled: true, priority: 1 }, 'gcc-investments': { name: 'GCC Resource Investments', enabled: true, priority: 2 }, + 'consumer-prices': { name: 'Consumer Prices', enabled: true, priority: 2 }, 'airline-intel': { name: 'Airline Intelligence', enabled: true, priority: 2 }, polymarket: { name: 'Commodity Predictions', enabled: true, priority: 2 }, 'world-clock': { name: 'World Clock', enabled: true, priority: 2 }, @@ -954,7 +957,7 @@ export const LAYER_TO_SOURCE: Partial> = // ============================================ // Maps category keys to panel keys. Only categories with at least one // matching panel in the user's active panel settings are shown. -export const PANEL_CATEGORY_MAP: Record = { +export const PANEL_CATEGORY_MAP: Record = { // All variants — essential panels core: { labelKey: 'header.panelCatCore', @@ -1032,7 +1035,8 @@ export const PANEL_CATEGORY_MAP: Record; +} + +export interface ConsumerPricesServiceCallOptions { + headers?: Record; + signal?: AbortSignal; +} + +export class ConsumerPricesServiceClient { + private baseURL: string; + private fetchFn: typeof fetch; + private defaultHeaders: Record; + + constructor(baseURL: string, options?: ConsumerPricesServiceClientOptions) { + this.baseURL = baseURL.replace(/\/+$/, ""); + this.fetchFn = options?.fetch ?? globalThis.fetch; + this.defaultHeaders = { ...options?.defaultHeaders }; + } + + async getConsumerPriceOverview(req: GetConsumerPriceOverviewRequest, options?: ConsumerPricesServiceCallOptions): Promise { + let path = "/api/consumer-prices/v1/get-consumer-price-overview"; + const params = new URLSearchParams(); + if (req.marketCode != null && req.marketCode !== "") params.set("market_code", String(req.marketCode)); + if (req.basketSlug != null && req.basketSlug !== "") params.set("basket_slug", String(req.basketSlug)); + const url = this.baseURL + path + (params.toString() ? "?" + params.toString() : ""); + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as GetConsumerPriceOverviewResponse; + } + + async getConsumerPriceBasketSeries(req: GetConsumerPriceBasketSeriesRequest, options?: ConsumerPricesServiceCallOptions): Promise { + let path = "/api/consumer-prices/v1/get-consumer-price-basket-series"; + const params = new URLSearchParams(); + if (req.marketCode != null && req.marketCode !== "") params.set("market_code", String(req.marketCode)); + if (req.basketSlug != null && req.basketSlug !== "") params.set("basket_slug", String(req.basketSlug)); + if (req.range != null && req.range !== "") params.set("range", String(req.range)); + const url = this.baseURL + path + (params.toString() ? "?" + params.toString() : ""); + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as GetConsumerPriceBasketSeriesResponse; + } + + async listConsumerPriceCategories(req: ListConsumerPriceCategoriesRequest, options?: ConsumerPricesServiceCallOptions): Promise { + let path = "/api/consumer-prices/v1/list-consumer-price-categories"; + const params = new URLSearchParams(); + if (req.marketCode != null && req.marketCode !== "") params.set("market_code", String(req.marketCode)); + if (req.basketSlug != null && req.basketSlug !== "") params.set("basket_slug", String(req.basketSlug)); + if (req.range != null && req.range !== "") params.set("range", String(req.range)); + const url = this.baseURL + path + (params.toString() ? "?" + params.toString() : ""); + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as ListConsumerPriceCategoriesResponse; + } + + async listConsumerPriceMovers(req: ListConsumerPriceMoversRequest, options?: ConsumerPricesServiceCallOptions): Promise { + let path = "/api/consumer-prices/v1/list-consumer-price-movers"; + const params = new URLSearchParams(); + if (req.marketCode != null && req.marketCode !== "") params.set("market_code", String(req.marketCode)); + if (req.range != null && req.range !== "") params.set("range", String(req.range)); + if (req.limit != null && req.limit !== 0) params.set("limit", String(req.limit)); + if (req.categorySlug != null && req.categorySlug !== "") params.set("category_slug", String(req.categorySlug)); + const url = this.baseURL + path + (params.toString() ? "?" + params.toString() : ""); + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as ListConsumerPriceMoversResponse; + } + + async listRetailerPriceSpreads(req: ListRetailerPriceSpreadsRequest, options?: ConsumerPricesServiceCallOptions): Promise { + let path = "/api/consumer-prices/v1/list-retailer-price-spreads"; + const params = new URLSearchParams(); + if (req.marketCode != null && req.marketCode !== "") params.set("market_code", String(req.marketCode)); + if (req.basketSlug != null && req.basketSlug !== "") params.set("basket_slug", String(req.basketSlug)); + const url = this.baseURL + path + (params.toString() ? "?" + params.toString() : ""); + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as ListRetailerPriceSpreadsResponse; + } + + async getConsumerPriceFreshness(req: GetConsumerPriceFreshnessRequest, options?: ConsumerPricesServiceCallOptions): Promise { + let path = "/api/consumer-prices/v1/get-consumer-price-freshness"; + const params = new URLSearchParams(); + if (req.marketCode != null && req.marketCode !== "") params.set("market_code", String(req.marketCode)); + const url = this.baseURL + path + (params.toString() ? "?" + params.toString() : ""); + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as GetConsumerPriceFreshnessResponse; + } + + private async handleError(resp: Response): Promise { + const body = await resp.text(); + if (resp.status === 400) { + try { + const parsed = JSON.parse(body); + if (parsed.violations) { + throw new ValidationError(parsed.violations); + } + } catch (e) { + if (e instanceof ValidationError) throw e; + } + } + throw new ApiError(resp.status, `Request failed with status ${resp.status}`, body); + } +} + diff --git a/src/generated/server/worldmonitor/consumer_prices/v1/service_server.ts b/src/generated/server/worldmonitor/consumer_prices/v1/service_server.ts new file mode 100644 index 000000000..4abfd269a --- /dev/null +++ b/src/generated/server/worldmonitor/consumer_prices/v1/service_server.ts @@ -0,0 +1,497 @@ +// @ts-nocheck +// Code generated by protoc-gen-ts-server. DO NOT EDIT. +// source: worldmonitor/consumer_prices/v1/service.proto + +export interface GetConsumerPriceOverviewRequest { + marketCode: string; + basketSlug: string; +} + +export interface GetConsumerPriceOverviewResponse { + marketCode: string; + asOf: string; + currencyCode: string; + essentialsIndex: number; + valueBasketIndex: number; + wowPct: number; + momPct: number; + retailerSpreadPct: number; + coveragePct: number; + freshnessLagMin: number; + topCategories: CategorySnapshot[]; + upstreamUnavailable: boolean; +} + +export interface CategorySnapshot { + slug: string; + name: string; + wowPct: number; + momPct: number; + currentIndex: number; + sparkline: number[]; + coveragePct: number; + itemCount: number; +} + +export interface GetConsumerPriceBasketSeriesRequest { + marketCode: string; + basketSlug: string; + range: string; +} + +export interface GetConsumerPriceBasketSeriesResponse { + marketCode: string; + basketSlug: string; + asOf: string; + currencyCode: string; + range: string; + essentialsSeries: BasketPoint[]; + valueSeries: BasketPoint[]; + upstreamUnavailable: boolean; +} + +export interface BasketPoint { + date: string; + index: number; +} + +export interface ListConsumerPriceCategoriesRequest { + marketCode: string; + basketSlug: string; + range: string; +} + +export interface ListConsumerPriceCategoriesResponse { + marketCode: string; + asOf: string; + range: string; + categories: CategorySnapshot[]; + upstreamUnavailable: boolean; +} + +export interface ListConsumerPriceMoversRequest { + marketCode: string; + range: string; + limit: number; + categorySlug: string; +} + +export interface ListConsumerPriceMoversResponse { + marketCode: string; + asOf: string; + range: string; + risers: PriceMover[]; + fallers: PriceMover[]; + upstreamUnavailable: boolean; +} + +export interface PriceMover { + productId: string; + title: string; + category: string; + retailerSlug: string; + changePct: number; + currentPrice: number; + currencyCode: string; +} + +export interface ListRetailerPriceSpreadsRequest { + marketCode: string; + basketSlug: string; +} + +export interface ListRetailerPriceSpreadsResponse { + marketCode: string; + asOf: string; + basketSlug: string; + currencyCode: string; + retailers: RetailerSpread[]; + spreadPct: number; + upstreamUnavailable: boolean; +} + +export interface RetailerSpread { + slug: string; + name: string; + basketTotal: number; + deltaVsCheapest: number; + deltaVsCheapestPct: number; + itemCount: number; + freshnessMin: number; + currencyCode: string; +} + +export interface GetConsumerPriceFreshnessRequest { + marketCode: string; +} + +export interface GetConsumerPriceFreshnessResponse { + marketCode: string; + asOf: string; + retailers: RetailerFreshnessInfo[]; + overallFreshnessMin: number; + stalledCount: number; + upstreamUnavailable: boolean; +} + +export interface RetailerFreshnessInfo { + slug: string; + name: string; + lastRunAt: string; + status: string; + parseSuccessRate: number; + freshnessMin: number; +} + +export interface FieldViolation { + field: string; + description: string; +} + +export class ValidationError extends Error { + violations: FieldViolation[]; + + constructor(violations: FieldViolation[]) { + super("Validation failed"); + this.name = "ValidationError"; + this.violations = violations; + } +} + +export class ApiError extends Error { + statusCode: number; + body: string; + + constructor(statusCode: number, message: string, body: string) { + super(message); + this.name = "ApiError"; + this.statusCode = statusCode; + this.body = body; + } +} + +export interface ServerContext { + request: Request; + pathParams: Record; + headers: Record; +} + +export interface ServerOptions { + onError?: (error: unknown, req: Request) => Response | Promise; + validateRequest?: (methodName: string, body: unknown) => FieldViolation[] | undefined; +} + +export interface RouteDescriptor { + method: string; + path: string; + handler: (req: Request) => Promise; +} + +export interface ConsumerPricesServiceHandler { + getConsumerPriceOverview(ctx: ServerContext, req: GetConsumerPriceOverviewRequest): Promise; + getConsumerPriceBasketSeries(ctx: ServerContext, req: GetConsumerPriceBasketSeriesRequest): Promise; + listConsumerPriceCategories(ctx: ServerContext, req: ListConsumerPriceCategoriesRequest): Promise; + listConsumerPriceMovers(ctx: ServerContext, req: ListConsumerPriceMoversRequest): Promise; + listRetailerPriceSpreads(ctx: ServerContext, req: ListRetailerPriceSpreadsRequest): Promise; + getConsumerPriceFreshness(ctx: ServerContext, req: GetConsumerPriceFreshnessRequest): Promise; +} + +export function createConsumerPricesServiceRoutes( + handler: ConsumerPricesServiceHandler, + options?: ServerOptions, +): RouteDescriptor[] { + return [ + { + method: "GET", + path: "/api/consumer-prices/v1/get-consumer-price-overview", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const url = new URL(req.url, "http://localhost"); + const params = url.searchParams; + const body: GetConsumerPriceOverviewRequest = { + marketCode: params.get("market_code") ?? "", + basketSlug: params.get("basket_slug") ?? "", + }; + if (options?.validateRequest) { + const bodyViolations = options.validateRequest("getConsumerPriceOverview", body); + if (bodyViolations) { + throw new ValidationError(bodyViolations); + } + } + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.getConsumerPriceOverview(ctx, body); + return new Response(JSON.stringify(result as GetConsumerPriceOverviewResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, + { + method: "GET", + path: "/api/consumer-prices/v1/get-consumer-price-basket-series", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const url = new URL(req.url, "http://localhost"); + const params = url.searchParams; + const body: GetConsumerPriceBasketSeriesRequest = { + marketCode: params.get("market_code") ?? "", + basketSlug: params.get("basket_slug") ?? "", + range: params.get("range") ?? "", + }; + if (options?.validateRequest) { + const bodyViolations = options.validateRequest("getConsumerPriceBasketSeries", body); + if (bodyViolations) { + throw new ValidationError(bodyViolations); + } + } + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.getConsumerPriceBasketSeries(ctx, body); + return new Response(JSON.stringify(result as GetConsumerPriceBasketSeriesResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, + { + method: "GET", + path: "/api/consumer-prices/v1/list-consumer-price-categories", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const url = new URL(req.url, "http://localhost"); + const params = url.searchParams; + const body: ListConsumerPriceCategoriesRequest = { + marketCode: params.get("market_code") ?? "", + basketSlug: params.get("basket_slug") ?? "", + range: params.get("range") ?? "", + }; + if (options?.validateRequest) { + const bodyViolations = options.validateRequest("listConsumerPriceCategories", body); + if (bodyViolations) { + throw new ValidationError(bodyViolations); + } + } + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.listConsumerPriceCategories(ctx, body); + return new Response(JSON.stringify(result as ListConsumerPriceCategoriesResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, + { + method: "GET", + path: "/api/consumer-prices/v1/list-consumer-price-movers", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const url = new URL(req.url, "http://localhost"); + const params = url.searchParams; + const body: ListConsumerPriceMoversRequest = { + marketCode: params.get("market_code") ?? "", + range: params.get("range") ?? "", + limit: Number(params.get("limit") ?? "0"), + categorySlug: params.get("category_slug") ?? "", + }; + if (options?.validateRequest) { + const bodyViolations = options.validateRequest("listConsumerPriceMovers", body); + if (bodyViolations) { + throw new ValidationError(bodyViolations); + } + } + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.listConsumerPriceMovers(ctx, body); + return new Response(JSON.stringify(result as ListConsumerPriceMoversResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, + { + method: "GET", + path: "/api/consumer-prices/v1/list-retailer-price-spreads", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const url = new URL(req.url, "http://localhost"); + const params = url.searchParams; + const body: ListRetailerPriceSpreadsRequest = { + marketCode: params.get("market_code") ?? "", + basketSlug: params.get("basket_slug") ?? "", + }; + if (options?.validateRequest) { + const bodyViolations = options.validateRequest("listRetailerPriceSpreads", body); + if (bodyViolations) { + throw new ValidationError(bodyViolations); + } + } + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.listRetailerPriceSpreads(ctx, body); + return new Response(JSON.stringify(result as ListRetailerPriceSpreadsResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, + { + method: "GET", + path: "/api/consumer-prices/v1/get-consumer-price-freshness", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const url = new URL(req.url, "http://localhost"); + const params = url.searchParams; + const body: GetConsumerPriceFreshnessRequest = { + marketCode: params.get("market_code") ?? "", + }; + if (options?.validateRequest) { + const bodyViolations = options.validateRequest("getConsumerPriceFreshness", body); + if (bodyViolations) { + throw new ValidationError(bodyViolations); + } + } + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.getConsumerPriceFreshness(ctx, body); + return new Response(JSON.stringify(result as GetConsumerPriceFreshnessResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, + ]; +} + diff --git a/src/locales/en.json b/src/locales/en.json index 52f70e1a9..768135deb 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -355,7 +355,8 @@ "gulfIndices": "Gulf Indices", "gulfCurrencies": "Gulf Currencies", "gulfOil": "Gulf Oil", - "airlineIntel": "✈️ Airline Intelligence" + "airlineIntel": "✈️ Airline Intelligence", + "consumerPrices": "Consumer Prices" }, "commands": { "prefixes": { @@ -940,6 +941,42 @@ "colFytd": "FY YTD", "infoTooltip": "Trade Policy WTO baseline and tariff-impact monitoring:
  • Overview: WTO MFN baseline rates with US effective-rate context when available
  • Tariffs: WTO MFN tariff trends vs the US effective tariff estimate
  • Trade Flows: Export/import volumes with year-over-year changes
  • Barriers: Technical barriers to trade (TBT/SPS notifications)
  • Revenue: Monthly US customs duties revenue (US Treasury MTS data)
" }, + "consumerPrices": { + "title": "Consumer Prices", + "subtitle": "Basket price tracking across key markets", + "tabs": { + "overview": "Overview", + "categories": "Categories", + "movers": "Movers", + "spread": "Retailer Spread", + "health": "Data Health" + }, + "essentialsIndex": "Essentials Index", + "valueBasketIndex": "Value Basket", + "wowChange": "WoW", + "momChange": "MoM", + "retailerSpread": "Retailer Spread", + "coveragePct": "Coverage", + "freshnessLag": "Freshness Lag", + "noCategories": "No category data available", + "noMovers": "No price movers available", + "noSpread": "No retailer spread data available", + "noHealth": "No data health information", + "upstreamUnavailable": "Consumer price data temporarily unavailable", + "allCategories": "All Categories", + "risers": "Rising", + "fallers": "Falling", + "unitPrice": "Unit price", + "freshLabel": "Fresh", + "staleLabel": "Stale", + "laggingLabel": "Lagging", + "stalledLabel": "Stalled", + "retailer": "Retailer", + "lastUpdated": "Last updated", + "retailers": "retailers", + "items": "items", + "infoTooltip": "Consumer Prices Real-time basket price tracking:
  • Overview: Essentials index, value basket, and week-on-week change
  • Categories: Per-category price trends with 30-day range
  • Movers: Biggest rising and falling items this week
  • Spread: Price variance across retailers for the same basket
Data sourced from live retailer price scraping." + }, "gdelt": { "empty": "No recent articles for this topic" }, diff --git a/src/services/consumer-prices/index.ts b/src/services/consumer-prices/index.ts new file mode 100644 index 000000000..30627f276 --- /dev/null +++ b/src/services/consumer-prices/index.ts @@ -0,0 +1,229 @@ +import { getRpcBaseUrl } from '@/services/rpc-client'; +import { + ConsumerPricesServiceClient, + type GetConsumerPriceOverviewResponse, + type GetConsumerPriceBasketSeriesResponse, + type ListConsumerPriceCategoriesResponse, + type ListConsumerPriceMoversResponse, + type ListRetailerPriceSpreadsResponse, + type GetConsumerPriceFreshnessResponse, + type CategorySnapshot, + type PriceMover, + type RetailerSpread, + type BasketPoint, + type RetailerFreshnessInfo, +} from '@/generated/client/worldmonitor/consumer_prices/v1/service_client'; +import { createCircuitBreaker } from '@/utils'; +import { getHydratedData } from '@/services/bootstrap'; + +export type { + GetConsumerPriceOverviewResponse, + GetConsumerPriceBasketSeriesResponse, + ListConsumerPriceCategoriesResponse, + ListConsumerPriceMoversResponse, + ListRetailerPriceSpreadsResponse, + GetConsumerPriceFreshnessResponse, + CategorySnapshot, + PriceMover, + RetailerSpread, + BasketPoint, + RetailerFreshnessInfo, +}; + +export const DEFAULT_MARKET = 'ae'; +export const DEFAULT_BASKET = 'essentials-ae'; + +const client = new ConsumerPricesServiceClient(getRpcBaseUrl(), { + fetch: (...args) => globalThis.fetch(...args), +}); + +const overviewBreaker = createCircuitBreaker({ + name: 'Consumer Prices Overview', + cacheTtlMs: 30 * 60 * 1000, + persistCache: true, +}); +const seriesBreaker = createCircuitBreaker({ + name: 'Consumer Prices Series', + cacheTtlMs: 60 * 60 * 1000, + persistCache: true, +}); +const categoriesBreaker = createCircuitBreaker({ + name: 'Consumer Prices Categories', + cacheTtlMs: 30 * 60 * 1000, + persistCache: true, +}); +const moversBreaker = createCircuitBreaker({ + name: 'Consumer Prices Movers', + cacheTtlMs: 30 * 60 * 1000, + persistCache: true, +}); +const spreadBreaker = createCircuitBreaker({ + name: 'Consumer Prices Spread', + cacheTtlMs: 30 * 60 * 1000, + persistCache: true, +}); +const freshnessBreaker = createCircuitBreaker({ + name: 'Consumer Prices Freshness', + cacheTtlMs: 10 * 60 * 1000, + persistCache: true, +}); + +const emptyOverview: GetConsumerPriceOverviewResponse = { + marketCode: DEFAULT_MARKET, + asOf: '0', + currencyCode: 'AED', + essentialsIndex: 0, + valueBasketIndex: 0, + wowPct: 0, + momPct: 0, + retailerSpreadPct: 0, + coveragePct: 0, + freshnessLagMin: 0, + topCategories: [], + upstreamUnavailable: true, +}; + +const emptySeries: GetConsumerPriceBasketSeriesResponse = { + marketCode: DEFAULT_MARKET, + basketSlug: DEFAULT_BASKET, + asOf: '0', + currencyCode: 'AED', + range: '30d', + essentialsSeries: [], + valueSeries: [], + upstreamUnavailable: true, +}; + +const emptyCategories: ListConsumerPriceCategoriesResponse = { + marketCode: DEFAULT_MARKET, + asOf: '0', + range: '30d', + categories: [], + upstreamUnavailable: true, +}; + +const emptyMovers: ListConsumerPriceMoversResponse = { + marketCode: DEFAULT_MARKET, + asOf: '0', + range: '30d', + risers: [], + fallers: [], + upstreamUnavailable: true, +}; + +const emptySpread: ListRetailerPriceSpreadsResponse = { + marketCode: DEFAULT_MARKET, + asOf: '0', + basketSlug: DEFAULT_BASKET, + currencyCode: 'AED', + retailers: [], + spreadPct: 0, + upstreamUnavailable: true, +}; + +const emptyFreshness: GetConsumerPriceFreshnessResponse = { + marketCode: DEFAULT_MARKET, + asOf: '0', + retailers: [], + overallFreshnessMin: 0, + stalledCount: 0, + upstreamUnavailable: true, +}; + +export async function fetchConsumerPriceOverview( + marketCode = DEFAULT_MARKET, + basketSlug = DEFAULT_BASKET, +): Promise { + const hydrated = getHydratedData('consumerPricesOverview') as GetConsumerPriceOverviewResponse | undefined; + if (hydrated?.asOf) return hydrated; + + try { + return await overviewBreaker.execute( + () => client.getConsumerPriceOverview({ marketCode, basketSlug }), + emptyOverview, + ); + } catch { + return emptyOverview; + } +} + +export async function fetchConsumerPriceBasketSeries( + marketCode = DEFAULT_MARKET, + basketSlug = DEFAULT_BASKET, + range = '30d', +): Promise { + try { + return await seriesBreaker.execute( + () => client.getConsumerPriceBasketSeries({ marketCode, basketSlug, range }), + emptySeries, + ); + } catch { + return { ...emptySeries, range }; + } +} + +export async function fetchConsumerPriceCategories( + marketCode = DEFAULT_MARKET, + basketSlug = DEFAULT_BASKET, + range = '30d', +): Promise { + const hydrated = getHydratedData('consumerPricesCategories') as ListConsumerPriceCategoriesResponse | undefined; + if (hydrated?.categories?.length) return hydrated; + + try { + return await categoriesBreaker.execute( + () => client.listConsumerPriceCategories({ marketCode, basketSlug, range }), + emptyCategories, + ); + } catch { + return emptyCategories; + } +} + +export async function fetchConsumerPriceMovers( + marketCode = DEFAULT_MARKET, + range = '30d', + categorySlug?: string, +): Promise { + const hydrated = getHydratedData('consumerPricesMovers') as ListConsumerPriceMoversResponse | undefined; + if (hydrated?.risers?.length || hydrated?.fallers?.length) return hydrated; + + try { + return await moversBreaker.execute( + () => client.listConsumerPriceMovers({ marketCode, range, categorySlug: categorySlug ?? '', limit: 10 }), + emptyMovers, + ); + } catch { + return emptyMovers; + } +} + +export async function fetchRetailerPriceSpreads( + marketCode = DEFAULT_MARKET, + basketSlug = DEFAULT_BASKET, +): Promise { + const hydrated = getHydratedData('consumerPricesSpread') as ListRetailerPriceSpreadsResponse | undefined; + if (hydrated?.retailers?.length) return hydrated; + + try { + return await spreadBreaker.execute( + () => client.listRetailerPriceSpreads({ marketCode, basketSlug }), + emptySpread, + ); + } catch { + return emptySpread; + } +} + +export async function fetchConsumerPriceFreshness( + marketCode = DEFAULT_MARKET, +): Promise { + try { + return await freshnessBreaker.execute( + () => client.getConsumerPriceFreshness({ marketCode }), + emptyFreshness, + ); + } catch { + return emptyFreshness; + } +}