diff --git a/scripts/seed-forecasts.mjs b/scripts/seed-forecasts.mjs index 91673b23a..9179496b2 100644 --- a/scripts/seed-forecasts.mjs +++ b/scripts/seed-forecasts.mjs @@ -36,6 +36,10 @@ const MAX_PUBLISHED_FORECASTS_PER_SITUATION = 3; const MAX_PUBLISHED_FORECASTS_PER_SITUATION_DOMAIN = 2; const MAX_PUBLISHED_FORECASTS_PER_FAMILY = 4; const MAX_PUBLISHED_FORECASTS_PER_FAMILY_DOMAIN = 2; +const MIN_TARGET_PUBLISHED_FORECASTS = 10; +const MAX_TARGET_PUBLISHED_FORECASTS = 14; +const MAX_PRESELECTED_FORECASTS_PER_FAMILY = 3; +const MAX_PRESELECTED_FORECASTS_PER_SITUATION = 2; const CYBER_MIN_THREATS_PER_COUNTRY = 5; const CYBER_MAX_FORECASTS = 12; const CYBER_SCORE_TYPE_MULTIPLIER = 1.5; // bonus per distinct threat type @@ -4563,6 +4567,7 @@ function summarizePublishFiltering(predictions) { ); return { + suppressedFamilySelection: reasonCounts.family_selection || 0, suppressedWeakFallback: reasonCounts.weak_fallback || 0, suppressedSituationOverlap: reasonCounts.situation_overlap || 0, suppressedSituationCap: reasonCounts.situation_cap || 0, @@ -4581,6 +4586,173 @@ function summarizePublishFiltering(predictions) { }; } +function getPublishSelectionTarget(predictions = []) { + const familyCount = new Set(predictions.map((pred) => pred.familyContext?.id).filter(Boolean)).size; + const situationCount = new Set(predictions.map((pred) => pred.situationContext?.id).filter(Boolean)).size; + const dynamicTarget = Math.ceil((familyCount * 1.5) + Math.min(4, situationCount * 0.15)); + return Math.max( + Math.min(predictions.length, MIN_TARGET_PUBLISHED_FORECASTS), + Math.min(predictions.length, MAX_TARGET_PUBLISHED_FORECASTS, dynamicTarget || MIN_TARGET_PUBLISHED_FORECASTS), + ); +} + +function computePublishSelectionScore(pred) { + const readiness = pred?.readiness?.overall ?? scoreForecastReadiness(pred).overall; + const priority = typeof pred?.analysisPriority === 'number' ? pred.analysisPriority : computeAnalysisPriority(pred); + const narrativeSource = pred?.traceMeta?.narrativeSource || 'fallback'; + const familyBreadth = Math.min(1, ((pred.familyContext?.forecastCount || 1) - 1) / 6); + const situationBreadth = Math.min(1, ((pred.situationContext?.forecastCount || 1) - 1) / 4); + const signalBreadth = Math.min(1, ((pred.situationContext?.topSignals || []).length || 0) / 4); + const domainLift = ['market', 'military', 'supply_chain', 'infrastructure'].includes(pred.domain) ? 0.02 : 0; + const enrichedLift = narrativeSource.startsWith('llm_') ? 0.025 : 0; + return +( + (priority * 0.55) + + (readiness * 0.2) + + ((pred.probability || 0) * 0.15) + + ((pred.confidence || 0) * 0.07) + + (familyBreadth * 0.015) + + (situationBreadth * 0.01) + + (signalBreadth * 0.01) + + domainLift + + enrichedLift + ).toFixed(6); +} + +function selectPublishedForecastPool(predictions, options = {}) { + const eligible = (predictions || []).filter((pred) => (pred?.probability || 0) > (options.minProbability ?? PUBLISH_MIN_PROBABILITY)); + const targetCount = options.targetCount ?? getPublishSelectionTarget(eligible); + const selected = []; + const selectedIds = new Set(); + const familyCounts = new Map(); + const familyDomainCounts = new Map(); + const situationCounts = new Map(); + const domainCounts = new Map(); + + for (const pred of predictions || []) pred.publishSelectionScore = computePublishSelectionScore(pred); + + const ranked = eligible + .slice() + .sort((a, b) => (b.publishSelectionScore || 0) - (a.publishSelectionScore || 0) + || (b.analysisPriority || 0) - (a.analysisPriority || 0) + || (b.probability || 0) - (a.probability || 0)); + + const familyBuckets = new Map(); + for (const pred of ranked) { + const familyId = pred.familyContext?.id || `solo:${pred.situationContext?.id || pred.id}`; + if (!familyBuckets.has(familyId)) familyBuckets.set(familyId, []); + familyBuckets.get(familyId).push(pred); + } + + const orderedFamilyIds = [...familyBuckets.keys()].sort((leftId, rightId) => { + const left = familyBuckets.get(leftId) || []; + const right = familyBuckets.get(rightId) || []; + const leftTop = left[0]; + const rightTop = right[0]; + const leftScore = (leftTop?.publishSelectionScore || 0) + Math.min(0.05, ((leftTop?.familyContext?.forecastCount || 1) - 1) * 0.005); + const rightScore = (rightTop?.publishSelectionScore || 0) + Math.min(0.05, ((rightTop?.familyContext?.forecastCount || 1) - 1) * 0.005); + return rightScore - leftScore || leftId.localeCompare(rightId); + }); + + function canSelect(pred, mode = 'fill') { + if (!pred || selectedIds.has(pred.id)) return false; + const familyId = pred.familyContext?.id || `solo:${pred.situationContext?.id || pred.id}`; + const familyTotal = familyCounts.get(familyId) || 0; + const familyDomainKey = `${familyId}:${pred.domain}`; + const familyDomainTotal = familyDomainCounts.get(familyDomainKey) || 0; + const situationId = pred.situationContext?.id || pred.id; + const situationTotal = situationCounts.get(situationId) || 0; + if (familyTotal >= Math.min(MAX_PUBLISHED_FORECASTS_PER_FAMILY, MAX_PRESELECTED_FORECASTS_PER_FAMILY)) return false; + if (familyDomainTotal >= MAX_PUBLISHED_FORECASTS_PER_FAMILY_DOMAIN) return false; + if (situationTotal >= MAX_PRESELECTED_FORECASTS_PER_SITUATION) return false; + if (mode === 'diversity') { + const domainTotal = domainCounts.get(pred.domain) || 0; + if (domainTotal >= 2 && !['market', 'military', 'supply_chain', 'infrastructure'].includes(pred.domain)) return false; + } + return true; + } + + function take(pred) { + const familyId = pred.familyContext?.id || `solo:${pred.situationContext?.id || pred.id}`; + const familyDomainKey = `${familyId}:${pred.domain}`; + const situationId = pred.situationContext?.id || pred.id; + selected.push(pred); + selectedIds.add(pred.id); + familyCounts.set(familyId, (familyCounts.get(familyId) || 0) + 1); + familyDomainCounts.set(familyDomainKey, (familyDomainCounts.get(familyDomainKey) || 0) + 1); + situationCounts.set(situationId, (situationCounts.get(situationId) || 0) + 1); + domainCounts.set(pred.domain, (domainCounts.get(pred.domain) || 0) + 1); + } + + for (const familyId of orderedFamilyIds) { + if (selected.length >= targetCount) break; + const bucket = familyBuckets.get(familyId) || []; + const choice = bucket.find((pred) => canSelect(pred, 'diversity')); + if (choice) take(choice); + } + + for (const familyId of orderedFamilyIds) { + if (selected.length >= targetCount) break; + const bucket = familyBuckets.get(familyId) || []; + const selectedDomains = new Set(selected.filter((pred) => (pred.familyContext?.id || `solo:${pred.situationContext?.id || pred.id}`) === familyId).map((pred) => pred.domain)); + const choice = bucket.find((pred) => !selectedDomains.has(pred.domain) && canSelect(pred, 'diversity')); + if (choice) take(choice); + } + + for (const pred of ranked) { + if (selected.length >= targetCount) break; + if (canSelect(pred, 'fill')) take(pred); + } + + const deferredCandidates = ranked.filter((pred) => !selectedIds.has(pred.id)); + if (deferredCandidates.length > 0) { + console.log(` [filterPublished] Deferred ${deferredCandidates.length} forecast(s) in family selection`); + } + + const result = selected + .slice() + .sort((a, b) => (b.analysisPriority || 0) - (a.analysisPriority || 0) + || (b.publishSelectionScore || 0) - (a.publishSelectionScore || 0) + || (b.probability || 0) - (a.probability || 0)); + result.deferredCandidates = deferredCandidates; + result.targetCount = targetCount; + return result; +} + +function buildPublishedForecastArtifacts(candidatePool, fullRunSituationClusters) { + const filteredPredictions = filterPublishedForecasts(candidatePool); + const filteredSituationClusters = projectSituationClusters(fullRunSituationClusters, filteredPredictions); + attachSituationContext(filteredPredictions, filteredSituationClusters); + const filteredSituationFamilies = attachSituationFamilyContext(filteredPredictions, buildSituationFamilies(filteredSituationClusters)); + const publishedPredictions = applySituationFamilyCaps(filteredPredictions, filteredSituationFamilies); + const publishedSituationClusters = projectSituationClusters(fullRunSituationClusters, publishedPredictions); + attachSituationContext(publishedPredictions, publishedSituationClusters); + const publishedSituationFamilies = attachSituationFamilyContext(publishedPredictions, buildSituationFamilies(publishedSituationClusters)); + refreshPublishedNarratives(publishedPredictions); + return { + filteredPredictions, + filteredSituationClusters, + filteredSituationFamilies, + publishedPredictions, + publishedSituationClusters, + publishedSituationFamilies, + }; +} + +function markDeferredFamilySelection(predictions, selectedPool) { + const selectedIds = new Set((selectedPool || []).map((pred) => pred.id)); + for (const pred of predictions || []) { + if ((pred?.probability || 0) <= PUBLISH_MIN_PROBABILITY) continue; + if (selectedIds.has(pred.id)) continue; + if (pred.publishDiagnostics?.reason) continue; + pred.publishDiagnostics = { + reason: 'family_selection', + familyId: pred.familyContext?.id || '', + situationId: pred.situationContext?.id || '', + targetCount: selectedPool?.targetCount || 0, + }; + } +} + function filterPublishedForecasts(predictions, minProbability = PUBLISH_MIN_PROBABILITY) { let weakFallbackCount = 0; let overlapSuppressedCount = 0; @@ -5583,16 +5755,23 @@ async function fetchForecasts() { const enrichmentMeta = await enrichScenariosWithLLM(predictions); populateFallbackNarratives(predictions); - const initiallyPublishedPredictions = filterPublishedForecasts(predictions); - const initiallyPublishedSituationClusters = projectSituationClusters(fullRunSituationClusters, initiallyPublishedPredictions); - attachSituationContext(initiallyPublishedPredictions, initiallyPublishedSituationClusters); - const initiallyPublishedSituationFamilies = attachSituationFamilyContext(initiallyPublishedPredictions, buildSituationFamilies(initiallyPublishedSituationClusters)); - const publishedPredictions = applySituationFamilyCaps(initiallyPublishedPredictions, initiallyPublishedSituationFamilies); + const publishSelectionPool = selectPublishedForecastPool(predictions); + let finalSelectionPool = [...publishSelectionPool]; + finalSelectionPool.targetCount = publishSelectionPool.targetCount || finalSelectionPool.length; + const deferredCandidates = [...(publishSelectionPool.deferredCandidates || [])]; + let publishArtifacts = buildPublishedForecastArtifacts(finalSelectionPool, fullRunSituationClusters); + while (publishArtifacts.publishedPredictions.length < (finalSelectionPool.targetCount || 0) && deferredCandidates.length > 0) { + finalSelectionPool.push(deferredCandidates.shift()); + publishArtifacts = buildPublishedForecastArtifacts(finalSelectionPool, fullRunSituationClusters); + } + markDeferredFamilySelection(predictions, finalSelectionPool); + const initiallyPublishedPredictions = publishArtifacts.filteredPredictions; + const initiallyPublishedSituationClusters = publishArtifacts.filteredSituationClusters; + const initiallyPublishedSituationFamilies = publishArtifacts.filteredSituationFamilies; + const publishedPredictions = publishArtifacts.publishedPredictions; const publishTelemetry = summarizePublishFiltering(predictions); - const publishedSituationClusters = projectSituationClusters(fullRunSituationClusters, publishedPredictions); - attachSituationContext(publishedPredictions, publishedSituationClusters); - const publishedSituationFamilies = attachSituationFamilyContext(publishedPredictions, buildSituationFamilies(publishedSituationClusters)); - refreshPublishedNarratives(publishedPredictions); + const publishedSituationClusters = publishArtifacts.publishedSituationClusters; + const publishedSituationFamilies = publishArtifacts.publishedSituationFamilies; if (publishedPredictions.length !== predictions.length) { console.log(` Filtered ${predictions.length - publishedPredictions.length} forecasts at publish floor > ${PUBLISH_MIN_PROBABILITY}`); } @@ -5603,6 +5782,7 @@ async function fetchForecasts() { generatedAt: Date.now(), enrichmentMeta, publishTelemetry, + publishSelectionPool, situationClusters: publishedSituationClusters, situationFamilies: publishedSituationFamilies, fullRunSituationClusters, @@ -5778,6 +5958,8 @@ export { scoreForecastReadiness, computeAnalysisPriority, rankForecastsForAnalysis, + selectPublishedForecastPool, + buildPublishedForecastArtifacts, filterPublishedForecasts, applySituationFamilyCaps, summarizePublishFiltering, diff --git a/tests/forecast-detectors.test.mjs b/tests/forecast-detectors.test.mjs index 87923f330..25c1cb574 100644 --- a/tests/forecast-detectors.test.mjs +++ b/tests/forecast-detectors.test.mjs @@ -50,6 +50,8 @@ import { scoreForecastReadiness, computeAnalysisPriority, rankForecastsForAnalysis, + selectPublishedForecastPool, + buildPublishedForecastArtifacts, filterPublishedForecasts, applySituationFamilyCaps, selectForecastsForEnrichment, @@ -2006,6 +2008,107 @@ describe('forecast quality gating', () => { assert.equal(telemetry.cappedFamilies, 1); }); + it('preselects published forecasts across families before overlap suppression', () => { + const preds = [ + makePrediction('conflict', 'Iran', 'Escalation risk: Iran', 0.72, 0.65, '7d', [{ type: 'ucdp', value: 'Iran events elevated', weight: 0.4 }]), + makePrediction('political', 'Iran', 'Political instability: Iran', 0.58, 0.59, '14d', [{ type: 'news_corroboration', value: 'Emergency meetings continue', weight: 0.35 }]), + makePrediction('market', 'Middle East', 'Oil repricing risk: Gulf', 0.55, 0.57, '30d', [{ type: 'prediction_market', value: 'Oil reprices higher', weight: 0.3 }]), + makePrediction('supply_chain', 'Persian Gulf', 'Shipping disruption: Persian Gulf', 0.53, 0.56, '14d', [{ type: 'chokepoint', value: 'Routing delays persist', weight: 0.35 }]), + makePrediction('conflict', 'Ukraine', 'Escalation risk: Ukraine', 0.64, 0.61, '7d', [{ type: 'ucdp', value: 'Ukraine conflict remains active', weight: 0.42 }]), + makePrediction('market', 'Black Sea', 'Grain pricing pressure: Black Sea', 0.5, 0.54, '30d', [{ type: 'prediction_market', value: 'Grain risk premium widens', weight: 0.28 }]), + ]; + + buildForecastCases(preds); + for (const [index, pred] of preds.entries()) { + pred.traceMeta = { narrativeSource: index < 2 ? 'llm_combined' : 'fallback' }; + pred.readiness = { overall: 0.7 - (index * 0.04) }; + pred.analysisPriority = 0.24 - (index * 0.02); + } + + const familyA = { id: 'fam-middle-east', label: 'Middle East pressure family', forecastCount: 4, situationCount: 4, situationIds: ['sit-iran-conflict', 'sit-iran-political', 'sit-gulf-market', 'sit-gulf-shipping'] }; + const familyB = { id: 'fam-black-sea', label: 'Black Sea pressure family', forecastCount: 2, situationCount: 2, situationIds: ['sit-ukraine-conflict', 'sit-blacksea-market'] }; + const contexts = [ + ['sit-iran-conflict', 'Iran conflict situation', familyA], + ['sit-iran-political', 'Iran political situation', familyA], + ['sit-gulf-market', 'Gulf market situation', familyA], + ['sit-gulf-shipping', 'Persian Gulf shipping situation', familyA], + ['sit-ukraine-conflict', 'Ukraine conflict situation', familyB], + ['sit-blacksea-market', 'Black Sea market situation', familyB], + ]; + for (const [index, pred] of preds.entries()) { + const [id, label, family] = contexts[index]; + pred.situationContext = { id, label, forecastCount: 1, topSignals: [{ type: 'news_corroboration', count: 1 }] }; + pred.caseFile.situationContext = pred.situationContext; + pred.familyContext = family; + pred.caseFile.familyContext = family; + } + + const selected = selectPublishedForecastPool(preds); + assert.ok(selected.some((pred) => pred.familyContext?.id === familyA.id)); + assert.ok(selected.some((pred) => pred.familyContext?.id === familyB.id)); + assert.ok(selected.some((pred) => pred.domain === 'market')); + assert.ok((selected.deferredCandidates || []).length >= 1); + }); + + it('backfills deferred forecasts when filtering drops a preselected duplicate', () => { + const primary = makePrediction('conflict', 'Iran', 'Escalation risk: Iran', 0.74, 0.66, '7d', [{ type: 'ucdp', value: 'Iran events elevated', weight: 0.4 }]); + const duplicate = makePrediction('conflict', 'Iran', 'Retaliatory conflict risk: Iran', 0.69, 0.58, '7d', [{ type: 'ucdp', value: 'Iran events elevated', weight: 0.36 }]); + const political = makePrediction('political', 'Iran', 'Political instability: Iran', 0.59, 0.57, '14d', [{ type: 'news_corroboration', value: 'Emergency cabinet meetings continue', weight: 0.35 }]); + const supply = makePrediction('supply_chain', 'Persian Gulf', 'Shipping disruption: Persian Gulf', 0.54, 0.56, '14d', [{ type: 'chokepoint', value: 'Routing delays persist', weight: 0.34 }]); + + buildForecastCases([primary, duplicate, political, supply]); + const fullRunSituationClusters = [ + { id: 'sit-iran-conflict', label: 'Iran conflict situation', dominantRegion: 'Iran', dominantDomain: 'conflict', regions: ['Iran'], domains: ['conflict'], actors: ['Iran'], branchKinds: ['base'], forecastIds: [primary.id, duplicate.id], forecastCount: 2, avgProbability: 0.715, avgConfidence: 0.62, topSignals: [{ type: 'ucdp', count: 2 }], sampleTitles: [primary.title, duplicate.title] }, + { id: 'sit-iran-political', label: 'Iran political situation', dominantRegion: 'Iran', dominantDomain: 'political', regions: ['Iran'], domains: ['political'], actors: ['Iran'], branchKinds: ['base'], forecastIds: [political.id], forecastCount: 1, avgProbability: 0.59, avgConfidence: 0.57, topSignals: [{ type: 'news_corroboration', count: 1 }], sampleTitles: [political.title] }, + { id: 'sit-gulf-shipping', label: 'Persian Gulf shipping situation', dominantRegion: 'Persian Gulf', dominantDomain: 'supply_chain', regions: ['Persian Gulf'], domains: ['supply_chain'], actors: ['Shipping'], branchKinds: ['base'], forecastIds: [supply.id], forecastCount: 1, avgProbability: 0.54, avgConfidence: 0.56, topSignals: [{ type: 'chokepoint', count: 1 }], sampleTitles: [supply.title] }, + ]; + + const familyA = { id: 'fam-middle-east', label: 'Middle East pressure family', forecastCount: 3, situationCount: 2, situationIds: ['sit-iran-conflict', 'sit-iran-political'] }; + const familyB = { id: 'fam-gulf', label: 'Persian Gulf pressure family', forecastCount: 1, situationCount: 1, situationIds: ['sit-gulf-shipping'] }; + for (const pred of [primary, duplicate, political, supply]) { + pred.traceMeta = { narrativeSource: 'fallback' }; + pred.readiness = { overall: 0.7 }; + } + primary.analysisPriority = 0.25; + duplicate.analysisPriority = 0.2; + political.analysisPriority = 0.18; + supply.analysisPriority = 0.14; + + primary.situationContext = fullRunSituationClusters[0]; + duplicate.situationContext = fullRunSituationClusters[0]; + political.situationContext = fullRunSituationClusters[1]; + supply.situationContext = fullRunSituationClusters[2]; + primary.caseFile.situationContext = primary.situationContext; + duplicate.caseFile.situationContext = duplicate.situationContext; + political.caseFile.situationContext = political.situationContext; + supply.caseFile.situationContext = supply.situationContext; + primary.familyContext = familyA; + duplicate.familyContext = familyA; + political.familyContext = familyA; + supply.familyContext = familyB; + primary.caseFile.familyContext = familyA; + duplicate.caseFile.familyContext = familyA; + political.caseFile.familyContext = familyA; + supply.caseFile.familyContext = familyB; + + const pool = selectPublishedForecastPool([primary, duplicate, political], { targetCount: 3 }); + assert.equal(pool.length, 3); + assert.equal(pool.deferredCandidates.length, 0); + + const expandedPool = selectPublishedForecastPool([primary, duplicate, political, supply], { targetCount: 3 }); + let candidatePool = [...expandedPool]; + let deferred = [...expandedPool.deferredCandidates]; + let artifacts = buildPublishedForecastArtifacts(candidatePool, fullRunSituationClusters); + while (artifacts.publishedPredictions.length < expandedPool.targetCount && deferred.length > 0) { + candidatePool.push(deferred.shift()); + artifacts = buildPublishedForecastArtifacts(candidatePool, fullRunSituationClusters); + } + + assert.equal(artifacts.publishedPredictions.length, 3); + assert.ok(artifacts.publishedPredictions.some((pred) => pred.id === supply.id)); + assert.ok(!artifacts.publishedPredictions.some((pred) => pred.id === duplicate.id)); + }); + it('does not report capped situations when a situation only reaches the cap without dropping anything', () => { const preds = [ makePrediction('conflict', 'Iran', 'Escalation risk: Iran', 0.66, 0.6, '7d', [ diff --git a/tests/forecast-trace-export.test.mjs b/tests/forecast-trace-export.test.mjs index 7d8a3a9a9..afea84e3a 100644 --- a/tests/forecast-trace-export.test.mjs +++ b/tests/forecast-trace-export.test.mjs @@ -80,6 +80,7 @@ describe('forecast trace artifact builder', () => { predictions: [a, b], fullRunPredictions: [a, b, c], publishTelemetry: { + suppressedFamilySelection: 2, suppressedWeakFallback: 1, suppressedSituationOverlap: 2, suppressedSituationCap: 1, @@ -149,6 +150,7 @@ describe('forecast trace artifact builder', () => { assert.equal(artifacts.summary.quality.traced.fallbackRate, 1); assert.equal(artifacts.summary.quality.traced.enrichedRate, 0); assert.equal(artifacts.summary.quality.publish.suppressedSituationOverlap, 2); + assert.equal(artifacts.summary.quality.publish.suppressedFamilySelection, 2); assert.equal(artifacts.summary.quality.publish.suppressedSituationCap, 1); assert.equal(artifacts.summary.quality.publish.suppressedSituationDomainCap, 1); assert.equal(artifacts.summary.quality.publish.cappedSituations, 1);