fix(ai-builder): Use placeholders for user-provided values instead of hardcoding fake addresses (#28407)

This commit is contained in:
Albert Alises
2026-04-13 15:29:31 +02:00
committed by GitHub
parent 6217d08ce9
commit 39c6217109
16 changed files with 223 additions and 9 deletions

View File

@@ -200,7 +200,7 @@ Always pass \`conversationContext\` when spawning background agents (\`build-wor
**Credentials**: Call \`list-credentials\` first to know what's available. Build the workflow immediately — the builder auto-resolves available credentials and auto-mocks missing ones. Planned builder tasks handle their own verification and credential finalization flow.
**Post-build flow** (for direct builds via \`build-workflow-with-agent\`):
1. Builder finishes → check if the workflow has mocked credentials, missing parameters, or unconfigured triggers.
1. Builder finishes → check if the workflow has mocked credentials, missing parameters, unresolved placeholders, or unconfigured triggers.
2. If yes → call \`setup-workflow\` with the workflowId so the user can configure them through the setup UI.
3. When \`setup-workflow\` returns \`deferred: true\`, respect the user's decision — do not retry with \`setup-credentials\` or any other setup tool. The user chose to set things up later.
4. Ask the user if they want to test the workflow.

View File

@@ -27,13 +27,23 @@ import {
WORKFLOW_SDK_PATTERNS,
} from '@n8n/workflow-sdk/prompts/sdk-reference';
// ── Shared placeholder guidance (single source of truth) ────────────────────
// prettier-ignore
const PLACEHOLDER_RULE =
"**Do NOT use `placeholder()` for discoverable resources** (spreadsheet IDs, calendar IDs, channel IDs, folder IDs) — resolve real IDs via `explore-node-resources` or create them via setup workflows. For **user-provided values** that cannot be discovered or created (email recipients, phone numbers, custom URLs, notification targets), use `placeholder('descriptive hint')` so the setup wizard prompts the user after the build. Never hardcode fake values like `user@example.com`.";
// prettier-ignore
const PLACEHOLDER_ESCALATION =
'When the user says "send me", "email me", "notify me", or similar and you don\'t know their specific address, use `placeholder(\'Your email address\')` for the recipient field rather than hardcoding a fake address like `user@example.com`. The setup wizard will collect this from the user after the build.';
// ── Shared SDK reference sections ────────────────────────────────────────────
const SDK_CODE_RULES = `## SDK Code Rules
- Do NOT specify node positions — they are auto-calculated by the layout engine.
- For credentials, see the credential rules in your specific workflow process section below.
- **Do NOT use \`placeholder()\`** — always resolve real resource IDs via \`explore-node-resources\` or create resources via setup workflows. If a resource truly cannot be created (external system), use a descriptive string comment like \`'NEEDS: Slack channel #engineering'\` and explain in your summary.
- ${PLACEHOLDER_RULE}
- Use \`expr('{{ $json.field }}')\` for n8n expressions. Variables MUST be inside \`{{ }}\`.
- Do NOT use \`as const\` assertions — the workflow parser only supports JavaScript syntax, not TypeScript-only features. Just use plain string literals.
- Use string values directly for discriminator fields like \`resource\` and \`operation\` (e.g., \`resource: 'message'\` not \`resource: 'message' as const\`).
@@ -445,6 +455,7 @@ When called with failure details for an existing workflow, start from the pre-lo
## Escalation
- If you are stuck or need information only a human can provide (e.g., a chat ID, API key, external resource name), use the \`ask-user\` tool to ask a clear question.
- Do NOT retry the same failing approach more than twice — ask the user instead.
- ${PLACEHOLDER_ESCALATION}
## Mandatory Process
1. **Research**: If the workflow fits a known category (notification, chatbot, scheduling, data_transformation, etc.), call \`get-suggested-nodes\` first for curated recommendations. Then use \`search-nodes\` for service-specific nodes (use short service names: "Gmail", "Slack", not "send email SMTP"). The results include \`discriminators\` (available resources and operations) for nodes that need them. Then call \`get-node-type-definition\` with the appropriate resource/operation to get the TypeScript schema with exact parameter names and types. **Pay attention to @builderHint annotations** in search results and type definitions — they prevent common configuration mistakes.
@@ -631,7 +642,7 @@ Replace \`CHUNK_WORKFLOW_ID\` with the actual ID returned by \`submit-workflow\`
## Setup Workflows (Create Missing Resources)
**NEVER use \`placeholder()\` or hardcoded placeholder strings like "YOUR_SPREADSHEET_ID".** If a resource doesn't exist, create it.
${PLACEHOLDER_RULE}
When \`explore-node-resources\` returns no results for a required resource:
@@ -648,6 +659,7 @@ When called with failure details for an existing workflow, start from the pre-lo
## Escalation
- If you are stuck or need information only a human can provide (e.g., a chat ID, API key, external resource name), use the \`ask-user\` tool to ask a clear question.
- Do NOT retry the same failing approach more than twice — ask the user instead.
- ${PLACEHOLDER_ESCALATION}
## Sandbox Isolation
@@ -714,7 +726,7 @@ n8n normalizes column names to snake_case (e.g., \`dayName\` → \`day_name\`).
4. **Resolve real resource IDs**: Check the node schemas from step 3 for parameters with \`searchListMethod\` or \`loadOptionsMethod\`. For EACH one, call \`explore-node-resources\` with the node type, method name, and the matching credential from step 1 to discover real resource IDs.
- **This is mandatory for: calendars, spreadsheets, channels, folders, models, databases, and any other list-based parameter.** Do NOT assume values like "primary", "default", or "General" — always look up the real ID.
- Example: Google Calendar's \`calendar\` parameter uses \`searchListMethod: getCalendars\`. Call \`explore-node-resources\` with \`methodName: "getCalendars"\` to get the actual calendar ID (e.g., "user@example.com"), not "primary".
- **NEVER use \`placeholder()\` or fake IDs.** If a resource doesn't exist, build a setup workflow to create it (see "Setup Workflows" section).
- **Never use \`placeholder()\` or fake IDs for discoverable resources.** Create them via a setup workflow instead (see "Setup Workflows" section). For user-provided values, follow the placeholder rules in "SDK Code Rules".
- If the resource can't be created via n8n (e.g., Slack channels), explain clearly in your summary what the user needs to set up.
5. **Write workflow code** to \`${workspaceRoot}/src/workflow.ts\`.

View File

@@ -94,12 +94,13 @@ function buildOutcome(
taskId,
workflowId: attempt.workflowId,
submitted: true,
triggerType: detectTriggerType(attempt),
triggerType: attempt.hasUnresolvedPlaceholders ? 'trigger_only' : detectTriggerType(attempt),
needsUserInput: false,
mockedNodeNames: attempt.mockedNodeNames,
mockedCredentialTypes: attempt.mockedCredentialTypes,
mockedCredentialsByNode: attempt.mockedCredentialsByNode,
verificationPinData: attempt.verificationPinData,
hasUnresolvedPlaceholders: attempt.hasUnresolvedPlaceholders,
summary: finalText,
};
}

View File

@@ -5,6 +5,7 @@
* Separated from the tool definition so the tool stays a thin suspend/resume
* state machine, and this logic is testable independently.
*/
import { hasPlaceholderDeep } from '@n8n/utils';
import type { IDataObject, NodeJSON, DisplayOptions } from '@n8n/workflow-sdk';
import { matchesDisplayOptions } from '@n8n/workflow-sdk';
import { nanoid } from 'nanoid';
@@ -67,6 +68,13 @@ export async function buildSetupRequests(
.catch(() => ({}));
}
// Also treat placeholder values as parameter issues so the setup wizard surfaces them
for (const [paramName, paramValue] of Object.entries(parameters)) {
if (!parameterIssues[paramName] && hasPlaceholderDeep(paramValue)) {
parameterIssues[paramName] = ['Contains a placeholder value - please provide the real value'];
}
}
// Build editable parameter definitions for parameters that have issues
let editableParameters: SetupRequest['editableParameters'];
if (Object.keys(parameterIssues).length > 0 && nodeDesc?.properties) {

View File

@@ -8,6 +8,7 @@
import { createTool } from '@mastra/core/tools';
import type { Workspace } from '@mastra/core/workspace';
import { hasPlaceholderDeep } from '@n8n/utils';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { validateWorkflow, layoutWorkflowJSON } from '@n8n/workflow-sdk';
import { createHash, randomUUID } from 'node:crypto';
@@ -36,6 +37,8 @@ export interface SubmitWorkflowAttempt {
mockedCredentialsByNode?: Record<string, string[]>;
/** Verification-only pin data — scoped to this build, never persisted to workflow. */
verificationPinData?: Record<string, Array<Record<string, unknown>>>;
/** Whether any node parameters contain unresolved placeholder values. */
hasUnresolvedPlaceholders?: boolean;
errors?: string[];
}
@@ -346,6 +349,10 @@ export function createSubmitWorkflowTool(
(n) => n.type?.endsWith?.('Trigger') || n.type?.endsWith?.('trigger'),
);
const triggerNodeTypes = triggers.map((t) => t.type).filter(Boolean);
// Scan node parameters for unresolved placeholder values
const hasPlaceholders = (json.nodes ?? []).some((n) => hasPlaceholderDeep(n.parameters));
await reportAttempt({
success: true,
workflowId: savedId,
@@ -359,6 +366,7 @@ export function createSubmitWorkflowTool(
hasMockedCredentials && Object.keys(mockResult.verificationPinData).length > 0
? mockResult.verificationPinData
: undefined,
hasUnresolvedPlaceholders: hasPlaceholders || undefined,
});
return {
success: true,

View File

@@ -79,6 +79,30 @@ describe('formatWorkflowLoopGuidance', () => {
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('"unknown"');
});
it('should trigger setup-workflow guidance when hasUnresolvedPlaceholders is true (no mocked credentials)', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'Built with placeholders',
workflowId: 'wf-ph-1',
hasUnresolvedPlaceholders: true,
};
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('setup-workflow');
expect(result).toContain('wf-ph-1');
});
it('should trigger setup-workflow guidance when both mocked credentials and placeholders exist', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'Built with mocks and placeholders',
mockedCredentialTypes: ['gmailOAuth2'],
hasUnresolvedPlaceholders: true,
workflowId: 'wf-ph-2',
};
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('setup-workflow');
});
});
// ── verify ─────────────────────────────────────────────────────────────────

View File

@@ -148,6 +148,38 @@ describe('handleBuildOutcome', () => {
expect(attempt.attempt).toBe(2);
});
it('persists hasUnresolvedPlaceholders in state and done action', () => {
const state = makeState();
const outcome = makeOutcome({
workflowId: 'wf_123',
triggerType: 'trigger_only',
hasUnresolvedPlaceholders: true,
});
const { state: next, action } = handleBuildOutcome(state, [], outcome);
expect(next.hasUnresolvedPlaceholders).toBe(true);
expect(action.type).toBe('done');
if (action.type === 'done') {
expect(action.hasUnresolvedPlaceholders).toBe(true);
}
});
it('does not set hasUnresolvedPlaceholders when not present in outcome', () => {
const state = makeState();
const outcome = makeOutcome({
workflowId: 'wf_123',
triggerType: 'trigger_only',
});
const { state: next, action } = handleBuildOutcome(state, [], outcome);
expect(next.hasUnresolvedPlaceholders).toBeUndefined();
if (action.type === 'done') {
expect(action.hasUnresolvedPlaceholders).toBeUndefined();
}
});
});
// ── handleVerificationVerdict ───────────────────────────────────────────────
@@ -175,6 +207,22 @@ describe('handleVerificationVerdict', () => {
expect(action.type).toBe('done');
});
it('passes through hasUnresolvedPlaceholders from state on verified', () => {
const state = makeState({
phase: 'verifying',
workflowId: 'wf_123',
hasUnresolvedPlaceholders: true,
});
const verdict = makeVerdict({ verdict: 'verified' });
const { action } = handleVerificationVerdict(state, [], verdict);
expect(action.type).toBe('done');
if (action.type === 'done') {
expect(action.hasUnresolvedPlaceholders).toBe(true);
}
});
it('transitions to blocked on needs_user_input', () => {
const state = makeState({ phase: 'verifying', workflowId: 'wf_123' });
const verdict = makeVerdict({

View File

@@ -10,7 +10,7 @@ export function formatWorkflowLoopGuidance(
): string {
switch (action.type) {
case 'done': {
if (action.mockedCredentialTypes?.length) {
if (action.mockedCredentialTypes?.length || action.hasUnresolvedPlaceholders) {
return (
'Workflow verified successfully with temporary mock data. ' +
`Call \`setup-workflow\` with workflowId "${action.workflowId ?? 'unknown'}" ` +

View File

@@ -75,11 +75,13 @@ export function handleBuildOutcome(
outcome.mockedCredentialTypes && outcome.mockedCredentialTypes.length > 0
? outcome.mockedCredentialTypes
: undefined;
const hasUnresolvedPlaceholders = outcome.hasUnresolvedPlaceholders ?? undefined;
const updatedState: WorkflowLoopState = {
...state,
workflowId: outcome.workflowId ?? state.workflowId,
lastTaskId: outcome.taskId,
mockedCredentialTypes: mockedCredentialTypes ?? state.mockedCredentialTypes,
hasUnresolvedPlaceholders: hasUnresolvedPlaceholders ?? state.hasUnresolvedPlaceholders,
};
if (outcome.triggerType === 'trigger_only') {
@@ -90,6 +92,7 @@ export function handleBuildOutcome(
workflowId: outcome.workflowId,
summary: outcome.summary,
mockedCredentialTypes,
hasUnresolvedPlaceholders: updatedState.hasUnresolvedPlaceholders,
},
attempt,
};
@@ -138,6 +141,7 @@ export function handleVerificationVerdict(
workflowId: verdict.workflowId,
summary: verdict.summary,
mockedCredentialTypes: state.mockedCredentialTypes,
hasUnresolvedPlaceholders: state.hasUnresolvedPlaceholders,
},
attempt,
};
@@ -152,6 +156,7 @@ export function handleVerificationVerdict(
workflowId: verdict.workflowId,
summary: verdict.summary,
mockedCredentialTypes: state.mockedCredentialTypes,
hasUnresolvedPlaceholders: state.hasUnresolvedPlaceholders,
},
attempt,
};

View File

@@ -29,6 +29,8 @@ export const workflowLoopStateSchema = z.object({
rebuildAttempts: z.number().int().min(0),
/** Credential types that were mocked during build (persisted across phases). */
mockedCredentialTypes: z.array(z.string()).optional(),
/** Whether the submitted workflow contains unresolved placeholder values (persisted across phases). */
hasUnresolvedPlaceholders: z.boolean().optional(),
});
export type WorkflowLoopPhase = z.infer<typeof workflowLoopPhaseSchema>;
@@ -80,6 +82,8 @@ export const workflowBuildOutcomeSchema = z.object({
mockedCredentialsByNode: z.record(z.array(z.string())).optional(),
/** Verification-only pin data — scoped to this build, never persisted to workflow. */
verificationPinData: z.record(z.array(z.record(z.unknown()))).optional(),
/** Whether any node parameters contain unresolved placeholder values. */
hasUnresolvedPlaceholders: z.boolean().optional(),
summary: z.string(),
});
@@ -124,5 +128,11 @@ export type WorkflowLoopAction =
diagnosis: string;
patch?: Record<string, unknown>;
}
| { type: 'done'; workflowId?: string; summary: string; mockedCredentialTypes?: string[] }
| {
type: 'done';
workflowId?: string;
summary: string;
mockedCredentialTypes?: string[];
hasUnresolvedPlaceholders?: boolean;
}
| { type: 'blocked'; reason: string };

View File

@@ -10,3 +10,4 @@ export * from './sort/sortByProperty';
export * from './string/truncate';
export * from './files/sanitize';
export * from './files/path';
export * from './placeholder';

View File

@@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest';
import { isPlaceholderString, hasPlaceholderDeep } from './placeholder';
describe('isPlaceholderString', () => {
it('returns true for a valid placeholder sentinel', () => {
expect(isPlaceholderString('<__PLACEHOLDER_VALUE__Your email address__>')).toBe(true);
});
it('returns false for a regular string', () => {
expect(isPlaceholderString('user@example.com')).toBe(false);
});
it('returns false for non-string values', () => {
expect(isPlaceholderString(42)).toBe(false);
expect(isPlaceholderString(null)).toBe(false);
expect(isPlaceholderString(undefined)).toBe(false);
expect(isPlaceholderString({})).toBe(false);
});
it('returns false for partial matches', () => {
expect(isPlaceholderString('<__PLACEHOLDER_VALUE__no suffix')).toBe(false);
expect(isPlaceholderString('no prefix__>')).toBe(false);
});
});
describe('hasPlaceholderDeep', () => {
it('detects placeholder in a flat string', () => {
expect(hasPlaceholderDeep('<__PLACEHOLDER_VALUE__hint__>')).toBe(true);
});
it('returns false for a regular string', () => {
expect(hasPlaceholderDeep('hello')).toBe(false);
});
it('detects placeholder nested in an array', () => {
expect(hasPlaceholderDeep(['a', '<__PLACEHOLDER_VALUE__hint__>'])).toBe(true);
});
it('detects placeholder nested in an object', () => {
expect(hasPlaceholderDeep({ to: '<__PLACEHOLDER_VALUE__email__>' })).toBe(true);
});
it('detects deeply nested placeholder', () => {
expect(hasPlaceholderDeep({ a: { b: [{ c: '<__PLACEHOLDER_VALUE__x__>' }] } })).toBe(true);
});
it('returns false when no placeholders exist', () => {
expect(hasPlaceholderDeep({ a: { b: [1, 'hello', null] } })).toBe(false);
});
it('returns false for null and undefined', () => {
expect(hasPlaceholderDeep(null)).toBe(false);
expect(hasPlaceholderDeep(undefined)).toBe(false);
});
});

View File

@@ -0,0 +1,21 @@
const PLACEHOLDER_PREFIX = '<__PLACEHOLDER_VALUE__';
const PLACEHOLDER_SUFFIX = '__>';
/** Check if a value is a placeholder sentinel string (format: `<__PLACEHOLDER_VALUE__hint__>`). */
export function isPlaceholderString(value: unknown): boolean {
return (
typeof value === 'string' &&
value.startsWith(PLACEHOLDER_PREFIX) &&
value.endsWith(PLACEHOLDER_SUFFIX)
);
}
/** Recursively check if a value (string, array, or object) contains any placeholder sentinel strings. */
export function hasPlaceholderDeep(value: unknown): boolean {
if (typeof value === 'string') return isPlaceholderString(value);
if (Array.isArray(value)) return value.some(hasPlaceholderDeep);
if (value !== null && typeof value === 'object') {
return Object.values(value as Record<string, unknown>).some(hasPlaceholderDeep);
}
return false;
}

View File

@@ -1,11 +1,17 @@
import type { ComputedRef, Ref } from 'vue';
import { ref } from 'vue';
import { hasPlaceholderDeep } from '@n8n/utils';
import { NodeHelpers, type INodeProperties } from 'n8n-workflow';
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
import type { IUpdateInformation } from '@/Interface';
import { isNestedParam, isParamValueSet, type SetupCard } from '../instanceAiWorkflowSetup.utils';
/** Check if the original node parameter value was a placeholder sentinel. */
function isOriginalValuePlaceholder(req: SetupCard['nodes'][0], paramName: string): boolean {
return hasPlaceholderDeep(req.node.parameters[paramName]);
}
export function useSetupCardParameters(
cards: ComputedRef<SetupCard[]>,
trackedParamNames: Ref<Map<string, Set<string>>>,
@@ -112,6 +118,10 @@ export function useSetupCardParameters(
if (isParamValueSet(val)) {
merged[paramName] = val;
hasValues = true;
} else if (isOriginalValuePlaceholder(req, paramName)) {
// Explicitly send empty string to clear the placeholder sentinel on the backend
merged[paramName] = '';
hasValues = true;
}
}
if (Object.keys(merged).length > 0) {

View File

@@ -1,6 +1,7 @@
import type { Ref } from 'vue';
import { computed, ref, watch } from 'vue';
import type { InstanceAiWorkflowSetupNode } from '@n8n/api-types';
import { hasPlaceholderDeep } from '@n8n/utils';
import { NodeConnectionTypes } from 'n8n-workflow';
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
@@ -309,6 +310,8 @@ export function useSetupCards(
if (storeNode) {
const liveIssues = getNodeParametersIssues(nodeTypesStore, storeNode);
if (Object.keys(liveIssues).length > 0) return false;
// Also check for remaining placeholder values in node parameters
if (hasPlaceholderDeep(storeNode.parameters)) return false;
}
}
}

View File

@@ -1,4 +1,5 @@
import type { InstanceAiWorkflowSetupNode } from '@n8n/api-types';
import { isPlaceholderString } from '@n8n/utils';
import type { INodeProperties } from 'n8n-workflow';
import { isResourceLocatorValue } from 'n8n-workflow';
import type { INodeUi } from '@/Interface';
@@ -65,11 +66,17 @@ export function credGroupKey(req: InstanceAiWorkflowSetupNode): string {
return credType;
}
/** Check if a parameter value is meaningfully set (not empty, null, or an empty resource locator). */
/** Check if a parameter value is meaningfully set (not empty, null, placeholder, or an empty resource locator). */
export function isParamValueSet(val: unknown): boolean {
if (val === undefined || val === null || val === '') return false;
if (isPlaceholderString(val)) return false;
if (isResourceLocatorValue(val)) {
return val.value !== '' && val.value !== null && val.value !== undefined;
return (
val.value !== '' &&
val.value !== null &&
val.value !== undefined &&
!isPlaceholderString(val.value)
);
}
return true;
}