marketing-shibata50/multica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(agent): enforce cross-turn web fetch evidence

Browse source
This commit is contained in:
Jiayuan Zhang 2026-02-17 01:48:53 +08:00
parent ce6291e9eb
commit b5b65c6bae
3 changed files with 249 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

67
packages/core/src/agent/runner.ts
View file

@ -43,6 +43,7 @@ import {
} from "./system-prompt/index.js";
import type { AuthProfileFailureReason } from "./auth-profiles/index.js";
import {
analyzeCrossTurnWebFetchNeed,
shouldEnforceWebFetchAfterSearch,
summarizeWebToolUsage,
type ToolExecutionRecord,
@ -142,6 +143,16 @@ const WEB_SEARCH_FETCH_ENFORCEMENT_PROMPT = [
"If all fetch attempts fail, explicitly say so and avoid relying on snippets for specific claims.",
].join("\n");
const CROSS_TURN_WEB_FETCH_ENFORCEMENT_PROMPT = [
"You are about to finalize a web-dependent answer, but no successful web_fetch happened in this turn.",
"Do not rely only on snippets or prior-turn memory for fresh factual claims.",
"Before finalizing your answer, you MUST:",
"1) If relevant URLs are already available in this conversation, call web_fetch on 1-3 of them.",
"2) If no URLs are available, call web_search to find candidates, then web_fetch on 1-3 relevant URLs.",
"3) Revise your answer using fetched page content as primary evidence.",
"If all fetch attempts fail, explicitly report that limitation and avoid specific claims not backed by fetched content.",
].join("\n");
export class Agent {
private readonly agent: PiAgentCore;
private output;
@ -580,6 +591,10 @@ export class Agent {
});
await this.agent.prompt(prompt);
await this.enforceWebFetchAfterSearchIfNeeded(toolExecutionStartIndex);
await this.enforceCrossTurnWebFetchIfNeeded({
toolExecutionStartIndex,
userPrompt: prompt,
});
this.runLog.log("llm_result", {
duration_ms: Date.now() - llmStart,
});
@ -851,6 +866,58 @@ export class Agent {
}
}
private async enforceCrossTurnWebFetchIfNeeded(params: {
toolExecutionStartIndex: number;
userPrompt: string;
}): Promise<void> {
if (this._internalRun) return;
const activeTools = new Set(
(this.agent.state.tools ?? []).map((tool) => tool.name.toLowerCase()),
);
const webFetchAvailable = activeTools.has("web_fetch");
const currentTurnExecutions = this.currentRunToolExecutions.slice(
params.toolExecutionStartIndex,
);
const usage = summarizeWebToolUsage(currentTurnExecutions);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable,
userPrompt: params.userPrompt,
assistantText: this.output.state.lastAssistantText ?? "",
});
if (!analysis.shouldEnforce) return;
this.runLog.log("web_cross_turn_fetch_guard", {
fetch_calls: usage.fetchCalls,
fetch_success: usage.fetchSuccess,
explicit_fetch_request: analysis.explicitFetchRequest,
user_provides_url: analysis.userProvidesUrl,
freshness_cue: analysis.freshnessCue,
web_cue: analysis.webCue,
user_needs_fresh_web_evidence: analysis.userNeedsFreshWebEvidence,
user_blocks_web_fetch: analysis.userBlocksWebFetch,
assistant_web_claim_signal: analysis.assistantHasWebClaimSignal,
});
try {
await this.agent.prompt(CROSS_TURN_WEB_FETCH_ENFORCEMENT_PROMPT);
this.runLog.log("web_cross_turn_fetch_guard_applied", {
explicit_fetch_request: analysis.explicitFetchRequest,
user_provides_url: analysis.userProvidesUrl,
});
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
this.runLog.log("web_cross_turn_fetch_guard_failed", {
error: message.slice(0, 200),
});
if (this.debug) {
this.stderr.write(`[web-cross-turn-guard] Failed to enforce fetch: ${message}\n`);
}
}
}
private handleRunLogEvent(event: AgentEvent) {
if (event.type === "tool_execution_start") {
const toolName = (event as any).toolName ?? "unknown";

88
packages/core/src/agent/web-tools-policy.test.ts
View file

@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import {
analyzeCrossTurnWebFetchNeed,
shouldEnforceWebFetchAfterSearch,
summarizeWebToolUsage,
type ToolExecutionRecord,
@ -142,4 +143,91 @@ describe("web-tools-policy", () => {
).toBe(true);
});
});
describe("analyzeCrossTurnWebFetchNeed", () => {
it("enforces when user explicitly asks to refetch page content", () => {
const usage = summarizeWebToolUsage([]);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable: true,
userPrompt: "Please refetch the page body this turn and verify with sources.",
assistantText: "Here is a quick summary.",
});
expect(analysis.shouldEnforce).toBe(true);
expect(analysis.explicitFetchRequest).toBe(true);
});
it("enforces for freshness requests when assistant makes web-style claims", () => {
const usage = summarizeWebToolUsage([]);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable: true,
userPrompt: "Give me the latest web news about OpenAI with sources.",
assistantText: "According to Reuters, OpenAI announced a new release.",
});
expect(analysis.shouldEnforce).toBe(true);
expect(analysis.freshnessCue).toBe(true);
expect(analysis.webCue).toBe(true);
expect(analysis.assistantHasWebClaimSignal).toBe(true);
});
it("does not enforce when a fetch was already attempted in this turn", () => {
const usage = summarizeWebToolUsage([
buildRecord({
toolName: "web_fetch",
details: { error: true, code: "fetch_failed" },
}),
]);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable: true,
userPrompt: "Please verify with the latest web sources.",
assistantText: "According to Reuters, ...",
});
expect(analysis.shouldEnforce).toBe(false);
});
it("does not enforce when user explicitly blocks web fetch", () => {
const usage = summarizeWebToolUsage([]);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable: true,
userPrompt: "Do not browse the web, only use snippets.",
assistantText: "According to Reuters, ...",
});
expect(analysis.shouldEnforce).toBe(false);
expect(analysis.userBlocksWebFetch).toBe(true);
});
it("enforces when user provides a direct URL but no fetch happened", () => {
const usage = summarizeWebToolUsage([]);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable: true,
userPrompt: "Summarize https://example.com/article and include key takeaways.",
assistantText: "I can summarize it for you.",
});
expect(analysis.shouldEnforce).toBe(true);
expect(analysis.userProvidesUrl).toBe(true);
});
it("does not enforce for non-web freshness requests", () => {
const usage = summarizeWebToolUsage([]);
const analysis = analyzeCrossTurnWebFetchNeed({
usage,
webFetchAvailable: true,
userPrompt: "What is the latest version in this repository?",
assistantText: "The latest version is 1.2.3.",
});
expect(analysis.shouldEnforce).toBe(false);
expect(analysis.freshnessCue).toBe(true);
expect(analysis.webCue).toBe(false);
});
});
});

94
packages/core/src/agent/web-tools-policy.ts
View file

@ -12,6 +12,57 @@ export type WebToolUsage = {
fetchSuccess: number;
};
export type CrossTurnWebFetchGuardAnalysis = {
shouldEnforce: boolean;
explicitFetchRequest: boolean;
userProvidesUrl: boolean;
freshnessCue: boolean;
webCue: boolean;
userNeedsFreshWebEvidence: boolean;
userBlocksWebFetch: boolean;
assistantHasWebClaimSignal: boolean;
};
const URL_PATTERN = /https?:\/\/[^\s)]+/i;
const USER_EXPLICIT_FETCH_PATTERNS: RegExp[] = [
/\b(re[-\s]?fetch|fetch (again|fresh)|verify with sources?|cite sources?|provide (sources?|links?))\b/i,
/\b(revisit|revalidate|double-check)\b.*\b(source|link|url|web|website)\b/i,
/(?:\u672c\u8f6e|\u8fd9\u4e00\u8f6e).*(?:\u91cd\u65b0|\u518d\u6b21).*(?:\u6293\u53d6|\u83b7\u53d6|\u62c9\u53d6)/,
/(?:\u91cd\u65b0|\u518d\u6b21).*(?:\u6293\u53d6|\u83b7\u53d6).*(?:\u7f51\u9875|\u6b63\u6587|\u539f\u6587|\u94fe\u63a5)/,
/(?:\u7ed9\u51fa|\u63d0\u4f9b).*(?:\u6765\u6e90|\u94fe\u63a5|\u5f15\u7528)/,
/(?:\u6838\u5b9e|\u67e5\u8bc1|\u9a8c\u8bc1).*(?:\u6765\u6e90|\u7f51\u9875)/,
];
const USER_FRESHNESS_PATTERNS: RegExp[] = [
/\b(latest|most recent|recent|today|current|up-to-date|newest|breaking)\b/i,
/\b(news|update|updates)\b/i,
/(?:\u6700\u65b0|\u6700\u8fd1|\u4eca\u5929|\u5f53\u524d|\u8fd1\u671f|\u52a8\u6001|\u65b0\u95fb|\u8d44\u8baf)/,
];
const USER_WEB_CONTEXT_PATTERNS: RegExp[] = [
/\b(web|internet|online|url|urls|link|links|website|article|source|sources|news)\b/i,
/(?:\u7f51\u9875|\u7f51\u7ad9|\u7f51\u7edc|\u4e92\u8054\u7f51|\u94fe\u63a5|\u6765\u6e90|\u65b0\u95fb|\u62a5\u9053|\u6587\u7ae0)/,
];
const USER_WEB_BLOCK_PATTERNS: RegExp[] = [
/\b(do not|don't|no|without)\s+(browse|web|internet|web_search|web_fetch|fetch)\b/i,
/\bonly\b.*\b(snippet|snippets)\b/i,
/(?:\u4e0d\u8981|\u4e0d\u9700)\s*(?:\u8054\u7f51|\u6293\u53d6|\u641c\u7d22|\u83b7\u53d6\u7f51\u9875|web_fetch|web_search)/,
/(?:\u4ec5|\u53ea).*(?:snippet|\u6458\u8981)/i,
];
const ASSISTANT_WEB_CLAIM_PATTERNS: RegExp[] = [
/\b(according to|reported by|as reported|source|sources|citation|cited|press release)\b/i,
/\b(reuters|bloomberg|associated press|ap news|financial times|wall street journal)\b/i,
/(?:\u636e[^。\n]{0,24}(?:\u62a5\u9053|\u663e\u793a|\u79f0)|\u6765\u6e90|\u62a5\u9053\u79f0|\u516c\u544a|\u53d1\u5e03|\u5ba3\u5e03)/,
];
function hasAnyPattern(text: string, patterns: RegExp[]): boolean {
if (!text.trim()) return false;
return patterns.some((pattern) => pattern.test(text));
}
function hasToolError(details: Record<string, unknown> | null): boolean {
return details?.error === true;
}
@ -84,3 +135,46 @@ export function shouldEnforceWebFetchAfterSearch(params: {
return true;
}
export function analyzeCrossTurnWebFetchNeed(params: {
usage: WebToolUsage;
webFetchAvailable: boolean;
userPrompt: string;
assistantText: string;
}): CrossTurnWebFetchGuardAnalysis {
const userPrompt = params.userPrompt ?? "";
const assistantText = params.assistantText ?? "";
const explicitFetchRequest = hasAnyPattern(
userPrompt,
USER_EXPLICIT_FETCH_PATTERNS,
);
const userProvidesUrl = URL_PATTERN.test(userPrompt);
const freshnessCue = hasAnyPattern(userPrompt, USER_FRESHNESS_PATTERNS);
const webCue = userProvidesUrl || hasAnyPattern(userPrompt, USER_WEB_CONTEXT_PATTERNS);
const userNeedsFreshWebEvidence =
explicitFetchRequest || userProvidesUrl || (freshnessCue && webCue);
const userBlocksWebFetch = hasAnyPattern(userPrompt, USER_WEB_BLOCK_PATTERNS);
const assistantHasWebClaimSignal =
URL_PATTERN.test(assistantText) ||
hasAnyPattern(assistantText, ASSISTANT_WEB_CLAIM_PATTERNS);
const shouldEnforce =
params.webFetchAvailable &&
params.usage.fetchCalls === 0 &&
params.usage.fetchSuccess === 0 &&
!userBlocksWebFetch &&
userNeedsFreshWebEvidence &&
(explicitFetchRequest || userProvidesUrl || assistantHasWebClaimSignal);
return {
shouldEnforce,
explicitFetchRequest,
userProvidesUrl,
freshnessCue,
webCue,
userNeedsFreshWebEvidence,
userBlocksWebFetch,
assistantHasWebClaimSignal,
};
}