diff --git a/packages/core/src/agent/run-log.error-detection.test.ts b/packages/core/src/agent/run-log.error-detection.test.ts new file mode 100644 index 00000000..653f1f0d --- /dev/null +++ b/packages/core/src/agent/run-log.error-detection.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect } from "vitest"; +import { inferRunLogToolIsError } from "./runner.js"; + +describe("inferRunLogToolIsError", () => { + it("returns true when event explicitly marks error", () => { + expect(inferRunLogToolIsError(true, undefined, null)).toBe(true); + }); + + it("returns true when details.error is present", () => { + expect( + inferRunLogToolIsError( + false, + "{\"error\":true}", + { error: "Financial Datasets API error", message: "Invalid ticker" }, + ), + ).toBe(true); + }); + + it("returns true when details.error_type uses boolean-like marker", () => { + expect(inferRunLogToolIsError(false, undefined, { error_type: true })).toBe(true); + expect(inferRunLogToolIsError(false, undefined, { error_type: "true" })).toBe(true); + }); + + it("returns true when text payload starts with error prefix", () => { + expect(inferRunLogToolIsError(false, "error: ENOENT", null)).toBe(true); + }); + + it("returns false for successful tool responses", () => { + expect( + inferRunLogToolIsError( + false, + "{\"domain\":\"finance\",\"action\":\"get_price_snapshot\"}", + { domain: "finance", action: "get_price_snapshot" }, + ), + ).toBe(false); + }); +}); diff --git a/packages/core/src/agent/runner.ts b/packages/core/src/agent/runner.ts index 47716152..5e6c1843 100644 --- a/packages/core/src/agent/runner.ts +++ b/packages/core/src/agent/runner.ts @@ -236,6 +236,36 @@ export function evaluateCustomSkillAuthoringConsent( } return { allowAuthoring: false, declined: false }; +/** + * Infer whether a tool call should be classified as error in run-log. + * + * Some tool adapters encode failures in payload fields (`error`, `error_type`) + * without setting `event.isError=true`. This helper keeps run-log semantics + * consistent for E2E health checks. + */ +export function inferRunLogToolIsError( + eventIsError: unknown, + resultText: string | undefined, + details: Record | null, +): boolean { + if (eventIsError === true) return true; + if (!details) { + return typeof resultText === "string" && /^error[:\s]/i.test(resultText.trim()); + } + + const errorType = details.error_type; + if (typeof errorType === "boolean") return errorType; + if (typeof errorType === "string") { + const normalized = errorType.trim().toLowerCase(); + if (normalized === "true" || normalized === "error") return true; + } + + const errorValue = details.error; + if (typeof errorValue === "string") return errorValue.trim().length > 0; + if (errorValue === true) return true; + if (errorValue && typeof errorValue === "object") return true; + + return typeof resultText === "string" && /^error[:\s]/i.test(resultText.trim()); } // ── Run-log result extraction helpers ────────────────────────────────────── @@ -1292,8 +1322,7 @@ export class Agent { const resultText = extractRunLogResultText(result); const resultChars = resultText?.length ?? 0; const details = extractRunLogResultDetails(result); - const isError = Boolean((event as any).isError ?? false); - + const isError = inferRunLogToolIsError((event as any).isError, resultText, details); this.currentRunToolExecutions.push({ toolName, isError,