From 278334474f3fd94e1dcd88c9cad1a8cfb1ba08be Mon Sep 17 00:00:00 2001 From: Jiayuan Zhang Date: Fri, 13 Feb 2026 15:12:30 +0800 Subject: [PATCH] feat(session): externalize base64 images from session JSONL Session JSONL files were bloated because base64 image data was stored inline (a real session had 6.7MB of images in a 9.8MB file). Images are now extracted to per-session media/ directories as binary files, with compact $ref references stored in the JSONL. Images are restored transparently on read. Old sessions with inline base64 remain backward compatible and auto-migrate on next compaction. Co-Authored-By: Claude Opus 4.6 --- .../core/src/agent/session/storage.test.ts | 242 ++++++++++++++++++ packages/core/src/agent/session/storage.ts | 214 +++++++++++++++- 2 files changed, 452 insertions(+), 4 deletions(-) diff --git a/packages/core/src/agent/session/storage.test.ts b/packages/core/src/agent/session/storage.test.ts index 6f9498c9..37c88bef 100644 --- a/packages/core/src/agent/session/storage.test.ts +++ b/packages/core/src/agent/session/storage.test.ts @@ -6,6 +6,7 @@ import { resolveBaseDir, resolveSessionDir, resolveSessionPath, + resolveMediaDir, ensureSessionDir, readEntries, appendEntry, @@ -274,4 +275,245 @@ describe("session/storage", () => { expect(readFileSync(filePath, "utf8")).toBe(""); }); }); + + describe("image externalization", () => { + // Generate a large base64 string (>43KB to exceed MIN_EXTERNALIZE_B64_LENGTH) + const largeBinarySize = 50_000; // 50KB binary + const largeBuffer = Buffer.alloc(largeBinarySize, 0x42); // fill with 'B' + const largeBase64 = largeBuffer.toString("base64"); + + // Small base64 that should stay inline + const smallBase64 = Buffer.alloc(100, 0x41).toString("base64"); + + function makeImageEntry(imageData: string, sessionId = "img-session"): SessionEntry { + return { + type: "message", + message: { + role: "user", + content: [ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: imageData }, + ], + } as any, + timestamp: Date.now(), + }; + } + + function makeFormatBEntry(imageData: string): SessionEntry { + return { + type: "message", + message: { + role: "user", + content: [ + { type: "image", source: { type: "base64", data: imageData } }, + ], + } as any, + timestamp: Date.now(), + }; + } + + function makeToolResultEntry(imageData: string): SessionEntry { + return { + type: "message", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "test-id", + content: [ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: imageData }, + ], + }, + ], + } as any, + timestamp: Date.now(), + }; + } + + it("should externalize Format A image and create media file", async () => { + const sessionId = "ext-format-a"; + const entry = makeImageEntry(largeBase64); + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + // Read raw JSONL — should have $ref, not data + const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8"); + const rawEntry = JSON.parse(rawContent.trim()); + expect(rawEntry.message.content[1].$ref).toMatch(/^media\/[a-f0-9]+\.bin$/); + expect(rawEntry.message.content[1].data).toBeUndefined(); + + // Media file should exist + const mediaDir = resolveMediaDir(sessionId, { baseDir: testBaseDir }); + const files = existsSync(mediaDir) + ? require("node:fs").readdirSync(mediaDir) as string[] + : []; + expect(files).toHaveLength(1); + expect(files[0]).toMatch(/^[a-f0-9]+\.bin$/); + + // Binary content should match original + const binPath = join(mediaDir, files[0]!); + const savedBuffer = readFileSync(binPath); + expect(savedBuffer).toEqual(largeBuffer); + }); + + it("should externalize Format B image (Anthropic source style)", async () => { + const sessionId = "ext-format-b"; + const entry = makeFormatBEntry(largeBase64); + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8"); + const rawEntry = JSON.parse(rawContent.trim()); + expect(rawEntry.message.content[0].source.type).toBe("$ref"); + expect(rawEntry.message.content[0].source.path).toMatch(/^media\/[a-f0-9]+\.bin$/); + }); + + it("should restore externalized images on read (round-trip)", async () => { + const sessionId = "ext-roundtrip"; + const entry = makeImageEntry(largeBase64); + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + const entries = readEntries(sessionId, { baseDir: testBaseDir }); + expect(entries).toHaveLength(1); + const content = (entries[0] as any).message.content; + expect(content[1].type).toBe("image"); + expect(content[1].data).toBe(largeBase64); + expect(content[1].$ref).toBeUndefined(); + }); + + it("should restore Format B images on read", async () => { + const sessionId = "ext-roundtrip-b"; + const entry = makeFormatBEntry(largeBase64); + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + const entries = readEntries(sessionId, { baseDir: testBaseDir }); + expect(entries).toHaveLength(1); + const block = (entries[0] as any).message.content[0]; + expect(block.source.type).toBe("base64"); + expect(block.source.data).toBe(largeBase64); + }); + + it("should handle old sessions with inline base64 (backward compat)", () => { + const sessionId = "old-inline"; + const dir = join(testBaseDir, sessionId); + mkdirSync(dir, { recursive: true }); + + // Write raw JSONL with inline base64 (old format, no $ref) + const entry = makeImageEntry(largeBase64); + writeFileSync(join(dir, "session.jsonl"), `${JSON.stringify(entry)}\n`); + + const entries = readEntries(sessionId, { baseDir: testBaseDir }); + expect(entries).toHaveLength(1); + const content = (entries[0] as any).message.content; + expect(content[1].data).toBe(largeBase64); + }); + + it("should return placeholder for missing media file", () => { + const sessionId = "missing-media"; + const dir = join(testBaseDir, sessionId); + mkdirSync(dir, { recursive: true }); + + // Write JSONL with $ref but no media file + const rawEntry = { + type: "message", + message: { + role: "user", + content: [ + { type: "image", $ref: "media/deadbeef.bin" }, + ], + }, + timestamp: Date.now(), + }; + writeFileSync(join(dir, "session.jsonl"), `${JSON.stringify(rawEntry)}\n`); + + const entries = readEntries(sessionId, { baseDir: testBaseDir }); + expect(entries).toHaveLength(1); + const block = (entries[0] as any).message.content[0]; + expect(block.type).toBe("text"); + expect(block.text).toContain("unavailable"); + }); + + it("should deduplicate same image data", async () => { + const sessionId = "ext-dedup"; + const entry1 = makeImageEntry(largeBase64); + const entry2 = makeImageEntry(largeBase64); + + await appendEntry(sessionId, entry1, { baseDir: testBaseDir }); + await appendEntry(sessionId, entry2, { baseDir: testBaseDir }); + + const mediaDir = resolveMediaDir(sessionId, { baseDir: testBaseDir }); + const files = require("node:fs").readdirSync(mediaDir) as string[]; + expect(files).toHaveLength(1); // same hash = same file + }); + + it("should keep small images inline", async () => { + const sessionId = "ext-small"; + const entry = makeImageEntry(smallBase64); + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + // Read raw JSONL — small image should still have data, not $ref + const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8"); + const rawEntry = JSON.parse(rawContent.trim()); + expect(rawEntry.message.content[1].data).toBe(smallBase64); + expect(rawEntry.message.content[1].$ref).toBeUndefined(); + + // No media dir should be created + const mediaDir = resolveMediaDir(sessionId, { baseDir: testBaseDir }); + expect(existsSync(mediaDir)).toBe(false); + }); + + it("should not affect non-image entries", async () => { + const sessionId = "ext-noimg"; + const entry: SessionEntry = { + type: "message", + message: { role: "assistant", content: "Just text response" } as any, + timestamp: 1000, + }; + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8"); + expect(rawContent.trim()).toBe(JSON.stringify(entry)); + }); + + it("should handle images inside nested tool_result content", async () => { + const sessionId = "ext-tool-result"; + const entry = makeToolResultEntry(largeBase64); + + await appendEntry(sessionId, entry, { baseDir: testBaseDir }); + + // Raw JSONL should have $ref inside tool_result + const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8"); + const rawEntry = JSON.parse(rawContent.trim()); + const toolResult = rawEntry.message.content[0]; + expect(toolResult.content[1].$ref).toMatch(/^media\/[a-f0-9]+\.bin$/); + + // Round-trip should restore + const entries = readEntries(sessionId, { baseDir: testBaseDir }); + const restored = (entries[0] as any).message.content[0].content[1]; + expect(restored.data).toBe(largeBase64); + expect(restored.$ref).toBeUndefined(); + }); + + it("should externalize via writeEntries (compaction path)", async () => { + const sessionId = "ext-write-entries"; + const entry = makeImageEntry(largeBase64); + + await writeEntries(sessionId, [entry], { baseDir: testBaseDir }); + + // Should be externalized + const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8"); + const rawEntry = JSON.parse(rawContent.trim()); + expect(rawEntry.message.content[1].$ref).toMatch(/^media\/[a-f0-9]+\.bin$/); + + // Round-trip + const entries = readEntries(sessionId, { baseDir: testBaseDir }); + expect((entries[0] as any).message.content[1].data).toBe(largeBase64); + }); + }); }); diff --git a/packages/core/src/agent/session/storage.ts b/packages/core/src/agent/session/storage.ts index 75ff9239..51c8245c 100644 --- a/packages/core/src/agent/session/storage.ts +++ b/packages/core/src/agent/session/storage.ts @@ -1,6 +1,7 @@ import { join } from "path"; -import { existsSync, mkdirSync, readFileSync } from "fs"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { appendFile, writeFile } from "fs/promises"; +import { createHash } from "node:crypto"; import type { SessionEntry } from "./types.js"; import { DATA_DIR } from "@multica/utils"; import { acquireSessionWriteLock } from "./session-write-lock.js"; @@ -9,6 +10,9 @@ export type SessionStorageOptions = { baseDir?: string | undefined; }; +/** Minimum base64 data length to externalize (32KB decoded ≈ 43KB base64) */ +const MIN_EXTERNALIZE_B64_LENGTH = 43_000; + export function resolveBaseDir(options?: SessionStorageOptions) { return options?.baseDir ?? join(DATA_DIR, "sessions"); } @@ -21,6 +25,10 @@ export function resolveSessionPath(sessionId: string, options?: SessionStorageOp return join(resolveSessionDir(sessionId, options), "session.jsonl"); } +export function resolveMediaDir(sessionId: string, options?: SessionStorageOptions) { + return join(resolveSessionDir(sessionId, options), "media"); +} + export function ensureSessionDir(sessionId: string, options?: SessionStorageOptions) { const dir = resolveSessionDir(sessionId, options); // mkdirSync with recursive is idempotent (no-op if dir exists), @@ -37,6 +45,200 @@ export function ensureSessionDir(sessionId: string, options?: SessionStorageOpti } } +// ─── Image Externalization ────────────────────────────────────────────────── + +function contentHash(base64Data: string): string { + const buffer = Buffer.from(base64Data, "base64"); + return createHash("sha256").update(buffer).digest("hex").slice(0, 32); +} + +function ensureMediaDir(sessionId: string, options?: SessionStorageOptions): void { + const dir = resolveMediaDir(sessionId, options); + try { + mkdirSync(dir, { recursive: true }); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + mkdirSync(dir, { recursive: true }); + } else { + throw err; + } + } +} + +function saveImageBinary( + sessionId: string, + hash: string, + base64Data: string, + options?: SessionStorageOptions, +): void { + ensureMediaDir(sessionId, options); + const filePath = join(resolveMediaDir(sessionId, options), `${hash}.bin`); + if (existsSync(filePath)) return; // dedup + const buffer = Buffer.from(base64Data, "base64"); + writeFileSync(filePath, buffer); +} + +/** + * Replace a single image content block with an external file reference. + * Returns the original block unchanged if it's not an externalizable image. + */ +function externalizeBlock( + block: any, + sessionId: string, + options?: SessionStorageOptions, +): any { + if (!block || typeof block !== "object" || block.type !== "image") return block; + + // Format A: { type: "image", data: "" } + if (typeof block.data === "string" && block.data.length > MIN_EXTERNALIZE_B64_LENGTH) { + const hash = contentHash(block.data); + const relPath = `media/${hash}.bin`; + saveImageBinary(sessionId, hash, block.data, options); + const { data: _removed, ...rest } = block; + return { ...rest, $ref: relPath }; + } + + // Format B: { type: "image", source: { type: "base64", data: "" } } + if ( + block.source && + typeof block.source === "object" && + block.source.type === "base64" && + typeof block.source.data === "string" && + block.source.data.length > MIN_EXTERNALIZE_B64_LENGTH + ) { + const hash = contentHash(block.source.data); + const relPath = `media/${hash}.bin`; + saveImageBinary(sessionId, hash, block.source.data, options); + return { ...block, source: { type: "$ref", path: relPath } }; + } + + return block; +} + +/** + * Restore an externalized image reference back to inline base64 data. + */ +function internalizeBlock( + block: any, + sessionId: string, + options?: SessionStorageOptions, +): any { + if (!block || typeof block !== "object" || block.type !== "image") return block; + + // Format A ref: { type: "image", $ref: "media/.bin" } + if (typeof block.$ref === "string") { + const filePath = join(resolveSessionDir(sessionId, options), block.$ref); + try { + const buffer = readFileSync(filePath); + const data = buffer.toString("base64"); + const { $ref: _removed, ...rest } = block; + return { ...rest, data }; + } catch { + return { type: "text", text: "[Image unavailable: referenced media file not found]" }; + } + } + + // Format B ref: { type: "image", source: { type: "$ref", path: "media/.bin" } } + if (block.source && typeof block.source === "object" && block.source.type === "$ref") { + const filePath = join(resolveSessionDir(sessionId, options), block.source.path); + try { + const buffer = readFileSync(filePath); + const data = buffer.toString("base64"); + return { ...block, source: { type: "base64", data } }; + } catch { + return { type: "text", text: "[Image unavailable: referenced media file not found]" }; + } + } + + return block; +} + +/** + * Walk content blocks (including nested tool_result.content) and apply a transform. + */ +function transformContentBlocks( + content: any[], + transformBlock: (block: any) => any, +): { content: any[]; changed: boolean } { + let changed = false; + const result: any[] = []; + + for (const block of content) { + // Handle nested tool_result content + if (block && typeof block === "object" && block.type === "tool_result" && Array.isArray(block.content)) { + const inner = transformContentBlocks(block.content, transformBlock); + if (inner.changed) { + changed = true; + result.push({ ...block, content: inner.content }); + } else { + result.push(block); + } + continue; + } + + const transformed = transformBlock(block); + if (transformed !== block) changed = true; + result.push(transformed); + } + + return { content: result, changed }; +} + +/** + * Extract base64 image data from a session entry, save as binary files, + * and replace with file references. + */ +function externalizeImages( + entry: SessionEntry, + sessionId: string, + options?: SessionStorageOptions, +): SessionEntry { + if (entry.type !== "message") return entry; + + const message = entry.message as any; + const content = message.content; + if (!Array.isArray(content)) return entry; + + const result = transformContentBlocks(content, (block) => + externalizeBlock(block, sessionId, options), + ); + + if (!result.changed) return entry; + + return { + ...entry, + message: { ...message, content: result.content }, + } as SessionEntry; +} + +/** + * Resolve external file references in a session entry back to inline base64 data. + */ +function internalizeImages( + entry: SessionEntry, + sessionId: string, + options?: SessionStorageOptions, +): SessionEntry { + if (entry.type !== "message") return entry; + + const message = entry.message as any; + const content = message.content; + if (!Array.isArray(content)) return entry; + + const result = transformContentBlocks(content, (block) => + internalizeBlock(block, sessionId, options), + ); + + if (!result.changed) return entry; + + return { + ...entry, + message: { ...message, content: result.content }, + } as SessionEntry; +} + +// ─── Public API ───────────────────────────────────────────────────────────── + export function readEntries(sessionId: string, options?: SessionStorageOptions): SessionEntry[] { const filePath = resolveSessionPath(sessionId, options); if (!existsSync(filePath)) return []; @@ -45,7 +247,8 @@ export function readEntries(sessionId: string, options?: SessionStorageOptions): const entries: SessionEntry[] = []; for (const line of lines) { try { - entries.push(JSON.parse(line) as SessionEntry); + const raw = JSON.parse(line) as SessionEntry; + entries.push(internalizeImages(raw, sessionId, options)); } catch { // Skip malformed lines } @@ -62,7 +265,8 @@ export async function appendEntry( const filePath = resolveSessionPath(sessionId, options); const lock = await acquireSessionWriteLock({ sessionFile: filePath }); try { - await appendFile(filePath, `${JSON.stringify(entry)}\n`, "utf8"); + const externalized = externalizeImages(entry, sessionId, options); + await appendFile(filePath, `${JSON.stringify(externalized)}\n`, "utf8"); } finally { await lock.release(); } @@ -77,7 +281,9 @@ export async function writeEntries( const filePath = resolveSessionPath(sessionId, options); const lock = await acquireSessionWriteLock({ sessionFile: filePath }); try { - const content = entries.map((entry) => JSON.stringify(entry)).join("\n"); + const content = entries + .map((entry) => JSON.stringify(externalizeImages(entry, sessionId, options))) + .join("\n"); await writeFile(filePath, content ? `${content}\n` : "", "utf8"); } finally { await lock.release();