Merge remote-tracking branch 'origin/main' into feat/desktop-api-client
This commit is contained in:
yushen
commit
5ee08e9368
21 changed files with 975 additions and 125 deletions
|
|
@ -2,31 +2,73 @@
|
|||
# Multica Desktop Environment Configuration
|
||||
# =============================================================================
|
||||
#
|
||||
# Local Development:
|
||||
# Just run `pnpm dev` - no .env file needed (uses defaults)
|
||||
# This file documents all available environment variables for Desktop builds.
|
||||
# Copy this file and rename it based on your target environment.
|
||||
#
|
||||
# =============================================================================
|
||||
# Build Commands
|
||||
# =============================================================================
|
||||
#
|
||||
# Development (no .env file needed):
|
||||
# pnpm dev # Uses hardcoded dev defaults
|
||||
#
|
||||
# Staging Build:
|
||||
# `pnpm build:staging` uses .env.staging
|
||||
# pnpm build:staging # Uses .env.staging
|
||||
#
|
||||
# Production Build:
|
||||
# `pnpm build` uses .env.production
|
||||
# pnpm build:production # Uses .env.production
|
||||
#
|
||||
# Default Build (uses .env.production):
|
||||
# pnpm build:desktop # Same as build:production
|
||||
#
|
||||
# =============================================================================
|
||||
# Build Output
|
||||
# =============================================================================
|
||||
#
|
||||
# After build completes, installer packages are located at:
|
||||
#
|
||||
# apps/desktop/release/{version}/
|
||||
# ├── Multica-{version}-arm64.dmg # macOS Apple Silicon
|
||||
# ├── Multica-{version}-x64.dmg # macOS Intel
|
||||
# ├── Multica-Windows-{version}-Setup.exe # Windows
|
||||
# ├── Multica-Linux-{version}.AppImage # Linux AppImage
|
||||
# └── Multica-Linux-{version}.deb # Linux deb
|
||||
#
|
||||
# Compiled code (before packaging) is in:
|
||||
# apps/desktop/out/
|
||||
#
|
||||
# =============================================================================
|
||||
# Configuration Guide
|
||||
# =============================================================================
|
||||
#
|
||||
# For production builds, create .env.production with the following variables:
|
||||
#
|
||||
# Required:
|
||||
# MAIN_VITE_GATEWAY_URL - WebSocket Gateway URL for remote connections
|
||||
# MAIN_VITE_WEB_URL - Web App URL for OAuth login flow
|
||||
#
|
||||
# Example .env.production:
|
||||
# MAIN_VITE_GATEWAY_URL=https://gateway.multica.ai
|
||||
# MAIN_VITE_WEB_URL=https://www.multica.ai
|
||||
#
|
||||
# =============================================================================
|
||||
# Variable Naming Convention
|
||||
# =============================================================================
|
||||
#
|
||||
# Variable naming convention:
|
||||
# MAIN_VITE_* - Main process only (Node.js, full system access)
|
||||
# RENDERER_VITE_* - Renderer process only (browser context)
|
||||
# VITE_* - All processes
|
||||
#
|
||||
# =============================================================================
|
||||
# URL Configuration
|
||||
# =============================================================================
|
||||
#
|
||||
# MAIN_VITE_GATEWAY_URL - WebSocket Gateway
|
||||
# Hub connects to Gateway for remote device access (QR code pairing)
|
||||
#
|
||||
# MAIN_VITE_WEB_URL - Web App URL
|
||||
# Desktop opens this URL for user login (OAuth flow)
|
||||
#
|
||||
# Environment Variables
|
||||
# =============================================================================
|
||||
|
||||
# MAIN_VITE_GATEWAY_URL
|
||||
# WebSocket Gateway URL - Hub connects here for remote device access (QR code pairing)
|
||||
# Production example: https://gateway.multica.ai
|
||||
MAIN_VITE_GATEWAY_URL=http://localhost:3000
|
||||
|
||||
# MAIN_VITE_WEB_URL
|
||||
# Web App URL - Desktop opens this URL for user login (OAuth flow)
|
||||
# Production example: https://www.multica.ai
|
||||
MAIN_VITE_WEB_URL=http://localhost:3000
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
*/
|
||||
|
||||
import http from "node:http";
|
||||
import crypto from "node:crypto";
|
||||
import { ipcMain, shell, BrowserWindow } from "electron";
|
||||
import {
|
||||
existsSync,
|
||||
|
|
@ -18,7 +17,7 @@ import {
|
|||
mkdirSync,
|
||||
} from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { DATA_DIR } from "@multica/utils";
|
||||
import { DATA_DIR, generateEncryptedId, isValidEncryptedId } from "@multica/utils";
|
||||
import type { AuthUser } from "@multica/types";
|
||||
|
||||
// ============================================================================
|
||||
|
|
@ -46,37 +45,6 @@ interface AuthFileData {
|
|||
|
||||
const AUTH_FILE_PATH = join(DATA_DIR, "auth.json");
|
||||
|
||||
/**
|
||||
* SHA-256 hash function.
|
||||
*/
|
||||
function sha256(text: string): string {
|
||||
return crypto.createHash("sha256").update(text, "utf8").digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate encrypted Device ID.
|
||||
* Algorithm (consistent with devv-sdk and Web):
|
||||
* 1. Generate UUID
|
||||
* 2. SHA-256 hash of UUID, take first 32 chars
|
||||
* 3. SHA-256 hash of step 2 result, take first 8 chars
|
||||
* 4. Return: step3[0:8] + step2[0:32] = 40 chars
|
||||
*
|
||||
* This encrypted format is stored directly (not the raw UUID).
|
||||
*/
|
||||
function generateEncryptedDeviceId(): string {
|
||||
const uuid = crypto.randomUUID();
|
||||
const firstHash = sha256(uuid).slice(0, 32);
|
||||
const finalId = sha256(firstHash).slice(0, 8) + firstHash;
|
||||
return finalId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate device ID format (40 hex characters).
|
||||
*/
|
||||
function isValidDeviceId(deviceId: string): boolean {
|
||||
return typeof deviceId === "string" && /^[a-f0-9]{40}$/i.test(deviceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read raw auth file data, handling all edge cases.
|
||||
* Returns null if file doesn't exist or is invalid.
|
||||
|
|
@ -123,32 +91,26 @@ function writeAuthFile(data: Partial<AuthFileData>): boolean {
|
|||
/**
|
||||
* Get or create a persistent Device ID.
|
||||
* Device ID persists across logins/logouts - it represents the device, not the user.
|
||||
* The stored value is already encrypted (40 hex chars), not the raw UUID.
|
||||
* The stored value is encrypted (40 hex chars).
|
||||
*/
|
||||
export function getOrCreateDeviceId(): string {
|
||||
const existing = readAuthFile();
|
||||
|
||||
// If we have a valid encrypted deviceId (40 hex chars), return it
|
||||
if (existing?.deviceId && isValidDeviceId(existing.deviceId)) {
|
||||
// If we have a valid encrypted deviceId, return it
|
||||
if (existing?.deviceId && isValidEncryptedId(existing.deviceId)) {
|
||||
return existing.deviceId;
|
||||
}
|
||||
|
||||
// Generate new encrypted deviceId
|
||||
const newDeviceId = generateEncryptedDeviceId();
|
||||
const newDeviceId = generateEncryptedId();
|
||||
console.log("[Auth] Generated new Device ID:", newDeviceId.slice(0, 8) + "...");
|
||||
|
||||
// If there was an old-format deviceId (UUID), we'll replace it
|
||||
if (existing?.deviceId && !isValidDeviceId(existing.deviceId)) {
|
||||
console.log("[Auth] Migrating old-format Device ID to encrypted format");
|
||||
}
|
||||
|
||||
// Preserve any existing auth data while adding/updating deviceId
|
||||
// Preserve any existing auth data while adding deviceId
|
||||
const dataToSave: Partial<AuthFileData> = existing
|
||||
? { ...existing, deviceId: newDeviceId }
|
||||
: { deviceId: newDeviceId };
|
||||
|
||||
if (!writeAuthFile(dataToSave)) {
|
||||
// Write failed, but we can still return the generated ID for this session
|
||||
console.error("[Auth] Failed to persist new Device ID");
|
||||
}
|
||||
|
||||
|
|
@ -189,7 +151,7 @@ function saveAuthData(sid: string, user: AuthUser, passedDeviceId?: string): boo
|
|||
try {
|
||||
// Use passed deviceId from Web if valid, otherwise use local one
|
||||
let deviceId: string;
|
||||
if (passedDeviceId && isValidDeviceId(passedDeviceId)) {
|
||||
if (passedDeviceId && isValidEncryptedId(passedDeviceId)) {
|
||||
deviceId = passedDeviceId;
|
||||
console.log("[Auth] Using Device ID from Web browser:", deviceId.slice(0, 8) + "...");
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import { TooltipProvider } from '@multica/ui/components/ui/tooltip'
|
|||
import { Toaster } from './components/toaster'
|
||||
import Layout from './pages/layout'
|
||||
import HomePage from './pages/home'
|
||||
import ChatPage from './pages/chat'
|
||||
import ProfilePage from './pages/agent/profile'
|
||||
import SkillsPage from './pages/agent/skills'
|
||||
import ToolsPage from './pages/agent/tools'
|
||||
|
|
@ -73,7 +72,7 @@ const router = createHashRouter([
|
|||
</OnboardingGuard>
|
||||
),
|
||||
},
|
||||
{ path: 'chat', element: <ChatPage /> },
|
||||
{ path: 'chat', element: null },
|
||||
{ path: 'agent/profile', element: <ProfilePage /> },
|
||||
{ path: 'agent/skills', element: <SkillsPage /> },
|
||||
{ path: 'agent/tools', element: <ToolsPage /> },
|
||||
|
|
|
|||
|
|
@ -73,12 +73,13 @@ export function LocalChat({ initialPrompt }: LocalChatProps) {
|
|||
const currentMeta = current ? providers.find((p) => p.id === current.provider) : null
|
||||
|
||||
// Auto-send initial prompt after a short delay
|
||||
const hasSentInitialPrompt = useRef(false)
|
||||
const lastPromptRef = useRef<string | undefined>(undefined)
|
||||
useEffect(() => {
|
||||
if (!agentId || !initialPrompt || hasSentInitialPrompt.current) return
|
||||
if (!agentId || !initialPrompt) return
|
||||
if (initialPrompt === lastPromptRef.current) return
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
hasSentInitialPrompt.current = true
|
||||
lastPromptRef.current = initialPrompt
|
||||
sendMessage(initialPrompt)
|
||||
// Remove prompt from URL to prevent re-sending on back navigation
|
||||
navigate('/chat', { replace: true })
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { useState, useEffect } from 'react'
|
||||
import { Outlet, NavLink, useLocation, useNavigate } from 'react-router-dom'
|
||||
import { Button } from '@multica/ui/components/ui/button'
|
||||
import { MulticaIcon } from '@multica/ui/components/multica-icon'
|
||||
|
|
@ -43,6 +44,7 @@ import {
|
|||
} from '@multica/ui/components/ui/sidebar'
|
||||
import { cn } from '@multica/ui/lib/utils'
|
||||
import { ModeToggle } from '../components/mode-toggle'
|
||||
import { LocalChat } from '../components/local-chat'
|
||||
import { DeviceConfirmDialog } from '../components/device-confirm-dialog'
|
||||
import { UpdateNotification } from '../components/update-notification'
|
||||
import { useAuthStore } from '../stores/auth'
|
||||
|
|
@ -151,8 +153,20 @@ export default function Layout() {
|
|||
const location = useLocation()
|
||||
const navigate = useNavigate()
|
||||
const isAgentActive = location.pathname.startsWith('/agent')
|
||||
const isOnChat = location.pathname === '/chat'
|
||||
const { user, clearAuth } = useAuthStore()
|
||||
|
||||
// Lazy mount: only mount Chat on first visit, then keep it mounted forever
|
||||
const [chatMounted, setChatMounted] = useState(false)
|
||||
useEffect(() => {
|
||||
if (isOnChat && !chatMounted) setChatMounted(true)
|
||||
}, [isOnChat, chatMounted])
|
||||
|
||||
// Extract initialPrompt from URL search params when navigating to /chat?prompt=...
|
||||
const initialPrompt = isOnChat
|
||||
? new URLSearchParams(location.search).get('prompt') ?? undefined
|
||||
: undefined
|
||||
|
||||
const handleLogout = async () => {
|
||||
await clearAuth()
|
||||
navigate('/login')
|
||||
|
|
@ -285,7 +299,14 @@ export default function Layout() {
|
|||
<SidebarInset className="overflow-hidden">
|
||||
<MainHeader />
|
||||
<main className="flex-1 overflow-hidden min-h-1">
|
||||
<Outlet />
|
||||
<div className={cn('h-full', isOnChat && 'hidden')}>
|
||||
<Outlet />
|
||||
</div>
|
||||
{chatMounted && (
|
||||
<div className={cn('h-full flex flex-col overflow-hidden', !isOnChat && 'hidden')}>
|
||||
<LocalChat initialPrompt={initialPrompt} />
|
||||
</div>
|
||||
)}
|
||||
</main>
|
||||
</SidebarInset>
|
||||
|
||||
|
|
|
|||
|
|
@ -23,14 +23,20 @@ export default function LoginPage() {
|
|||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-screen items-center justify-center bg-background">
|
||||
<div
|
||||
className="flex h-screen items-center justify-center bg-background"
|
||||
style={{ WebkitAppRegion: 'drag' } as React.CSSProperties}
|
||||
>
|
||||
<Loading className="size-6" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-screen flex-col items-center justify-center bg-background p-8 animate-in fade-in duration-300">
|
||||
<div
|
||||
className="flex h-screen flex-col items-center justify-center bg-background p-8 animate-in fade-in duration-300"
|
||||
style={{ WebkitAppRegion: 'drag' } as React.CSSProperties}
|
||||
>
|
||||
<div className="w-full max-w-sm flex flex-col items-center text-center space-y-6">
|
||||
{/* Brand */}
|
||||
<div className="flex items-center gap-2">
|
||||
|
|
@ -44,7 +50,12 @@ export default function LoginPage() {
|
|||
</p>
|
||||
|
||||
{/* Sign In */}
|
||||
<Button onClick={startLogin} size="lg" className="px-8">
|
||||
<Button
|
||||
onClick={startLogin}
|
||||
size="lg"
|
||||
className="px-8"
|
||||
style={{ WebkitAppRegion: 'no-drag' } as React.CSSProperties}
|
||||
>
|
||||
Sign In to Continue
|
||||
</Button>
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
*/
|
||||
|
||||
import { Inject, Injectable, Logger } from "@nestjs/common";
|
||||
import { v7 as uuidv7 } from "uuid";
|
||||
import { generateEncryptedId } from "@multica/utils";
|
||||
import type { RowDataPacket } from "mysql2/promise";
|
||||
import { DatabaseService } from "../database/database.service.js";
|
||||
import type { TelegramUser, TelegramUserCreate } from "./types.js";
|
||||
|
|
@ -88,7 +88,7 @@ export class TelegramUserStore {
|
|||
}
|
||||
|
||||
// Create new user with provided or generated device ID
|
||||
const deviceId = data.deviceId ?? `tg-${uuidv7()}`;
|
||||
const deviceId = data.deviceId ?? `tg-${generateEncryptedId()}`;
|
||||
|
||||
await this.db.execute(
|
||||
`INSERT INTO telegram_users (
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import type { OnModuleInit } from "@nestjs/common";
|
|||
import { Bot, InputFile, webhookCallback } from "grammy";
|
||||
import type { Context } from "grammy";
|
||||
import { v7 as uuidv7 } from "uuid";
|
||||
import { generateEncryptedId } from "@multica/utils";
|
||||
import { parseConnectionCode } from "@multica/store/connection";
|
||||
import type { ConnectionInfo } from "@multica/store/connection";
|
||||
import {
|
||||
|
|
@ -233,7 +234,7 @@ export class TelegramService implements OnModuleInit {
|
|||
}
|
||||
|
||||
// 4. Generate device ID and register virtual device
|
||||
const deviceId = `tg-${uuidv7()}`;
|
||||
const deviceId = `tg-${generateEncryptedId()}`;
|
||||
this.registerVirtualDeviceForUser(deviceId, telegramUserId);
|
||||
|
||||
// 5. Send verify RPC
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { MulticaIcon } from '@multica/ui/components/multica-icon'
|
|||
import { LoginAuthType, UserInfo } from '@/lib/interface'
|
||||
import { saveSession, isAuthenticated } from '@/lib/auth'
|
||||
import { userLogin } from '@/service/user'
|
||||
import { getOrCreateDeviceId, generateDeviceIdHeader } from '@/lib/device'
|
||||
import { getOrCreateDeviceId } from '@/lib/device'
|
||||
|
||||
type LoginStep = 'email' | 'code'
|
||||
|
||||
|
|
@ -115,9 +115,8 @@ export function LoginForm() {
|
|||
const port = nextUrl.searchParams.get('port')
|
||||
const platform = nextUrl.searchParams.get('platform') || 'web'
|
||||
|
||||
// Get Device ID and encrypt for Desktop
|
||||
const rawDeviceId = getOrCreateDeviceId()
|
||||
const deviceId = await generateDeviceIdHeader(rawDeviceId)
|
||||
// Get Device ID (already encrypted 40-char format)
|
||||
const deviceId = await getOrCreateDeviceId()
|
||||
|
||||
const params = new URLSearchParams({
|
||||
sid,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/**
|
||||
* Device ID management for Multica Web
|
||||
* Consistent with copilot-search: stores raw UUID, encrypts when transmitting
|
||||
* Stores encrypted format directly (40 hex chars)
|
||||
*/
|
||||
|
||||
const DEVICE_ID_KEY = 'MULTICA_DEVICE_ID'
|
||||
|
|
@ -13,30 +13,43 @@ async function sha256(text: string): Promise<string> {
|
|||
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
|
||||
// Generate encrypted device ID (40 hex chars)
|
||||
async function generateEncryptedDeviceId(): Promise<string> {
|
||||
const uuid = crypto.randomUUID()
|
||||
const firstHash = (await sha256(uuid)).slice(0, 32)
|
||||
return (await sha256(firstHash)).slice(0, 8) + firstHash
|
||||
}
|
||||
|
||||
// Validate encrypted ID format (40 hex characters)
|
||||
function isValidEncryptedId(id: string): boolean {
|
||||
return typeof id === 'string' && /^[a-f0-9]{40}$/i.test(id)
|
||||
}
|
||||
|
||||
// Cached promise for async generation
|
||||
let deviceIdPromise: Promise<string> | null = null
|
||||
|
||||
/**
|
||||
* Get or create Device ID (raw UUID format)
|
||||
* Stored in localStorage, encrypted only when transmitting
|
||||
* Get or create Device ID (encrypted 40-char format)
|
||||
* Stored in localStorage, ready to use directly
|
||||
*/
|
||||
export function getOrCreateDeviceId(): string {
|
||||
export async function getOrCreateDeviceId(): Promise<string> {
|
||||
if (typeof window === 'undefined') return ''
|
||||
|
||||
let deviceId = localStorage.getItem(DEVICE_ID_KEY)
|
||||
const existing = localStorage.getItem(DEVICE_ID_KEY)
|
||||
|
||||
if (!deviceId) {
|
||||
deviceId = crypto.randomUUID()
|
||||
localStorage.setItem(DEVICE_ID_KEY, deviceId)
|
||||
// If already encrypted format, return as-is
|
||||
if (existing && isValidEncryptedId(existing)) {
|
||||
return existing
|
||||
}
|
||||
|
||||
return deviceId
|
||||
// Generate new encrypted ID
|
||||
if (!deviceIdPromise) {
|
||||
deviceIdPromise = generateEncryptedDeviceId().then((id) => {
|
||||
localStorage.setItem(DEVICE_ID_KEY, id)
|
||||
return id
|
||||
})
|
||||
}
|
||||
|
||||
return deviceIdPromise
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate encrypted Device-Id header value
|
||||
* Algorithm (consistent with copilot-search):
|
||||
* 1. sha256(uuid).slice(0, 32) = hashedDeviceId
|
||||
* 2. sha256(hashedDeviceId).slice(0, 8) + hashedDeviceId = 40 chars
|
||||
*/
|
||||
export async function generateDeviceIdHeader(deviceId: string): Promise<string> {
|
||||
const hashedDeviceId = (await sha256(deviceId)).slice(0, 32)
|
||||
return (await sha256(hashedDeviceId)).slice(0, 8) + hashedDeviceId
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
import { API_HOST } from '@/lib/constant';
|
||||
import { getOrCreateDeviceId, generateDeviceIdHeader } from '@/lib/device';
|
||||
import { getOrCreateDeviceId } from '@/lib/device';
|
||||
import { getSid } from '@/lib/auth';
|
||||
|
||||
// Fetch request wrapper
|
||||
export async function request<T = unknown>(url: string, options: RequestInit = {}): Promise<T> {
|
||||
// Get or generate Device ID, encrypt for header
|
||||
let deviceIdHeader = '';
|
||||
// Get or generate Device ID (already encrypted 40-char format)
|
||||
let deviceId = '';
|
||||
let sid: string | null = null;
|
||||
if (typeof window !== 'undefined') {
|
||||
const deviceId = getOrCreateDeviceId();
|
||||
deviceIdHeader = await generateDeviceIdHeader(deviceId);
|
||||
deviceId = await getOrCreateDeviceId();
|
||||
sid = getSid();
|
||||
}
|
||||
|
||||
|
|
@ -18,7 +17,7 @@ export async function request<T = unknown>(url: string, options: RequestInit = {
|
|||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'os-type': '3',
|
||||
...(deviceIdHeader && { 'Device-Id': deviceIdHeader }),
|
||||
...(deviceId && { 'Device-Id': deviceId }),
|
||||
...(sid && { 'Authorization': `Bearer ${sid}` }),
|
||||
...options.headers,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
resolveBaseDir,
|
||||
resolveSessionDir,
|
||||
resolveSessionPath,
|
||||
resolveMediaDir,
|
||||
ensureSessionDir,
|
||||
readEntries,
|
||||
appendEntry,
|
||||
|
|
@ -274,4 +275,245 @@ describe("session/storage", () => {
|
|||
expect(readFileSync(filePath, "utf8")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("image externalization", () => {
|
||||
// Generate a large base64 string (>43KB to exceed MIN_EXTERNALIZE_B64_LENGTH)
|
||||
const largeBinarySize = 50_000; // 50KB binary
|
||||
const largeBuffer = Buffer.alloc(largeBinarySize, 0x42); // fill with 'B'
|
||||
const largeBase64 = largeBuffer.toString("base64");
|
||||
|
||||
// Small base64 that should stay inline
|
||||
const smallBase64 = Buffer.alloc(100, 0x41).toString("base64");
|
||||
|
||||
function makeImageEntry(imageData: string, sessionId = "img-session"): SessionEntry {
|
||||
return {
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Read image file [image/png]" },
|
||||
{ type: "image", data: imageData },
|
||||
],
|
||||
} as any,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function makeFormatBEntry(imageData: string): SessionEntry {
|
||||
return {
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image", source: { type: "base64", data: imageData } },
|
||||
],
|
||||
} as any,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function makeToolResultEntry(imageData: string): SessionEntry {
|
||||
return {
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "test-id",
|
||||
content: [
|
||||
{ type: "text", text: "Read image file [image/png]" },
|
||||
{ type: "image", data: imageData },
|
||||
],
|
||||
},
|
||||
],
|
||||
} as any,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
it("should externalize Format A image and create media file", async () => {
|
||||
const sessionId = "ext-format-a";
|
||||
const entry = makeImageEntry(largeBase64);
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
// Read raw JSONL — should have $ref, not data
|
||||
const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8");
|
||||
const rawEntry = JSON.parse(rawContent.trim());
|
||||
expect(rawEntry.message.content[1].$ref).toMatch(/^media\/[a-f0-9]+\.bin$/);
|
||||
expect(rawEntry.message.content[1].data).toBeUndefined();
|
||||
|
||||
// Media file should exist
|
||||
const mediaDir = resolveMediaDir(sessionId, { baseDir: testBaseDir });
|
||||
const files = existsSync(mediaDir)
|
||||
? require("node:fs").readdirSync(mediaDir) as string[]
|
||||
: [];
|
||||
expect(files).toHaveLength(1);
|
||||
expect(files[0]).toMatch(/^[a-f0-9]+\.bin$/);
|
||||
|
||||
// Binary content should match original
|
||||
const binPath = join(mediaDir, files[0]!);
|
||||
const savedBuffer = readFileSync(binPath);
|
||||
expect(savedBuffer).toEqual(largeBuffer);
|
||||
});
|
||||
|
||||
it("should externalize Format B image (Anthropic source style)", async () => {
|
||||
const sessionId = "ext-format-b";
|
||||
const entry = makeFormatBEntry(largeBase64);
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8");
|
||||
const rawEntry = JSON.parse(rawContent.trim());
|
||||
expect(rawEntry.message.content[0].source.type).toBe("$ref");
|
||||
expect(rawEntry.message.content[0].source.path).toMatch(/^media\/[a-f0-9]+\.bin$/);
|
||||
});
|
||||
|
||||
it("should restore externalized images on read (round-trip)", async () => {
|
||||
const sessionId = "ext-roundtrip";
|
||||
const entry = makeImageEntry(largeBase64);
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
const entries = readEntries(sessionId, { baseDir: testBaseDir });
|
||||
expect(entries).toHaveLength(1);
|
||||
const content = (entries[0] as any).message.content;
|
||||
expect(content[1].type).toBe("image");
|
||||
expect(content[1].data).toBe(largeBase64);
|
||||
expect(content[1].$ref).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should restore Format B images on read", async () => {
|
||||
const sessionId = "ext-roundtrip-b";
|
||||
const entry = makeFormatBEntry(largeBase64);
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
const entries = readEntries(sessionId, { baseDir: testBaseDir });
|
||||
expect(entries).toHaveLength(1);
|
||||
const block = (entries[0] as any).message.content[0];
|
||||
expect(block.source.type).toBe("base64");
|
||||
expect(block.source.data).toBe(largeBase64);
|
||||
});
|
||||
|
||||
it("should handle old sessions with inline base64 (backward compat)", () => {
|
||||
const sessionId = "old-inline";
|
||||
const dir = join(testBaseDir, sessionId);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Write raw JSONL with inline base64 (old format, no $ref)
|
||||
const entry = makeImageEntry(largeBase64);
|
||||
writeFileSync(join(dir, "session.jsonl"), `${JSON.stringify(entry)}\n`);
|
||||
|
||||
const entries = readEntries(sessionId, { baseDir: testBaseDir });
|
||||
expect(entries).toHaveLength(1);
|
||||
const content = (entries[0] as any).message.content;
|
||||
expect(content[1].data).toBe(largeBase64);
|
||||
});
|
||||
|
||||
it("should return placeholder for missing media file", () => {
|
||||
const sessionId = "missing-media";
|
||||
const dir = join(testBaseDir, sessionId);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Write JSONL with $ref but no media file
|
||||
const rawEntry = {
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image", $ref: "media/deadbeef.bin" },
|
||||
],
|
||||
},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
writeFileSync(join(dir, "session.jsonl"), `${JSON.stringify(rawEntry)}\n`);
|
||||
|
||||
const entries = readEntries(sessionId, { baseDir: testBaseDir });
|
||||
expect(entries).toHaveLength(1);
|
||||
const block = (entries[0] as any).message.content[0];
|
||||
expect(block.type).toBe("text");
|
||||
expect(block.text).toContain("unavailable");
|
||||
});
|
||||
|
||||
it("should deduplicate same image data", async () => {
|
||||
const sessionId = "ext-dedup";
|
||||
const entry1 = makeImageEntry(largeBase64);
|
||||
const entry2 = makeImageEntry(largeBase64);
|
||||
|
||||
await appendEntry(sessionId, entry1, { baseDir: testBaseDir });
|
||||
await appendEntry(sessionId, entry2, { baseDir: testBaseDir });
|
||||
|
||||
const mediaDir = resolveMediaDir(sessionId, { baseDir: testBaseDir });
|
||||
const files = require("node:fs").readdirSync(mediaDir) as string[];
|
||||
expect(files).toHaveLength(1); // same hash = same file
|
||||
});
|
||||
|
||||
it("should keep small images inline", async () => {
|
||||
const sessionId = "ext-small";
|
||||
const entry = makeImageEntry(smallBase64);
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
// Read raw JSONL — small image should still have data, not $ref
|
||||
const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8");
|
||||
const rawEntry = JSON.parse(rawContent.trim());
|
||||
expect(rawEntry.message.content[1].data).toBe(smallBase64);
|
||||
expect(rawEntry.message.content[1].$ref).toBeUndefined();
|
||||
|
||||
// No media dir should be created
|
||||
const mediaDir = resolveMediaDir(sessionId, { baseDir: testBaseDir });
|
||||
expect(existsSync(mediaDir)).toBe(false);
|
||||
});
|
||||
|
||||
it("should not affect non-image entries", async () => {
|
||||
const sessionId = "ext-noimg";
|
||||
const entry: SessionEntry = {
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Just text response" } as any,
|
||||
timestamp: 1000,
|
||||
};
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8");
|
||||
expect(rawContent.trim()).toBe(JSON.stringify(entry));
|
||||
});
|
||||
|
||||
it("should handle images inside nested tool_result content", async () => {
|
||||
const sessionId = "ext-tool-result";
|
||||
const entry = makeToolResultEntry(largeBase64);
|
||||
|
||||
await appendEntry(sessionId, entry, { baseDir: testBaseDir });
|
||||
|
||||
// Raw JSONL should have $ref inside tool_result
|
||||
const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8");
|
||||
const rawEntry = JSON.parse(rawContent.trim());
|
||||
const toolResult = rawEntry.message.content[0];
|
||||
expect(toolResult.content[1].$ref).toMatch(/^media\/[a-f0-9]+\.bin$/);
|
||||
|
||||
// Round-trip should restore
|
||||
const entries = readEntries(sessionId, { baseDir: testBaseDir });
|
||||
const restored = (entries[0] as any).message.content[0].content[1];
|
||||
expect(restored.data).toBe(largeBase64);
|
||||
expect(restored.$ref).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should externalize via writeEntries (compaction path)", async () => {
|
||||
const sessionId = "ext-write-entries";
|
||||
const entry = makeImageEntry(largeBase64);
|
||||
|
||||
await writeEntries(sessionId, [entry], { baseDir: testBaseDir });
|
||||
|
||||
// Should be externalized
|
||||
const rawContent = readFileSync(join(testBaseDir, sessionId, "session.jsonl"), "utf8");
|
||||
const rawEntry = JSON.parse(rawContent.trim());
|
||||
expect(rawEntry.message.content[1].$ref).toMatch(/^media\/[a-f0-9]+\.bin$/);
|
||||
|
||||
// Round-trip
|
||||
const entries = readEntries(sessionId, { baseDir: testBaseDir });
|
||||
expect((entries[0] as any).message.content[1].data).toBe(largeBase64);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { join } from "path";
|
||||
import { existsSync, mkdirSync, readFileSync } from "fs";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
||||
import { appendFile, writeFile } from "fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
import { DATA_DIR } from "@multica/utils";
|
||||
import { acquireSessionWriteLock } from "./session-write-lock.js";
|
||||
|
|
@ -9,6 +10,9 @@ export type SessionStorageOptions = {
|
|||
baseDir?: string | undefined;
|
||||
};
|
||||
|
||||
/** Minimum base64 data length to externalize (32KB decoded ≈ 43KB base64) */
|
||||
const MIN_EXTERNALIZE_B64_LENGTH = 43_000;
|
||||
|
||||
export function resolveBaseDir(options?: SessionStorageOptions) {
|
||||
return options?.baseDir ?? join(DATA_DIR, "sessions");
|
||||
}
|
||||
|
|
@ -21,6 +25,10 @@ export function resolveSessionPath(sessionId: string, options?: SessionStorageOp
|
|||
return join(resolveSessionDir(sessionId, options), "session.jsonl");
|
||||
}
|
||||
|
||||
export function resolveMediaDir(sessionId: string, options?: SessionStorageOptions) {
|
||||
return join(resolveSessionDir(sessionId, options), "media");
|
||||
}
|
||||
|
||||
export function ensureSessionDir(sessionId: string, options?: SessionStorageOptions) {
|
||||
const dir = resolveSessionDir(sessionId, options);
|
||||
// mkdirSync with recursive is idempotent (no-op if dir exists),
|
||||
|
|
@ -37,6 +45,200 @@ export function ensureSessionDir(sessionId: string, options?: SessionStorageOpti
|
|||
}
|
||||
}
|
||||
|
||||
// ─── Image Externalization ──────────────────────────────────────────────────
|
||||
|
||||
function contentHash(base64Data: string): string {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
return createHash("sha256").update(buffer).digest("hex").slice(0, 32);
|
||||
}
|
||||
|
||||
function ensureMediaDir(sessionId: string, options?: SessionStorageOptions): void {
|
||||
const dir = resolveMediaDir(sessionId, options);
|
||||
try {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function saveImageBinary(
|
||||
sessionId: string,
|
||||
hash: string,
|
||||
base64Data: string,
|
||||
options?: SessionStorageOptions,
|
||||
): void {
|
||||
ensureMediaDir(sessionId, options);
|
||||
const filePath = join(resolveMediaDir(sessionId, options), `${hash}.bin`);
|
||||
if (existsSync(filePath)) return; // dedup
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
writeFileSync(filePath, buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a single image content block with an external file reference.
|
||||
* Returns the original block unchanged if it's not an externalizable image.
|
||||
*/
|
||||
function externalizeBlock(
|
||||
block: any,
|
||||
sessionId: string,
|
||||
options?: SessionStorageOptions,
|
||||
): any {
|
||||
if (!block || typeof block !== "object" || block.type !== "image") return block;
|
||||
|
||||
// Format A: { type: "image", data: "<base64>" }
|
||||
if (typeof block.data === "string" && block.data.length > MIN_EXTERNALIZE_B64_LENGTH) {
|
||||
const hash = contentHash(block.data);
|
||||
const relPath = `media/${hash}.bin`;
|
||||
saveImageBinary(sessionId, hash, block.data, options);
|
||||
const { data: _removed, ...rest } = block;
|
||||
return { ...rest, $ref: relPath };
|
||||
}
|
||||
|
||||
// Format B: { type: "image", source: { type: "base64", data: "<base64>" } }
|
||||
if (
|
||||
block.source &&
|
||||
typeof block.source === "object" &&
|
||||
block.source.type === "base64" &&
|
||||
typeof block.source.data === "string" &&
|
||||
block.source.data.length > MIN_EXTERNALIZE_B64_LENGTH
|
||||
) {
|
||||
const hash = contentHash(block.source.data);
|
||||
const relPath = `media/${hash}.bin`;
|
||||
saveImageBinary(sessionId, hash, block.source.data, options);
|
||||
return { ...block, source: { type: "$ref", path: relPath } };
|
||||
}
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore an externalized image reference back to inline base64 data.
|
||||
*/
|
||||
function internalizeBlock(
|
||||
block: any,
|
||||
sessionId: string,
|
||||
options?: SessionStorageOptions,
|
||||
): any {
|
||||
if (!block || typeof block !== "object" || block.type !== "image") return block;
|
||||
|
||||
// Format A ref: { type: "image", $ref: "media/<hash>.bin" }
|
||||
if (typeof block.$ref === "string") {
|
||||
const filePath = join(resolveSessionDir(sessionId, options), block.$ref);
|
||||
try {
|
||||
const buffer = readFileSync(filePath);
|
||||
const data = buffer.toString("base64");
|
||||
const { $ref: _removed, ...rest } = block;
|
||||
return { ...rest, data };
|
||||
} catch {
|
||||
return { type: "text", text: "[Image unavailable: referenced media file not found]" };
|
||||
}
|
||||
}
|
||||
|
||||
// Format B ref: { type: "image", source: { type: "$ref", path: "media/<hash>.bin" } }
|
||||
if (block.source && typeof block.source === "object" && block.source.type === "$ref") {
|
||||
const filePath = join(resolveSessionDir(sessionId, options), block.source.path);
|
||||
try {
|
||||
const buffer = readFileSync(filePath);
|
||||
const data = buffer.toString("base64");
|
||||
return { ...block, source: { type: "base64", data } };
|
||||
} catch {
|
||||
return { type: "text", text: "[Image unavailable: referenced media file not found]" };
|
||||
}
|
||||
}
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk content blocks (including nested tool_result.content) and apply a transform.
|
||||
*/
|
||||
function transformContentBlocks(
|
||||
content: any[],
|
||||
transformBlock: (block: any) => any,
|
||||
): { content: any[]; changed: boolean } {
|
||||
let changed = false;
|
||||
const result: any[] = [];
|
||||
|
||||
for (const block of content) {
|
||||
// Handle nested tool_result content
|
||||
if (block && typeof block === "object" && block.type === "tool_result" && Array.isArray(block.content)) {
|
||||
const inner = transformContentBlocks(block.content, transformBlock);
|
||||
if (inner.changed) {
|
||||
changed = true;
|
||||
result.push({ ...block, content: inner.content });
|
||||
} else {
|
||||
result.push(block);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const transformed = transformBlock(block);
|
||||
if (transformed !== block) changed = true;
|
||||
result.push(transformed);
|
||||
}
|
||||
|
||||
return { content: result, changed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract base64 image data from a session entry, save as binary files,
|
||||
* and replace with file references.
|
||||
*/
|
||||
function externalizeImages(
|
||||
entry: SessionEntry,
|
||||
sessionId: string,
|
||||
options?: SessionStorageOptions,
|
||||
): SessionEntry {
|
||||
if (entry.type !== "message") return entry;
|
||||
|
||||
const message = entry.message as any;
|
||||
const content = message.content;
|
||||
if (!Array.isArray(content)) return entry;
|
||||
|
||||
const result = transformContentBlocks(content, (block) =>
|
||||
externalizeBlock(block, sessionId, options),
|
||||
);
|
||||
|
||||
if (!result.changed) return entry;
|
||||
|
||||
return {
|
||||
...entry,
|
||||
message: { ...message, content: result.content },
|
||||
} as SessionEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve external file references in a session entry back to inline base64 data.
|
||||
*/
|
||||
function internalizeImages(
|
||||
entry: SessionEntry,
|
||||
sessionId: string,
|
||||
options?: SessionStorageOptions,
|
||||
): SessionEntry {
|
||||
if (entry.type !== "message") return entry;
|
||||
|
||||
const message = entry.message as any;
|
||||
const content = message.content;
|
||||
if (!Array.isArray(content)) return entry;
|
||||
|
||||
const result = transformContentBlocks(content, (block) =>
|
||||
internalizeBlock(block, sessionId, options),
|
||||
);
|
||||
|
||||
if (!result.changed) return entry;
|
||||
|
||||
return {
|
||||
...entry,
|
||||
message: { ...message, content: result.content },
|
||||
} as SessionEntry;
|
||||
}
|
||||
|
||||
// ─── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
export function readEntries(sessionId: string, options?: SessionStorageOptions): SessionEntry[] {
|
||||
const filePath = resolveSessionPath(sessionId, options);
|
||||
if (!existsSync(filePath)) return [];
|
||||
|
|
@ -45,7 +247,8 @@ export function readEntries(sessionId: string, options?: SessionStorageOptions):
|
|||
const entries: SessionEntry[] = [];
|
||||
for (const line of lines) {
|
||||
try {
|
||||
entries.push(JSON.parse(line) as SessionEntry);
|
||||
const raw = JSON.parse(line) as SessionEntry;
|
||||
entries.push(internalizeImages(raw, sessionId, options));
|
||||
} catch {
|
||||
// Skip malformed lines
|
||||
}
|
||||
|
|
@ -62,7 +265,8 @@ export async function appendEntry(
|
|||
const filePath = resolveSessionPath(sessionId, options);
|
||||
const lock = await acquireSessionWriteLock({ sessionFile: filePath });
|
||||
try {
|
||||
await appendFile(filePath, `${JSON.stringify(entry)}\n`, "utf8");
|
||||
const externalized = externalizeImages(entry, sessionId, options);
|
||||
await appendFile(filePath, `${JSON.stringify(externalized)}\n`, "utf8");
|
||||
} finally {
|
||||
await lock.release();
|
||||
}
|
||||
|
|
@ -77,7 +281,9 @@ export async function writeEntries(
|
|||
const filePath = resolveSessionPath(sessionId, options);
|
||||
const lock = await acquireSessionWriteLock({ sessionFile: filePath });
|
||||
try {
|
||||
const content = entries.map((entry) => JSON.stringify(entry)).join("\n");
|
||||
const content = entries
|
||||
.map((entry) => JSON.stringify(externalizeImages(entry, sessionId, options)))
|
||||
.join("\n");
|
||||
await writeFile(filePath, content ? `${content}\n` : "", "utf8");
|
||||
} finally {
|
||||
await lock.release();
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { createDataTool } from "./tools/data/index.js";
|
|||
import { createSendFileTool } from "./tools/send-file.js";
|
||||
import type { SendFileCallback } from "./tools/send-file.js";
|
||||
import { filterTools } from "./tools/policy.js";
|
||||
import { wrapReadToolWithImageResize } from "./tools/image-resize.js";
|
||||
import { isMulticaError, isRetryableError } from "@multica/utils";
|
||||
import type { ExecApprovalCallback } from "./tools/exec-approval-types.js";
|
||||
|
||||
|
|
@ -106,9 +107,9 @@ export function createAllTools(options: CreateToolsOptions | string): AgentTool<
|
|||
const opts: CreateToolsOptions = typeof options === "string" ? { cwd: options } : options;
|
||||
const { cwd, profileDir, isSubagent, sessionId } = opts;
|
||||
|
||||
const baseTools = createCodingTools(cwd).filter(
|
||||
(tool) => tool.name !== "bash",
|
||||
) as AgentTool<any>[];
|
||||
const baseTools = createCodingTools(cwd)
|
||||
.filter((tool) => tool.name !== "bash")
|
||||
.map((tool) => tool.name === "read" ? wrapReadToolWithImageResize(tool) : tool) as AgentTool<any>[];
|
||||
|
||||
const execTool = createExecTool(cwd, opts.onExecApprovalNeeded);
|
||||
const processTool = createProcessTool(cwd);
|
||||
|
|
|
|||
56
packages/core/src/agent/tools/image-resize.test.ts
Normal file
56
packages/core/src/agent/tools/image-resize.test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import { wrapReadToolWithImageResize } from "./image-resize.js";
|
||||
|
||||
describe("image-resize", () => {
|
||||
function makeMockReadTool(content: any[]) {
|
||||
return {
|
||||
name: "read",
|
||||
description: "test",
|
||||
parameters: {} as any,
|
||||
execute: async () => ({ content }),
|
||||
};
|
||||
}
|
||||
|
||||
it("should pass through non-image content unchanged", async () => {
|
||||
const tool = makeMockReadTool([
|
||||
{ type: "text", text: "Hello world" },
|
||||
]);
|
||||
const wrapped = wrapReadToolWithImageResize(tool as any);
|
||||
const result = await wrapped.execute({} as any) as any;
|
||||
expect(result.content).toHaveLength(1);
|
||||
expect(result.content[0].text).toBe("Hello world");
|
||||
});
|
||||
|
||||
it("should pass through small images unchanged", async () => {
|
||||
const smallBase64 = Buffer.alloc(100, 0x41).toString("base64");
|
||||
const tool = makeMockReadTool([
|
||||
{ type: "image", data: smallBase64 },
|
||||
]);
|
||||
const wrapped = wrapReadToolWithImageResize(tool as any);
|
||||
const result = await wrapped.execute({} as any) as any;
|
||||
expect(result.content[0].data).toBe(smallBase64);
|
||||
});
|
||||
|
||||
it("should pass through results without content arrays", async () => {
|
||||
const tool = {
|
||||
name: "read",
|
||||
description: "test",
|
||||
parameters: {} as any,
|
||||
execute: async () => ({ text: "plain" }),
|
||||
};
|
||||
const wrapped = wrapReadToolWithImageResize(tool as any);
|
||||
const result = await wrapped.execute({} as any) as any;
|
||||
expect(result.text).toBe("plain");
|
||||
});
|
||||
|
||||
it("should handle execution errors gracefully", async () => {
|
||||
const tool = {
|
||||
name: "read",
|
||||
description: "test",
|
||||
parameters: {} as any,
|
||||
execute: async () => { throw new Error("file not found"); },
|
||||
};
|
||||
const wrapped = wrapReadToolWithImageResize(tool as any);
|
||||
await expect(wrapped.execute({} as any)).rejects.toThrow("file not found");
|
||||
});
|
||||
});
|
||||
211
packages/core/src/agent/tools/image-resize.ts
Normal file
211
packages/core/src/agent/tools/image-resize.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
/**
|
||||
* Image resize wrapper for the read tool.
|
||||
*
|
||||
* Wraps the read tool from pi-coding-agent to automatically downscale
|
||||
* oversized images returned in tool results. Uses macOS `sips` for resize
|
||||
* (no extra dependencies required).
|
||||
*/
|
||||
|
||||
import type { AgentTool, AgentToolResult } from "@mariozechner/pi-agent-core";
|
||||
import { execFile } from "node:child_process";
|
||||
import { writeFile, readFile, mkdtemp, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
/** Max decoded image binary size (1MB) */
|
||||
const MAX_IMAGE_BYTES = 1 * 1024 * 1024;
|
||||
|
||||
/** Max image dimension in pixels per side */
|
||||
const MAX_IMAGE_DIMENSION_PX = 2000;
|
||||
|
||||
/** JPEG quality for resized output */
|
||||
const JPEG_QUALITY = 80;
|
||||
|
||||
type ContentBlock = AgentToolResult<unknown>["content"][number];
|
||||
|
||||
function isImageBlock(block: unknown): block is { type: "image"; data: string; [key: string]: unknown } {
|
||||
return (
|
||||
!!block &&
|
||||
typeof block === "object" &&
|
||||
(block as any).type === "image" &&
|
||||
typeof (block as any).data === "string"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sips command and return output buffer.
|
||||
* Only available on macOS.
|
||||
*/
|
||||
function runSips(args: string[]): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile("/usr/bin/sips", args, { timeout: 20_000, maxBuffer: 1024 * 1024 }, (err, stdout) => {
|
||||
if (err) reject(err);
|
||||
else resolve(stdout);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get image dimensions via sips.
|
||||
*/
|
||||
async function getImageDimensions(
|
||||
buffer: Buffer,
|
||||
tmpDir: string,
|
||||
): Promise<{ width: number; height: number } | null> {
|
||||
const input = join(tmpDir, "in.img");
|
||||
await writeFile(input, buffer);
|
||||
|
||||
try {
|
||||
const stdout = await runSips(["-g", "pixelWidth", "-g", "pixelHeight", input]);
|
||||
const w = stdout.match(/pixelWidth:\s*(\d+)/);
|
||||
const h = stdout.match(/pixelHeight:\s*(\d+)/);
|
||||
if (w?.[1] && h?.[1]) {
|
||||
return { width: parseInt(w[1], 10), height: parseInt(h[1], 10) };
|
||||
}
|
||||
} catch {
|
||||
// sips not available or failed
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize image to JPEG via sips.
|
||||
*/
|
||||
async function resizeWithSips(
|
||||
buffer: Buffer,
|
||||
maxSide: number,
|
||||
quality: number,
|
||||
tmpDir: string,
|
||||
): Promise<Buffer> {
|
||||
const input = join(tmpDir, "in.img");
|
||||
const output = join(tmpDir, "out.jpg");
|
||||
await writeFile(input, buffer);
|
||||
|
||||
await runSips([
|
||||
"-Z", String(maxSide),
|
||||
"-s", "format", "jpeg",
|
||||
"-s", "formatOptions", String(quality),
|
||||
input,
|
||||
"--out", output,
|
||||
]);
|
||||
|
||||
return readFile(output);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if image needs resize and perform it if necessary.
|
||||
* Returns the original base64 if no resize needed or if resize fails.
|
||||
*/
|
||||
async function maybeResizeImage(base64Data: string): Promise<{ base64: string; mimeType?: string; resized: boolean }> {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
const overSize = buffer.byteLength > MAX_IMAGE_BYTES;
|
||||
|
||||
// Quick check: if small enough by bytes and we can't check dimensions, pass through
|
||||
if (!overSize && process.platform !== "darwin") {
|
||||
return { base64: base64Data, resized: false };
|
||||
}
|
||||
|
||||
// On macOS, use sips to check dimensions and resize if needed
|
||||
if (process.platform === "darwin") {
|
||||
const tmpDir = await mkdtemp(join(tmpdir(), "multica-img-"));
|
||||
try {
|
||||
const dims = await getImageDimensions(buffer, tmpDir);
|
||||
|
||||
// If we can get dimensions and everything is within limits, pass through
|
||||
if (dims && !overSize && dims.width <= MAX_IMAGE_DIMENSION_PX && dims.height <= MAX_IMAGE_DIMENSION_PX) {
|
||||
return { base64: base64Data, resized: false };
|
||||
}
|
||||
|
||||
// Need resize
|
||||
const maxDim = dims ? Math.max(dims.width, dims.height) : MAX_IMAGE_DIMENSION_PX;
|
||||
const targetSide = Math.min(MAX_IMAGE_DIMENSION_PX, maxDim);
|
||||
const resized = await resizeWithSips(buffer, targetSide, JPEG_QUALITY, tmpDir);
|
||||
|
||||
// If still too large, try progressively smaller sizes
|
||||
if (resized.byteLength > MAX_IMAGE_BYTES) {
|
||||
for (const side of [1600, 1200, 800]) {
|
||||
const smaller = await resizeWithSips(buffer, side, JPEG_QUALITY, tmpDir);
|
||||
if (smaller.byteLength <= MAX_IMAGE_BYTES) {
|
||||
return { base64: smaller.toString("base64"), mimeType: "image/jpeg", resized: true };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { base64: resized.toString("base64"), mimeType: "image/jpeg", resized: true };
|
||||
} catch {
|
||||
// sips failed, pass through original
|
||||
return { base64: base64Data, resized: false };
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// Non-macOS: pass through (future: add sharp support)
|
||||
return { base64: base64Data, resized: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Process tool result content blocks, resizing oversized images.
|
||||
*/
|
||||
async function processResultContent(content: ContentBlock[]): Promise<ContentBlock[]> {
|
||||
const result: ContentBlock[] = [];
|
||||
|
||||
for (const block of content) {
|
||||
if (!isImageBlock(block)) {
|
||||
result.push(block);
|
||||
continue;
|
||||
}
|
||||
|
||||
const decoded = Buffer.from(block.data, "base64");
|
||||
// Skip small images entirely
|
||||
if (decoded.byteLength <= MAX_IMAGE_BYTES) {
|
||||
result.push(block);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const resized = await maybeResizeImage(block.data);
|
||||
if (resized.resized) {
|
||||
result.push({ ...block, data: resized.base64 } as ContentBlock);
|
||||
} else {
|
||||
result.push(block);
|
||||
}
|
||||
} catch {
|
||||
result.push(block);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap the read tool to automatically resize oversized images in results.
|
||||
*/
|
||||
export function wrapReadToolWithImageResize(
|
||||
tool: AgentTool<any, any>,
|
||||
): AgentTool<any, any> {
|
||||
const originalExecute = tool.execute;
|
||||
|
||||
return {
|
||||
...tool,
|
||||
execute: async (...args: Parameters<typeof originalExecute>) => {
|
||||
const result = await originalExecute(...args);
|
||||
|
||||
// Only process results with content arrays
|
||||
const resultAny = result as any;
|
||||
if (!resultAny?.content || !Array.isArray(resultAny.content)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Check if there are any image blocks worth processing
|
||||
const hasLargeImages = resultAny.content.some(
|
||||
(block: unknown) =>
|
||||
isImageBlock(block) && Buffer.from((block as any).data, "base64").byteLength > MAX_IMAGE_BYTES,
|
||||
);
|
||||
if (!hasLargeImages) return result;
|
||||
|
||||
const processed = await processResultContent(resultAny.content);
|
||||
return { ...resultAny, content: processed } as typeof result;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -1,22 +1,27 @@
|
|||
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { v7 as uuidv7 } from "uuid";
|
||||
import { DATA_DIR } from "@multica/utils";
|
||||
import { DATA_DIR, generateEncryptedId, isValidEncryptedId } from "@multica/utils";
|
||||
|
||||
const HUB_ID_FILE = join(DATA_DIR, "hub-id");
|
||||
|
||||
/**
|
||||
* 获取当前 Hub 的 ID。
|
||||
* 首次调用时生成 UUIDv7 并持久化到 ~/.super-multica/hub-id，
|
||||
* 获取当前 Hub 的 ID（加密后的 40 字符格式）。
|
||||
* 首次调用时生成加密 ID 并持久化到 ~/.super-multica/hub-id，
|
||||
* 后续调用直接读取。
|
||||
*/
|
||||
export function getHubId(): string {
|
||||
try {
|
||||
return readFileSync(HUB_ID_FILE, "utf-8").trim();
|
||||
const existing = readFileSync(HUB_ID_FILE, "utf-8").trim();
|
||||
if (isValidEncryptedId(existing)) {
|
||||
return existing;
|
||||
}
|
||||
} catch {
|
||||
const id = uuidv7();
|
||||
mkdirSync(DATA_DIR, { recursive: true });
|
||||
writeFileSync(HUB_ID_FILE, id, "utf-8");
|
||||
return id;
|
||||
// File doesn't exist or read error
|
||||
}
|
||||
|
||||
// Generate new encrypted ID
|
||||
const id = generateEncryptedId();
|
||||
mkdirSync(DATA_DIR, { recursive: true });
|
||||
writeFileSync(HUB_ID_FILE, id, "utf-8");
|
||||
return id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { useState, useEffect, useCallback, useRef } from "react";
|
||||
import { v7 as uuidv7 } from "uuid";
|
||||
import {
|
||||
GatewayClient,
|
||||
type ConnectionState,
|
||||
|
|
@ -37,13 +36,43 @@ function clearIdentity(): void {
|
|||
localStorage.removeItem(STORAGE_KEY);
|
||||
}
|
||||
|
||||
function getDeviceId(): string {
|
||||
let id = localStorage.getItem(DEVICE_KEY);
|
||||
if (!id) {
|
||||
id = uuidv7();
|
||||
localStorage.setItem(DEVICE_KEY, id);
|
||||
// SHA-256 hash (Web Crypto API)
|
||||
async function sha256(text: string): Promise<string> {
|
||||
const buffer = new TextEncoder().encode(text);
|
||||
const hashBuffer = await crypto.subtle.digest("SHA-256", buffer);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
// Generate encrypted device ID (40 hex chars, consistent with copilot-search)
|
||||
async function generateEncryptedDeviceId(): Promise<string> {
|
||||
const uuid = crypto.randomUUID();
|
||||
const firstHash = (await sha256(uuid)).slice(0, 32);
|
||||
return (await sha256(firstHash)).slice(0, 8) + firstHash;
|
||||
}
|
||||
|
||||
// Validate encrypted ID format (40 hex characters)
|
||||
function isValidEncryptedId(id: string): boolean {
|
||||
return typeof id === "string" && /^[a-f0-9]{40}$/i.test(id);
|
||||
}
|
||||
|
||||
// Cached promise for device ID generation
|
||||
let deviceIdPromise: Promise<string> | null = null;
|
||||
|
||||
async function getDeviceId(): Promise<string> {
|
||||
const existing = localStorage.getItem(DEVICE_KEY);
|
||||
// If already encrypted format, return as-is
|
||||
if (existing && isValidEncryptedId(existing)) {
|
||||
return existing;
|
||||
}
|
||||
return id;
|
||||
// Generate new encrypted ID (or migrate old UUID)
|
||||
if (!deviceIdPromise) {
|
||||
deviceIdPromise = generateEncryptedDeviceId().then((id) => {
|
||||
localStorage.setItem(DEVICE_KEY, id);
|
||||
return id;
|
||||
});
|
||||
}
|
||||
return deviceIdPromise;
|
||||
}
|
||||
|
||||
export type PageState = "loading" | "not-connected" | "connecting" | "connected";
|
||||
|
|
@ -72,12 +101,12 @@ export function useGatewayConnection(): UseGatewayConnectionReturn {
|
|||
|
||||
const connectToGateway = useCallback(
|
||||
(id: ConnectionIdentity, token?: string) => {
|
||||
const doConnect = () => {
|
||||
const doConnect = async () => {
|
||||
disconnectingRef.current = false;
|
||||
setPageState("connecting");
|
||||
setError(null);
|
||||
|
||||
const deviceId = getDeviceId();
|
||||
const deviceId = await getDeviceId();
|
||||
|
||||
const client = new GatewayClient({
|
||||
url: id.gateway,
|
||||
|
|
|
|||
50
packages/utils/src/device-id.ts
Normal file
50
packages/utils/src/device-id.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* Encrypted Device/Hub ID generation utilities
|
||||
*
|
||||
* All device identifiers (Device ID, Hub ID, etc.) use the same encryption format:
|
||||
* 1. Generate UUID
|
||||
* 2. sha256(uuid).slice(0, 32) = firstHash
|
||||
* 3. sha256(firstHash).slice(0, 8) + firstHash = 40 hex chars
|
||||
*
|
||||
* This is consistent with copilot-search/devv-sdk.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { v7 as uuidv7 } from "uuid";
|
||||
|
||||
/**
|
||||
* SHA-256 hash function (Node.js)
|
||||
*/
|
||||
function sha256(text: string): string {
|
||||
return createHash("sha256").update(text, "utf8").digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an encrypted device/hub ID (40 hex characters)
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. Generate UUIDv7
|
||||
* 2. sha256(uuid).slice(0, 32) = firstHash
|
||||
* 3. sha256(firstHash).slice(0, 8) + firstHash = 40 chars
|
||||
*/
|
||||
export function generateEncryptedId(): string {
|
||||
const uuid = uuidv7();
|
||||
const firstHash = sha256(uuid).slice(0, 32);
|
||||
return sha256(firstHash).slice(0, 8) + firstHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate encrypted ID format (40 hex characters)
|
||||
*/
|
||||
export function isValidEncryptedId(id: string): boolean {
|
||||
return typeof id === "string" && /^[a-f0-9]{40}$/i.test(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypt a raw UUID to the 40-char format
|
||||
* Used when migrating old UUIDs to encrypted format
|
||||
*/
|
||||
export function encryptRawId(rawId: string): string {
|
||||
const firstHash = sha256(rawId).slice(0, 32);
|
||||
return sha256(firstHash).slice(0, 8) + firstHash;
|
||||
}
|
||||
|
|
@ -3,3 +3,4 @@ export * from "./paths.js";
|
|||
export * from "./errors.js";
|
||||
export * from "./retry.js";
|
||||
export * from "./cancellation.js";
|
||||
export * from "./device-id.js";
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
{
|
||||
"$schema": "https://turbo.build/schema.json",
|
||||
"globalDependencies": ["tsconfig.json", "tsconfig.base.json"],
|
||||
"globalEnv": ["API_URL"],
|
||||
"tasks": {
|
||||
"build": {
|
||||
"dependsOn": ["^build"],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue