marketing-shibata50/9router
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(db): migrate from lowdb to SQLite with repos pattern

Browse source 
- Add modular DB layer (adapters, migrations, repos, helpers)
- Replace localDb/usageDb/requestDetailsDb monoliths with repos
- Add Tailscale tunnel integration & status check API
- Add /api/cli-tools/all-statuses aggregated endpoint
- Add settingsStore (Zustand) and mitm/dbReader
- Add DB unit tests (benchmark, concurrent, migration, vs-lowdb)
This commit is contained in:
decolua 2026-05-09 17:48:20 +07:00
parent 145f588cc0
commit bee8dad946
63 changed files with 4223 additions and 2330 deletions
Download patch file Download diff file Expand all files Collapse all files

7
package.json
View file

@ -1,10 +1,10 @@
{
"name": "9router-app",
"version": "0.4.20",
"version": "0.4.25",
"description": "9Router web dashboard",
"private": true,
"scripts": {
"dev": "next dev --webpack --port 20128",
"dev": "next dev --webpack --hostname 127.0.0.1 --port 20128",
"build": "NODE_ENV=production next build --webpack",
"start": "NODE_ENV=production next start",
"dev:bun": "bun --bun next dev --webpack --port 20128",
@ -20,7 +20,6 @@
"fs": "^0.0.1-security",
"http-proxy-middleware": "^3.0.5",
"jose": "^6.1.3",
"lowdb": "^7.0.1",
"marked": "^18.0.1",
"monaco-editor": "^0.55.1",
"next": "^16.1.6",
@ -28,7 +27,6 @@
"node-machine-id": "^1.1.12",
"open": "^11.0.0",
"ora": "^9.1.0",
"proper-lockfile": "^4.1.2",
"react": "19.2.4",
"react-dom": "19.2.4",
"react-is": "^16.13.1",
@ -43,6 +41,7 @@
"optionalDependencies": {
"better-sqlite3": "^12.6.2"
},
"comment_better_sqlite3": "kept in optionalDependencies so npm install doesn't fail on systems without build tools — sql.js is used as fallback at runtime",
"devDependencies": {
"@tailwindcss/postcss": "^4.1.18",
"eslint": "^9",

31
src/app/(dashboard)/dashboard/cli-tools/CLIToolsPageClient.js
View file

@ -4,21 +4,12 @@ import { useState, useEffect, useCallback } from "react";
import { Card, CardSkeleton } from "@/shared/components";
import { CLI_TOOLS } from "@/shared/constants/cliTools";
import { getModelsByProviderId, PROVIDER_ID_TO_ALIAS } from "@/shared/constants/models";
import { ClaudeToolCard, CodexToolCard, DroidToolCard, OpenClawToolCard, HermesToolCard, DefaultToolCard, OpenCodeToolCard, CoworkToolCard, MitmLinkCard } from "./components";
import { ClaudeToolCard, CodexToolCard, DroidToolCard, OpenClawToolCard, HermesToolCard, DefaultToolCard, OpenCodeToolCard, CoworkToolCard, CopilotToolCard, MitmLinkCard } from "./components";
import { MITM_TOOLS } from "@/shared/constants/cliTools";
const CLOUD_URL = process.env.NEXT_PUBLIC_CLOUD_URL;
const STATUS_ENDPOINTS = {
claude: "/api/cli-tools/claude-settings",
codex: "/api/cli-tools/codex-settings",
opencode: "/api/cli-tools/opencode-settings",
droid: "/api/cli-tools/droid-settings",
openclaw: "/api/cli-tools/openclaw-settings",
hermes: "/api/cli-tools/hermes-settings",
cowork: "/api/cli-tools/cowork-settings",
};
const ALL_STATUSES_URL = "/api/cli-tools/all-statuses";
export default function CLIToolsPageClient({ machineId }) {
const [connections, setConnections] = useState([]);
@ -42,18 +33,8 @@ export default function CLIToolsPageClient({ machineId }) {
const fetchAllStatuses = async () => {
try {
const entries = await Promise.all(
Object.entries(STATUS_ENDPOINTS).map(async ([toolId, url]) => {
try {
const res = await fetch(url);
const data = await res.json();
return [toolId, data];
} catch {
return [toolId, null];
}
})
);
setToolStatuses(Object.fromEntries(entries));
const res = await fetch(ALL_STATUSES_URL);
if (res.ok) setToolStatuses(await res.json());
} catch (error) {
console.log("Error fetching tool statuses:", error);
}
@ -138,7 +119,7 @@ export default function CLIToolsPageClient({ machineId }) {
if (tunnelEnabled && tunnelPublicUrl) return tunnelPublicUrl;
if (cloudEnabled && CLOUD_URL) return CLOUD_URL;
if (typeof window !== "undefined") return window.location.origin;
return "http://localhost:20128";
return "http://127.0.0.1:20128";
};
if (loading) {
@ -207,6 +188,8 @@ export default function CLIToolsPageClient({ machineId }) {
return <OpenClawToolCard key={toolId} {...commonProps} activeProviders={getActiveProviders()} hasActiveProviders={hasActiveProviders} cloudEnabled={cloudEnabled} initialStatus={toolStatuses.openclaw} />;
case "hermes":
return <HermesToolCard key={toolId} {...commonProps} activeProviders={getActiveProviders()} hasActiveProviders={hasActiveProviders} cloudEnabled={cloudEnabled} initialStatus={toolStatuses.hermes} />;
case "copilot":
return <CopilotToolCard key={toolId} {...commonProps} activeProviders={getActiveProviders()} cloudEnabled={cloudEnabled} initialStatus={toolStatuses.copilot} />;
default:
return <DefaultToolCard key={toolId} toolId={toolId} {...commonProps} activeProviders={getActiveProviders()} cloudEnabled={cloudEnabled} tunnelEnabled={tunnelEnabled} />;
}

128
src/app/(dashboard)/dashboard/cli-tools/components/CopilotToolCard.js
View file

@ -16,10 +16,7 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
const [customBaseUrl, setCustomBaseUrl] = useState("");
const [modelAliases, setModelAliases] = useState({});
const [showManualConfigModal, setShowManualConfigModal] = useState(false);
// Model list management
const [modelInput, setModelInput] = useState("");
const [modelList, setModelList] = useState([]);
const [selectedModels, setSelectedModels] = useState([]);
const [modalOpen, setModalOpen] = useState(false);
useEffect(() => {
@ -40,12 +37,12 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
if (isExpanded) fetchModelAliases();
}, [isExpanded]);
// Pre-fill model list from existing config
// Pre-fill from existing config
useEffect(() => {
if (status?.config && Array.isArray(status.config) && modelList.length === 0) {
if (status?.config && Array.isArray(status.config) && selectedModels.length === 0) {
const entry = status.config.find((e) => e.name === "9Router");
if (entry?.models?.length > 0) {
setModelList(entry.models.map((m) => m.id));
setSelectedModels(entry.models.map((m) => m.id));
}
}
}, [status]);
@ -68,20 +65,16 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
};
const configStatus = getConfigStatus();
const getEffectiveBaseUrl = () => {
const url = customBaseUrl || baseUrl;
return url.endsWith("/v1") ? url : `${url}/v1`;
};
const getDisplayUrl = () => customBaseUrl || `${baseUrl}/v1`;
const hasCustomSelectedApiKey = selectedApiKey && !apiKeys.some((key) => key.key === selectedApiKey);
const addModel = () => {
const val = modelInput.trim();
if (!val || modelList.includes(val)) return;
setModelList((prev) => [...prev, val]);
setModelInput("");
};
const removeModel = (id) => setModelList((prev) => prev.filter((m) => m !== id));
const removeModel = (id) => setSelectedModels((prev) => prev.filter((m) => m !== id));
const checkStatus = async () => {
setChecking(true);
@ -107,11 +100,11 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
const res = await fetch("/api/cli-tools/copilot-settings", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ baseUrl: getEffectiveBaseUrl(), apiKey: keyToUse, models: modelList }),
body: JSON.stringify({ baseUrl: getEffectiveBaseUrl(), apiKey: keyToUse, models: selectedModels }),
});
const data = await res.json();
if (res.ok) {
setMessage({ type: "success", text: data.message || "Settings applied successfully!" });
setMessage({ type: "success", text: data.message || "Settings applied! Reload VS Code." });
checkStatus();
} else {
setMessage({ type: "error", text: data.error || "Failed to apply settings" });
@ -131,7 +124,7 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
const data = await res.json();
if (res.ok) {
setMessage({ type: "success", text: "Settings reset successfully!" });
setModelList([]);
setSelectedModels([]);
checkStatus();
} else {
setMessage({ type: "error", text: data.error || "Failed to reset settings" });
@ -148,6 +141,7 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
? selectedApiKey
: (!cloudEnabled ? "sk_9router" : "<API_KEY_FROM_DASHBOARD>");
const effectiveBaseUrl = getEffectiveBaseUrl();
const modelsToShow = selectedModels.length > 0 ? selectedModels : ["provider/model-id"];
return [{
filename: "~/Library/Application Support/Code/User/chatLanguageModels.json",
@ -155,7 +149,7 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
name: "9Router",
vendor: "azure",
apiKey: keyToUse,
models: modelList.map((id) => ({
models: modelsToShow.map((id) => ({
id, name: id,
url: `${effectiveBaseUrl}/chat/completions#models.ai.azure.com`,
toolCalling: true, vision: false,
@ -166,14 +160,14 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
};
return (
<Card padding="sm" className="overflow-hidden">
<div className="flex items-center justify-between hover:cursor-pointer" onClick={onToggle}>
<div className="flex items-center gap-3">
<Card padding="xs" className="overflow-hidden">
<div className="flex items-start justify-between gap-3 hover:cursor-pointer sm:items-center" onClick={onToggle}>
<div className="flex min-w-0 items-center gap-3">
<div className="size-8 flex items-center justify-center shrink-0">
<Image src="/providers/copilot.png" alt={tool.name} width={32} height={32} className="size-8 object-contain rounded-lg" sizes="32px" onError={(e) => { e.target.style.display = "none"; }} />
</div>
<div className="min-w-0">
<div className="flex items-center gap-2">
<div className="flex min-w-0 flex-wrap items-center gap-2">
<h3 className="font-medium text-sm">{tool.name}</h3>
{configStatus === "configured" && <span className="px-1.5 py-0.5 text-[10px] font-medium bg-green-500/10 text-green-600 dark:text-green-400 rounded-full">Connected</span>}
{configStatus === "not_configured" && <span className="px-1.5 py-0.5 text-[10px] font-medium bg-yellow-500/10 text-yellow-600 dark:text-yellow-400 rounded-full">Not configured</span>}
@ -196,7 +190,6 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
{!checking && (
<>
{/* Info */}
<div className="flex items-start gap-3 p-3 bg-blue-500/10 border border-blue-500/30 rounded-lg">
<span className="material-symbols-outlined text-blue-500 text-lg">info</span>
<div className="text-xs text-blue-700 dark:text-blue-300">
@ -205,11 +198,13 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
</div>
</div>
<div className="flex flex-col gap-3">
<div className="flex flex-col gap-1">
<label className="text-xs font-medium text-text-muted">Select Endpoint</label>
<div className="flex flex-col gap-2">
{/* Endpoint */}
<div className="grid grid-cols-1 gap-1.5 sm:grid-cols-[8rem_auto_1fr] sm:items-center sm:gap-2">
<span className="text-xs font-semibold text-text-main sm:text-right sm:text-sm">Select Endpoint</span>
<span className="material-symbols-outlined hidden text-text-muted text-[14px] sm:inline">arrow_forward</span>
<BaseUrlSelect
value={customBaseUrl || getEffectiveBaseUrl()}
value={customBaseUrl || getDisplayUrl()}
onChange={setCustomBaseUrl}
requiresExternalUrl={tool.requiresExternalUrl}
tunnelEnabled={tunnelEnabled}
@ -220,53 +215,43 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
</div>
{/* API Key */}
<div className="flex flex-col gap-1">
<label className="text-xs font-medium text-text-muted">API Key</label>
<div className="grid grid-cols-1 gap-1.5 sm:grid-cols-[8rem_auto_1fr_auto] sm:items-center sm:gap-2">
<span className="text-xs font-semibold text-text-main sm:text-right sm:text-sm">API Key</span>
<span className="material-symbols-outlined hidden text-text-muted text-[14px] sm:inline">arrow_forward</span>
{apiKeys.length > 0 || selectedApiKey ? (
<select value={selectedApiKey} onChange={(e) => setSelectedApiKey(e.target.value)} className="px-3 py-2 bg-bg-secondary rounded-lg text-sm border border-border focus:outline-none focus:ring-1 focus:ring-primary/50">
<select value={selectedApiKey} onChange={(e) => setSelectedApiKey(e.target.value)} className="w-full min-w-0 px-2 py-2 bg-surface rounded text-xs border border-border focus:outline-none focus:ring-1 focus:ring-primary/50 sm:py-1.5">
{hasCustomSelectedApiKey && <option value={selectedApiKey}>{selectedApiKey}</option>}
{apiKeys.map((key) => <option key={key.id} value={key.key}>{key.key}</option>)}
</select>
) : (
<span className="text-sm text-text-muted">
<span className="min-w-0 rounded bg-surface/40 px-2 py-2 text-xs text-text-muted sm:py-1.5">
{cloudEnabled ? "No API keys - Create one in Keys page" : "sk_9router (default)"}
</span>
)}
</div>
{/* Model input + Add */}
<div className="flex flex-col gap-1">
<label className="text-xs font-medium text-text-muted">
Models {modelList.length > 0 && <span className="text-primary">({modelList.length} added)</span>}
</label>
{/* Model list */}
{modelList.length > 0 && (
<div className="flex flex-col gap-1 mb-1">
{modelList.map((id) => (
<div key={id} className="flex items-center gap-2 px-3 py-1.5 bg-bg-secondary rounded-lg border border-border">
<span className="flex-1 text-sm font-mono truncate">{id}</span>
<button onClick={() => removeModel(id)} className="text-text-muted hover:text-red-500 transition-colors" title="Remove">
<span className="material-symbols-outlined text-[14px]">close</span>
</button>
</div>
))}
{/* Models */}
<div className="grid grid-cols-1 gap-1.5 sm:grid-cols-[8rem_auto_1fr] sm:items-start sm:gap-2">
<span className="w-32 shrink-0 text-sm font-semibold text-text-main text-right pt-1">Models</span>
<span className="material-symbols-outlined text-text-muted text-[14px] mt-1.5">arrow_forward</span>
<div className="flex-1 flex flex-col gap-2">
<div className="flex flex-wrap gap-1.5 min-h-[28px] px-2 py-1.5 bg-surface rounded border border-border">
{selectedModels.length === 0 ? (
<span className="text-xs text-text-muted">No models selected</span>
) : (
selectedModels.map((model) => (
<span key={model} className="inline-flex items-center gap-1 px-2 py-0.5 rounded text-xs bg-black/5 dark:bg-white/5 text-text-muted border border-transparent hover:border-border">
{model}
<button onClick={(e) => { e.stopPropagation(); removeModel(model); }} className="ml-0.5 hover:text-red-500">
<span className="material-symbols-outlined text-[12px]">close</span>
</button>
</span>
))
)}
</div>
<div>
<button onClick={() => setModalOpen(true)} disabled={!activeProviders?.length} className={`px-2 py-1 rounded border text-xs transition-colors ${activeProviders?.length ? "bg-surface border-border text-text-main hover:border-primary cursor-pointer" : "opacity-50 cursor-not-allowed border-border"}`}>Add Model</button>
</div>
)}
<div className="grid grid-cols-1 gap-2 sm:grid-cols-[1fr_auto_auto] sm:items-center">
<input
type="text"
value={modelInput}
onChange={(e) => setModelInput(e.target.value)}
onKeyDown={(e) => e.key === "Enter" && addModel()}
placeholder="provider/model-id"
className="min-w-0 px-3 py-2 bg-bg-secondary rounded-lg text-sm border border-border focus:outline-none focus:ring-1 focus:ring-primary/50"
/>
<button onClick={() => setModalOpen(true)} disabled={!activeProviders?.length} className={`rounded-lg border px-3 py-2 text-sm transition-colors sm:shrink-0 ${activeProviders?.length ? "bg-bg-secondary border-border hover:border-primary cursor-pointer" : "opacity-50 cursor-not-allowed border-border"}`}>Select</button>
<button onClick={addModel} disabled={!modelInput.trim()} className="rounded-lg border border-border bg-bg-secondary px-3 py-2 text-sm transition-colors hover:border-primary disabled:opacity-50 sm:shrink-0" title="Add model">
<span className="material-symbols-outlined text-[16px]">add</span>
</button>
</div>
</div>
</div>
@ -279,13 +264,13 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
)}
<div className="grid grid-cols-1 gap-2 sm:flex sm:items-center">
<Button variant="primary" size="sm" onClick={handleApply} disabled={modelList.length === 0} loading={applying}>
<Button variant="primary" size="sm" onClick={handleApply} disabled={selectedModels.length === 0} loading={applying}>
<span className="material-symbols-outlined text-[14px] mr-1">save</span>Apply
</Button>
<Button variant="outline" size="sm" onClick={handleReset} disabled={!status?.has9Router} loading={restoring}>
<span className="material-symbols-outlined text-[14px] mr-1">restore</span>Reset
</Button>
<Button variant="ghost" size="sm" onClick={() => setShowManualConfigModal(true)} disabled={modelList.length === 0}>
<Button variant="ghost" size="sm" onClick={() => setShowManualConfigModal(true)} disabled={selectedModels.length === 0}>
<span className="material-symbols-outlined text-[14px] mr-1">content_copy</span>Manual Config
</Button>
</div>
@ -297,11 +282,16 @@ export default function CopilotToolCard({ tool, isExpanded, onToggle, baseUrl, a
<ModelSelectModal
isOpen={modalOpen}
onClose={() => setModalOpen(false)}
onSelect={(model) => { setModelInput(model.value); setModalOpen(false); }}
selectedModel={modelInput}
onSelect={(model) => {
if (!selectedModels.includes(model.value)) {
setSelectedModels([...selectedModels, model.value]);
}
setModalOpen(false);
}}
selectedModel={null}
activeProviders={activeProviders}
modelAliases={modelAliases}
title="Select Model for GitHub Copilot"
title="Add Model for GitHub Copilot"
/>
<ManualConfigModal

2
src/app/(dashboard)/dashboard/cli-tools/components/DefaultToolCard.js
View file

@ -21,7 +21,7 @@ export default function DefaultToolCard({ toolId, tool, isExpanded, onToggle, ba
: (!cloudEnabled ? "sk_9router" : "your-api-key");
// Add /v1 suffix only if not already present (DRY - avoid duplicate)
const normalizedBaseUrl = baseUrl || "http://localhost:20128";
const normalizedBaseUrl = baseUrl || "http://127.0.0.1:20128";
const baseUrlWithV1 = normalizedBaseUrl.endsWith("/v1")
? normalizedBaseUrl
: `${normalizedBaseUrl}/v1`;

2
src/app/(dashboard)/dashboard/cli-tools/components/MitmServerCard.js
View file

@ -3,7 +3,7 @@
import { useState, useEffect, useCallback } from "react";
import { Card, Button, Badge, Input } from "@/shared/components";
const DEFAULT_MITM_ROUTER_BASE = "http://localhost:20128";
const DEFAULT_MITM_ROUTER_BASE = "http://127.0.0.1:20128";
/**
* Shared MITM infrastructure card — manages SSL cert + server start/stop.

146
src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js
View file

@ -1,6 +1,6 @@
"use client";
import { useState, useEffect, useRef } from "react";
import { useState, useEffect, useRef, useCallback } from "react";
import PropTypes from "prop-types";
import { Card, Button, Input, Modal, CardSkeleton, Toggle } from "@/shared/components";
import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard";
@ -15,6 +15,7 @@ const TUNNEL_BENEFITS = [
const TUNNEL_PING_INTERVAL_MS = 2000;
const TUNNEL_PING_MAX_MS = 300000;
const STATUS_POLL_INTERVAL_MS = 5000;
const REACHABLE_MISS_THRESHOLD = 2;
const CAVEMAN_LEVELS = [
{ id: "lite", label: "Lite", desc: "Drop filler, keep grammar" },
@ -39,6 +40,7 @@ export default function APIPageClient({ machineId }) {
// Cloudflare Tunnel state
const [tunnelChecking, setTunnelChecking] = useState(true);
const [tunnelEnabled, setTunnelEnabled] = useState(false);
const [tunnelReachable, setTunnelReachable] = useState(false);
const [tunnelUrl, setTunnelUrl] = useState("");
const [tunnelPublicUrl, setTunnelPublicUrl] = useState("");
const [tunnelLoading, setTunnelLoading] = useState(false);
@ -49,6 +51,7 @@ export default function APIPageClient({ machineId }) {
// Tailscale state
const [tsEnabled, setTsEnabled] = useState(false);
const [tsReachable, setTsReachable] = useState(false);
const [tsUrl, setTsUrl] = useState("");
const [tsLoading, setTsLoading] = useState(false);
const [tsProgress, setTsProgress] = useState("");
@ -62,6 +65,17 @@ export default function APIPageClient({ machineId }) {
const [showDisableTsModal, setShowDisableTsModal] = useState(false);
const tsLogRef = useRef(null);
// Debounce reachable=false: server may briefly return false during background refresh.
// Only flip UI to "reconnecting" after N consecutive misses to avoid spinner flicker.
const tunnelMissRef = useRef(0);
const tsMissRef = useRef(0);
// Track whether reachable=true was ever observed in this session.
// Distinguishes "Checking..." (initial cold cache) from "Reconnecting..." (lost connection).
const tunnelEverReachableRef = useRef(false);
const tsEverReachableRef = useRef(false);
const [tunnelEverReachable, setTunnelEverReachable] = useState(false);
const [tsEverReachable, setTsEverReachable] = useState(false);
// API key visibility toggle state
const [visibleKeys, setVisibleKeys] = useState(new Set());
@ -85,6 +99,23 @@ export default function APIPageClient({ machineId }) {
};
}, []);
// Update reachable state with miss-debounce: avoids spinner flicker when server
// briefly returns reachable=false during background probe refresh.
// Also flips everReachable on first success (UI uses it to distinguish Checking vs Reconnecting).
const updateReachable = useCallback((reachable, missRef, setter, everRef, everSetter) => {
if (reachable) {
missRef.current = 0;
setter(true);
if (!everRef.current) {
everRef.current = true;
everSetter(true);
}
} else {
missRef.current += 1;
if (missRef.current >= REACHABLE_MISS_THRESHOLD) setter(false);
}
}, []);
// Trust user intent (settingsEnabled): UI stays "enabled" while watchdog restarts process
const syncTunnelStatus = async () => {
try {
@ -97,11 +128,13 @@ export default function APIPageClient({ machineId }) {
setTunnelUrl(tUrl);
setTunnelPublicUrl(tPublicUrl);
setTunnelEnabled(tEnabled);
updateReachable(!!data.tunnel?.reachable, tunnelMissRef, setTunnelReachable, tunnelEverReachableRef, setTunnelEverReachable);
const tsEn = data.tailscale?.settingsEnabled ?? data.tailscale?.enabled ?? false;
const tsUrlVal = data.tailscale?.tunnelUrl || "";
setTsUrl(tsUrlVal);
setTsEnabled(tsEn);
updateReachable(!!data.tailscale?.reachable, tsMissRef, setTsReachable, tsEverReachableRef, setTsEverReachable);
} catch { /* ignore poll errors */ }
};
@ -129,26 +162,14 @@ export default function APIPageClient({ machineId }) {
const tPublicUrl = data.tunnel?.publicUrl || "";
setTunnelUrl(tUrl);
setTunnelPublicUrl(tPublicUrl);
// Trust user intent: stays enabled while watchdog restores process
setTunnelEnabled(tEnabled);
updateReachable(!!data.tunnel?.reachable, tunnelMissRef, setTunnelReachable, tunnelEverReachableRef, setTunnelEverReachable);
const tsEn = data.tailscale?.settingsEnabled ?? data.tailscale?.enabled ?? false;
const tsUrlVal = data.tailscale?.tunnelUrl || "";
setTsUrl(tsUrlVal);
setTsEnabled(tsEn);
// Background reachability probes (non-blocking, only show warning)
if (tEnabled && (tPublicUrl || tUrl)) {
const healthUrl = `${tPublicUrl || tUrl}/api/health`;
fetch(healthUrl, { cache: "no-store" })
.then((r) => { if (!r.ok) setTunnelStatus({ type: "warning", message: "Tunnel reconnecting..." }); })
.catch(() => setTunnelStatus({ type: "warning", message: "Tunnel reconnecting..." }));
}
if (tsEn && tsUrlVal) {
fetch(`${tsUrlVal}/api/health`, { mode: "no-cors", cache: "no-store" })
.then((r) => { if (!(r.ok || r.type === "opaque")) setTsStatus({ type: "warning", message: "Tailscale reconnecting..." }); })
.catch(() => setTsStatus({ type: "warning", message: "Tailscale reconnecting..." }));
}
updateReachable(!!data.tailscale?.reachable, tsMissRef, setTsReachable, tsEverReachableRef, setTsEverReachable);
}
} catch (error) {
console.log("Error loading settings:", error);
@ -428,8 +449,15 @@ export default function APIPageClient({ machineId }) {
return false;
};
const handleConnectTailscale = async (preOpenedTab) => {
const tab = preOpenedTab || null;
// Open auth URL only when actually needed (avoids blank popup flash on success path).
// Falls back to status message with clickable link if popup blocker prevents opening.
const openAuthUrl = (url) => {
const w = window.open(url, "tailscale_auth", "width=600,height=700");
if (!w) setTsStatus({ type: "warning", message: `Popup blocked. Open manually: ${url}` });
return w;
};
const handleConnectTailscale = async () => {
setShowTsModal(false);
setTsConnecting(true);
setTsLoading(true);
@ -440,23 +468,15 @@ export default function APIPageClient({ machineId }) {
const data = await res.json();
if (res.ok && data.success) {
if (tab) tab.close();
setTsUrl(data.tunnelUrl || "");
const reachable = await pingTsHealth(data.tunnelUrl);
if (reachable) {
setTsEnabled(true);
setTsStatus(null);
} else {
setTsEnabled(true);
setTsStatus({ type: "warning", message: "Connected but not reachable yet." });
}
setTsEnabled(true);
setTsStatus(reachable ? null : { type: "warning", message: "Connected but not reachable yet." });
return;
}
// Needs login: redirect pre-opened tab or open new
if (data.needsLogin && data.authUrl) {
if (tab) tab.location.href = data.authUrl;
else window.open(data.authUrl, "tailscale_auth", "width=600,height=700");
openAuthUrl(data.authUrl);
setTsProgress("Waiting for login...");
for (let i = 0; i < 40; i++) {
await new Promise((r) => setTimeout(r, 3000));
@ -469,18 +489,12 @@ export default function APIPageClient({ machineId }) {
const res2 = await fetch("/api/tunnel/tailscale-enable", { method: "POST" });
const data2 = await res2.json();
if (res2.ok && data2.success) {
if (tab) tab.close();
setTsUrl(data2.tunnelUrl || "");
const ok2 = await pingTsHealth(data2.tunnelUrl);
if (ok2) {
setTsEnabled(true);
setTsStatus(null);
} else {
setTsEnabled(true);
setTsStatus({ type: "warning", message: "Connected but not reachable yet." });
}
setTsEnabled(true);
setTsStatus(ok2 ? null : { type: "warning", message: "Connected but not reachable yet." });
} else if (data2.funnelNotEnabled && data2.enableUrl) {
await pollFunnelEnable(data2.enableUrl, tab);
await pollFunnelEnable(data2.enableUrl);
} else {
setTsStatus({ type: "error", message: data2.error || "Failed to start funnel" });
}
@ -493,16 +507,13 @@ export default function APIPageClient({ machineId }) {
return;
}
// Funnel not enabled: redirect pre-opened tab
if (data.funnelNotEnabled && data.enableUrl) {
await pollFunnelEnable(data.enableUrl, tab);
await pollFunnelEnable(data.enableUrl);
return;
}
if (tab) tab.close();
setTsStatus({ type: "error", message: data.error || "Failed to connect" });
} catch (error) {
if (tab) tab.close();
setTsStatus({ type: "error", message: error.message });
} finally {
setTsLoading(false);
@ -511,9 +522,8 @@ export default function APIPageClient({ machineId }) {
}
};
const pollFunnelEnable = async (enableUrl, tab) => {
if (tab) tab.location.href = enableUrl;
else window.open(enableUrl, "tailscale_auth", "width=600,height=700");
const pollFunnelEnable = async (enableUrl) => {
openAuthUrl(enableUrl);
setTsProgress("Enable Funnel in browser, waiting...");
for (let i = 0; i < 40; i++) {
await new Promise((r) => setTimeout(r, 3000));
@ -521,16 +531,10 @@ export default function APIPageClient({ machineId }) {
const res = await fetch("/api/tunnel/tailscale-enable", { method: "POST" });
const data = await res.json();
if (res.ok && data.success) {
if (tab) tab.close();
setTsUrl(data.tunnelUrl || "");
const ok3 = await pingTsHealth(data.tunnelUrl);
if (ok3) {
setTsEnabled(true);
setTsStatus(null);
} else {
setTsEnabled(true);
setTsStatus({ type: "warning", message: "Connected but not reachable yet." });
}
setTsEnabled(true);
setTsStatus(ok3 ? null : { type: "warning", message: "Connected but not reachable yet." });
return;
}
if (data.funnelNotEnabled) continue;
@ -685,7 +689,7 @@ export default function APIPageClient({ machineId }) {
<span className={`text-xs font-mono px-1.5 py-0.5 rounded shrink-0 min-w-[88px] text-center ${
tunnelEnabled ? "bg-primary/10 text-primary" : "bg-surface-2 text-text-muted"
}`}>Tunnel</span>
{tunnelEnabled && !tunnelLoading ? (
{tunnelEnabled && !tunnelLoading && tunnelReachable ? (
<>
<Input value={`${tunnelPublicUrl || tunnelUrl}/v1`} readOnly className="flex-1 font-mono text-sm" />
<button
@ -702,6 +706,20 @@ export default function APIPageClient({ machineId }) {
<span className="material-symbols-outlined text-[18px]">power_settings_new</span>
</button>
</>
) : tunnelEnabled && !tunnelLoading && !tunnelReachable ? (
<>
<div className="flex-1 flex items-center gap-2 px-3 py-1.5 rounded border border-amber-300 dark:border-amber-800 bg-amber-500/5 text-sm text-amber-600 dark:text-amber-400">
<span className="material-symbols-outlined animate-spin text-sm">progress_activity</span>
{tunnelEverReachable ? "Tunnel reconnecting..." : "Tunnel checking..."}
</div>
<button
onClick={() => setShowDisableTunnelModal(true)}
className="p-2 hover:bg-red-500/10 rounded text-red-500 transition-colors shrink-0"
title="Disable Tunnel"
>
<span className="material-symbols-outlined text-[18px]">power_settings_new</span>
</button>
</>
) : tunnelLoading ? (
<>
<div className="flex-1 flex items-center gap-2 px-3 py-1.5 rounded border border-border bg-input text-sm text-text-muted">
@ -759,7 +777,7 @@ export default function APIPageClient({ machineId }) {
<span className={`text-xs font-mono px-1.5 py-0.5 rounded shrink-0 min-w-[88px] text-center ${
tsEnabled ? "bg-primary/10 text-primary" : "bg-surface-2 text-text-muted"
}`}>Tailscale</span>
{tsEnabled && !tsLoading ? (
{tsEnabled && !tsLoading && tsReachable ? (
<>
<Input value={`${tsUrl}/v1`} readOnly className="flex-1 font-mono text-sm" />
<button
@ -776,6 +794,20 @@ export default function APIPageClient({ machineId }) {
<span className="material-symbols-outlined text-[18px]">power_settings_new</span>
</button>
</>
) : tsEnabled && !tsLoading && !tsReachable ? (
<>
<div className="flex-1 flex items-center gap-2 px-3 py-1.5 rounded border border-amber-300 dark:border-amber-800 bg-amber-500/5 text-sm text-amber-600 dark:text-amber-400">
<span className="material-symbols-outlined animate-spin text-sm">progress_activity</span>
{tsEverReachable ? "Tailscale reconnecting..." : "Tailscale checking..."}
</div>
<button
onClick={() => setShowDisableTsModal(true)}
className="p-2 hover:bg-red-500/10 rounded text-red-500 transition-colors shrink-0"
title="Disable Tailscale"
>
<span className="material-symbols-outlined text-[18px]">power_settings_new</span>
</button>
</>
) : (tsLoading || tsConnecting) ? (
<>
<div className="flex-1 flex items-center gap-2 px-3 py-1.5 rounded border border-border bg-input text-sm text-text-muted">
@ -1211,11 +1243,7 @@ export default function APIPageClient({ machineId }) {
</div>
<div className="flex gap-2">
<Button
onClick={() => {
const tab = window.open("", "tailscale_auth", "width=600,height=700");
if (tab) tab.document.write("<p style='font-family:sans-serif;text-align:center;margin-top:40px'>Connecting to Tailscale...</p>");
handleConnectTailscale(tab);
}}
onClick={() => handleConnectTailscale()}
fullWidth
>
Connect

81
src/app/(dashboard)/dashboard/media-providers/[kind]/page.js
View file

@ -3,7 +3,7 @@
import { useParams, notFound, useRouter } from "next/navigation";
import Link from "next/link";
import { useEffect, useState } from "react";
import { Card, Badge, Button, AddCustomEmbeddingModal } from "@/shared/components";
import { Card, Badge, Button, Toggle, AddCustomEmbeddingModal } from "@/shared/components";
import ProviderIcon from "@/shared/components/ProviderIcon";
import { MEDIA_PROVIDER_KINDS, AI_PROVIDERS, getProvidersByKind } from "@/shared/constants/providers";
@ -19,7 +19,7 @@ function getEffectiveStatus(conn) {
return conn.testStatus === "unavailable" && !isCooldown ? "active" : conn.testStatus;
}
function MediaProviderCard({ provider, kind, connections, isCustom }) {
function MediaProviderCard({ provider, kind, connections, isCustom, onToggle }) {
const providerInfo = AI_PROVIDERS[provider.id];
const isNoAuth = !!providerInfo?.noAuth;
@ -29,6 +29,12 @@ function MediaProviderCard({ provider, kind, connections, isCustom }) {
const total = providerConns.length;
const allDisabled = total > 0 && providerConns.every((c) => c.isActive === false);
const handleToggleClick = (e) => {
e.preventDefault();
e.stopPropagation();
if (onToggle) onToggle(provider.id, allDisabled);
};
const renderStatus = () => {
if (isNoAuth) return <Badge variant="success" size="sm">Ready</Badge>;
if (allDisabled) return <Badge variant="default" size="sm">Disabled</Badge>;
@ -48,27 +54,42 @@ function MediaProviderCard({ provider, kind, connections, isCustom }) {
padding="xs"
className={`h-full hover:bg-black/[0.01] dark:hover:bg-white/[0.01] transition-colors cursor-pointer ${allDisabled ? "opacity-50" : ""}`}
>
<div className="flex min-w-0 items-center gap-3">
<div
className="size-8 rounded-lg flex items-center justify-center shrink-0"
style={{ backgroundColor: `${provider.color?.length > 7 ? provider.color : (provider.color ?? "#888") + "15"}` }}
>
<ProviderIcon
src={`/providers/${provider.id}.png`}
alt={provider.name}
size={30}
className="object-contain rounded-lg max-w-[30px] max-h-[30px]"
fallbackText={provider.textIcon || provider.id.slice(0, 2).toUpperCase()}
fallbackColor={provider.color}
/>
</div>
<div>
<h3 className="font-semibold text-sm">{provider.name}</h3>
<div className="flex items-center gap-2 mt-0.5 flex-wrap">
{isCustom && <Badge variant="default" size="sm">Custom</Badge>}
{renderStatus()}
<div className="flex min-w-0 items-center justify-between gap-3">
<div className="flex min-w-0 items-center gap-3">
<div
className="size-8 rounded-lg flex items-center justify-center shrink-0"
style={{ backgroundColor: `${provider.color?.length > 7 ? provider.color : (provider.color ?? "#888") + "15"}` }}
>
<ProviderIcon
src={`/providers/${provider.id}.png`}
alt={provider.name}
size={30}
className="object-contain rounded-lg max-w-[30px] max-h-[30px]"
fallbackText={provider.textIcon || provider.id.slice(0, 2).toUpperCase()}
fallbackColor={provider.color}
/>
</div>
<div className="min-w-0">
<h3 className="font-semibold text-sm">{provider.name}</h3>
<div className="flex items-center gap-2 mt-0.5 flex-wrap">
{isCustom && <Badge variant="default" size="sm">Custom</Badge>}
{renderStatus()}
</div>
</div>
</div>
{total > 0 && (
<div
className="shrink-0 opacity-100 transition-opacity sm:opacity-0 sm:group-hover:opacity-100"
onClick={handleToggleClick}
>
<Toggle
size="sm"
checked={!allDisabled}
onChange={() => {}}
title={allDisabled ? "Enable provider" : "Disable provider"}
/>
</div>
)}
</div>
</Card>
</Link>
@ -170,6 +191,22 @@ export default function MediaProviderKindPage() {
const allProviders = [...providers, ...customProviders];
const handleToggleProvider = async (providerId, newActive) => {
const providerConns = connections.filter((c) => c.provider === providerId);
setConnections((prev) =>
prev.map((c) => (c.provider === providerId ? { ...c, isActive: newActive } : c))
);
await Promise.allSettled(
providerConns.map((c) =>
fetch(`/api/providers/${c.id}`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ isActive: newActive }),
})
)
);
};
const handleCreateCombo = async () => {
const base = COMBO_BASE_NAMES[kind] || `${kind}-combo`;
let name = base;
@ -221,6 +258,7 @@ export default function MediaProviderKindPage() {
provider={provider}
kind={kind}
connections={connections}
onToggle={handleToggleProvider}
/>
))}
{customProviders.map((provider) => (
@ -230,6 +268,7 @@ export default function MediaProviderKindPage() {
kind={kind}
connections={connections}
isCustom
onToggle={handleToggleProvider}
/>
))}
</div>

2
src/app/(dashboard)/dashboard/media-providers/combo/[id]/page.js
View file

@ -235,7 +235,7 @@ export default function ComboDetailPage() {
const examplePath = EXAMPLE_PATHS[combo.kind];
const exampleBody = combo.kind && EXAMPLE_BODIES[combo.kind] ? EXAMPLE_BODIES[combo.kind](combo.name) : null;
const curlExample = examplePath
? `curl -X POST http://localhost:20128${examplePath} \\\n -H "Content-Type: application/json" \\\n -H "Authorization: Bearer ${apiKey || "YOUR_KEY"}" \\\n -d '${JSON.stringify(exampleBody)}'`
? `curl -X POST http://127.0.0.1:20128${examplePath} \\\n -H "Content-Type: application/json" \\\n -H "Authorization: Bearer ${apiKey || "YOUR_KEY"}" \\\n -d '${JSON.stringify(exampleBody)}'`
: "";
const backHref = getListingHref(combo.kind);

2
src/app/(dashboard)/dashboard/profile/page.js
View file

@ -389,7 +389,7 @@ export default function ProfilePage() {
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between p-3 rounded-lg bg-bg border border-border gap-2">
<div>
<p className="font-medium text-sm sm:text-base">Database Location</p>
<p className="text-xs sm:text-sm text-text-muted font-mono break-all">~/.9router/db.json</p>
<p className="text-xs sm:text-sm text-text-muted font-mono break-all">~/.9router/db/data.sqlite</p>
</div>
</div>
<div className="flex flex-col sm:flex-row gap-2">

38
src/app/api/cli-tools/all-statuses/route.js Normal file
View file

@ -0,0 +1,38 @@
"use server";
import { NextResponse } from "next/server";
import { GET as claudeGet } from "../claude-settings/route";
import { GET as codexGet } from "../codex-settings/route";
import { GET as opencodeGet } from "../opencode-settings/route";
import { GET as droidGet } from "../droid-settings/route";
import { GET as openclawGet } from "../openclaw-settings/route";
import { GET as hermesGet } from "../hermes-settings/route";
import { GET as coworkGet } from "../cowork-settings/route";
import { GET as copilotGet } from "../copilot-settings/route";
const STATUS_GETTERS = {
claude: claudeGet,
codex: codexGet,
opencode: opencodeGet,
droid: droidGet,
openclaw: openclawGet,
hermes: hermesGet,
cowork: coworkGet,
copilot: copilotGet,
};
// Batch endpoint: gather all CLI tool statuses in one round-trip
export async function GET() {
const entries = await Promise.all(
Object.entries(STATUS_GETTERS).map(async ([toolId, getter]) => {
try {
const res = await getter();
const data = await res.json();
return [toolId, data];
} catch {
return [toolId, null];
}
})
);
return NextResponse.json(Object.fromEntries(entries));
}

2
src/app/api/cli-tools/antigravity-mitm/route.js
View file

@ -16,7 +16,7 @@ import { getSettings, updateSettings } from "@/lib/localDb";
initDbHooks(getSettings, updateSettings);
const DEFAULT_MITM_ROUTER_BASE = "http://localhost:20128";
const DEFAULT_MITM_ROUTER_BASE = "http://127.0.0.1:20128";
function normalizeMitmRouterBaseUrlInput(input) {
if (input == null || String(input).trim() === "") {

30
src/app/api/tunnel/tailscale-check/route.js
View file

@ -1,28 +1,31 @@
import os from "os";
import { execSync } from "child_process";
import { exec } from "child_process";
import { promisify } from "util";
import { NextResponse } from "next/server";
import { isTailscaleInstalled, isTailscaleLoggedIn, TAILSCALE_SOCKET } from "@/lib/tunnel/tailscale";
const execAsync = promisify(exec);
const EXTENDED_PATH = `/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:${process.env.PATH || ""}`;
const PROBE_TIMEOUT_MS = 1500;
function hasBrew() {
try { execSync("which brew", { stdio: "ignore", windowsHide: true, env: { ...process.env, PATH: EXTENDED_PATH } }); return true; } catch { return false; }
async function hasBrew() {
try {
await execAsync("which brew", { windowsHide: true, env: { ...process.env, PATH: EXTENDED_PATH }, timeout: PROBE_TIMEOUT_MS });
return true;
} catch { return false; }
}
function isDaemonRunning() {
async function isDaemonRunning() {
try {
// Use custom socket + --json; exit 0 even when not logged in
execSync(`tailscale --socket ${TAILSCALE_SOCKET} status --json`, {
stdio: "ignore",
await execAsync(`tailscale --socket ${TAILSCALE_SOCKET} status --json`, {
windowsHide: true,
env: { ...process.env, PATH: EXTENDED_PATH },
timeout: 3000
timeout: PROBE_TIMEOUT_MS
});
return true;
} catch {
// Fallback: check if tailscaled process is alive
try {
execSync("pgrep -x tailscaled", { stdio: "ignore", windowsHide: true, timeout: 2000 });
await execAsync("pgrep -x tailscaled", { windowsHide: true, timeout: PROBE_TIMEOUT_MS });
return true;
} catch { return false; }
}
@ -32,8 +35,11 @@ export async function GET() {
try {
const installed = isTailscaleInstalled();
const platform = os.platform();
const brewAvailable = platform === "darwin" && hasBrew();
const daemonRunning = installed ? isDaemonRunning() : false;
// Run independent probes in parallel — none blocks the event loop
const [brewAvailable, daemonRunning] = await Promise.all([
platform === "darwin" ? hasBrew() : Promise.resolve(false),
installed ? isDaemonRunning() : Promise.resolve(false),
]);
const loggedIn = daemonRunning ? isTailscaleLoggedIn() : false;
return NextResponse.json({ installed, loggedIn, platform, brewAvailable, daemonRunning });
} catch (error) {

10
src/app/landing/components/GetStarted.js
View file

@ -40,7 +40,7 @@ export default function GetStarted() {
<div className="flex-none w-8 h-8 rounded-full bg-[#f97815]/20 text-[#f97815] flex items-center justify-center font-bold">3</div>
<div>
<h4 className="font-bold text-lg">Route Requests</h4>
<p className="text-sm text-gray-500 mt-1">Point your CLI tools to http://localhost:20128</p>
<p className="text-sm text-gray-500 mt-1">Point your CLI tools to http://127.0.0.1:20128</p>
</div>
</div>
</div>
@ -72,8 +72,8 @@ export default function GetStarted() {
<div className="text-gray-400 mb-6">
<span className="text-[#f97815]">&gt;</span> Starting 9Router...<br/>
<span className="text-[#f97815]">&gt;</span> Server running on <span className="text-blue-400">http://localhost:20128</span><br/>
<span className="text-[#f97815]">&gt;</span> Dashboard: <span className="text-blue-400">http://localhost:20128/dashboard</span><br/>
<span className="text-[#f97815]">&gt;</span> Server running on <span className="text-blue-400">http://127.0.0.1:20128</span><br/>
<span className="text-[#f97815]">&gt;</span> Dashboard: <span className="text-blue-400">http://127.0.0.1:20128/dashboard</span><br/>
<span className="text-green-400">&gt;</span> Ready to route! ✓
</div>
@ -83,8 +83,8 @@ export default function GetStarted() {
<div className="text-gray-400 text-xs">
<span className="text-purple-400">Data Location:</span><br/>
<span className="text-gray-500"> macOS/Linux:</span> ~/.9router/db.json<br/>
<span className="text-gray-500"> Windows:</span> %APPDATA%/9router/db.json
<span className="text-gray-500"> macOS/Linux:</span> ~/.9router/db/data.sqlite<br/>
<span className="text-gray-500"> Windows:</span> %APPDATA%/9router/db/data.sqlite
</div>
</div>
</div>

18
src/app/layout.js
View file

@ -30,13 +30,25 @@ export default function RootLayout({ children }) {
return (
<html lang="en" suppressHydrationWarning>
<head>
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossOrigin="anonymous" />
{/* eslint-disable-next-line @next/next/no-page-custom-font */}
{/* Non-blocking icon font: preload + inject stylesheet via script */}
<link
rel="preload"
as="style"
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200&display=swap"
rel="stylesheet"
/>
<script
dangerouslySetInnerHTML={{
__html: `(function(){var l=document.createElement('link');l.rel='stylesheet';l.href='https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200&display=swap';document.head.appendChild(l);})();`,
}}
/>
<noscript>
{/* eslint-disable-next-line @next/next/no-page-custom-font */}
<link
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200&display=swap"
rel="stylesheet"
/>
</noscript>
</head>
<body className={`${inter.variable} font-sans antialiased`}>
<ThemeProvider>

55
src/lib/db/adapters/betterSqliteAdapter.js Normal file
View file

@ -0,0 +1,55 @@
import Database from "better-sqlite3";
import { PRAGMA_SQL } from "../schema.js";
// Periodic checkpoint to keep WAL file small (avoid huge -wal/-shm growth)
const CHECKPOINT_INTERVAL_MS = 60 * 1000;
export function createBetterSqliteAdapter(filePath) {
const db = new Database(filePath);
db.exec(PRAGMA_SQL);
// Schema is created/synced by migrate.js after adapter init
const stmtCache = new Map();
function prepare(sql) {
let stmt = stmtCache.get(sql);
if (!stmt) {
stmt = db.prepare(sql);
stmtCache.set(sql, stmt);
}
return stmt;
}
// Truncate WAL periodically so file stays small for backup/copy
const checkpointTimer = setInterval(() => {
try { db.pragma("wal_checkpoint(TRUNCATE)"); } catch {}
}, CHECKPOINT_INTERVAL_MS);
if (typeof checkpointTimer.unref === "function") checkpointTimer.unref();
function gracefulClose() {
try { db.pragma("wal_checkpoint(TRUNCATE)"); } catch {}
try { stmtCache.clear(); } catch {}
try { db.close(); } catch {}
}
// Ensure WAL is flushed and -wal/-shm files removed on shutdown
const onShutdown = () => gracefulClose();
process.once("beforeExit", onShutdown);
process.once("SIGINT", () => { onShutdown(); process.exit(0); });
process.once("SIGTERM", () => { onShutdown(); process.exit(0); });
return {
driver: "better-sqlite3",
run(sql, params = []) { return prepare(sql).run(params); },
get(sql, params = []) { return prepare(sql).get(params); },
all(sql, params = []) { return prepare(sql).all(params); },
exec(sql) { return db.exec(sql); },
transaction(fn) { return db.transaction(fn)(); },
checkpoint() { try { db.pragma("wal_checkpoint(TRUNCATE)"); } catch {} },
close() {
clearInterval(checkpointTimer);
gracefulClose();
},
raw: db,
};
}

114
src/lib/db/adapters/sqljsAdapter.js Normal file
View file

@ -0,0 +1,114 @@
import fs from "node:fs";
import initSqlJs from "sql.js";
import { PRAGMA_SQL } from "../schema.js";
let SQL = null;
async function loadSql() {
if (SQL) return SQL;
SQL = await initSqlJs();
return SQL;
}
export async function createSqlJsAdapter(filePath) {
const SQLLib = await loadSql();
const buf = fs.existsSync(filePath) ? fs.readFileSync(filePath) : null;
const db = new SQLLib.Database(buf);
db.exec(PRAGMA_SQL);
// Schema is created/synced by migrate.js after adapter init
let dirty = false;
let saveTimer = null;
const SAVE_DEBOUNCE_MS = 100;
function persist() {
const data = db.export();
fs.writeFileSync(filePath, Buffer.from(data));
dirty = false;
}
function scheduleSave() {
dirty = true;
if (saveTimer) clearTimeout(saveTimer);
saveTimer = setTimeout(() => {
saveTimer = null;
if (dirty) {
try { persist(); } catch (e) { console.error("[sqljs] save failed:", e); }
}
}, SAVE_DEBOUNCE_MS);
}
function paramsObj(params) {
if (!params || (Array.isArray(params) && params.length === 0)) return undefined;
return params;
}
function run(sql, params = []) {
const stmt = db.prepare(sql);
try {
stmt.bind(paramsObj(params));
stmt.step();
const changes = db.getRowsModified();
const lastInsertRowid = db.exec("SELECT last_insert_rowid() as id")[0]?.values?.[0]?.[0] ?? null;
scheduleSave();
return { changes, lastInsertRowid };
} finally {
stmt.free();
}
}
function get(sql, params = []) {
const stmt = db.prepare(sql);
try {
stmt.bind(paramsObj(params));
if (stmt.step()) return stmt.getAsObject();
return undefined;
} finally {
stmt.free();
}
}
function all(sql, params = []) {
const stmt = db.prepare(sql);
try {
stmt.bind(paramsObj(params));
const rows = [];
while (stmt.step()) rows.push(stmt.getAsObject());
return rows;
} finally {
stmt.free();
}
}
function exec(sql) {
db.exec(sql);
scheduleSave();
}
function transaction(fn) {
db.exec("BEGIN");
try {
const result = fn();
db.exec("COMMIT");
scheduleSave();
return result;
} catch (e) {
db.exec("ROLLBACK");
throw e;
}
}
function close() {
if (saveTimer) clearTimeout(saveTimer);
if (dirty) persist();
db.close();
}
// Flush on shutdown
const flush = () => { if (dirty) try { persist(); } catch {} };
process.on("beforeExit", flush);
process.on("SIGINT", flush);
process.on("SIGTERM", flush);
return { driver: "sql.js", run, get, all, exec, transaction, close, raw: db };
}

35
src/lib/db/backup.js Normal file
View file

@ -0,0 +1,35 @@
import fs from "node:fs";
import path from "node:path";
import { BACKUPS_DIR, ensureDirs } from "./paths.js";
import { timestampSlug, getAppVersion } from "./version.js";
const KEEP_BACKUPS = 5;
export function makeBackupDir(label) {
ensureDirs();
const ver = getAppVersion();
const slug = `${label}-${ver}-${timestampSlug()}`;
const dir = path.join(BACKUPS_DIR, slug);
fs.mkdirSync(dir, { recursive: true });
return dir;
}
export function backupFile(srcPath, destDir, destName = null) {
if (!fs.existsSync(srcPath)) return null;
const name = destName || path.basename(srcPath);
const dest = path.join(destDir, name);
fs.copyFileSync(srcPath, dest);
return dest;
}
export function pruneOldBackups() {
if (!fs.existsSync(BACKUPS_DIR)) return;
const entries = fs.readdirSync(BACKUPS_DIR, { withFileTypes: true })
.filter((e) => e.isDirectory())
.map((e) => ({ name: e.name, full: path.join(BACKUPS_DIR, e.name), mtime: fs.statSync(path.join(BACKUPS_DIR, e.name)).mtimeMs }))
.sort((a, b) => b.mtime - a.mtime);
for (const old of entries.slice(KEEP_BACKUPS)) {
try { fs.rmSync(old.full, { recursive: true, force: true }); } catch {}
}
}

52
src/lib/db/driver.js Normal file
View file

@ -0,0 +1,52 @@
import { ensureDirs, DATA_FILE } from "./paths.js";
// Use global to survive Next.js dev hot-reload (module state resets on reload)
if (!global._dbAdapter) global._dbAdapter = { instance: null, initPromise: null, logged: false };
const state = global._dbAdapter;
async function tryBetterSqlite() {
try {
const { createBetterSqliteAdapter } = await import("./adapters/betterSqliteAdapter.js");
return createBetterSqliteAdapter(DATA_FILE);
} catch (e) {
console.warn(`[DB] better-sqlite3 unavailable: ${e.message}`);
return null;
}
}
async function trySqlJs() {
try {
const { createSqlJsAdapter } = await import("./adapters/sqljsAdapter.js");
return await createSqlJsAdapter(DATA_FILE);
} catch (e) {
console.warn(`[DB] sql.js unavailable: ${e.message}`);
return null;
}
}
async function initAdapter() {
ensureDirs();
let adapter = await tryBetterSqlite();
if (!adapter) adapter = await trySqlJs();
if (!adapter) throw new Error("[DB] No SQLite driver available (better-sqlite3 + sql.js both failed)");
if (!state.logged) {
console.log(`[DB] Driver: ${adapter.driver} | file: ${DATA_FILE}`);
state.logged = true;
}
const { runMigrationOnce } = await import("./migrate.js");
await runMigrationOnce(adapter);
return adapter;
}
export async function getAdapter() {
if (state.instance) return state.instance;
if (!state.initPromise) state.initPromise = initAdapter().then((a) => { state.instance = a; return a; });
return state.initPromise;
}
export function getAdapterSync() {
if (!state.instance) throw new Error("[DB] adapter not initialized — await getAdapter() first");
return state.instance;
}

9
src/lib/db/helpers/jsonCol.js Normal file
View file

@ -0,0 +1,9 @@
export function parseJson(str, fallback = null) {
if (str == null) return fallback;
if (typeof str !== "string") return str;
try { return JSON.parse(str); } catch { return fallback; }
}
export function stringifyJson(value) {
return JSON.stringify(value ?? null);
}

39
src/lib/db/helpers/kvStore.js Normal file
View file

@ -0,0 +1,39 @@
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "./jsonCol.js";
export function makeKv(scope) {
return {
async get(key, fallback = null) {
const db = await getAdapter();
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [scope, key]);
return row ? parseJson(row.value, fallback) : fallback;
},
async getAll() {
const db = await getAdapter();
const rows = db.all(`SELECT key, value FROM kv WHERE scope = ?`, [scope]);
const out = {};
for (const r of rows) out[r.key] = parseJson(r.value);
return out;
},
async set(key, value) {
const db = await getAdapter();
db.run(`INSERT INTO kv(scope, key, value) VALUES(?, ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`, [scope, key, stringifyJson(value)]);
},
async setMany(obj) {
const db = await getAdapter();
db.transaction(() => {
for (const [k, v] of Object.entries(obj)) {
db.run(`INSERT INTO kv(scope, key, value) VALUES(?, ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`, [scope, k, stringifyJson(v)]);
}
});
},
async remove(key) {
const db = await getAdapter();
db.run(`DELETE FROM kv WHERE scope = ? AND key = ?`, [scope, key]);
},
async clear() {
const db = await getAdapter();
db.run(`DELETE FROM kv WHERE scope = ?`, [scope]);
},
};
}

22
src/lib/db/helpers/metaStore.js Normal file
View file

@ -0,0 +1,22 @@
import { getAdapter } from "../driver.js";
export async function getMeta(key, fallback = null) {
const db = await getAdapter();
const row = db.get(`SELECT value FROM _meta WHERE key = ?`, [key]);
return row ? row.value : fallback;
}
export async function setMeta(key, value) {
const db = await getAdapter();
db.run(`INSERT INTO _meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, [key, String(value)]);
}
// Sync versions for use during migration (adapter passed directly)
export function getMetaSync(adapter, key, fallback = null) {
const row = adapter.get(`SELECT value FROM _meta WHERE key = ?`, [key]);
return row ? row.value : fallback;
}
export function setMetaSync(adapter, key, value) {
adapter.run(`INSERT INTO _meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, [key, String(value)]);
}

171
src/lib/db/index.js Normal file
View file

@ -0,0 +1,171 @@
// Public API barrel — all DB functions
import { getAdapter } from "./driver.js";
import { stringifyJson, parseJson } from "./helpers/jsonCol.js";
// Settings
export {
getSettings, updateSettings, isCloudEnabled, getCloudUrl, exportSettings,
} from "./repos/settingsRepo.js";
// Provider connections
export {
getProviderConnections, getProviderConnectionById,
createProviderConnection, updateProviderConnection,
deleteProviderConnection, deleteProviderConnectionsByProvider,
reorderProviderConnections, cleanupProviderConnections,
} from "./repos/connectionsRepo.js";
// Provider nodes
export {
getProviderNodes, getProviderNodeById,
createProviderNode, updateProviderNode, deleteProviderNode,
} from "./repos/nodesRepo.js";
// Proxy pools
export {
getProxyPools, getProxyPoolById,
createProxyPool, updateProxyPool, deleteProxyPool,
} from "./repos/proxyPoolsRepo.js";
// API keys
export {
getApiKeys, getApiKeyById, createApiKey, updateApiKey, deleteApiKey, validateApiKey,
} from "./repos/apiKeysRepo.js";
// Combos
export {
getCombos, getComboById, getComboByName,
createCombo, updateCombo, deleteCombo,
} from "./repos/combosRepo.js";
// Aliases (model + custom + mitm)
export {
getModelAliases, setModelAlias, deleteModelAlias,
getCustomModels, addCustomModel, deleteCustomModel,
getMitmAlias, setMitmAliasAll,
} from "./repos/aliasRepo.js";
// Pricing
export {
getPricing, getPricingForModel, updatePricing, resetPricing, resetAllPricing,
} from "./repos/pricingRepo.js";
// Disabled models
export {
getDisabledModels, getDisabledByProvider, disableModels, enableModels,
} from "./repos/disabledModelsRepo.js";
// Usage
export {
statsEmitter, trackPendingRequest, getActiveRequests,
saveRequestUsage, getUsageHistory, getUsageStats, getChartData,
appendRequestLog, getRecentLogs,
} from "./repos/usageRepo.js";
// Request details
export {
saveRequestDetail, getRequestDetails, getRequestDetailById,
} from "./repos/requestDetailsRepo.js";
// Export/import full DB
export async function exportDb() {
const db = await getAdapter();
const { exportSettings } = await import("./repos/settingsRepo.js");
const out = {
settings: await exportSettings(),
providerConnections: db.all(`SELECT * FROM providerConnections`).map((r) => ({ ...parseJson(r.data, {}), id: r.id, provider: r.provider, authType: r.authType, name: r.name, email: r.email, priority: r.priority, isActive: r.isActive === 1, createdAt: r.createdAt, updatedAt: r.updatedAt })),
providerNodes: db.all(`SELECT * FROM providerNodes`).map((r) => ({ ...parseJson(r.data, {}), id: r.id, type: r.type, name: r.name, createdAt: r.createdAt, updatedAt: r.updatedAt })),
proxyPools: db.all(`SELECT * FROM proxyPools`).map((r) => ({ ...parseJson(r.data, {}), id: r.id, isActive: r.isActive === 1, testStatus: r.testStatus, createdAt: r.createdAt, updatedAt: r.updatedAt })),
apiKeys: db.all(`SELECT * FROM apiKeys`).map((r) => ({ id: r.id, key: r.key, name: r.name, machineId: r.machineId, isActive: r.isActive === 1, createdAt: r.createdAt })),
combos: db.all(`SELECT * FROM combos`).map((r) => ({ id: r.id, name: r.name, kind: r.kind, models: parseJson(r.models, []), createdAt: r.createdAt, updatedAt: r.updatedAt })),
modelAliases: {},
customModels: [],
mitmAlias: {},
pricing: {},
};
for (const r of db.all(`SELECT key, value FROM kv WHERE scope = 'modelAliases'`)) out.modelAliases[r.key] = parseJson(r.value);
for (const r of db.all(`SELECT key, value FROM kv WHERE scope = 'customModels'`)) out.customModels.push(parseJson(r.value));
for (const r of db.all(`SELECT key, value FROM kv WHERE scope = 'mitmAlias'`)) out.mitmAlias[r.key] = parseJson(r.value);
for (const r of db.all(`SELECT key, value FROM kv WHERE scope = 'pricing'`)) out.pricing[r.key] = parseJson(r.value);
return out;
}
export async function importDb(payload) {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
throw new Error("Invalid database payload");
}
const db = await getAdapter();
db.transaction(() => {
// Wipe all tables (keep _meta)
db.run(`DELETE FROM settings`);
db.run(`DELETE FROM providerConnections`);
db.run(`DELETE FROM providerNodes`);
db.run(`DELETE FROM proxyPools`);
db.run(`DELETE FROM apiKeys`);
db.run(`DELETE FROM combos`);
db.run(`DELETE FROM kv WHERE scope IN ('modelAliases', 'customModels', 'mitmAlias', 'pricing')`);
// Settings
if (payload.settings) {
db.run(`INSERT INTO settings(id, data) VALUES(1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data`, [stringifyJson(payload.settings)]);
}
for (const c of payload.providerConnections || []) {
const { id, provider, authType, name, email, priority, isActive, createdAt, updatedAt, ...rest } = c;
db.run(
`INSERT OR REPLACE INTO providerConnections(id, provider, authType, name, email, priority, isActive, data, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[id, provider, authType || "oauth", name || null, email || null, priority || null, isActive === false ? 0 : 1, stringifyJson(rest), createdAt || new Date().toISOString(), updatedAt || new Date().toISOString()]
);
}
for (const n of payload.providerNodes || []) {
const { id, type, name, createdAt, updatedAt, ...rest } = n;
db.run(
`INSERT OR REPLACE INTO providerNodes(id, type, name, data, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[id, type || null, name || null, stringifyJson(rest), createdAt || new Date().toISOString(), updatedAt || new Date().toISOString()]
);
}
for (const p of payload.proxyPools || []) {
const { id, isActive, testStatus, createdAt, updatedAt, ...rest } = p;
db.run(
`INSERT OR REPLACE INTO proxyPools(id, isActive, testStatus, data, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[id, isActive === false ? 0 : 1, testStatus || "unknown", stringifyJson(rest), createdAt || new Date().toISOString(), updatedAt || new Date().toISOString()]
);
}
for (const k of payload.apiKeys || []) {
db.run(
`INSERT OR REPLACE INTO apiKeys(id, key, name, machineId, isActive, createdAt) VALUES(?, ?, ?, ?, ?, ?)`,
[k.id, k.key, k.name || null, k.machineId || null, k.isActive === false ? 0 : 1, k.createdAt || new Date().toISOString()]
);
}
for (const c of payload.combos || []) {
db.run(
`INSERT OR REPLACE INTO combos(id, name, kind, models, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[c.id, c.name, c.kind || null, stringifyJson(c.models || []), c.createdAt || new Date().toISOString(), c.updatedAt || new Date().toISOString()]
);
}
for (const [a, m] of Object.entries(payload.modelAliases || {})) {
db.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('modelAliases', ?, ?)`, [a, stringifyJson(m)]);
}
for (const m of payload.customModels || []) {
const k = `${m.providerAlias}|${m.id}|${m.type || "llm"}`;
db.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('customModels', ?, ?)`, [k, stringifyJson(m)]);
}
for (const [tool, mappings] of Object.entries(payload.mitmAlias || {})) {
db.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('mitmAlias', ?, ?)`, [tool, stringifyJson(mappings || {})]);
}
for (const [provider, models] of Object.entries(payload.pricing || {})) {
db.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('pricing', ?, ?)`, [provider, stringifyJson(models || {})]);
}
});
return await exportDb();
}
// Eager init helper (optional)
export async function initDb() {
await getAdapter();
}

248
src/lib/db/migrate.js Normal file
View file

@ -0,0 +1,248 @@
import fs from "node:fs";
import path from "node:path";
import { LEGACY_FILES, DB_DIR, DATA_FILE } from "./paths.js";
import { TABLES, buildCreateTableSql } from "./schema.js";
import { MIGRATIONS, latestVersion } from "./migrations/index.js";
import { getMetaSync, setMetaSync } from "./helpers/metaStore.js";
import { makeBackupDir, backupFile, pruneOldBackups } from "./backup.js";
import { getAppVersion } from "./version.js";
import { stringifyJson } from "./helpers/jsonCol.js";
// Marker file: prevents re-importing legacy JSON when user wipes data.sqlite.
const MIGRATED_MARKER = path.join(DB_DIR, ".migrated-from-json");
// Track per-adapter so reusing same adapter skips re-run, but new adapter (after reset) re-runs.
const _migratedAdapters = new WeakSet();
function readJsonSafe(file) {
if (!fs.existsSync(file)) return null;
try { return JSON.parse(fs.readFileSync(file, "utf-8")); } catch { return null; }
}
function isFreshDb(adapter) {
// Table _meta may not exist yet on truly fresh DB
try {
const row = adapter.get(`SELECT COUNT(*) as c FROM _meta`);
return !row || row.c === 0;
} catch {
return true;
}
}
// ─── Versioned migrations runner (skip-version safe) ─────────────────────
function runVersionedMigrations(adapter) {
// Bootstrap _meta first so we can read schemaVersion
adapter.exec(buildCreateTableSql("_meta", TABLES._meta));
const current = parseInt(getMetaSync(adapter, "schemaVersion", "0"), 10) || 0;
const target = latestVersion();
if (current >= target) return { applied: 0, from: current, to: current };
const pending = MIGRATIONS.filter((m) => m.version > current);
let lastApplied = current;
for (const m of pending) {
adapter.transaction(() => {
m.up(adapter);
setMetaSync(adapter, "schemaVersion", m.version);
});
lastApplied = m.version;
console.log(`[DB][migrate] applied #${m.version} ${m.name}`);
}
return { applied: pending.length, from: current, to: lastApplied };
}
// ─── Auto-sync (additive only): add missing tables/columns/indexes ───────
function syncSchemaFromTables(adapter) {
for (const [tableName, def] of Object.entries(TABLES)) {
// Create table if absent
adapter.exec(buildCreateTableSql(tableName, def));
// Diff columns
const existing = adapter.all(`PRAGMA table_info(${tableName})`);
const existingNames = new Set(existing.map((r) => r.name));
for (const [colName, colDef] of Object.entries(def.columns)) {
if (!existingNames.has(colName)) {
// SQLite ADD COLUMN restrictions: no PRIMARY KEY / UNIQUE w/o NULL ok.
// We strip PRIMARY KEY / UNIQUE since those are only valid at create time.
const safeDef = colDef
.replace(/PRIMARY KEY( AUTOINCREMENT)?/i, "")
.replace(/UNIQUE/i, "")
.trim();
try {
adapter.exec(`ALTER TABLE ${tableName} ADD COLUMN ${colName} ${safeDef}`);
console.log(`[DB][sync] +column ${tableName}.${colName}`);
} catch (e) {
console.warn(`[DB][sync] add column ${tableName}.${colName} failed: ${e.message}`);
}
}
}
// Indexes (idempotent)
for (const idx of def.indexes || []) {
try { adapter.exec(idx); } catch {}
}
}
}
// ─── Legacy JSON import (one-time) ───────────────────────────────────────
function importLegacyMain(adapter, data) {
if (!data || typeof data !== "object") return;
if (data.settings) {
adapter.run(`INSERT INTO settings(id, data) VALUES(1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data`, [stringifyJson(data.settings)]);
}
for (const c of data.providerConnections || []) {
const { id, provider, authType, name, email, priority, isActive, createdAt, updatedAt, ...rest } = c;
adapter.run(
`INSERT OR REPLACE INTO providerConnections(id, provider, authType, name, email, priority, isActive, data, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[id, provider, authType || "oauth", name || null, email || null, priority || null, isActive === false ? 0 : 1, stringifyJson(rest), createdAt || new Date().toISOString(), updatedAt || new Date().toISOString()]
);
}
for (const n of data.providerNodes || []) {
const { id, type, name, createdAt, updatedAt, ...rest } = n;
adapter.run(
`INSERT OR REPLACE INTO providerNodes(id, type, name, data, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[id, type || null, name || null, stringifyJson(rest), createdAt || new Date().toISOString(), updatedAt || new Date().toISOString()]
);
}
for (const p of data.proxyPools || []) {
const { id, isActive, testStatus, createdAt, updatedAt, ...rest } = p;
adapter.run(
`INSERT OR REPLACE INTO proxyPools(id, isActive, testStatus, data, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[id, isActive === false ? 0 : 1, testStatus || "unknown", stringifyJson(rest), createdAt || new Date().toISOString(), updatedAt || new Date().toISOString()]
);
}
for (const k of data.apiKeys || []) {
adapter.run(
`INSERT OR REPLACE INTO apiKeys(id, key, name, machineId, isActive, createdAt) VALUES(?, ?, ?, ?, ?, ?)`,
[k.id, k.key, k.name || null, k.machineId || null, k.isActive === false ? 0 : 1, k.createdAt || new Date().toISOString()]
);
}
for (const c of data.combos || []) {
adapter.run(
`INSERT OR REPLACE INTO combos(id, name, kind, models, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[c.id, c.name, c.kind || null, stringifyJson(c.models || []), c.createdAt || new Date().toISOString(), c.updatedAt || new Date().toISOString()]
);
}
for (const [alias, model] of Object.entries(data.modelAliases || {})) {
adapter.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('modelAliases', ?, ?)`, [alias, stringifyJson(model)]);
}
for (const m of data.customModels || []) {
const k = `${m.providerAlias}|${m.id}|${m.type || "llm"}`;
adapter.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('customModels', ?, ?)`, [k, stringifyJson(m)]);
}
for (const [tool, mappings] of Object.entries(data.mitmAlias || {})) {
adapter.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('mitmAlias', ?, ?)`, [tool, stringifyJson(mappings || {})]);
}
for (const [provider, models] of Object.entries(data.pricing || {})) {
adapter.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('pricing', ?, ?)`, [provider, stringifyJson(models || {})]);
}
}
function importLegacyUsage(adapter, data) {
if (!data || typeof data !== "object") return;
for (const e of data.history || []) {
const t = e.tokens || {};
adapter.run(
`INSERT INTO usageHistory(timestamp, provider, model, connectionId, apiKey, endpoint, promptTokens, completionTokens, cost, status, tokens, meta) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
e.timestamp || new Date().toISOString(),
e.provider || null, e.model || null, e.connectionId || null, e.apiKey || null, e.endpoint || null,
t.prompt_tokens || t.input_tokens || 0,
t.completion_tokens || t.output_tokens || 0,
e.cost || 0,
e.status || "ok",
stringifyJson(t),
stringifyJson({}),
]
);
}
for (const [dateKey, day] of Object.entries(data.dailySummary || {})) {
adapter.run(`INSERT OR REPLACE INTO usageDaily(dateKey, data) VALUES(?, ?)`, [dateKey, stringifyJson(day)]);
}
if (typeof data.totalRequestsLifetime === "number") {
setMetaSync(adapter, "totalRequestsLifetime", data.totalRequestsLifetime);
}
}
function importLegacyDisabled(adapter, data) {
if (!data || typeof data.disabled !== "object") return;
for (const [provider, ids] of Object.entries(data.disabled)) {
adapter.run(`INSERT OR REPLACE INTO kv(scope, key, value) VALUES('disabledModels', ?, ?)`, [provider, stringifyJson(ids || [])]);
}
}
function importLegacyDetails(adapter, data) {
if (!data || !Array.isArray(data.records)) return;
for (const r of data.records) {
adapter.run(
`INSERT OR REPLACE INTO requestDetails(id, timestamp, provider, model, connectionId, status, data) VALUES(?, ?, ?, ?, ?, ?, ?)`,
[r.id, r.timestamp || new Date().toISOString(), r.provider || null, r.model || null, r.connectionId || null, r.status || null, stringifyJson(r)]
);
}
}
// ─── Main entry ──────────────────────────────────────────────────────────
export async function runMigrationOnce(adapter) {
if (_migratedAdapters.has(adapter)) return;
_migratedAdapters.add(adapter);
// Capture freshness BEFORE migrations stamp _meta (otherwise we'd misclassify
// a brand-new DB as non-fresh once schemaVersion is written).
const fresh = isFreshDb(adapter);
// 1. Always run versioned migrations chain (skip-version safe)
const migInfo = runVersionedMigrations(adapter);
// 2. Additive sync (auto add missing columns/indexes declared in TABLES)
syncSchemaFromTables(adapter);
// 3. One-time legacy JSON import (only if DB was fresh on entry)
const alreadyImported = fs.existsSync(MIGRATED_MARKER);
const legacyMain = readJsonSafe(LEGACY_FILES.main);
const legacyUsage = readJsonSafe(LEGACY_FILES.usage);
const legacyDisabled = readJsonSafe(LEGACY_FILES.disabled);
const legacyDetails = readJsonSafe(LEGACY_FILES.details);
const hasLegacy = !!(legacyMain || legacyUsage || legacyDisabled || legacyDetails);
if (fresh && hasLegacy && !alreadyImported) {
const t0 = Date.now();
const backupDir = makeBackupDir("migrate-from-json");
for (const f of Object.values(LEGACY_FILES)) backupFile(f, backupDir);
adapter.transaction(() => {
importLegacyMain(adapter, legacyMain);
importLegacyUsage(adapter, legacyUsage);
importLegacyDisabled(adapter, legacyDisabled);
importLegacyDetails(adapter, legacyDetails);
setMetaSync(adapter, "appVersion", getAppVersion());
setMetaSync(adapter, "migratedAt", new Date().toISOString());
});
try { fs.writeFileSync(MIGRATED_MARKER, new Date().toISOString()); } catch {}
pruneOldBackups();
console.log(`[DB][migrate] JSON → SQLite in ${Date.now() - t0}ms | legacy JSON kept at DATA_DIR | backup: ${backupDir}`);
return;
}
if (fresh) {
setMetaSync(adapter, "appVersion", getAppVersion());
return;
}
// 4. App version bump → backup data.sqlite (safety net before user-side upgrade)
const oldVer = getMetaSync(adapter, "appVersion", null);
const newVer = getAppVersion();
if (oldVer && oldVer !== newVer) {
const backupDir = makeBackupDir(`upgrade-${oldVer}-to-${newVer}`);
try { backupFile(DATA_FILE, backupDir); } catch {}
setMetaSync(adapter, "appVersion", newVer);
pruneOldBackups();
console.log(`[DB][migrate] App ${oldVer} → ${newVer} | schema ${migInfo.from} → ${migInfo.to} | backup: ${backupDir}`);
} else if (migInfo.applied > 0) {
// Schema upgrade without app version bump — still backup
const backupDir = makeBackupDir(`schema-${migInfo.from}-to-${migInfo.to}`);
try { backupFile(DATA_FILE, backupDir); } catch {}
pruneOldBackups();
}
}

14
src/lib/db/migrations/001-initial.js Normal file
View file

@ -0,0 +1,14 @@
// Initial schema bootstrap. For fresh DB this creates all tables/indexes.
// For existing DB at version 0 (legacy unstamped), it's idempotent (IF NOT EXISTS).
import { TABLES, buildCreateTableSql } from "../schema.js";
export default {
version: 1,
name: "initial",
up(db) {
for (const [name, def] of Object.entries(TABLES)) {
db.exec(buildCreateTableSql(name, def));
for (const idx of def.indexes || []) db.exec(idx);
}
},
};

10
src/lib/db/migrations/index.js Normal file
View file

@ -0,0 +1,10 @@
// Migration registry — append new entries when schema changes.
// Each migration: { version: number, name: string, up(db): void }
// Versions MUST be unique and monotonically increasing.
import m001 from "./001-initial.js";
export const MIGRATIONS = [m001].sort((a, b) => a.version - b.version);
export function latestVersion() {
return MIGRATIONS.length ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
}

18
src/lib/db/paths.js Normal file
View file

@ -0,0 +1,18 @@
import path from "node:path";
import fs from "node:fs";
import { DATA_DIR } from "@/lib/dataDir.js";
export const DB_DIR = path.join(DATA_DIR, "db");
export const DATA_FILE = path.join(DB_DIR, "data.sqlite");
export const BACKUPS_DIR = path.join(DB_DIR, "backups");
export const LEGACY_FILES = {
main: path.join(DATA_DIR, "db.json"),
usage: path.join(DATA_DIR, "usage.json"),
disabled: path.join(DATA_DIR, "disabledModels.json"),
details: path.join(DATA_DIR, "request-details.json"),
};
export function ensureDirs() {
for (const dir of [DATA_DIR, DB_DIR, BACKUPS_DIR]) {
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}
}

62
src/lib/db/repos/aliasRepo.js Normal file
View file

@ -0,0 +1,62 @@
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
import { makeKv } from "../helpers/kvStore.js";
const aliasKv = makeKv("modelAliases");
const customKv = makeKv("customModels");
const mitmKv = makeKv("mitmAlias");
// modelAliases: key=alias, value=modelString
export async function getModelAliases() {
return await aliasKv.getAll();
}
export async function setModelAlias(alias, model) {
await aliasKv.set(alias, model);
}
export async function deleteModelAlias(alias) {
await aliasKv.remove(alias);
}
// customModels: key=`${providerAlias}|${id}|${type}`, value=full model object
function customKey(providerAlias, id, type) {
return `${providerAlias}|${id}|${type}`;
}
export async function getCustomModels() {
const all = await customKv.getAll();
return Object.values(all);
}
// Atomic check-then-insert inside transaction to prevent duplicate races
export async function addCustomModel({ providerAlias, id, type = "llm", name }) {
const k = customKey(providerAlias, id, type);
const db = await getAdapter();
let added = false;
db.transaction(() => {
const row = db.get(`SELECT 1 FROM kv WHERE scope = 'customModels' AND key = ?`, [k]);
if (row) return;
const value = stringifyJson({ providerAlias, id, type, name: name || id });
db.run(`INSERT INTO kv(scope, key, value) VALUES('customModels', ?, ?)`, [k, value]);
added = true;
});
return added;
}
export async function deleteCustomModel({ providerAlias, id, type = "llm" }) {
await customKv.remove(customKey(providerAlias, id, type));
}
// mitmAlias: key=toolName, value=mappings object
export async function getMitmAlias(toolName) {
if (toolName) {
const v = await mitmKv.get(toolName);
return v || {};
}
return await mitmKv.getAll();
}
export async function setMitmAliasAll(toolName, mappings) {
await mitmKv.set(toolName, mappings || {});
}

75
src/lib/db/repos/apiKeysRepo.js Normal file
View file

@ -0,0 +1,75 @@
import { v4 as uuidv4 } from "uuid";
import { getAdapter } from "../driver.js";
function rowToKey(row) {
if (!row) return null;
return {
id: row.id,
key: row.key,
name: row.name,
machineId: row.machineId,
isActive: row.isActive === 1 || row.isActive === true,
createdAt: row.createdAt,
};
}
export async function getApiKeys() {
const db = await getAdapter();
const rows = db.all(`SELECT * FROM apiKeys ORDER BY createdAt ASC`);
return rows.map(rowToKey);
}
export async function getApiKeyById(id) {
const db = await getAdapter();
const row = db.get(`SELECT * FROM apiKeys WHERE id = ?`, [id]);
return rowToKey(row);
}
export async function createApiKey(name, machineId) {
if (!machineId) throw new Error("machineId is required");
const db = await getAdapter();
const { generateApiKeyWithMachine } = await import("@/shared/utils/apiKey");
const result = generateApiKeyWithMachine(machineId);
const apiKey = {
id: uuidv4(),
name,
key: result.key,
machineId,
isActive: true,
createdAt: new Date().toISOString(),
};
db.run(
`INSERT INTO apiKeys(id, key, name, machineId, isActive, createdAt) VALUES(?, ?, ?, ?, ?, ?)`,
[apiKey.id, apiKey.key, apiKey.name, apiKey.machineId, 1, apiKey.createdAt]
);
return apiKey;
}
export async function updateApiKey(id, data) {
const db = await getAdapter();
let result = null;
db.transaction(() => {
const row = db.get(`SELECT * FROM apiKeys WHERE id = ?`, [id]);
if (!row) return;
const merged = { ...rowToKey(row), ...data };
db.run(
`UPDATE apiKeys SET key = ?, name = ?, machineId = ?, isActive = ? WHERE id = ?`,
[merged.key, merged.name, merged.machineId, merged.isActive ? 1 : 0, id]
);
result = merged;
});
return result;
}
export async function deleteApiKey(id) {
const db = await getAdapter();
const res = db.run(`DELETE FROM apiKeys WHERE id = ?`, [id]);
return (res?.changes ?? 0) > 0;
}
export async function validateApiKey(key) {
const db = await getAdapter();
const row = db.get(`SELECT isActive FROM apiKeys WHERE key = ?`, [key]);
if (!row) return false;
return row.isActive === 1 || row.isActive === true;
}

73
src/lib/db/repos/combosRepo.js Normal file
View file

@ -0,0 +1,73 @@
import { v4 as uuidv4 } from "uuid";
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
function rowToCombo(row) {
if (!row) return null;
return {
id: row.id,
name: row.name,
kind: row.kind,
models: parseJson(row.models, []),
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
export async function getCombos() {
const db = await getAdapter();
const rows = db.all(`SELECT * FROM combos ORDER BY createdAt ASC`);
return rows.map(rowToCombo);
}
export async function getComboById(id) {
const db = await getAdapter();
const row = db.get(`SELECT * FROM combos WHERE id = ?`, [id]);
return rowToCombo(row);
}
export async function getComboByName(name) {
const db = await getAdapter();
const row = db.get(`SELECT * FROM combos WHERE name = ?`, [name]);
return rowToCombo(row);
}
export async function createCombo(data) {
const db = await getAdapter();
const now = new Date().toISOString();
const combo = {
id: uuidv4(),
name: data.name,
kind: data.kind || null,
models: data.models || [],
createdAt: now,
updatedAt: now,
};
db.run(
`INSERT INTO combos(id, name, kind, models, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
[combo.id, combo.name, combo.kind, stringifyJson(combo.models), combo.createdAt, combo.updatedAt]
);
return combo;
}
export async function updateCombo(id, data) {
const db = await getAdapter();
let result = null;
db.transaction(() => {
const row = db.get(`SELECT * FROM combos WHERE id = ?`, [id]);
if (!row) return;
const merged = { ...rowToCombo(row), ...data, updatedAt: new Date().toISOString() };
db.run(
`UPDATE combos SET name = ?, kind = ?, models = ?, updatedAt = ? WHERE id = ?`,
[merged.name, merged.kind, stringifyJson(merged.models || []), merged.updatedAt, id]
);
result = merged;
});
return result;
}
export async function deleteCombo(id) {
const db = await getAdapter();
const res = db.run(`DELETE FROM combos WHERE id = ?`, [id]);
return (res?.changes ?? 0) > 0;
}

218
src/lib/db/repos/connectionsRepo.js Normal file
View file

@ -0,0 +1,218 @@
import { v4 as uuidv4 } from "uuid";
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
const OPTIONAL_FIELDS = [
"displayName", "email", "globalPriority", "defaultModel",
"accessToken", "refreshToken", "expiresAt", "tokenType",
"scope", "projectId", "apiKey", "testStatus",
"lastTested", "lastError", "lastErrorAt", "rateLimitedUntil", "expiresIn", "errorCode",
"consecutiveUseCount",
];
function rowToConn(row) {
if (!row) return null;
const extra = parseJson(row.data, {});
return {
...extra,
id: row.id,
provider: row.provider,
authType: row.authType,
name: row.name,
email: row.email,
priority: row.priority,
isActive: row.isActive === 1 || row.isActive === true,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
function connToRow(c) {
const { id, provider, authType, name, email, priority, isActive, createdAt, updatedAt, ...rest } = c;
return {
id,
provider,
authType,
name: name ?? null,
email: email ?? null,
priority: priority ?? null,
isActive: isActive === false ? 0 : 1,
data: stringifyJson(rest),
createdAt,
updatedAt,
};
}
function upsert(db, c) {
const r = connToRow(c);
db.run(
`INSERT INTO providerConnections(id, provider, authType, name, email, priority, isActive, data, createdAt, updatedAt)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
provider=excluded.provider, authType=excluded.authType, name=excluded.name,
email=excluded.email, priority=excluded.priority, isActive=excluded.isActive,
data=excluded.data, updatedAt=excluded.updatedAt`,
[r.id, r.provider, r.authType, r.name, r.email, r.priority, r.isActive, r.data, r.createdAt, r.updatedAt]
);
}
export async function getProviderConnections(filter = {}) {
const db = await getAdapter();
const where = [];
const params = [];
if (filter.provider) { where.push("provider = ?"); params.push(filter.provider); }
if (filter.isActive !== undefined) { where.push("isActive = ?"); params.push(filter.isActive ? 1 : 0); }
const sql = `SELECT * FROM providerConnections${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
const rows = db.all(sql, params);
const list = rows.map(rowToConn);
list.sort((a, b) => (a.priority || 999) - (b.priority || 999));
return list;
}
export async function getProviderConnectionById(id) {
const db = await getAdapter();
const row = db.get(`SELECT * FROM providerConnections WHERE id = ?`, [id]);
return rowToConn(row);
}
// Internal sync reorder — must be called INSIDE a transaction
function reorderInTx(db, providerId) {
const list = db.all(`SELECT * FROM providerConnections WHERE provider = ?`, [providerId]).map(rowToConn);
list.sort((a, b) => {
const pDiff = (a.priority || 0) - (b.priority || 0);
if (pDiff !== 0) return pDiff;
return new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0);
});
list.forEach((c, i) => {
db.run(`UPDATE providerConnections SET priority = ? WHERE id = ?`, [i + 1, c.id]);
});
}
export async function createProviderConnection(data) {
const db = await getAdapter();
const now = new Date().toISOString();
let result;
db.transaction(() => {
const all = db.all(`SELECT * FROM providerConnections WHERE provider = ?`, [data.provider]).map(rowToConn);
let existing = null;
if (data.authType === "oauth" && data.email) {
existing = all.find(c => c.authType === "oauth" && c.email === data.email);
} else if (data.authType === "apikey" && data.name) {
existing = all.find(c => c.authType === "apikey" && c.name === data.name);
}
if (existing) {
const merged = { ...existing, ...data, updatedAt: now };
upsert(db, merged);
result = merged;
return;
}
let connectionName = data.name || null;
if (!connectionName && data.authType === "oauth") {
connectionName = data.email || `Account ${all.length + 1}`;
}
let connectionPriority = data.priority;
if (!connectionPriority) {
connectionPriority = all.reduce((m, c) => Math.max(m, c.priority || 0), 0) + 1;
}
const conn = {
id: uuidv4(),
provider: data.provider,
authType: data.authType || "oauth",
name: connectionName,
priority: connectionPriority,
isActive: data.isActive !== undefined ? data.isActive : true,
createdAt: now,
updatedAt: now,
};
for (const f of OPTIONAL_FIELDS) {
if (data[f] !== undefined && data[f] !== null) conn[f] = data[f];
}
if (data.providerSpecificData && Object.keys(data.providerSpecificData).length > 0) {
conn.providerSpecificData = data.providerSpecificData;
}
if (data.email !== undefined) conn.email = data.email;
upsert(db, conn);
reorderInTx(db, data.provider);
result = conn;
});
return result;
}
// Critical: OAuth refresh token race — atomic merge inside transaction
export async function updateProviderConnection(id, data) {
const db = await getAdapter();
let result;
db.transaction(() => {
const row = db.get(`SELECT * FROM providerConnections WHERE id = ?`, [id]);
if (!row) { result = null; return; }
const existing = rowToConn(row);
const merged = { ...existing, ...data, updatedAt: new Date().toISOString() };
upsert(db, merged);
if (data.priority !== undefined) reorderInTx(db, existing.provider);
result = merged;
});
return result;
}
export async function deleteProviderConnection(id) {
const db = await getAdapter();
let ok = false;
db.transaction(() => {
const row = db.get(`SELECT provider FROM providerConnections WHERE id = ?`, [id]);
if (!row) return;
db.run(`DELETE FROM providerConnections WHERE id = ?`, [id]);
reorderInTx(db, row.provider);
ok = true;
});
return ok;
}
export async function deleteProviderConnectionsByProvider(providerId) {
const db = await getAdapter();
const before = db.get(`SELECT COUNT(*) AS n FROM providerConnections WHERE provider = ?`, [providerId]);
db.run(`DELETE FROM providerConnections WHERE provider = ?`, [providerId]);
return before?.n || 0;
}
export async function reorderProviderConnections(providerId) {
const db = await getAdapter();
db.transaction(() => reorderInTx(db, providerId));
}
export async function cleanupProviderConnections() {
const db = await getAdapter();
const fieldsToCheck = [
"displayName", "email", "globalPriority", "defaultModel",
"accessToken", "refreshToken", "expiresAt", "tokenType",
"scope", "projectId", "apiKey", "testStatus",
"lastTested", "lastError", "lastErrorAt", "rateLimitedUntil", "expiresIn",
"consecutiveUseCount",
];
let cleaned = 0;
db.transaction(() => {
const rows = db.all(`SELECT * FROM providerConnections`);
for (const row of rows) {
const conn = rowToConn(row);
let dirty = false;
for (const f of fieldsToCheck) {
if (conn[f] === null || conn[f] === undefined) {
if (f in conn) { delete conn[f]; cleaned++; dirty = true; }
}
}
if (conn.providerSpecificData && Object.keys(conn.providerSpecificData).length === 0) {
delete conn.providerSpecificData;
cleaned++;
dirty = true;
}
if (dirty) upsert(db, conn);
}
});
return cleaned;
}

56
src/lib/db/repos/disabledModelsRepo.js Normal file
View file

@ -0,0 +1,56 @@
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
const SCOPE = "disabledModels";
export async function getDisabledModels() {
const db = await getAdapter();
const rows = db.all(`SELECT key, value FROM kv WHERE scope = ?`, [SCOPE]);
const out = {};
for (const r of rows) out[r.key] = parseJson(r.value, []);
return out;
}
export async function getDisabledByProvider(providerAlias) {
const db = await getAdapter();
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
return row ? (parseJson(row.value, []) || []) : [];
}
// Atomic read-merge-write inside a transaction (no JS yield mid-transaction).
export async function disableModels(providerAlias, ids) {
if (!providerAlias || !Array.isArray(ids)) return;
const db = await getAdapter();
db.transaction(() => {
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
const current = row ? (parseJson(row.value, []) || []) : [];
const merged = [...new Set([...current, ...ids])];
db.run(
`INSERT INTO kv(scope, key, value) VALUES(?, ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
[SCOPE, providerAlias, stringifyJson(merged)]
);
});
}
export async function enableModels(providerAlias, ids) {
if (!providerAlias) return;
const db = await getAdapter();
db.transaction(() => {
if (!Array.isArray(ids) || ids.length === 0) {
db.run(`DELETE FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
return;
}
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
const current = row ? (parseJson(row.value, []) || []) : [];
const removeSet = new Set(ids);
const next = current.filter((id) => !removeSet.has(id));
if (next.length === 0) {
db.run(`DELETE FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
} else {
db.run(
`INSERT INTO kv(scope, key, value) VALUES(?, ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
[SCOPE, providerAlias, stringifyJson(next)]
);
}
});
}

95
src/lib/db/repos/nodesRepo.js Normal file
View file

@ -0,0 +1,95 @@
import { v4 as uuidv4 } from "uuid";
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
function rowToNode(row) {
if (!row) return null;
const extra = parseJson(row.data, {});
return {
...extra,
id: row.id,
type: row.type,
name: row.name,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
function nodeToRow(n) {
const { id, type, name, createdAt, updatedAt, ...rest } = n;
return {
id,
type: type ?? null,
name: name ?? null,
data: stringifyJson(rest),
createdAt,
updatedAt,
};
}
function upsert(db, n) {
const r = nodeToRow(n);
db.run(
`INSERT INTO providerNodes(id, type, name, data, createdAt, updatedAt)
VALUES(?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
type=excluded.type, name=excluded.name, data=excluded.data, updatedAt=excluded.updatedAt`,
[r.id, r.type, r.name, r.data, r.createdAt, r.updatedAt]
);
}
export async function getProviderNodes(filter = {}) {
const db = await getAdapter();
const where = [];
const params = [];
if (filter.type) { where.push("type = ?"); params.push(filter.type); }
const sql = `SELECT * FROM providerNodes${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
return db.all(sql, params).map(rowToNode);
}
export async function getProviderNodeById(id) {
const db = await getAdapter();
return rowToNode(db.get(`SELECT * FROM providerNodes WHERE id = ?`, [id]));
}
export async function createProviderNode(data) {
const db = await getAdapter();
const now = new Date().toISOString();
const node = {
id: data.id || uuidv4(),
type: data.type,
name: data.name,
prefix: data.prefix,
apiType: data.apiType,
baseUrl: data.baseUrl,
createdAt: now,
updatedAt: now,
};
upsert(db, node);
return node;
}
export async function updateProviderNode(id, data) {
const db = await getAdapter();
let result = null;
db.transaction(() => {
const row = db.get(`SELECT * FROM providerNodes WHERE id = ?`, [id]);
if (!row) return;
const merged = { ...rowToNode(row), ...data, updatedAt: new Date().toISOString() };
upsert(db, merged);
result = merged;
});
return result;
}
export async function deleteProviderNode(id) {
const db = await getAdapter();
let removed = null;
db.transaction(() => {
const row = db.get(`SELECT * FROM providerNodes WHERE id = ?`, [id]);
if (!row) return;
removed = rowToNode(row);
db.run(`DELETE FROM providerNodes WHERE id = ?`, [id]);
});
return removed;
}

108
src/lib/db/repos/pricingRepo.js Normal file
View file

@ -0,0 +1,108 @@
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
import { makeKv } from "../helpers/kvStore.js";
const pricingKv = makeKv("pricing");
const CACHE_TTL_MS = 5000;
let cache = { value: null, expiresAt: 0 };
function invalidate() {
cache = { value: null, expiresAt: 0 };
}
async function getUserPricing() {
return await pricingKv.getAll();
}
export async function getPricing() {
const now = Date.now();
if (cache.value && cache.expiresAt > now) return cache.value;
const userPricing = await getUserPricing();
const { PROVIDER_PRICING } = await import("@/shared/constants/pricing.js");
const merged = {};
for (const [provider, models] of Object.entries(PROVIDER_PRICING)) {
merged[provider] = { ...models };
if (userPricing[provider]) {
for (const [model, pricing] of Object.entries(userPricing[provider])) {
merged[provider][model] = merged[provider][model]
? { ...merged[provider][model], ...pricing }
: pricing;
}
}
}
for (const [provider, models] of Object.entries(userPricing)) {
if (!merged[provider]) {
merged[provider] = { ...models };
} else {
for (const [model, pricing] of Object.entries(models)) {
if (!merged[provider][model]) merged[provider][model] = pricing;
}
}
}
cache = { value: merged, expiresAt: now + CACHE_TTL_MS };
return merged;
}
export async function getPricingForModel(provider, model) {
if (!model) return null;
const userPricing = await getUserPricing();
if (provider && userPricing[provider]?.[model]) return userPricing[provider][model];
const { getPricingForModel: resolveConst } = await import("@/shared/constants/pricing.js");
return resolveConst(provider, model);
}
// Atomic merge inside transaction (per-provider read-modify-write)
export async function updatePricing(pricingData) {
const db = await getAdapter();
db.transaction(() => {
for (const [provider, models] of Object.entries(pricingData)) {
const row = db.get(`SELECT value FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
const current = row ? (parseJson(row.value, {}) || {}) : {};
const merged = { ...current };
for (const [model, pricing] of Object.entries(models)) {
merged[model] = pricing;
}
db.run(
`INSERT INTO kv(scope, key, value) VALUES('pricing', ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
[provider, stringifyJson(merged)]
);
}
});
invalidate();
return await getUserPricing();
}
export async function resetPricing(provider, model) {
if (!provider) return await getUserPricing();
const db = await getAdapter();
db.transaction(() => {
if (!model) {
db.run(`DELETE FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
return;
}
const row = db.get(`SELECT value FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
const current = row ? (parseJson(row.value, {}) || {}) : {};
delete current[model];
if (Object.keys(current).length === 0) {
db.run(`DELETE FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
} else {
db.run(
`INSERT INTO kv(scope, key, value) VALUES('pricing', ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
[provider, stringifyJson(current)]
);
}
});
invalidate();
return await getUserPricing();
}
export async function resetAllPricing() {
await pricingKv.clear();
invalidate();
return {};
}

103
src/lib/db/repos/proxyPoolsRepo.js Normal file
View file

@ -0,0 +1,103 @@
import { v4 as uuidv4 } from "uuid";
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
function rowToPool(row) {
if (!row) return null;
const extra = parseJson(row.data, {});
return {
...extra,
id: row.id,
isActive: row.isActive === 1 || row.isActive === true,
testStatus: row.testStatus,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
function poolToRow(p) {
const { id, isActive, testStatus, createdAt, updatedAt, ...rest } = p;
return {
id,
isActive: isActive === false ? 0 : 1,
testStatus: testStatus ?? null,
data: stringifyJson(rest),
createdAt,
updatedAt,
};
}
function upsert(db, p) {
const r = poolToRow(p);
db.run(
`INSERT INTO proxyPools(id, isActive, testStatus, data, createdAt, updatedAt)
VALUES(?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
isActive=excluded.isActive, testStatus=excluded.testStatus,
data=excluded.data, updatedAt=excluded.updatedAt`,
[r.id, r.isActive, r.testStatus, r.data, r.createdAt, r.updatedAt]
);
}
export async function getProxyPools(filter = {}) {
const db = await getAdapter();
const where = [];
const params = [];
if (filter.isActive !== undefined) { where.push("isActive = ?"); params.push(filter.isActive ? 1 : 0); }
if (filter.testStatus) { where.push("testStatus = ?"); params.push(filter.testStatus); }
const sql = `SELECT * FROM proxyPools${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
const list = db.all(sql, params).map(rowToPool);
list.sort((a, b) => new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0));
return list;
}
export async function getProxyPoolById(id) {
const db = await getAdapter();
return rowToPool(db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]));
}
export async function createProxyPool(data) {
const db = await getAdapter();
const now = new Date().toISOString();
const pool = {
id: data.id || uuidv4(),
name: data.name,
proxyUrl: data.proxyUrl,
noProxy: data.noProxy || "",
type: data.type || "http",
isActive: data.isActive !== undefined ? data.isActive : true,
strictProxy: data.strictProxy === true,
testStatus: data.testStatus || "unknown",
lastTestedAt: data.lastTestedAt || null,
lastError: data.lastError || null,
createdAt: now,
updatedAt: now,
};
upsert(db, pool);
return pool;
}
export async function updateProxyPool(id, data) {
const db = await getAdapter();
let result = null;
db.transaction(() => {
const row = db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]);
if (!row) return;
const merged = { ...rowToPool(row), ...data, updatedAt: new Date().toISOString() };
upsert(db, merged);
result = merged;
});
return result;
}
export async function deleteProxyPool(id) {
const db = await getAdapter();
let removed = null;
db.transaction(() => {
const row = db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]);
if (!row) return;
removed = rowToPool(row);
db.run(`DELETE FROM proxyPools WHERE id = ?`, [id]);
});
return removed;
}

200
src/lib/db/repos/requestDetailsRepo.js Normal file
View file

@ -0,0 +1,200 @@
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
const DEFAULT_MAX_RECORDS = 200;
const DEFAULT_BATCH_SIZE = 20;
const DEFAULT_FLUSH_INTERVAL_MS = 5000;
const DEFAULT_MAX_JSON_SIZE = 5 * 1024;
const CONFIG_CACHE_TTL_MS = 5000;
let cachedConfig = null;
let cachedConfigTs = 0;
async function getObservabilityConfig() {
if (cachedConfig && (Date.now() - cachedConfigTs) < CONFIG_CACHE_TTL_MS) return cachedConfig;
try {
const { getSettings } = await import("./settingsRepo.js");
const settings = await getSettings();
const envEnabled = process.env.OBSERVABILITY_ENABLED !== "false";
const enabled = typeof settings.enableObservability === "boolean"
? settings.enableObservability
: envEnabled;
cachedConfig = {
enabled,
maxRecords: settings.observabilityMaxRecords || parseInt(process.env.OBSERVABILITY_MAX_RECORDS || String(DEFAULT_MAX_RECORDS), 10),
batchSize: settings.observabilityBatchSize || parseInt(process.env.OBSERVABILITY_BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10),
flushIntervalMs: settings.observabilityFlushIntervalMs || parseInt(process.env.OBSERVABILITY_FLUSH_INTERVAL_MS || String(DEFAULT_FLUSH_INTERVAL_MS), 10),
maxJsonSize: (settings.observabilityMaxJsonSize || parseInt(process.env.OBSERVABILITY_MAX_JSON_SIZE || "5", 10)) * 1024,
};
} catch {
cachedConfig = {
enabled: false,
maxRecords: DEFAULT_MAX_RECORDS,
batchSize: DEFAULT_BATCH_SIZE,
flushIntervalMs: DEFAULT_FLUSH_INTERVAL_MS,
maxJsonSize: DEFAULT_MAX_JSON_SIZE,
};
}
cachedConfigTs = Date.now();
return cachedConfig;
}
let writeBuffer = [];
let flushTimer = null;
let isFlushing = false;
function sanitizeHeaders(headers) {
if (!headers || typeof headers !== "object") return {};
const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token", "api-key"];
const sanitized = { ...headers };
for (const key of Object.keys(sanitized)) {
if (sensitiveKeys.some((s) => key.toLowerCase().includes(s))) delete sanitized[key];
}
return sanitized;
}
function generateDetailId(model) {
const timestamp = new Date().toISOString();
const random = Math.random().toString(36).substring(2, 8);
const modelPart = model ? model.replace(/[^a-zA-Z0-9-]/g, "-") : "unknown";
return `${timestamp}-${random}-${modelPart}`;
}
function truncateField(obj, maxSize) {
const str = JSON.stringify(obj || {});
if (str.length > maxSize) {
return { _truncated: true, _originalSize: str.length, _preview: str.substring(0, 200) };
}
return obj || {};
}
async function flushToDatabase() {
if (isFlushing) return;
if (writeBuffer.length === 0) return;
isFlushing = true;
try {
// Drain entire buffer (loop in case more pushed during await)
while (writeBuffer.length > 0) {
const items = writeBuffer.splice(0, writeBuffer.length);
const db = await getAdapter();
const config = await getObservabilityConfig();
db.transaction(() => {
for (const item of items) {
if (!item.id) item.id = generateDetailId(item.model);
if (!item.timestamp) item.timestamp = new Date().toISOString();
if (item.request?.headers) item.request.headers = sanitizeHeaders(item.request.headers);
const record = {
id: item.id,
provider: item.provider || null,
model: item.model || null,
connectionId: item.connectionId || null,
timestamp: item.timestamp,
status: item.status || null,
latency: item.latency || {},
tokens: item.tokens || {},
request: truncateField(item.request, config.maxJsonSize),
providerRequest: truncateField(item.providerRequest, config.maxJsonSize),
providerResponse: truncateField(item.providerResponse, config.maxJsonSize),
response: truncateField(item.response, config.maxJsonSize),
};
db.run(
`INSERT INTO requestDetails(id, timestamp, provider, model, connectionId, status, data) VALUES(?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET timestamp = excluded.timestamp, provider = excluded.provider, model = excluded.model, connectionId = excluded.connectionId, status = excluded.status, data = excluded.data`,
[record.id, record.timestamp, record.provider, record.model, record.connectionId, record.status, stringifyJson(record)]
);
}
const cnt = db.get(`SELECT COUNT(*) as c FROM requestDetails`);
if (cnt && cnt.c > config.maxRecords) {
db.run(
`DELETE FROM requestDetails WHERE id IN (SELECT id FROM requestDetails ORDER BY timestamp ASC LIMIT ?)`,
[cnt.c - config.maxRecords]
);
}
});
}
} catch (e) {
console.error("[requestDetailsRepo] Batch write failed:", e);
} finally {
isFlushing = false;
}
}
export async function saveRequestDetail(detail) {
const config = await getObservabilityConfig();
if (!config.enabled) return;
writeBuffer.push(detail);
// Trigger immediate flush if batch threshold reached.
// flushToDatabase() drains entire buffer in a loop, so all pushes during await are persisted.
if (writeBuffer.length >= config.batchSize) {
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
flushToDatabase().catch((e) => console.error("[requestDetailsRepo] flush err:", e));
} else if (!flushTimer) {
flushTimer = setTimeout(() => {
flushTimer = null;
flushToDatabase().catch(() => {});
}, config.flushIntervalMs);
}
}
export async function getRequestDetails(filter = {}) {
const db = await getAdapter();
const conds = [];
const params = [];
if (filter.provider) { conds.push("provider = ?"); params.push(filter.provider); }
if (filter.model) { conds.push("model = ?"); params.push(filter.model); }
if (filter.connectionId) { conds.push("connectionId = ?"); params.push(filter.connectionId); }
if (filter.status) { conds.push("status = ?"); params.push(filter.status); }
if (filter.startDate) { conds.push("timestamp >= ?"); params.push(new Date(filter.startDate).toISOString()); }
if (filter.endDate) { conds.push("timestamp <= ?"); params.push(new Date(filter.endDate).toISOString()); }
const where = conds.length ? `WHERE ${conds.join(" AND ")}` : "";
const cntRow = db.get(`SELECT COUNT(*) as c FROM requestDetails ${where}`, params);
const totalItems = cntRow ? cntRow.c : 0;
const page = filter.page || 1;
const pageSize = filter.pageSize || 50;
const totalPages = Math.ceil(totalItems / pageSize);
const offset = (page - 1) * pageSize;
const rows = db.all(
`SELECT data FROM requestDetails ${where} ORDER BY timestamp DESC LIMIT ? OFFSET ?`,
[...params, pageSize, offset]
);
const details = rows.map((r) => parseJson(r.data, {}));
return {
details,
pagination: { page, pageSize, totalItems, totalPages, hasNext: page < totalPages, hasPrev: page > 1 },
};
}
export async function getRequestDetailById(id) {
const db = await getAdapter();
const row = db.get(`SELECT data FROM requestDetails WHERE id = ?`, [id]);
return row ? parseJson(row.data, null) : null;
}
const _shutdownHandler = async () => {
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
if (writeBuffer.length > 0) await flushToDatabase();
};
function ensureShutdownHandler() {
process.off("beforeExit", _shutdownHandler);
process.off("SIGINT", _shutdownHandler);
process.off("SIGTERM", _shutdownHandler);
process.off("exit", _shutdownHandler);
process.on("beforeExit", _shutdownHandler);
process.on("SIGINT", _shutdownHandler);
process.on("SIGTERM", _shutdownHandler);
process.on("exit", _shutdownHandler);
}
ensureShutdownHandler();

98
src/lib/db/repos/settingsRepo.js Normal file
View file

@ -0,0 +1,98 @@
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
const DEFAULT_MITM_ROUTER_BASE = "http://127.0.0.1:20128";
const DEFAULT_SETTINGS = {
cloudEnabled: false,
tunnelEnabled: false,
tunnelUrl: "",
tunnelProvider: "cloudflare",
tailscaleEnabled: false,
tailscaleUrl: "",
stickyRoundRobinLimit: 3,
providerStrategies: {},
comboStrategy: "fallback",
comboStickyRoundRobinLimit: 1,
comboStrategies: {},
requireLogin: true,
tunnelDashboardAccess: true,
enableObservability: true,
observabilityMaxRecords: 1000,
observabilityBatchSize: 20,
observabilityFlushIntervalMs: 5000,
observabilityMaxJsonSize: 5,
outboundProxyEnabled: false,
outboundProxyUrl: "",
outboundNoProxy: "",
mitmRouterBaseUrl: DEFAULT_MITM_ROUTER_BASE,
dnsToolEnabled: {},
rtkEnabled: true,
cavemanEnabled: false,
cavemanLevel: "full",
};
async function readRaw() {
const db = await getAdapter();
const row = db.get(`SELECT data FROM settings WHERE id = 1`);
return row ? parseJson(row.data, {}) : {};
}
// Merge raw settings with defaults; backward-compat for missing keys
function mergeWithDefaults(raw) {
const merged = { ...DEFAULT_SETTINGS, ...(raw || {}) };
for (const [key, defVal] of Object.entries(DEFAULT_SETTINGS)) {
if (merged[key] === undefined) {
if (
key === "outboundProxyEnabled" &&
typeof merged.outboundProxyUrl === "string" &&
merged.outboundProxyUrl.trim()
) {
merged[key] = true;
} else {
merged[key] = defVal;
}
}
}
return merged;
}
export async function getSettings() {
const raw = await readRaw();
return mergeWithDefaults(raw);
}
// Atomic read-merge-write inside transaction (prevents losing concurrent updates)
export async function updateSettings(updates) {
const db = await getAdapter();
let next;
db.transaction(() => {
const row = db.get(`SELECT data FROM settings WHERE id = 1`);
const current = row ? parseJson(row.data, {}) : {};
next = { ...current, ...updates };
db.run(
`INSERT INTO settings(id, data) VALUES(1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data`,
[stringifyJson(next)]
);
});
return mergeWithDefaults(next);
}
export async function isCloudEnabled() {
const settings = await getSettings();
return settings.cloudEnabled === true;
}
export async function getCloudUrl() {
const settings = await getSettings();
return (
settings.cloudUrl ||
process.env.CLOUD_URL ||
process.env.NEXT_PUBLIC_CLOUD_URL ||
""
);
}
export async function exportSettings() {
return await readRaw();
}

698
src/lib/db/repos/usageRepo.js Normal file
View file

@ -0,0 +1,698 @@
import { EventEmitter } from "events";
import { getAdapter } from "../driver.js";
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
import { getMeta, setMeta } from "../helpers/metaStore.js";
const PENDING_TIMEOUT_MS = 60 * 1000;
const RING_CAP = 50;
const CONN_CACHE_TTL_MS = 30 * 1000;
const PERIOD_MS = { "24h": 86400000, "7d": 604800000, "30d": 2592000000, "60d": 5184000000 };
// In-memory state shared across Next.js modules
if (!global._pendingRequests) global._pendingRequests = { byModel: {}, byAccount: {} };
if (!global._lastErrorProvider) global._lastErrorProvider = { provider: "", ts: 0 };
if (!global._statsEmitter) {
global._statsEmitter = new EventEmitter();
global._statsEmitter.setMaxListeners(50);
}
if (!global._pendingTimers) global._pendingTimers = {};
if (!global._recentRing) global._recentRing = { items: [], initialized: false };
if (!global._connectionMapCache) global._connectionMapCache = { map: {}, ts: 0 };
const pendingRequests = global._pendingRequests;
const lastErrorProvider = global._lastErrorProvider;
const pendingTimers = global._pendingTimers;
const recentRing = global._recentRing;
const connCache = global._connectionMapCache;
export const statsEmitter = global._statsEmitter;
function getLocalDateKey(timestamp) {
const d = timestamp ? new Date(timestamp) : new Date();
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
}
function addToCounter(target, key, values) {
if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
target[key].requests += values.requests || 1;
target[key].promptTokens += values.promptTokens || 0;
target[key].completionTokens += values.completionTokens || 0;
target[key].cost += values.cost || 0;
if (values.meta) Object.assign(target[key], values.meta);
}
function aggregateEntryToDay(day, entry) {
const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0;
const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0;
const cost = entry.cost || 0;
const vals = { promptTokens, completionTokens, cost };
day.requests = (day.requests || 0) + 1;
day.promptTokens = (day.promptTokens || 0) + promptTokens;
day.completionTokens = (day.completionTokens || 0) + completionTokens;
day.cost = (day.cost || 0) + cost;
day.byProvider ||= {};
day.byModel ||= {};
day.byAccount ||= {};
day.byApiKey ||= {};
day.byEndpoint ||= {};
if (entry.provider) addToCounter(day.byProvider, entry.provider, vals);
const modelKey = entry.provider ? `${entry.model}|${entry.provider}` : entry.model;
addToCounter(day.byModel, modelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } });
if (entry.connectionId) {
addToCounter(day.byAccount, entry.connectionId, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } });
}
const apiKeyVal = entry.apiKey && typeof entry.apiKey === "string" ? entry.apiKey : "local-no-key";
const akModelKey = `${apiKeyVal}|${entry.model}|${entry.provider || "unknown"}`;
addToCounter(day.byApiKey, akModelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider, apiKey: entry.apiKey || null } });
const endpoint = entry.endpoint || "Unknown";
const epKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`;
addToCounter(day.byEndpoint, epKey, { ...vals, meta: { endpoint, rawModel: entry.model, provider: entry.provider } });
}
function pushToRing(entry) {
recentRing.items.push(entry);
if (recentRing.items.length > RING_CAP) {
recentRing.items = recentRing.items.slice(-RING_CAP);
}
}
async function getConnectionMapCached() {
if (Date.now() - connCache.ts < CONN_CACHE_TTL_MS) return connCache.map;
try {
const { getProviderConnections } = await import("./connectionsRepo.js");
const all = await getProviderConnections();
const map = {};
for (const c of all) map[c.id] = c.name || c.email || c.id;
connCache.map = map;
connCache.ts = Date.now();
} catch {}
return connCache.map;
}
async function ensureRingInitialized() {
if (recentRing.initialized) return;
recentRing.initialized = true;
try {
const db = await getAdapter();
const rows = db.all(`SELECT timestamp, provider, model, connectionId, apiKey, endpoint, cost, status, tokens FROM usageHistory ORDER BY id DESC LIMIT ?`, [RING_CAP]);
recentRing.items = rows.reverse().map((r) => ({
timestamp: r.timestamp, provider: r.provider, model: r.model, connectionId: r.connectionId,
apiKey: r.apiKey, endpoint: r.endpoint, cost: r.cost, status: r.status,
tokens: parseJson(r.tokens, {}),
}));
} catch {}
}
async function calculateCost(provider, model, tokens) {
if (!tokens || !provider || !model) return 0;
try {
const { getPricingForModel } = await import("./pricingRepo.js");
const pricing = await getPricingForModel(provider, model);
if (!pricing) return 0;
let cost = 0;
const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0;
const nonCachedInput = Math.max(0, inputTokens - cachedTokens);
cost += nonCachedInput * (pricing.input / 1000000);
if (cachedTokens > 0) {
const cachedRate = pricing.cached || pricing.input;
cost += cachedTokens * (cachedRate / 1000000);
}
const outputTokens = tokens.completion_tokens || tokens.output_tokens || 0;
cost += outputTokens * (pricing.output / 1000000);
const reasoningTokens = tokens.reasoning_tokens || 0;
if (reasoningTokens > 0) {
const rate = pricing.reasoning || pricing.output;
cost += reasoningTokens * (rate / 1000000);
}
const cacheCreationTokens = tokens.cache_creation_input_tokens || 0;
if (cacheCreationTokens > 0) {
const rate = pricing.cache_creation || pricing.input;
cost += cacheCreationTokens * (rate / 1000000);
}
return cost;
} catch (e) {
console.error("Error calculating cost:", e);
return 0;
}
}
export function trackPendingRequest(model, provider, connectionId, started, error = false) {
const modelKey = provider ? `${model} (${provider})` : model;
const timerKey = `${connectionId}|${modelKey}`;
if (!pendingRequests.byModel[modelKey]) pendingRequests.byModel[modelKey] = 0;
pendingRequests.byModel[modelKey] = Math.max(0, pendingRequests.byModel[modelKey] + (started ? 1 : -1));
if (pendingRequests.byModel[modelKey] === 0) delete pendingRequests.byModel[modelKey];
if (connectionId) {
if (!pendingRequests.byAccount[connectionId]) pendingRequests.byAccount[connectionId] = {};
if (!pendingRequests.byAccount[connectionId][modelKey]) pendingRequests.byAccount[connectionId][modelKey] = 0;
pendingRequests.byAccount[connectionId][modelKey] = Math.max(0, pendingRequests.byAccount[connectionId][modelKey] + (started ? 1 : -1));
if (pendingRequests.byAccount[connectionId][modelKey] === 0) {
delete pendingRequests.byAccount[connectionId][modelKey];
if (Object.keys(pendingRequests.byAccount[connectionId]).length === 0) {
delete pendingRequests.byAccount[connectionId];
}
}
}
if (started) {
clearTimeout(pendingTimers[timerKey]);
pendingTimers[timerKey] = setTimeout(() => {
delete pendingTimers[timerKey];
if (pendingRequests.byModel[modelKey] > 0) pendingRequests.byModel[modelKey] = 0;
if (connectionId && pendingRequests.byAccount[connectionId]?.[modelKey] > 0) {
pendingRequests.byAccount[connectionId][modelKey] = 0;
}
statsEmitter.emit("pending");
}, PENDING_TIMEOUT_MS);
} else {
clearTimeout(pendingTimers[timerKey]);
delete pendingTimers[timerKey];
}
if (!started && error && provider) {
lastErrorProvider.provider = provider.toLowerCase();
lastErrorProvider.ts = Date.now();
}
const t = new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" });
console.log(`[${t}] [PENDING] ${started ? "START" : "END"}${error ? " (ERROR)" : ""} | provider=${provider} | model=${model}`);
statsEmitter.emit("pending");
}
export async function getActiveRequests() {
const activeRequests = [];
const connectionMap = await getConnectionMapCached();
for (const [connectionId, models] of Object.entries(pendingRequests.byAccount)) {
for (const [modelKey, count] of Object.entries(models)) {
if (count > 0) {
const accountName = connectionMap[connectionId] || `Account ${connectionId.slice(0, 8)}...`;
const match = modelKey.match(/^(.*) \((.*)\)$/);
activeRequests.push({
model: match ? match[1] : modelKey,
provider: match ? match[2] : "unknown",
account: accountName, count,
});
}
}
}
await ensureRingInitialized();
const seen = new Set();
const recentRequests = [...recentRing.items]
.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp))
.map((e) => {
const t = e.tokens || {};
return {
timestamp: e.timestamp, model: e.model, provider: e.provider || "",
promptTokens: t.prompt_tokens || t.input_tokens || 0,
completionTokens: t.completion_tokens || t.output_tokens || 0,
status: e.status || "ok",
};
})
.filter((e) => {
if (e.promptTokens === 0 && e.completionTokens === 0) return false;
const minute = e.timestamp ? e.timestamp.slice(0, 16) : "";
const key = `${e.model}|${e.provider}|${e.promptTokens}|${e.completionTokens}|${minute}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
})
.slice(0, 20);
const errorProvider = (Date.now() - lastErrorProvider.ts < 10000) ? lastErrorProvider.provider : "";
return { activeRequests, recentRequests, errorProvider };
}
export async function saveRequestUsage(entry) {
try {
const db = await getAdapter();
if (!entry.timestamp) entry.timestamp = new Date().toISOString();
entry.cost = await calculateCost(entry.provider, entry.model, entry.tokens);
const tokens = entry.tokens || {};
const promptTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
const completionTokens = tokens.completion_tokens || tokens.output_tokens || 0;
// All 3 writes (history insert, daily upsert, lifetime counter) in ONE transaction.
// better-sqlite3 is sync → no JS yield mid-transaction → no race in same process.
db.transaction(() => {
db.run(
`INSERT INTO usageHistory(timestamp, provider, model, connectionId, apiKey, endpoint, promptTokens, completionTokens, cost, status, tokens, meta) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
entry.timestamp, entry.provider || null, entry.model || null,
entry.connectionId || null, entry.apiKey || null, entry.endpoint || null,
promptTokens, completionTokens, entry.cost || 0, entry.status || "ok",
stringifyJson(tokens), stringifyJson({}),
]
);
const dateKey = getLocalDateKey(entry.timestamp);
const row = db.get(`SELECT data FROM usageDaily WHERE dateKey = ?`, [dateKey]);
const day = row ? parseJson(row.data, {}) : {
requests: 0, promptTokens: 0, completionTokens: 0, cost: 0,
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
};
aggregateEntryToDay(day, entry);
db.run(`INSERT INTO usageDaily(dateKey, data) VALUES(?, ?) ON CONFLICT(dateKey) DO UPDATE SET data = excluded.data`, [dateKey, stringifyJson(day)]);
// Atomic counter increment in same transaction
const cur = db.get(`SELECT value FROM _meta WHERE key = 'totalRequestsLifetime'`);
const next = (cur ? parseInt(cur.value, 10) : 0) + 1;
db.run(`INSERT INTO _meta(key, value) VALUES('totalRequestsLifetime', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, [String(next)]);
});
pushToRing(entry);
statsEmitter.emit("update");
} catch (e) {
console.error("Failed to save usage stats:", e);
}
}
export async function getUsageHistory(filter = {}) {
const db = await getAdapter();
const conds = [];
const params = [];
if (filter.provider) { conds.push("provider = ?"); params.push(filter.provider); }
if (filter.model) { conds.push("model = ?"); params.push(filter.model); }
if (filter.startDate) { conds.push("timestamp >= ?"); params.push(new Date(filter.startDate).toISOString()); }
if (filter.endDate) { conds.push("timestamp <= ?"); params.push(new Date(filter.endDate).toISOString()); }
const where = conds.length ? `WHERE ${conds.join(" AND ")}` : "";
const rows = db.all(`SELECT timestamp, provider, model, connectionId, apiKey, endpoint, cost, status, tokens FROM usageHistory ${where} ORDER BY id ASC`, params);
return rows.map((r) => ({
timestamp: r.timestamp, provider: r.provider, model: r.model,
connectionId: r.connectionId, apiKey: r.apiKey, endpoint: r.endpoint,
cost: r.cost, status: r.status, tokens: parseJson(r.tokens, {}),
}));
}
function loadDaysInRange(adapter, maxDays) {
if (maxDays == null) {
return adapter.all(`SELECT dateKey, data FROM usageDaily`);
}
const today = new Date();
const cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - maxDays + 1);
const cutoffKey = `${cutoff.getFullYear()}-${String(cutoff.getMonth() + 1).padStart(2, "0")}-${String(cutoff.getDate()).padStart(2, "0")}`;
return adapter.all(`SELECT dateKey, data FROM usageDaily WHERE dateKey >= ?`, [cutoffKey]);
}
export async function getUsageStats(period = "all") {
const db = await getAdapter();
const [{ getProviderConnections }, { getApiKeys }, { getProviderNodes }] = await Promise.all([
import("./connectionsRepo.js"),
import("./apiKeysRepo.js"),
import("./nodesRepo.js"),
]);
let allConnections = [];
try { allConnections = await getProviderConnections(); } catch {}
const connectionMap = {};
for (const c of allConnections) connectionMap[c.id] = c.name || c.email || c.id;
const providerNodeNameMap = {};
try {
const nodes = await getProviderNodes();
for (const n of nodes) if (n.id && n.name) providerNodeNameMap[n.id] = n.name;
} catch {}
let allApiKeys = [];
try { allApiKeys = await getApiKeys(); } catch {}
const apiKeyMap = {};
for (const k of allApiKeys) apiKeyMap[k.key] = { name: k.name, id: k.id, createdAt: k.createdAt };
// recentRequests from live history (last 100 entries enough for 20 deduped)
const recentRows = db.all(`SELECT timestamp, provider, model, tokens, status FROM usageHistory ORDER BY id DESC LIMIT 100`);
const seen = new Set();
const recentRequests = recentRows
.map((r) => {
const t = parseJson(r.tokens, {}) || {};
return {
timestamp: r.timestamp, model: r.model, provider: r.provider || "",
promptTokens: t.prompt_tokens || t.input_tokens || 0,
completionTokens: t.completion_tokens || t.output_tokens || 0,
status: r.status || "ok",
};
})
.filter((e) => {
if (e.promptTokens === 0 && e.completionTokens === 0) return false;
const minute = e.timestamp ? e.timestamp.slice(0, 16) : "";
const key = `${e.model}|${e.provider}|${e.promptTokens}|${e.completionTokens}|${minute}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
})
.slice(0, 20);
const stats = {
totalRequests: 0,
totalPromptTokens: 0, totalCompletionTokens: 0, totalCost: 0,
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
last10Minutes: [],
pending: pendingRequests,
activeRequests: [],
recentRequests,
errorProvider: (Date.now() - lastErrorProvider.ts < 10000) ? lastErrorProvider.provider : "",
};
// Active requests
for (const [connectionId, models] of Object.entries(pendingRequests.byAccount)) {
for (const [modelKey, count] of Object.entries(models)) {
if (count > 0) {
const accountName = connectionMap[connectionId] || `Account ${connectionId.slice(0, 8)}...`;
const match = modelKey.match(/^(.*) \((.*)\)$/);
stats.activeRequests.push({
model: match ? match[1] : modelKey,
provider: match ? match[2] : "unknown",
account: accountName, count,
});
}
}
}
// last10Minutes — query 10min window
const now = new Date();
const currentMinuteStart = new Date(Math.floor(now.getTime() / 60000) * 60000);
const tenMinutesAgo = new Date(currentMinuteStart.getTime() - 9 * 60 * 1000);
const bucketMap = {};
for (let i = 0; i < 10; i++) {
const ts = currentMinuteStart.getTime() - (9 - i) * 60 * 1000;
bucketMap[ts] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
stats.last10Minutes.push(bucketMap[ts]);
}
const recent10 = db.all(
`SELECT timestamp, promptTokens, completionTokens, cost FROM usageHistory WHERE timestamp >= ? AND timestamp <= ?`,
[tenMinutesAgo.toISOString(), now.toISOString()]
);
for (const r of recent10) {
const tt = new Date(r.timestamp).getTime();
const minuteStart = Math.floor(tt / 60000) * 60000;
if (bucketMap[minuteStart]) {
bucketMap[minuteStart].requests++;
bucketMap[minuteStart].promptTokens += r.promptTokens || 0;
bucketMap[minuteStart].completionTokens += r.completionTokens || 0;
bucketMap[minuteStart].cost += r.cost || 0;
}
}
const useDailySummary = period !== "24h";
if (useDailySummary) {
const periodDays = { "7d": 7, "30d": 30, "60d": 60 };
const maxDays = periodDays[period] || null;
const dayRows = loadDaysInRange(db, maxDays);
for (const dr of dayRows) {
const dateKey = dr.dateKey;
const day = parseJson(dr.data, {});
stats.totalPromptTokens += day.promptTokens || 0;
stats.totalCompletionTokens += day.completionTokens || 0;
stats.totalCost += day.cost || 0;
for (const [prov, p] of Object.entries(day.byProvider || {})) {
if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
stats.byProvider[prov].requests += p.requests || 0;
stats.byProvider[prov].promptTokens += p.promptTokens || 0;
stats.byProvider[prov].completionTokens += p.completionTokens || 0;
stats.byProvider[prov].cost += p.cost || 0;
}
for (const [mk, m] of Object.entries(day.byModel || {})) {
const rawModel = m.rawModel || mk.split("|")[0];
const provider = m.provider || mk.split("|")[1] || "";
const statsKey = provider ? `${rawModel} (${provider})` : rawModel;
const providerDisplayName = providerNodeNameMap[provider] || provider;
if (!stats.byModel[statsKey]) {
stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey };
}
stats.byModel[statsKey].requests += m.requests || 0;
stats.byModel[statsKey].promptTokens += m.promptTokens || 0;
stats.byModel[statsKey].completionTokens += m.completionTokens || 0;
stats.byModel[statsKey].cost += m.cost || 0;
if (dateKey > (stats.byModel[statsKey].lastUsed || "")) stats.byModel[statsKey].lastUsed = dateKey;
}
for (const [connId, a] of Object.entries(day.byAccount || {})) {
const accountName = connectionMap[connId] || `Account ${connId.slice(0, 8)}...`;
const rawModel = a.rawModel || "";
const provider = a.provider || "";
const providerDisplayName = providerNodeNameMap[provider] || provider;
const accountKey = `${rawModel} (${provider} - ${accountName})`;
if (!stats.byAccount[accountKey]) {
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey };
}
stats.byAccount[accountKey].requests += a.requests || 0;
stats.byAccount[accountKey].promptTokens += a.promptTokens || 0;
stats.byAccount[accountKey].completionTokens += a.completionTokens || 0;
stats.byAccount[accountKey].cost += a.cost || 0;
if (dateKey > (stats.byAccount[accountKey].lastUsed || "")) stats.byAccount[accountKey].lastUsed = dateKey;
}
for (const [akKey, ak] of Object.entries(day.byApiKey || {})) {
const rawModel = ak.rawModel || "";
const provider = ak.provider || "";
const providerDisplayName = providerNodeNameMap[provider] || provider;
const apiKeyVal = ak.apiKey;
const keyInfo = apiKeyVal ? apiKeyMap[apiKeyVal] : null;
const keyName = keyInfo?.name || (apiKeyVal ? apiKeyVal.slice(0, 8) + "..." : "Local (No API Key)");
const apiKeyKey = apiKeyVal || "local-no-key";
if (!stats.byApiKey[akKey]) {
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKey: apiKeyVal, keyName, apiKeyKey, lastUsed: dateKey };
}
stats.byApiKey[akKey].requests += ak.requests || 0;
stats.byApiKey[akKey].promptTokens += ak.promptTokens || 0;
stats.byApiKey[akKey].completionTokens += ak.completionTokens || 0;
stats.byApiKey[akKey].cost += ak.cost || 0;
if (dateKey > (stats.byApiKey[akKey].lastUsed || "")) stats.byApiKey[akKey].lastUsed = dateKey;
}
for (const [epKey, ep] of Object.entries(day.byEndpoint || {})) {
const endpoint = ep.endpoint || epKey.split("|")[0] || "Unknown";
const rawModel = ep.rawModel || "";
const provider = ep.provider || "";
const providerDisplayName = providerNodeNameMap[provider] || provider;
if (!stats.byEndpoint[epKey]) {
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey };
}
stats.byEndpoint[epKey].requests += ep.requests || 0;
stats.byEndpoint[epKey].promptTokens += ep.promptTokens || 0;
stats.byEndpoint[epKey].completionTokens += ep.completionTokens || 0;
stats.byEndpoint[epKey].cost += ep.cost || 0;
if (dateKey > (stats.byEndpoint[epKey].lastUsed || "")) stats.byEndpoint[epKey].lastUsed = dateKey;
}
}
// Overlay precise lastUsed timestamps from history
const overlayCutoff = maxDays ? Date.now() - maxDays * 86400000 : 0;
const histRows = db.all(
`SELECT timestamp, provider, model, connectionId, apiKey, endpoint FROM usageHistory WHERE timestamp >= ?`,
[new Date(overlayCutoff).toISOString()]
);
for (const e of histRows) {
const ts = e.timestamp;
const modelKey = e.provider ? `${e.model} (${e.provider})` : e.model;
if (stats.byModel[modelKey] && new Date(ts) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = ts;
if (e.connectionId) {
const accountName = connectionMap[e.connectionId] || `Account ${e.connectionId.slice(0, 8)}...`;
const accountKey = `${e.model} (${e.provider} - ${accountName})`;
if (stats.byAccount[accountKey] && new Date(ts) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = ts;
}
const apiKeyKey = (e.apiKey && typeof e.apiKey === "string")
? `${e.apiKey}|${e.model}|${e.provider || "unknown"}`
: "local-no-key";
if (stats.byApiKey[apiKeyKey] && new Date(ts) > new Date(stats.byApiKey[apiKeyKey].lastUsed)) stats.byApiKey[apiKeyKey].lastUsed = ts;
const endpoint = e.endpoint || "Unknown";
const endpointKey = `${endpoint}|${e.model}|${e.provider || "unknown"}`;
if (stats.byEndpoint[endpointKey] && new Date(ts) > new Date(stats.byEndpoint[endpointKey].lastUsed)) stats.byEndpoint[endpointKey].lastUsed = ts;
}
} else {
// 24h: live history
const cutoff = new Date(Date.now() - PERIOD_MS["24h"]).toISOString();
const filtered = db.all(
`SELECT timestamp, provider, model, connectionId, apiKey, endpoint, promptTokens, completionTokens, cost, tokens FROM usageHistory WHERE timestamp >= ?`,
[cutoff]
);
for (const r of filtered) {
const tokens = parseJson(r.tokens, {}) || {};
const promptTokens = tokens.prompt_tokens || 0;
const completionTokens = tokens.completion_tokens || 0;
const entryCost = r.cost || 0;
const providerDisplayName = providerNodeNameMap[r.provider] || r.provider;
stats.totalPromptTokens += promptTokens;
stats.totalCompletionTokens += completionTokens;
stats.totalCost += entryCost;
if (!stats.byProvider[r.provider]) stats.byProvider[r.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
stats.byProvider[r.provider].requests++;
stats.byProvider[r.provider].promptTokens += promptTokens;
stats.byProvider[r.provider].completionTokens += completionTokens;
stats.byProvider[r.provider].cost += entryCost;
const modelKey = r.provider ? `${r.model} (${r.provider})` : r.model;
if (!stats.byModel[modelKey]) {
stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
}
stats.byModel[modelKey].requests++;
stats.byModel[modelKey].promptTokens += promptTokens;
stats.byModel[modelKey].completionTokens += completionTokens;
stats.byModel[modelKey].cost += entryCost;
if (new Date(r.timestamp) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = r.timestamp;
if (r.connectionId) {
const accountName = connectionMap[r.connectionId] || `Account ${r.connectionId.slice(0, 8)}...`;
const accountKey = `${r.model} (${r.provider} - ${accountName})`;
if (!stats.byAccount[accountKey]) {
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, connectionId: r.connectionId, accountName, lastUsed: r.timestamp };
}
stats.byAccount[accountKey].requests++;
stats.byAccount[accountKey].promptTokens += promptTokens;
stats.byAccount[accountKey].completionTokens += completionTokens;
stats.byAccount[accountKey].cost += entryCost;
if (new Date(r.timestamp) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = r.timestamp;
}
if (r.apiKey && typeof r.apiKey === "string") {
const keyInfo = apiKeyMap[r.apiKey];
const keyName = keyInfo?.name || r.apiKey.slice(0, 8) + "...";
const akKey = `${r.apiKey}|${r.model}|${r.provider || "unknown"}`;
if (!stats.byApiKey[akKey]) {
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKey: r.apiKey, keyName, apiKeyKey: r.apiKey, lastUsed: r.timestamp };
}
const ake = stats.byApiKey[akKey];
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp;
} else {
if (!stats.byApiKey["local-no-key"]) {
stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKey: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: r.timestamp };
}
const ake = stats.byApiKey["local-no-key"];
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp;
}
const endpoint = r.endpoint || "Unknown";
const epKey = `${endpoint}|${r.model}|${r.provider || "unknown"}`;
if (!stats.byEndpoint[epKey]) {
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
}
const epe = stats.byEndpoint[epKey];
epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cost += entryCost;
if (new Date(r.timestamp) > new Date(epe.lastUsed)) epe.lastUsed = r.timestamp;
}
}
stats.totalRequests = Object.values(stats.byProvider).reduce((sum, p) => sum + (p.requests || 0), 0);
return stats;
}
export async function getChartData(period = "7d") {
const db = await getAdapter();
const now = Date.now();
if (period === "24h") {
const bucketCount = 24;
const bucketMs = 3600000;
const labelFn = (ts) => new Date(ts).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false });
const startTime = now - bucketCount * bucketMs;
const buckets = Array.from({ length: bucketCount }, (_, i) => ({ label: labelFn(startTime + i * bucketMs), tokens: 0, cost: 0 }));
const rows = db.all(
`SELECT timestamp, promptTokens, completionTokens, cost FROM usageHistory WHERE timestamp >= ?`,
[new Date(startTime).toISOString()]
);
for (const r of rows) {
const t = new Date(r.timestamp).getTime();
if (t < startTime || t > now) continue;
const idx = Math.min(Math.floor((t - startTime) / bucketMs), bucketCount - 1);
buckets[idx].tokens += (r.promptTokens || 0) + (r.completionTokens || 0);
buckets[idx].cost += r.cost || 0;
}
return buckets;
}
const bucketCount = period === "7d" ? 7 : period === "30d" ? 30 : 60;
const today = new Date();
const labelFn = (d) => d.toLocaleDateString("en-US", { month: "short", day: "numeric" });
// Build map of dateKey → day data
const dayRows = loadDaysInRange(db, bucketCount);
const dayMap = {};
for (const r of dayRows) dayMap[r.dateKey] = parseJson(r.data, {});
return Array.from({ length: bucketCount }, (_, i) => {
const d = new Date(today);
d.setDate(d.getDate() - (bucketCount - 1 - i));
const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
const dayData = dayMap[dateKey];
return {
label: labelFn(d),
tokens: dayData ? (dayData.promptTokens || 0) + (dayData.completionTokens || 0) : 0,
cost: dayData ? (dayData.cost || 0) : 0,
};
});
}
function formatLogDate(date = new Date()) {
const pad = (n) => String(n).padStart(2, "0");
return `${pad(date.getDate())}-${pad(date.getMonth() + 1)}-${date.getFullYear()} ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
}
// No-op: request log is now derived from usageHistory table on read.
export async function appendRequestLog() {}
export async function getRecentLogs(limit = 200) {
try {
const db = getAdapter();
const rows = db.all(
`SELECT timestamp, provider, model, connectionId, promptTokens, completionTokens, status, tokens FROM usageHistory ORDER BY id DESC LIMIT ?`,
[limit],
);
if (!rows.length) return [];
const connMap = {};
try {
const { getProviderConnections } = await import("./connectionsRepo.js");
const connections = await getProviderConnections();
for (const c of connections) connMap[c.id] = c.name || c.email || "";
} catch {}
return rows.map((r) => {
const ts = formatLogDate(new Date(r.timestamp));
const p = r.provider?.toUpperCase() || "-";
const m = r.model || "-";
const account = connMap[r.connectionId] || (r.connectionId ? r.connectionId.slice(0, 8) : "-");
const tk = r.tokens ? parseJson(r.tokens, {}) : {};
const sent = r.promptTokens ?? tk.prompt_tokens ?? "-";
const received = r.completionTokens ?? tk.completion_tokens ?? "-";
return `${ts} | ${m} | ${p} | ${account} | ${sent} | ${received} | ${r.status || "-"}`;
});
} catch (e) {
console.error("[usageRepo] getRecentLogs failed:", e.message);
return [];
}
}

157
src/lib/db/schema.js Normal file
View file

@ -0,0 +1,157 @@
// Latest schema version — bumped when a migration is added in ./migrations/
export const SCHEMA_VERSION = 1;
export const PRAGMA_SQL = `
PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
PRAGMA temp_store = MEMORY;
PRAGMA mmap_size = 30000000;
PRAGMA cache_size = -64000;
PRAGMA foreign_keys = ON;
PRAGMA busy_timeout = 5000;
`;
// Declarative current schema. Used by syncSchemaFromTables() to
// auto-add missing tables/columns/indexes after versioned migrations.
// For destructive changes (drop/rename/type-change), write a migration file.
export const TABLES = {
_meta: {
columns: {
key: "TEXT PRIMARY KEY",
value: "TEXT NOT NULL",
},
},
settings: {
columns: {
id: "INTEGER PRIMARY KEY CHECK (id = 1)",
data: "TEXT NOT NULL",
},
},
providerConnections: {
columns: {
id: "TEXT PRIMARY KEY",
provider: "TEXT NOT NULL",
authType: "TEXT NOT NULL",
name: "TEXT",
email: "TEXT",
priority: "INTEGER",
isActive: "INTEGER DEFAULT 1",
data: "TEXT NOT NULL",
createdAt: "TEXT NOT NULL",
updatedAt: "TEXT NOT NULL",
},
indexes: [
"CREATE INDEX IF NOT EXISTS idx_pc_provider ON providerConnections(provider)",
"CREATE INDEX IF NOT EXISTS idx_pc_provider_active ON providerConnections(provider, isActive)",
"CREATE INDEX IF NOT EXISTS idx_pc_priority ON providerConnections(provider, priority)",
],
},
providerNodes: {
columns: {
id: "TEXT PRIMARY KEY",
type: "TEXT",
name: "TEXT",
data: "TEXT NOT NULL",
createdAt: "TEXT NOT NULL",
updatedAt: "TEXT NOT NULL",
},
indexes: ["CREATE INDEX IF NOT EXISTS idx_pn_type ON providerNodes(type)"],
},
proxyPools: {
columns: {
id: "TEXT PRIMARY KEY",
isActive: "INTEGER DEFAULT 1",
testStatus: "TEXT",
data: "TEXT NOT NULL",
createdAt: "TEXT NOT NULL",
updatedAt: "TEXT NOT NULL",
},
indexes: [
"CREATE INDEX IF NOT EXISTS idx_pp_active ON proxyPools(isActive)",
"CREATE INDEX IF NOT EXISTS idx_pp_status ON proxyPools(testStatus)",
],
},
apiKeys: {
columns: {
id: "TEXT PRIMARY KEY",
key: "TEXT UNIQUE NOT NULL",
name: "TEXT",
machineId: "TEXT",
isActive: "INTEGER DEFAULT 1",
createdAt: "TEXT NOT NULL",
},
indexes: ["CREATE INDEX IF NOT EXISTS idx_ak_key ON apiKeys(key)"],
},
combos: {
columns: {
id: "TEXT PRIMARY KEY",
name: "TEXT UNIQUE NOT NULL",
kind: "TEXT",
models: "TEXT NOT NULL",
createdAt: "TEXT NOT NULL",
updatedAt: "TEXT NOT NULL",
},
indexes: ["CREATE INDEX IF NOT EXISTS idx_combo_name ON combos(name)"],
},
kv: {
columns: {
scope: "TEXT NOT NULL",
key: "TEXT NOT NULL",
value: "TEXT NOT NULL",
},
primaryKey: "PRIMARY KEY (scope, key)",
indexes: ["CREATE INDEX IF NOT EXISTS idx_kv_scope ON kv(scope)"],
},
usageHistory: {
columns: {
id: "INTEGER PRIMARY KEY AUTOINCREMENT",
timestamp: "TEXT NOT NULL",
provider: "TEXT",
model: "TEXT",
connectionId: "TEXT",
apiKey: "TEXT",
endpoint: "TEXT",
promptTokens: "INTEGER DEFAULT 0",
completionTokens: "INTEGER DEFAULT 0",
cost: "REAL DEFAULT 0",
status: "TEXT",
tokens: "TEXT",
meta: "TEXT",
},
indexes: [
"CREATE INDEX IF NOT EXISTS idx_uh_ts ON usageHistory(timestamp DESC)",
"CREATE INDEX IF NOT EXISTS idx_uh_provider ON usageHistory(provider)",
"CREATE INDEX IF NOT EXISTS idx_uh_model ON usageHistory(model)",
"CREATE INDEX IF NOT EXISTS idx_uh_conn ON usageHistory(connectionId)",
],
},
usageDaily: {
columns: {
dateKey: "TEXT PRIMARY KEY",
data: "TEXT NOT NULL",
},
},
requestDetails: {
columns: {
id: "TEXT PRIMARY KEY",
timestamp: "TEXT NOT NULL",
provider: "TEXT",
model: "TEXT",
connectionId: "TEXT",
status: "TEXT",
data: "TEXT NOT NULL",
},
indexes: [
"CREATE INDEX IF NOT EXISTS idx_rd_ts ON requestDetails(timestamp DESC)",
"CREATE INDEX IF NOT EXISTS idx_rd_provider ON requestDetails(provider)",
"CREATE INDEX IF NOT EXISTS idx_rd_model ON requestDetails(model)",
"CREATE INDEX IF NOT EXISTS idx_rd_conn ON requestDetails(connectionId)",
],
},
};
export function buildCreateTableSql(name, def) {
const cols = Object.entries(def.columns).map(([k, v]) => `${k} ${v}`);
if (def.primaryKey) cols.push(def.primaryKey);
return `CREATE TABLE IF NOT EXISTS ${name} (${cols.join(", ")})`;
}

21
src/lib/db/version.js Normal file
View file

@ -0,0 +1,21 @@
import fs from "node:fs";
import path from "node:path";
let cachedVersion = null;
export function getAppVersion() {
if (cachedVersion) return cachedVersion;
try {
const pkgPath = path.join(process.cwd(), "package.json");
const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
cachedVersion = pkg.version || "0.0.0";
} catch {
cachedVersion = "0.0.0";
}
return cachedVersion;
}
export function timestampSlug(date = new Date()) {
const pad = (n) => String(n).padStart(2, "0");
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
}

71
src/lib/disabledModelsDb.js
View file

@ -1,67 +1,4 @@
import { Low } from "lowdb";
import { JSONFile } from "lowdb/node";
import path from "node:path";
import fs from "node:fs";
import { DATA_DIR } from "@/lib/dataDir.js";
const DB_FILE = path.join(DATA_DIR, "disabledModels.json");
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
const defaultData = { disabled: {} };
let dbInstance = null;
async function getDb() {
if (!dbInstance) {
const adapter = new JSONFile(DB_FILE);
dbInstance = new Low(adapter, defaultData);
try {
await dbInstance.read();
} catch (error) {
if (error instanceof SyntaxError) {
dbInstance.data = { ...defaultData };
await dbInstance.write();
} else {
throw error;
}
}
if (!dbInstance.data || typeof dbInstance.data !== "object") dbInstance.data = { ...defaultData };
if (!dbInstance.data.disabled) dbInstance.data.disabled = {};
}
return dbInstance;
}
export async function getDisabledModels() {
const db = await getDb();
return db.data.disabled || {};
}
export async function getDisabledByProvider(providerAlias) {
const all = await getDisabledModels();
return all[providerAlias] || [];
}
export async function disableModels(providerAlias, ids) {
if (!providerAlias || !Array.isArray(ids)) return;
const db = await getDb();
const current = new Set(db.data.disabled[providerAlias] || []);
ids.forEach((id) => current.add(id));
db.data.disabled[providerAlias] = [...current];
await db.write();
}
export async function enableModels(providerAlias, ids) {
if (!providerAlias) return;
const db = await getDb();
const current = db.data.disabled[providerAlias] || [];
if (!Array.isArray(ids) || ids.length === 0) {
delete db.data.disabled[providerAlias];
} else {
const removeSet = new Set(ids);
const next = current.filter((id) => !removeSet.has(id));
if (next.length === 0) delete db.data.disabled[providerAlias];
else db.data.disabled[providerAlias] = next;
}
await db.write();
}
// Shim → re-export from new SQLite-based DB layer (src/lib/db/)
export {
getDisabledModels, getDisabledByProvider, disableModels, enableModels,
} from "@/lib/db/index.js";

7
src/lib/initCloudSync.js
View file

@ -20,9 +20,12 @@ export async function ensureAppInitialized() {
return g.inProgress;
}
// Auto-initialize at runtime only, not during next build
// Auto-initialize at runtime only, not during next build.
// Defer to next tick so HTTP server can accept connections before heavy init runs.
if (process.env.NEXT_PHASE !== "phase-production-build") {
ensureAppInitialized().catch(console.log);
setImmediate(() => {
ensureAppInitialized().catch(console.log);
});
}
export default ensureAppInitialized;

863
src/lib/localDb.js
View file

@ -1,842 +1,21 @@
import { Low } from "lowdb";
import { JSONFile } from "lowdb/node";
import { v4 as uuidv4 } from "uuid";
import path from "node:path";
import fs from "node:fs";
import lockfile from "proper-lockfile";
import { DATA_DIR } from "@/lib/dataDir.js";
const DEFAULT_MITM_ROUTER_BASE = "http://localhost:20128";
const DB_FILE = path.join(DATA_DIR, "db.json");
if (!fs.existsSync(DATA_DIR)) {
fs.mkdirSync(DATA_DIR, { recursive: true });
}
const DEFAULT_SETTINGS = {
cloudEnabled: false,
tunnelEnabled: false,
tunnelUrl: "",
tunnelProvider: "cloudflare",
tailscaleEnabled: false,
tailscaleUrl: "",
stickyRoundRobinLimit: 3,
providerStrategies: {},
comboStrategy: "fallback",
comboStickyRoundRobinLimit: 1,
comboStrategies: {},
requireLogin: true,
tunnelDashboardAccess: true,
observabilityEnabled: true,
observabilityMaxRecords: 1000,
observabilityBatchSize: 20,
observabilityFlushIntervalMs: 5000,
observabilityMaxJsonSize: 1024,
outboundProxyEnabled: false,
outboundProxyUrl: "",
outboundNoProxy: "",
mitmRouterBaseUrl: DEFAULT_MITM_ROUTER_BASE,
dnsToolEnabled: {},
rtkEnabled: true,
cavemanEnabled: false,
cavemanLevel: "full",
};
function cloneDefaultData() {
return {
providerConnections: [],
providerNodes: [],
proxyPools: [],
modelAliases: {},
customModels: [],
mitmAlias: {},
combos: [],
apiKeys: [],
settings: { ...DEFAULT_SETTINGS },
pricing: {},
};
}
if (!fs.existsSync(DB_FILE)) {
fs.writeFileSync(DB_FILE, JSON.stringify(cloneDefaultData(), null, 2));
}
function ensureDbShape(data) {
const defaults = cloneDefaultData();
const next = data && typeof data === "object" ? data : {};
let changed = false;
for (const [key, defaultValue] of Object.entries(defaults)) {
if (next[key] === undefined || next[key] === null) {
next[key] = defaultValue;
changed = true;
continue;
}
if (key === "settings" && (typeof next.settings !== "object" || Array.isArray(next.settings))) {
next.settings = { ...defaultValue };
changed = true;
continue;
}
if (key === "settings" && typeof next.settings === "object" && !Array.isArray(next.settings)) {
for (const [settingKey, settingDefault] of Object.entries(defaultValue)) {
if (next.settings[settingKey] === undefined) {
// Backward-compat: if proxy URL was saved, default outboundProxyEnabled to true
if (
settingKey === "outboundProxyEnabled" &&
typeof next.settings.outboundProxyUrl === "string" &&
next.settings.outboundProxyUrl.trim()
) {
next.settings.outboundProxyEnabled = true;
} else {
next.settings[settingKey] = settingDefault;
}
changed = true;
}
}
}
// Migrate existing API keys to have isActive
if (key === "apiKeys" && Array.isArray(next.apiKeys)) {
for (const apiKey of next.apiKeys) {
if (apiKey.isActive === undefined || apiKey.isActive === null) {
apiKey.isActive = true;
changed = true;
}
}
}
}
return { data: next, changed };
}
let dbInstance = null;
const LOCK_OPTIONS = {
retries: { retries: 15, minTimeout: 50, maxTimeout: 3000 },
stale: 10000,
};
class LocalMutex {
constructor() {
this._queue = [];
this._locked = false;
}
async acquire() {
if (!this._locked) {
this._locked = true;
return () => this._release();
}
return new Promise((resolve) => {
this._queue.push(() => resolve(() => this._release()));
});
}
_release() {
const next = this._queue.shift();
if (next) next();
else this._locked = false;
}
}
const localMutex = new LocalMutex();
async function withFileLock(db, operation) {
const releaseLocal = await localMutex.acquire();
let release = null;
try {
release = await lockfile.lock(DB_FILE, LOCK_OPTIONS);
await operation();
} catch (error) {
if (error.code === "ELOCKED") {
console.warn(`[DB] File is locked, retrying...`);
}
throw error;
} finally {
if (release) {
try { await release(); } catch (_) { }
}
releaseLocal();
}
}
async function safeRead(db) {
await withFileLock(db, () => db.read());
}
async function safeWrite(db) {
await withFileLock(db, () => db.write());
}
export async function getDb() {
if (!dbInstance) {
dbInstance = new Low(new JSONFile(DB_FILE), cloneDefaultData());
}
try {
await safeRead(dbInstance);
} catch (error) {
if (error instanceof SyntaxError) {
console.warn('[DB] Corrupt JSON detected, resetting to defaults...');
dbInstance.data = cloneDefaultData();
await safeWrite(dbInstance);
} else {
throw error;
}
}
if (!dbInstance.data) {
dbInstance.data = cloneDefaultData();
await safeWrite(dbInstance);
} else {
const { data, changed } = ensureDbShape(dbInstance.data);
dbInstance.data = data;
if (changed) await safeWrite(dbInstance);
}
return dbInstance;
}
export async function getProviderConnections(filter = {}) {
const db = await getDb();
let connections = db.data.providerConnections || [];
if (filter.provider) connections = connections.filter(c => c.provider === filter.provider);
if (filter.isActive !== undefined) connections = connections.filter(c => c.isActive === filter.isActive);
connections.sort((a, b) => (a.priority || 999) - (b.priority || 999));
return connections;
}
export async function getProviderNodes(filter = {}) {
const db = await getDb();
let nodes = db.data.providerNodes || [];
if (filter.type) nodes = nodes.filter((node) => node.type === filter.type);
return nodes;
}
export async function getProviderNodeById(id) {
const db = await getDb();
return db.data.providerNodes.find((node) => node.id === id) || null;
}
export async function createProviderNode(data) {
const db = await getDb();
if (!db.data.providerNodes) db.data.providerNodes = [];
const now = new Date().toISOString();
const node = {
id: data.id || uuidv4(),
type: data.type,
name: data.name,
prefix: data.prefix,
apiType: data.apiType,
baseUrl: data.baseUrl,
createdAt: now,
updatedAt: now,
};
db.data.providerNodes.push(node);
await safeWrite(db);
return node;
}
export async function updateProviderNode(id, data) {
const db = await getDb();
if (!db.data.providerNodes) db.data.providerNodes = [];
const index = db.data.providerNodes.findIndex((node) => node.id === id);
if (index === -1) return null;
db.data.providerNodes[index] = {
...db.data.providerNodes[index],
...data,
updatedAt: new Date().toISOString(),
};
await safeWrite(db);
return db.data.providerNodes[index];
}
export async function deleteProviderNode(id) {
const db = await getDb();
if (!db.data.providerNodes) db.data.providerNodes = [];
const index = db.data.providerNodes.findIndex((node) => node.id === id);
if (index === -1) return null;
const [removed] = db.data.providerNodes.splice(index, 1);
await safeWrite(db);
return removed;
}
export async function getProxyPools(filter = {}) {
const db = await getDb();
let pools = db.data.proxyPools || [];
if (filter.isActive !== undefined) pools = pools.filter((pool) => pool.isActive === filter.isActive);
if (filter.testStatus) pools = pools.filter((pool) => pool.testStatus === filter.testStatus);
return pools.sort((a, b) => new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0));
}
export async function getProxyPoolById(id) {
const db = await getDb();
return (db.data.proxyPools || []).find((pool) => pool.id === id) || null;
}
export async function createProxyPool(data) {
const db = await getDb();
if (!db.data.proxyPools) db.data.proxyPools = [];
const now = new Date().toISOString();
const pool = {
id: data.id || uuidv4(),
name: data.name,
proxyUrl: data.proxyUrl,
noProxy: data.noProxy || "",
type: data.type || "http",
isActive: data.isActive !== undefined ? data.isActive : true,
strictProxy: data.strictProxy === true,
testStatus: data.testStatus || "unknown",
lastTestedAt: data.lastTestedAt || null,
lastError: data.lastError || null,
createdAt: now,
updatedAt: now,
};
db.data.proxyPools.push(pool);
await safeWrite(db);
return pool;
}
export async function updateProxyPool(id, data) {
const db = await getDb();
if (!db.data.proxyPools) db.data.proxyPools = [];
const index = db.data.proxyPools.findIndex((pool) => pool.id === id);
if (index === -1) return null;
db.data.proxyPools[index] = {
...db.data.proxyPools[index],
...data,
updatedAt: new Date().toISOString(),
};
await safeWrite(db);
return db.data.proxyPools[index];
}
export async function deleteProxyPool(id) {
const db = await getDb();
if (!db.data.proxyPools) db.data.proxyPools = [];
const index = db.data.proxyPools.findIndex((pool) => pool.id === id);
if (index === -1) return null;
const [removed] = db.data.proxyPools.splice(index, 1);
await safeWrite(db);
return removed;
}
export async function deleteProviderConnectionsByProvider(providerId) {
const db = await getDb();
const beforeCount = db.data.providerConnections.length;
db.data.providerConnections = db.data.providerConnections.filter(
(connection) => connection.provider !== providerId
);
const deletedCount = beforeCount - db.data.providerConnections.length;
await safeWrite(db);
return deletedCount;
}
export async function getProviderConnectionById(id) {
const db = await getDb();
return db.data.providerConnections.find(c => c.id === id) || null;
}
export async function createProviderConnection(data) {
const db = await getDb();
const now = new Date().toISOString();
// Upsert: check existing by provider + email (oauth) or provider + name (apikey)
let existingIndex = -1;
if (data.authType === "oauth" && data.email) {
existingIndex = db.data.providerConnections.findIndex(
c => c.provider === data.provider && c.authType === "oauth" && c.email === data.email
);
} else if (data.authType === "apikey" && data.name) {
existingIndex = db.data.providerConnections.findIndex(
c => c.provider === data.provider && c.authType === "apikey" && c.name === data.name
);
}
if (existingIndex !== -1) {
db.data.providerConnections[existingIndex] = {
...db.data.providerConnections[existingIndex],
...data,
updatedAt: now,
};
await safeWrite(db);
return db.data.providerConnections[existingIndex];
}
let connectionName = data.name || null;
if (!connectionName && data.authType === "oauth") {
if (data.email) {
connectionName = data.email;
} else {
const existingCount = db.data.providerConnections.filter(
c => c.provider === data.provider
).length;
connectionName = `Account ${existingCount + 1}`;
}
}
let connectionPriority = data.priority;
if (!connectionPriority) {
const providerConnections = db.data.providerConnections.filter(c => c.provider === data.provider);
const maxPriority = providerConnections.reduce((max, c) => Math.max(max, c.priority || 0), 0);
connectionPriority = maxPriority + 1;
}
const connection = {
id: uuidv4(),
provider: data.provider,
authType: data.authType || "oauth",
name: connectionName,
priority: connectionPriority,
isActive: data.isActive !== undefined ? data.isActive : true,
createdAt: now,
updatedAt: now,
};
const optionalFields = [
"displayName", "email", "globalPriority", "defaultModel",
"accessToken", "refreshToken", "expiresAt", "tokenType",
"scope", "projectId", "apiKey", "testStatus",
"lastTested", "lastError", "lastErrorAt", "rateLimitedUntil", "expiresIn", "errorCode",
"consecutiveUseCount"
];
for (const field of optionalFields) {
if (data[field] !== undefined && data[field] !== null) {
connection[field] = data[field];
}
}
if (data.providerSpecificData && Object.keys(data.providerSpecificData).length > 0) {
connection.providerSpecificData = data.providerSpecificData;
}
db.data.providerConnections.push(connection);
await safeWrite(db);
await reorderProviderConnections(data.provider);
return connection;
}
export async function updateProviderConnection(id, data) {
const db = await getDb();
const index = db.data.providerConnections.findIndex(c => c.id === id);
if (index === -1) return null;
const providerId = db.data.providerConnections[index].provider;
db.data.providerConnections[index] = {
...db.data.providerConnections[index],
...data,
updatedAt: new Date().toISOString(),
};
await safeWrite(db);
if (data.priority !== undefined) await reorderProviderConnections(providerId);
return db.data.providerConnections[index];
}
export async function deleteProviderConnection(id) {
const db = await getDb();
const index = db.data.providerConnections.findIndex(c => c.id === id);
if (index === -1) return false;
const providerId = db.data.providerConnections[index].provider;
db.data.providerConnections.splice(index, 1);
await safeWrite(db);
await reorderProviderConnections(providerId);
return true;
}
export async function reorderProviderConnections(providerId) {
const db = await getDb();
if (!db.data.providerConnections) return;
const providerConnections = db.data.providerConnections
.filter(c => c.provider === providerId)
.sort((a, b) => {
const pDiff = (a.priority || 0) - (b.priority || 0);
if (pDiff !== 0) return pDiff;
return new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0);
});
providerConnections.forEach((conn, index) => {
conn.priority = index + 1;
});
await safeWrite(db);
}
export async function getModelAliases() {
const db = await getDb();
return db.data.modelAliases || {};
}
export async function setModelAlias(alias, model) {
const db = await getDb();
db.data.modelAliases[alias] = model;
await safeWrite(db);
}
export async function deleteModelAlias(alias) {
const db = await getDb();
delete db.data.modelAliases[alias];
await safeWrite(db);
}
// Custom models — user-added models with explicit type (llm/image/tts/embedding/...)
export async function getCustomModels() {
const db = await getDb();
return db.data.customModels || [];
}
export async function addCustomModel({ providerAlias, id, type = "llm", name }) {
const db = await getDb();
if (!db.data.customModels) db.data.customModels = [];
const exists = db.data.customModels.some(
(m) => m.providerAlias === providerAlias && m.id === id && (m.type || "llm") === type
);
if (exists) return false;
db.data.customModels.push({ providerAlias, id, type, name: name || id });
await safeWrite(db);
return true;
}
export async function deleteCustomModel({ providerAlias, id, type = "llm" }) {
const db = await getDb();
if (!db.data.customModels) return;
db.data.customModels = db.data.customModels.filter(
(m) => !(m.providerAlias === providerAlias && m.id === id && (m.type || "llm") === type)
);
await safeWrite(db);
}
export async function getMitmAlias(toolName) {
const db = await getDb();
const all = db.data.mitmAlias || {};
if (toolName) return all[toolName] || {};
return all;
}
export async function setMitmAliasAll(toolName, mappings) {
const db = await getDb();
if (!db.data.mitmAlias) db.data.mitmAlias = {};
db.data.mitmAlias[toolName] = mappings || {};
await safeWrite(db);
}
export async function getCombos() {
const db = await getDb();
return db.data.combos || [];
}
export async function getComboById(id) {
const db = await getDb();
return (db.data.combos || []).find(c => c.id === id) || null;
}
export async function getComboByName(name) {
const db = await getDb();
return (db.data.combos || []).find(c => c.name === name) || null;
}
export async function createCombo(data) {
const db = await getDb();
if (!db.data.combos) db.data.combos = [];
const now = new Date().toISOString();
const combo = {
id: uuidv4(),
name: data.name,
models: data.models || [],
kind: data.kind || null,
createdAt: now,
updatedAt: now,
};
db.data.combos.push(combo);
await safeWrite(db);
return combo;
}
export async function updateCombo(id, data) {
const db = await getDb();
if (!db.data.combos) db.data.combos = [];
const index = db.data.combos.findIndex(c => c.id === id);
if (index === -1) return null;
db.data.combos[index] = {
...db.data.combos[index],
...data,
updatedAt: new Date().toISOString(),
};
await safeWrite(db);
return db.data.combos[index];
}
export async function deleteCombo(id) {
const db = await getDb();
if (!db.data.combos) return false;
const index = db.data.combos.findIndex(c => c.id === id);
if (index === -1) return false;
db.data.combos.splice(index, 1);
await safeWrite(db);
return true;
}
export async function getApiKeys() {
const db = await getDb();
return db.data.apiKeys || [];
}
function generateShortKey() {
const chars = "abcdefghijklmnopqrstuvwxyz0123456789";
let result = "";
for (let i = 0; i < 8; i++) {
result += chars.charAt(Math.floor(Math.random() * chars.length));
}
return result;
}
export async function createApiKey(name, machineId) {
if (!machineId) throw new Error("machineId is required");
const db = await getDb();
const now = new Date().toISOString();
const { generateApiKeyWithMachine } = await import("@/shared/utils/apiKey");
const result = generateApiKeyWithMachine(machineId);
const apiKey = {
id: uuidv4(),
name: name,
key: result.key,
machineId: machineId,
isActive: true,
createdAt: now,
};
db.data.apiKeys.push(apiKey);
await safeWrite(db);
return apiKey;
}
export async function deleteApiKey(id) {
const db = await getDb();
const index = db.data.apiKeys.findIndex(k => k.id === id);
if (index === -1) return false;
db.data.apiKeys.splice(index, 1);
await safeWrite(db);
return true;
}
export async function getApiKeyById(id) {
const db = await getDb();
return db.data.apiKeys.find(k => k.id === id) || null;
}
export async function updateApiKey(id, data) {
const db = await getDb();
const index = db.data.apiKeys.findIndex(k => k.id === id);
if (index === -1) return null;
db.data.apiKeys[index] = { ...db.data.apiKeys[index], ...data };
await safeWrite(db);
return db.data.apiKeys[index];
}
export async function validateApiKey(key) {
const db = await getDb();
const found = db.data.apiKeys.find(k => k.key === key);
return found && found.isActive !== false;
}
export async function cleanupProviderConnections() {
const db = await getDb();
const fieldsToCheck = [
"displayName", "email", "globalPriority", "defaultModel",
"accessToken", "refreshToken", "expiresAt", "tokenType",
"scope", "projectId", "apiKey", "testStatus",
"lastTested", "lastError", "lastErrorAt", "rateLimitedUntil", "expiresIn",
"consecutiveUseCount"
];
let cleaned = 0;
for (const connection of db.data.providerConnections) {
for (const field of fieldsToCheck) {
if (connection[field] === null || connection[field] === undefined) {
delete connection[field];
cleaned++;
}
}
if (connection.providerSpecificData && Object.keys(connection.providerSpecificData).length === 0) {
delete connection.providerSpecificData;
cleaned++;
}
}
if (cleaned > 0) await safeWrite(db);
return cleaned;
}
export async function getSettings() {
const db = await getDb();
return db.data.settings || { cloudEnabled: false };
}
export async function updateSettings(updates) {
const db = await getDb();
db.data.settings = { ...db.data.settings, ...updates };
await safeWrite(db);
return db.data.settings;
}
export async function exportDb() {
const db = await getDb();
return db.data || cloneDefaultData();
}
export async function importDb(payload) {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
throw new Error("Invalid database payload");
}
const nextData = {
...cloneDefaultData(),
...payload,
settings: {
...cloneDefaultData().settings,
...(payload.settings && typeof payload.settings === "object" && !Array.isArray(payload.settings)
? payload.settings
: {}),
},
};
const { data: normalized } = ensureDbShape(nextData);
const db = await getDb();
db.data = normalized;
await safeWrite(db);
return db.data;
}
export async function isCloudEnabled() {
const settings = await getSettings();
return settings.cloudEnabled === true;
}
export async function getCloudUrl() {
const settings = await getSettings();
return settings.cloudUrl || process.env.CLOUD_URL || process.env.NEXT_PUBLIC_CLOUD_URL || "";
}
export async function getPricing() {
const db = await getDb();
const userPricing = db.data.pricing || {};
const { PROVIDER_PRICING } = await import("@/shared/constants/pricing.js");
const merged = {};
for (const [provider, models] of Object.entries(PROVIDER_PRICING)) {
merged[provider] = { ...models };
if (userPricing[provider]) {
for (const [model, pricing] of Object.entries(userPricing[provider])) {
merged[provider][model] = merged[provider][model]
? { ...merged[provider][model], ...pricing }
: pricing;
}
}
}
for (const [provider, models] of Object.entries(userPricing)) {
if (!merged[provider]) {
merged[provider] = { ...models };
} else {
for (const [model, pricing] of Object.entries(models)) {
if (!merged[provider][model]) merged[provider][model] = pricing;
}
}
}
return merged;
}
export async function getPricingForModel(provider, model) {
if (!model) return null;
const db = await getDb();
const userPricing = db.data.pricing || {};
if (provider && userPricing[provider]?.[model]) {
return userPricing[provider][model];
}
const { getPricingForModel: resolve } = await import("@/shared/constants/pricing.js");
return resolve(provider, model);
}
export async function updatePricing(pricingData) {
const db = await getDb();
if (!db.data.pricing) db.data.pricing = {};
for (const [provider, models] of Object.entries(pricingData)) {
if (!db.data.pricing[provider]) db.data.pricing[provider] = {};
for (const [model, pricing] of Object.entries(models)) {
db.data.pricing[provider][model] = pricing;
}
}
await safeWrite(db);
return db.data.pricing;
}
export async function resetPricing(provider, model) {
const db = await getDb();
if (!db.data.pricing) db.data.pricing = {};
if (model) {
if (db.data.pricing[provider]) {
delete db.data.pricing[provider][model];
if (Object.keys(db.data.pricing[provider]).length === 0) {
delete db.data.pricing[provider];
}
}
} else {
delete db.data.pricing[provider];
}
await safeWrite(db);
return db.data.pricing;
}
export async function resetAllPricing() {
const db = await getDb();
db.data.pricing = {};
await safeWrite(db);
return db.data.pricing;
}
// Shim → re-export from new SQLite-based DB layer (src/lib/db/)
// Kept for backward compatibility with existing imports.
export {
getSettings, updateSettings, isCloudEnabled, getCloudUrl,
getProviderConnections, getProviderConnectionById,
createProviderConnection, updateProviderConnection,
deleteProviderConnection, deleteProviderConnectionsByProvider,
reorderProviderConnections, cleanupProviderConnections,
getProviderNodes, getProviderNodeById,
createProviderNode, updateProviderNode, deleteProviderNode,
getProxyPools, getProxyPoolById,
createProxyPool, updateProxyPool, deleteProxyPool,
getApiKeys, getApiKeyById, createApiKey, updateApiKey, deleteApiKey, validateApiKey,
getCombos, getComboById, getComboByName,
createCombo, updateCombo, deleteCombo,
getModelAliases, setModelAlias, deleteModelAlias,
getCustomModels, addCustomModel, deleteCustomModel,
getMitmAlias, setMitmAliasAll,
getPricing, getPricingForModel, updatePricing, resetPricing, resetAllPricing,
exportDb, importDb,
} from "@/lib/db/index.js";

5
src/lib/network/initOutboundProxy.js
View file

@ -17,6 +17,9 @@ export async function ensureOutboundProxyInitialized() {
return initialized;
}
ensureOutboundProxyInitialized().catch(console.log);
// Defer init so HTTP server accepts connections first
setImmediate(() => {
ensureOutboundProxyInitialized().catch(console.log);
});
export default ensureOutboundProxyInitialized;

249
src/lib/requestDetailsDb.js
View file

@ -1,245 +1,4 @@
import { Low } from "lowdb";
import { JSONFile } from "lowdb/node";
import path from "node:path";
import fs from "node:fs";
import { DATA_DIR } from "@/lib/dataDir.js";
const DEFAULT_MAX_RECORDS = 200;
const DEFAULT_BATCH_SIZE = 20;
const DEFAULT_FLUSH_INTERVAL_MS = 5000;
const DEFAULT_MAX_JSON_SIZE = 5 * 1024; // 5KB default, configurable via settings
const CONFIG_CACHE_TTL_MS = 5000;
const MAX_TOTAL_DB_SIZE = 50 * 1024 * 1024; // 50MB hard limit for total DB file
const DB_FILE = path.join(DATA_DIR, "request-details.json");
if (!fs.existsSync(DATA_DIR)) {
fs.mkdirSync(DATA_DIR, { recursive: true });
}
let dbInstance = null;
async function getDb() {
if (!dbInstance) {
const adapter = new JSONFile(DB_FILE);
const db = new Low(adapter, { records: [] });
await db.read();
if (!db.data?.records) db.data = { records: [] };
dbInstance = db;
}
return dbInstance;
}
// Config cache
let cachedConfig = null;
let cachedConfigTs = 0;
async function getObservabilityConfig() {
if (cachedConfig && (Date.now() - cachedConfigTs) < CONFIG_CACHE_TTL_MS) {
return cachedConfig;
}
try {
const { getSettings } = await import("@/lib/localDb");
const settings = await getSettings();
const envEnabled = process.env.OBSERVABILITY_ENABLED !== "false";
const enabled = typeof settings.enableObservability === "boolean"
? settings.enableObservability
: envEnabled;
cachedConfig = {
enabled,
maxRecords: settings.observabilityMaxRecords || parseInt(process.env.OBSERVABILITY_MAX_RECORDS || String(DEFAULT_MAX_RECORDS), 10),
batchSize: settings.observabilityBatchSize || parseInt(process.env.OBSERVABILITY_BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10),
flushIntervalMs: settings.observabilityFlushIntervalMs || parseInt(process.env.OBSERVABILITY_FLUSH_INTERVAL_MS || String(DEFAULT_FLUSH_INTERVAL_MS), 10),
maxJsonSize: (settings.observabilityMaxJsonSize || parseInt(process.env.OBSERVABILITY_MAX_JSON_SIZE || "5", 10)) * 1024,
};
} catch {
cachedConfig = {
enabled: false,
maxRecords: DEFAULT_MAX_RECORDS,
batchSize: DEFAULT_BATCH_SIZE,
flushIntervalMs: DEFAULT_FLUSH_INTERVAL_MS,
maxJsonSize: DEFAULT_MAX_JSON_SIZE,
};
}
cachedConfigTs = Date.now();
return cachedConfig;
}
// Batch write queue
let writeBuffer = [];
let flushTimer = null;
let isFlushing = false;
function safeJsonStringify(obj, maxSize) {
try {
const str = JSON.stringify(obj);
if (str.length > maxSize) {
return JSON.stringify({ _truncated: true, _originalSize: str.length, _preview: str.substring(0, 200) });
}
return str;
} catch {
return "{}";
}
}
function sanitizeHeaders(headers) {
if (!headers || typeof headers !== "object") return {};
const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token", "api-key"];
const sanitized = { ...headers };
for (const key of Object.keys(sanitized)) {
if (sensitiveKeys.some(s => key.toLowerCase().includes(s))) {
delete sanitized[key];
}
}
return sanitized;
}
function generateDetailId(model) {
const timestamp = new Date().toISOString();
const random = Math.random().toString(36).substring(2, 8);
const modelPart = model ? model.replace(/[^a-zA-Z0-9-]/g, "-") : "unknown";
return `${timestamp}-${random}-${modelPart}`;
}
async function flushToDatabase() {
if (isFlushing || writeBuffer.length === 0) return;
isFlushing = true;
try {
const itemsToSave = [...writeBuffer];
writeBuffer = [];
const db = await getDb();
const config = await getObservabilityConfig();
for (const item of itemsToSave) {
if (!item.id) item.id = generateDetailId(item.model);
if (!item.timestamp) item.timestamp = new Date().toISOString();
if (item.request?.headers) item.request.headers = sanitizeHeaders(item.request.headers);
// Serialize large fields
const record = {
id: item.id,
provider: item.provider || null,
model: item.model || null,
connectionId: item.connectionId || null,
timestamp: item.timestamp,
status: item.status || null,
latency: item.latency || {},
tokens: item.tokens || {},
request: item.request || {},
providerRequest: item.providerRequest || {},
providerResponse: item.providerResponse || {},
response: item.response || {},
};
// Truncate oversized JSON fields
const maxSize = config.maxJsonSize;
for (const field of ["request", "providerRequest", "providerResponse", "response"]) {
const str = JSON.stringify(record[field]);
if (str.length > maxSize) {
record[field] = { _truncated: true, _originalSize: str.length, _preview: str.substring(0, 200) };
}
}
// Upsert: replace existing record with same id
const idx = db.data.records.findIndex(r => r.id === record.id);
if (idx !== -1) {
db.data.records[idx] = record;
} else {
db.data.records.push(record);
}
}
// Keep only latest maxRecords (sorted by timestamp desc)
db.data.records.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
if (db.data.records.length > config.maxRecords) {
db.data.records = db.data.records.slice(0, config.maxRecords);
}
// Shrink records until total serialized size is within safe limit
while (db.data.records.length > 1) {
const totalSize = Buffer.byteLength(JSON.stringify(db.data), "utf8");
if (totalSize <= MAX_TOTAL_DB_SIZE) break;
db.data.records = db.data.records.slice(0, Math.floor(db.data.records.length / 2));
}
await db.write();
} catch (error) {
console.error("[requestDetailsDb] Batch write failed:", error);
} finally {
isFlushing = false;
}
}
export async function saveRequestDetail(detail) {
const config = await getObservabilityConfig();
if (!config.enabled) return;
writeBuffer.push(detail);
if (writeBuffer.length >= config.batchSize) {
await flushToDatabase();
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
} else if (!flushTimer) {
flushTimer = setTimeout(() => {
flushToDatabase().catch(() => {});
flushTimer = null;
}, config.flushIntervalMs);
}
}
export async function getRequestDetails(filter = {}) {
const db = await getDb();
let records = [...db.data.records];
// Apply filters
if (filter.provider) records = records.filter(r => r.provider === filter.provider);
if (filter.model) records = records.filter(r => r.model === filter.model);
if (filter.connectionId) records = records.filter(r => r.connectionId === filter.connectionId);
if (filter.status) records = records.filter(r => r.status === filter.status);
if (filter.startDate) records = records.filter(r => new Date(r.timestamp) >= new Date(filter.startDate));
if (filter.endDate) records = records.filter(r => new Date(r.timestamp) <= new Date(filter.endDate));
// Sort desc
records.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
const totalItems = records.length;
const page = filter.page || 1;
const pageSize = filter.pageSize || 50;
const totalPages = Math.ceil(totalItems / pageSize);
const details = records.slice((page - 1) * pageSize, page * pageSize);
return {
details,
pagination: { page, pageSize, totalItems, totalPages, hasNext: page < totalPages, hasPrev: page > 1 },
};
}
export async function getRequestDetailById(id) {
const db = await getDb();
return db.data.records.find(r => r.id === id) || null;
}
// Graceful shutdown — use named handler so we can remove it on re-registration
const _shutdownHandler = async () => {
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
if (writeBuffer.length > 0) await flushToDatabase();
};
function ensureShutdownHandler() {
// Remove any previously registered listeners from this module (hot-reload safety)
process.off("beforeExit", _shutdownHandler);
process.off("SIGINT", _shutdownHandler);
process.off("SIGTERM", _shutdownHandler);
process.off("exit", _shutdownHandler);
process.on("beforeExit", _shutdownHandler);
process.on("SIGINT", _shutdownHandler);
process.on("SIGTERM", _shutdownHandler);
process.on("exit", _shutdownHandler);
}
ensureShutdownHandler();
// Shim → re-export from new SQLite-based DB layer (src/lib/db/)
export {
saveRequestDetail, getRequestDetails, getRequestDetailById,
} from "@/lib/db/index.js";

19
src/lib/tunnel/networkProbe.js
View file

@ -26,15 +26,16 @@ export function checkInternet() {
}
async function resolveDns(hostname, timeoutMs) {
try {
await Promise.race([
resolver.resolve4(hostname),
new Promise((_, rej) => setTimeout(() => rej(new Error("dns timeout")), timeoutMs)),
]);
return true;
} catch {
return false;
}
// Try custom public DNS first (bypasses negative-cached NXDOMAIN on macOS).
// Fall back to OS resolver for hostnames blocked or unsupported by Cloudflare DNS
// (e.g. *.ts.net not always resolvable via 1.1.1.1).
const tryResolver = (fn) => Promise.race([
fn(),
new Promise((_, rej) => setTimeout(() => rej(new Error("dns timeout")), timeoutMs)),
]).then(() => true).catch(() => false);
if (await tryResolver(() => resolver.resolve4(hostname))) return true;
return tryResolver(() => dns.promises.resolve4(hostname));
}
// Single health probe: DNS via 1.1.1.1 → fetch /api/health

250
src/lib/tunnel/tailscale.js
View file

@ -1,11 +1,14 @@
import fs from "fs";
import path from "path";
import os from "os";
import { execSync, spawn } from "child_process";
import { execSync, exec, spawn } from "child_process";
import { promisify } from "util";
import { execWithPassword } from "@/mitm/dns/dnsConfig";
import { saveTailscalePid, loadTailscalePid, clearTailscalePid } from "./state.js";
import { DATA_DIR } from "@/lib/dataDir.js";
const execAsync = promisify(exec);
const BIN_DIR = path.join(DATA_DIR, "bin");
const IS_MAC = os.platform() === "darwin";
const IS_LINUX = os.platform() === "linux";
@ -20,17 +23,58 @@ const SOCKET_FLAG = IS_WINDOWS ? [] : ["--socket", TAILSCALE_SOCKET];
// Well-known Windows install path
const WINDOWS_TAILSCALE_BIN = "C:\\Program Files\\Tailscale\\tailscale.exe";
// Prefer system tailscale, fallback to local bin, then Windows default path
function getTailscaleBin() {
try {
const systemPath = execSync("which tailscale 2>/dev/null || where tailscale 2>nul", { encoding: "utf8", windowsHide: true }).trim();
if (systemPath) return systemPath;
} catch (e) { /* not in PATH */ }
// Common Unix install paths to probe synchronously (system tailscale)
const UNIX_TAILSCALE_CANDIDATES = [
"/usr/local/bin/tailscale",
"/opt/homebrew/bin/tailscale",
"/usr/bin/tailscale",
];
// ─── Cache + background refresh (avoid blocking event loop on dead daemon) ──
const PROBE_TTL_MS = 10000;
const PROBE_TIMEOUT_MS = 1500;
const binCache = { value: undefined, fetchedAt: 0, refreshing: false };
const runningCache = { value: false, fetchedAt: 0, refreshing: false };
const funnelUrlCache = { value: null, port: null, fetchedAt: 0, refreshing: false };
function fallbackBin() {
if (fs.existsSync(TAILSCALE_BIN)) return TAILSCALE_BIN;
if (IS_WINDOWS && fs.existsSync(WINDOWS_TAILSCALE_BIN)) return WINDOWS_TAILSCALE_BIN;
if (!IS_WINDOWS) return UNIX_TAILSCALE_CANDIDATES.find((p) => fs.existsSync(p)) || null;
return null;
}
function bgRefreshBin() {
if (binCache.refreshing) return;
binCache.refreshing = true;
execAsync("which tailscale 2>/dev/null || where tailscale 2>nul", { windowsHide: true, timeout: PROBE_TIMEOUT_MS })
.then(({ stdout }) => {
const sys = stdout.trim();
binCache.value = sys || fallbackBin();
})
.catch(() => { binCache.value = fallbackBin(); })
.finally(() => {
binCache.fetchedAt = Date.now();
binCache.refreshing = false;
});
}
// Sync getter: returns cached value, triggers background refresh if stale
function getTailscaleBin() {
if (Date.now() - binCache.fetchedAt > PROBE_TTL_MS) bgRefreshBin();
// First call: synchronously probe common install paths (no exec, no event-loop block)
if (binCache.value === undefined) {
if (fs.existsSync(TAILSCALE_BIN)) binCache.value = TAILSCALE_BIN;
else if (IS_WINDOWS && fs.existsSync(WINDOWS_TAILSCALE_BIN)) binCache.value = WINDOWS_TAILSCALE_BIN;
else if (!IS_WINDOWS) {
const found = UNIX_TAILSCALE_CANDIDATES.find((p) => fs.existsSync(p));
binCache.value = found || null;
} else binCache.value = null;
}
return binCache.value;
}
export function isTailscaleInstalled() {
return getTailscaleBin() !== null;
}
@ -58,29 +102,83 @@ export function isTailscaleLoggedIn() {
}
}
function bgRefreshRunning() {
if (runningCache.refreshing) return;
const bin = getTailscaleBin();
if (!bin) {
runningCache.value = false;
runningCache.fetchedAt = Date.now();
return;
}
runningCache.refreshing = true;
execAsync(`"${bin}" ${SOCKET_FLAG.join(" ")} funnel status --json`, { windowsHide: true, timeout: PROBE_TIMEOUT_MS })
.then(({ stdout }) => {
try {
const json = JSON.parse(stdout);
runningCache.value = Object.keys(json.AllowFunnel || {}).length > 0;
} catch { runningCache.value = false; }
})
.catch(() => { runningCache.value = false; })
.finally(() => {
runningCache.fetchedAt = Date.now();
runningCache.refreshing = false;
});
}
// Sync getter: never blocks; returns last known state, refreshes in background
export function isTailscaleRunning() {
if (Date.now() - runningCache.fetchedAt > PROBE_TTL_MS) bgRefreshRunning();
return runningCache.value;
}
// Synchronous strict probe for hot user-initiated paths (enable/connect flow).
// Blocks ~PROBE_TIMEOUT_MS at most; updates cache as a side effect.
export function isTailscaleRunningStrict() {
const bin = getTailscaleBin();
if (!bin) return false;
try {
const out = execSync(`"${bin}" ${SOCKET_FLAG.join(" ")} funnel status --json 2>/dev/null`, { encoding: "utf8", windowsHide: true });
const out = execSync(`"${bin}" ${SOCKET_FLAG.join(" ")} funnel status --json 2>/dev/null`, {
encoding: "utf8",
windowsHide: true,
timeout: PROBE_TIMEOUT_MS,
});
const json = JSON.parse(out);
return Object.keys(json.AllowFunnel || {}).length > 0;
} catch (e) {
const running = Object.keys(json.AllowFunnel || {}).length > 0;
runningCache.value = running;
runningCache.fetchedAt = Date.now();
return running;
} catch {
return false;
}
}
/** Get funnel URL from tailscale status */
export function getTailscaleFunnelUrl(port) {
function bgRefreshFunnelUrl(port) {
if (funnelUrlCache.refreshing) return;
const bin = getTailscaleBin();
if (!bin) return null;
try {
const out = execSync(`"${bin}" ${SOCKET_FLAG.join(" ")} status --json`, { encoding: "utf8", windowsHide: true });
const json = JSON.parse(out);
const dnsName = json.Self?.DNSName?.replace(/\.$/, "");
if (dnsName) return `https://${dnsName}`;
} catch (e) { /* ignore */ }
return null;
if (!bin) return;
funnelUrlCache.refreshing = true;
execAsync(`"${bin}" ${SOCKET_FLAG.join(" ")} status --json`, { windowsHide: true, timeout: PROBE_TIMEOUT_MS })
.then(({ stdout }) => {
try {
const json = JSON.parse(stdout);
const dnsName = json.Self?.DNSName?.replace(/\.$/, "");
funnelUrlCache.value = dnsName ? `https://${dnsName}` : null;
} catch { /* keep prev */ }
})
.catch(() => { /* keep prev */ })
.finally(() => {
funnelUrlCache.port = port;
funnelUrlCache.fetchedAt = Date.now();
funnelUrlCache.refreshing = false;
});
}
/** Get funnel URL from tailscale status (cached, non-blocking) */
export function getTailscaleFunnelUrl(port) {
if (Date.now() - funnelUrlCache.fetchedAt > PROBE_TTL_MS || funnelUrlCache.port !== port) {
bgRefreshFunnelUrl(port);
}
return funnelUrlCache.value;
}
/**
@ -280,8 +378,46 @@ async function installTailscaleWindows(log) {
throw new Error("Installation finished but tailscale.exe not found");
}
/** Start tailscaled with sudo (TUN mode required for Funnel) */
export async function startDaemonWithPassword(sudoPassword) {
// Self-heal: if state dir/files were previously created by root (e.g. legacy sudo daemon),
// reclaim ownership recursively so the user-mode daemon can read/write state files.
async function ensureUserOwnedDir(dir) {
try {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
return;
}
const uid = process.getuid();
const gid = process.getgid();
// Walk dir + all entries to find any non-user-owned items
const needsChown = (() => {
const stack = [dir];
while (stack.length) {
const cur = stack.pop();
try {
const st = fs.statSync(cur);
if (st.uid !== uid) return true;
if (st.isDirectory()) {
for (const name of fs.readdirSync(cur)) stack.push(path.join(cur, name));
}
} catch { /* ignore */ }
}
return false;
})();
if (!needsChown) return;
// Try direct chown first (works if already owned). Fallback to passwordless sudo.
try {
execSync(`chown -R ${uid}:${gid} "${dir}"`, { stdio: "ignore", timeout: 3000 });
} catch {
try { execSync(`sudo -n chown -R ${uid}:${gid} "${dir}"`, { stdio: "ignore", timeout: 3000 }); } catch { /* ignore */ }
}
} catch { /* ignore */ }
}
/** Start tailscaled in userspace-networking mode (no root, no sudo prompt). */
export async function startDaemonWithPassword(_sudoPasswordUnused) {
if (IS_WINDOWS) {
// Windows: tailscale runs as a Windows Service, try to start it
try {
@ -298,29 +434,63 @@ export async function startDaemonWithPassword(sudoPassword) {
return;
}
// Check if daemon already responds
// Detect unhealthy state: dir/files not owned by current user OR multiple daemons running.
// Either condition blocks userspace daemon → must kill all + reclaim ownership.
let needsRestart = false;
try {
const bin = getTailscaleBin() || "tailscale";
execSync(`"${bin}" ${SOCKET_FLAG.join(" ")} status --json`, {
stdio: "ignore",
windowsHide: true,
env: { ...process.env, PATH: EXTENDED_PATH },
timeout: 3000
});
return; // Already running
} catch { /* not running, start it */ }
const st = fs.statSync(TAILSCALE_DIR);
if (st.uid !== process.getuid()) needsRestart = true;
// Also check state file (the actual unhealthy resource)
const stateFile = path.join(TAILSCALE_DIR, "tailscaled.state");
if (fs.existsSync(stateFile) && fs.statSync(stateFile).uid !== process.getuid()) needsRestart = true;
} catch { /* dir doesn't exist yet */ }
// Ensure config dir exists
if (!fs.existsSync(TAILSCALE_DIR)) fs.mkdirSync(TAILSCALE_DIR, { recursive: true });
// Detect duplicate daemons on same socket → also requires restart
if (!needsRestart) {
try {
const ps = execSync(`pgrep -f "tailscaled.*${TAILSCALE_SOCKET}"`, { encoding: "utf8", timeout: 2000 }).trim();
if (ps && ps.split("\n").length > 1) needsRestart = true;
} catch { /* no match → ok */ }
}
// tailscaled requires root for TUN (needed for Funnel)
if (needsRestart) {
// Kill ALL tailscaled processes (root + user duplicates). Best-effort with/without sudo.
try { execSync("pkill -9 -x tailscaled", { stdio: "ignore", timeout: 3000 }); } catch { /* ignore */ }
try { execSync("sudo -n pkill -9 -x tailscaled", { stdio: "ignore", timeout: 3000 }); } catch { /* ignore */ }
await new Promise((r) => setTimeout(r, 1500));
} else {
// Check if our userspace daemon already responds
try {
const bin = getTailscaleBin() || "tailscale";
execSync(`"${bin}" ${SOCKET_FLAG.join(" ")} status --json`, {
stdio: "ignore",
windowsHide: true,
env: { ...process.env, PATH: EXTENDED_PATH },
timeout: 3000
});
return; // Already running and user-owned
} catch { /* not running, start it */ }
}
// Reclaim folder ownership if a previous root daemon left it locked
await ensureUserOwnedDir(TAILSCALE_DIR);
// Userspace-networking mode: no TUN device → no root needed → no sudo prompt
const tailscaledBin = IS_MAC ? "/usr/local/bin/tailscaled" : "tailscaled";
const daemonCmd = `${tailscaledBin} --socket=${TAILSCALE_SOCKET} --statedir=${TAILSCALE_DIR}`;
const args = [
`--socket=${TAILSCALE_SOCKET}`,
`--statedir=${TAILSCALE_DIR}`,
"--tun=userspace-networking",
];
// Start via sudo in background (nohup keeps it alive)
await execWithPassword(`nohup ${daemonCmd} > /dev/null 2>&1 &`, sudoPassword || "");
const child = spawn(tailscaledBin, args, {
detached: true,
stdio: "ignore",
env: { ...process.env, PATH: EXTENDED_PATH },
});
child.unref();
// Wait for daemon to be ready
// Wait for daemon socket to be ready
await new Promise((r) => setTimeout(r, 3000));
}
@ -403,7 +573,7 @@ export function startLogin(hostname) {
const url = parseAuthUrl(output);
if (url) resolve({ authUrl: url });
else if (code === 0 || isTailscaleLoggedIn()) resolve({ alreadyLoggedIn: true });
else reject(new Error(`tailscale up exited with code ${code}`));
else reject(new Error(`tailscale up exited with code ${code}: ${output.trim() || "no output"}`));
});
});
}

83
src/lib/tunnel/tunnelManager.js
View file

@ -1,7 +1,7 @@
import crypto from "crypto";
import { loadState, saveState, generateShortId } from "./state.js";
import { spawnQuickTunnel, killCloudflared, isCloudflaredRunning, setUnexpectedExitHandler } from "./cloudflared.js";
import { startFunnel, stopFunnel, isTailscaleRunning, isTailscaleLoggedIn, startLogin, startDaemonWithPassword } from "./tailscale.js";
import { startFunnel, stopFunnel, isTailscaleRunning, isTailscaleRunningStrict, isTailscaleLoggedIn, startLogin, startDaemonWithPassword } from "./tailscale.js";
import { getSettings, updateSettings } from "@/lib/localDb";
import { getCachedPassword, loadEncryptedPassword, initDbHooks } from "@/mitm/manager";
import { waitForHealth, probeUrlAlive } from "./networkProbe.js";
@ -33,6 +33,33 @@ export function isTunnelManuallyDisabled() { return tunnelSvc.cancelToken.cancel
export function isTunnelReconnecting() { return tunnelSvc.spawnInProgress; }
export function isTailscaleReconnecting() { return tailscaleSvc.spawnInProgress; }
// ─── Reachable cache: background probe of tunnel URL /api/health ─────────────
// UI uses this to know if the public URL actually serves content (not just process alive)
const REACHABLE_TTL_MS = 30000;
const tunnelReachable = { value: false, url: null, fetchedAt: 0, refreshing: false };
const tailscaleReachable = { value: false, url: null, fetchedAt: 0, refreshing: false };
function bgRefreshReachable(cache, url) {
if (cache.refreshing) return;
if (!url) { cache.value = false; cache.url = null; cache.fetchedAt = Date.now(); return; }
cache.refreshing = true;
probeUrlAlive(url)
.then((ok) => { cache.value = ok; })
.catch(() => { cache.value = false; })
.finally(() => {
cache.url = url;
cache.fetchedAt = Date.now();
cache.refreshing = false;
});
}
function readReachable(cache, url) {
// URL changed → invalidate
if (cache.url !== url) { cache.value = false; cache.fetchedAt = 0; }
if (Date.now() - cache.fetchedAt > REACHABLE_TTL_MS) bgRefreshReachable(cache, url);
return cache.value;
}
function getMachineId() {
try {
const { machineIdSync } = require("node-machine-id");
@ -94,9 +121,16 @@ export async function enableTunnel(localPort = 20128) {
saveState({ shortId, machineId, tunnelUrl });
await updateSettings({ tunnelEnabled: true, tunnelUrl });
// Block until /api/health responds via public URL — proves DNS propagated + tunnel works
// Verify direct tunnel URL is reachable first (avoid CDN-cache false positive on publicUrl)
await waitForHealth(tunnelUrl, token);
// Then verify public URL (DNS propagated through 9router.com worker)
await waitForHealth(publicUrl, token);
// Prime reachable cache so UI shows correct state immediately
tunnelReachable.value = true;
tunnelReachable.url = tunnelUrl;
tunnelReachable.fetchedAt = Date.now();
return { success: true, tunnelUrl, shortId, publicUrl };
} finally {
tunnelSvc.spawnInProgress = false;
@ -112,23 +146,31 @@ export async function disableTunnel() {
if (state) saveState({ shortId: state.shortId, machineId: state.machineId, tunnelUrl: null });
await updateSettings({ tunnelEnabled: false, tunnelUrl: "" });
tunnelReachable.value = false; tunnelReachable.url = null; tunnelReachable.fetchedAt = Date.now();
return { success: true };
}
export async function getTunnelStatus() {
const state = loadState();
const running = isCloudflaredRunning();
const settings = await getSettings();
const settingsEnabled = settings.tunnelEnabled === true;
const state = loadState();
const shortId = state?.shortId || "";
const publicUrl = shortId ? `https://r${shortId}.9router.com` : "";
const tunnelUrl = state?.tunnelUrl || "";
// Lazy: skip PID probe entirely when user disabled tunnel
const running = settingsEnabled ? isCloudflaredRunning() : false;
// Reachable: cached background probe (never blocks the request)
const reachable = settingsEnabled && running ? readReachable(tunnelReachable, tunnelUrl) : false;
return {
enabled: settings.tunnelEnabled === true && running,
settingsEnabled: settings.tunnelEnabled === true,
tunnelUrl: state?.tunnelUrl || "",
enabled: settingsEnabled && running,
settingsEnabled,
tunnelUrl,
shortId,
publicUrl,
running
running,
reachable
};
}
@ -163,7 +205,8 @@ export async function enableTailscale(localPort = 20128) {
return { success: false, funnelNotEnabled: true, enableUrl: result.enableUrl };
}
if (!isTailscaleLoggedIn() || !isTailscaleRunning()) {
// Strict probe: bypass cache so we don't false-negative on first invocation
if (!isTailscaleLoggedIn() || !isTailscaleRunningStrict()) {
stopFunnel();
return { success: false, error: "Tailscale not connected. Device may have been removed. Please re-login." };
}
@ -173,6 +216,11 @@ export async function enableTailscale(localPort = 20128) {
// Verify funnel actually serves /api/health
await waitForHealth(result.tunnelUrl, token);
// Prime reachable cache so UI shows correct state immediately
tailscaleReachable.value = true;
tailscaleReachable.url = result.tunnelUrl;
tailscaleReachable.fetchedAt = Date.now();
return { success: true, tunnelUrl: result.tunnelUrl };
} finally {
tailscaleSvc.spawnInProgress = false;
@ -183,16 +231,23 @@ export async function disableTailscale() {
tailscaleSvc.cancelToken.cancelled = true;
stopFunnel();
await updateSettings({ tailscaleEnabled: false, tailscaleUrl: "" });
tailscaleReachable.value = false; tailscaleReachable.url = null; tailscaleReachable.fetchedAt = Date.now();
return { success: true };
}
export async function getTailscaleStatus() {
const settings = await getSettings();
const running = isTailscaleRunning();
const settingsEnabled = settings.tailscaleEnabled === true;
const tunnelUrl = settings.tailscaleUrl || "";
// Lazy: skip execSync funnel-status probe when user disabled Tailscale
const running = settingsEnabled ? isTailscaleRunning() : false;
// Reachable: cached background probe (never blocks the request)
const reachable = settingsEnabled && running ? readReachable(tailscaleReachable, tunnelUrl) : false;
return {
enabled: settings.tailscaleEnabled === true && running,
settingsEnabled: settings.tailscaleEnabled === true,
tunnelUrl: settings.tailscaleUrl || "",
running
enabled: settingsEnabled && running,
settingsEnabled,
tunnelUrl,
running,
reachable
};
}

900
src/lib/usageDb.js
View file

@ -1,893 +1,7 @@
import { Low } from "lowdb";
import { JSONFile } from "lowdb/node";
import { EventEmitter } from "events";
import path from "path";
import fs from "fs";
import { DATA_DIR } from "@/lib/dataDir.js";
const DB_FILE = path.join(DATA_DIR, "usage.json");
const LOG_FILE = path.join(DATA_DIR, "log.txt");
// Ensure data directory exists
if (fs && typeof fs.existsSync === "function") {
try {
if (!fs.existsSync(DATA_DIR)) {
fs.mkdirSync(DATA_DIR, { recursive: true });
console.log(`[usageDb] Created data directory: ${DATA_DIR}`);
}
} catch (error) {
console.error("[usageDb] Failed to create data directory:", error.message);
}
}
const defaultData = {
history: [],
totalRequestsLifetime: 0,
dailySummary: {},
};
function getLocalDateKey(timestamp) {
const d = timestamp ? new Date(timestamp) : new Date();
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
}
function addToCounter(target, key, values) {
if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
target[key].requests += values.requests || 1;
target[key].promptTokens += values.promptTokens || 0;
target[key].completionTokens += values.completionTokens || 0;
target[key].cost += values.cost || 0;
if (values.meta) Object.assign(target[key], values.meta);
}
function aggregateEntryToDailySummary(dailySummary, entry) {
const dateKey = getLocalDateKey(entry.timestamp);
if (!dailySummary[dateKey]) {
dailySummary[dateKey] = {
requests: 0, promptTokens: 0, completionTokens: 0, cost: 0,
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
};
}
const day = dailySummary[dateKey];
const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0;
const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0;
const cost = entry.cost || 0;
const vals = { promptTokens, completionTokens, cost };
day.requests += 1;
day.promptTokens += promptTokens;
day.completionTokens += completionTokens;
day.cost += cost;
if (entry.provider) addToCounter(day.byProvider, entry.provider, vals);
const modelKey = entry.provider ? `${entry.model}|${entry.provider}` : entry.model;
addToCounter(day.byModel, modelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } });
if (entry.connectionId) {
addToCounter(day.byAccount, entry.connectionId, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } });
}
const apiKeyVal = entry.apiKey && typeof entry.apiKey === "string" ? entry.apiKey : "local-no-key";
const akModelKey = `${apiKeyVal}|${entry.model}|${entry.provider || "unknown"}`;
addToCounter(day.byApiKey, akModelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider, apiKey: entry.apiKey || null } });
const endpoint = entry.endpoint || "Unknown";
const epKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`;
addToCounter(day.byEndpoint, epKey, { ...vals, meta: { endpoint, rawModel: entry.model, provider: entry.provider } });
}
function migrateHistoryToDailySummary(db) {
const history = db.data.history || [];
if (!history.length) return false;
db.data.dailySummary = {};
for (const entry of history) {
aggregateEntryToDailySummary(db.data.dailySummary, entry);
}
console.log(`[usageDb] Migrated ${history.length} history entries to dailySummary (${Object.keys(db.data.dailySummary).length} days)`);
return true;
}
// Singleton instance
let dbInstance = null;
// Use global to share pending state across Next.js route modules
if (!global._pendingRequests) {
global._pendingRequests = { byModel: {}, byAccount: {} };
}
const pendingRequests = global._pendingRequests;
// Track last error provider for UI edge coloring (auto-clears after 10s)
if (!global._lastErrorProvider) {
global._lastErrorProvider = { provider: "", ts: 0 };
}
const lastErrorProvider = global._lastErrorProvider;
// Use global to share singleton across Next.js route modules
if (!global._statsEmitter) {
global._statsEmitter = new EventEmitter();
global._statsEmitter.setMaxListeners(50);
}
export const statsEmitter = global._statsEmitter;
// Safety timers — force-clear pending counts after 1 min if END was never called
if (!global._pendingTimers) global._pendingTimers = {};
const pendingTimers = global._pendingTimers;
const PENDING_TIMEOUT_MS = 60 * 1000; // 1 minute
// In-memory ring buffer for recent requests (avoids disk I/O on every SSE emit)
const RING_CAP = 50;
const CONN_CACHE_TTL_MS = 30 * 1000;
if (!global._recentRing) global._recentRing = { items: [], initialized: false };
if (!global._connectionMapCache) global._connectionMapCache = { map: {}, ts: 0 };
const recentRing = global._recentRing;
const connCache = global._connectionMapCache;
function pushToRing(entry) {
recentRing.items.push(entry);
if (recentRing.items.length > RING_CAP) {
recentRing.items = recentRing.items.slice(-RING_CAP);
}
}
async function getConnectionMapCached() {
if (Date.now() - connCache.ts < CONN_CACHE_TTL_MS) return connCache.map;
try {
const { getProviderConnections } = await import("@/lib/localDb.js");
const allConnections = await getProviderConnections();
const map = {};
for (const conn of allConnections) map[conn.id] = conn.name || conn.email || conn.id;
connCache.map = map;
connCache.ts = Date.now();
} catch {}
return connCache.map;
}
async function ensureRingInitialized() {
if (recentRing.initialized) return;
recentRing.initialized = true;
try {
const db = await getUsageDb();
const history = db.data.history || [];
recentRing.items = history.slice(-RING_CAP);
} catch {}
}
/**
* Track a pending request
* @param {string} model
* @param {string} provider
* @param {string} connectionId
* @param {boolean} started - true if started, false if finished
* @param {boolean} [error] - true if ended with error
*/
export function trackPendingRequest(model, provider, connectionId, started, error = false) {
const modelKey = provider ? `${model} (${provider})` : model;
const timerKey = `${connectionId}|${modelKey}`;
// Track by model
if (!pendingRequests.byModel[modelKey]) pendingRequests.byModel[modelKey] = 0;
pendingRequests.byModel[modelKey] = Math.max(0, pendingRequests.byModel[modelKey] + (started ? 1 : -1));
if (pendingRequests.byModel[modelKey] === 0) delete pendingRequests.byModel[modelKey];
// Track by account
if (connectionId) {
if (!pendingRequests.byAccount[connectionId]) pendingRequests.byAccount[connectionId] = {};
if (!pendingRequests.byAccount[connectionId][modelKey]) pendingRequests.byAccount[connectionId][modelKey] = 0;
pendingRequests.byAccount[connectionId][modelKey] = Math.max(0, pendingRequests.byAccount[connectionId][modelKey] + (started ? 1 : -1));
if (pendingRequests.byAccount[connectionId][modelKey] === 0) {
delete pendingRequests.byAccount[connectionId][modelKey];
if (Object.keys(pendingRequests.byAccount[connectionId]).length === 0) {
delete pendingRequests.byAccount[connectionId];
}
}
}
if (started) {
// Safety timeout: force-clear if END is never called (client disconnect, crash, etc.)
clearTimeout(pendingTimers[timerKey]);
pendingTimers[timerKey] = setTimeout(() => {
delete pendingTimers[timerKey];
if (pendingRequests.byModel[modelKey] > 0) {
pendingRequests.byModel[modelKey] = 0;
}
if (connectionId && pendingRequests.byAccount[connectionId]?.[modelKey] > 0) {
pendingRequests.byAccount[connectionId][modelKey] = 0;
}
statsEmitter.emit("pending");
}, PENDING_TIMEOUT_MS);
} else {
// END called normally — cancel the safety timer
clearTimeout(pendingTimers[timerKey]);
delete pendingTimers[timerKey];
}
// Track error provider (auto-clears after 10s)
if (!started && error && provider) {
lastErrorProvider.provider = provider.toLowerCase();
lastErrorProvider.ts = Date.now();
}
const t = new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" });
console.log(`[${t}] [PENDING] ${started ? "START" : "END"}${error ? " (ERROR)" : ""} | provider=${provider} | model=${model}`);
statsEmitter.emit("pending");
}
/**
* Lightweight: get only activeRequests + recentRequests without full stats recalc
*/
export async function getActiveRequests() {
const activeRequests = [];
const connectionMap = await getConnectionMapCached();
for (const [connectionId, models] of Object.entries(pendingRequests.byAccount)) {
for (const [modelKey, count] of Object.entries(models)) {
if (count > 0) {
const accountName = connectionMap[connectionId] || `Account ${connectionId.slice(0, 8)}...`;
const match = modelKey.match(/^(.*) \((.*)\)$/);
const modelName = match ? match[1] : modelKey;
const providerName = match ? match[2] : "unknown";
activeRequests.push({ model: modelName, provider: providerName, account: accountName, count });
}
}
}
// Recent requests from in-memory ring (zero disk I/O)
await ensureRingInitialized();
const seen = new Set();
const recentRequests = [...recentRing.items]
.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp))
.map((e) => {
const t = e.tokens || {};
const promptTokens = t.prompt_tokens || t.input_tokens || 0;
const completionTokens = t.completion_tokens || t.output_tokens || 0;
return { timestamp: e.timestamp, model: e.model, provider: e.provider || "", promptTokens, completionTokens, status: e.status || "ok" };
})
.filter((e) => {
if (e.promptTokens === 0 && e.completionTokens === 0) return false;
const minute = e.timestamp ? e.timestamp.slice(0, 16) : "";
const key = `${e.model}|${e.provider}|${e.promptTokens}|${e.completionTokens}|${minute}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
})
.slice(0, 20);
// Error provider (auto-clear after 10s)
const errorProvider = (Date.now() - lastErrorProvider.ts < 10000) ? lastErrorProvider.provider : "";
return { activeRequests, recentRequests, errorProvider };
}
/**
* Get usage database instance (singleton)
*/
export async function getUsageDb() {
if (!dbInstance) {
const adapter = new JSONFile(DB_FILE);
dbInstance = new Low(adapter, defaultData);
// Try to read DB with error recovery for corrupt JSON
try {
await dbInstance.read();
} catch (error) {
if (error instanceof SyntaxError) {
console.warn('[DB] Corrupt Usage JSON detected, resetting to defaults...');
dbInstance.data = defaultData;
await dbInstance.write();
} else {
throw error;
}
}
if (!dbInstance.data) {
dbInstance.data = { ...defaultData };
await dbInstance.write();
}
// Migration: build dailySummary from existing history (one-time)
if (!dbInstance.data.dailySummary) {
if (migrateHistoryToDailySummary(dbInstance)) {
await dbInstance.write();
} else {
dbInstance.data.dailySummary = {};
}
}
}
return dbInstance;
}
/**
* Save request usage
* @param {object} entry - Usage entry { provider, model, tokens: { prompt_tokens, completion_tokens, ... }, connectionId?, apiKey? }
*/
export async function saveRequestUsage(entry) {
try {
const db = await getUsageDb();
// Add timestamp if not present
if (!entry.timestamp) {
entry.timestamp = new Date().toISOString();
}
// Ensure history array exists
if (!Array.isArray(db.data.history)) {
db.data.history = [];
}
if (typeof db.data.totalRequestsLifetime !== "number") {
db.data.totalRequestsLifetime = db.data.history.length;
}
const entryCost = await calculateCost(entry.provider, entry.model, entry.tokens);
entry.cost = entryCost;
db.data.history.push(entry);
db.data.totalRequestsLifetime += 1;
if (!db.data.dailySummary) db.data.dailySummary = {};
aggregateEntryToDailySummary(db.data.dailySummary, entry);
const MAX_HISTORY = 2000;
if (db.data.history.length > MAX_HISTORY) {
db.data.history.splice(0, db.data.history.length - MAX_HISTORY);
}
await db.write();
pushToRing(entry);
statsEmitter.emit("update");
} catch (error) {
console.error("Failed to save usage stats:", error);
}
}
/**
* Get usage history
* @param {object} filter - Filter criteria
*/
export async function getUsageHistory(filter = {}) {
const db = await getUsageDb();
let history = db.data.history || [];
// Apply filters
if (filter.provider) {
history = history.filter(h => h.provider === filter.provider);
}
if (filter.model) {
history = history.filter(h => h.model === filter.model);
}
if (filter.startDate) {
const start = new Date(filter.startDate).getTime();
history = history.filter(h => new Date(h.timestamp).getTime() >= start);
}
if (filter.endDate) {
const end = new Date(filter.endDate).getTime();
history = history.filter(h => new Date(h.timestamp).getTime() <= end);
}
return history;
}
/**
* Format date as dd-mm-yyyy h:m:s
*/
function formatLogDate(date = new Date()) {
const pad = (n) => String(n).padStart(2, "0");
const d = pad(date.getDate());
const m = pad(date.getMonth() + 1);
const y = date.getFullYear();
const h = pad(date.getHours());
const min = pad(date.getMinutes());
const s = pad(date.getSeconds());
return `${d}-${m}-${y} ${h}:${min}:${s}`;
}
/**
* Append to log.txt
* Format: datetime(dd-mm-yyyy h:m:s) | model | provider | account | tokens sent | tokens received | status
*/
export async function appendRequestLog({ model, provider, connectionId, tokens, status }) {
try {
const timestamp = formatLogDate();
const p = provider?.toUpperCase() || "-";
const m = model || "-";
// Resolve account name
let account = connectionId ? connectionId.slice(0, 8) : "-";
try {
const { getProviderConnections } = await import("@/lib/localDb.js");
const connections = await getProviderConnections();
const conn = connections.find(c => c.id === connectionId);
if (conn) {
account = conn.name || conn.email || account;
}
} catch {}
const sent = tokens?.prompt_tokens !== undefined ? tokens.prompt_tokens : "-";
const received = tokens?.completion_tokens !== undefined ? tokens.completion_tokens : "-";
const line = `${timestamp} | ${m} | ${p} | ${account} | ${sent} | ${received} | ${status}\n`;
fs.appendFileSync(LOG_FILE, line);
// Trim to keep only last 200 lines
const content = fs.readFileSync(LOG_FILE, "utf-8");
const lines = content.trim().split("\n");
if (lines.length > 200) {
fs.writeFileSync(LOG_FILE, lines.slice(-200).join("\n") + "\n");
}
} catch (error) {
console.error("Failed to append to log.txt:", error.message);
}
}
/**
* Get last N lines of log.txt
*/
export async function getRecentLogs(limit = 200) {
// Runtime check: ensure fs module is available
if (!fs || typeof fs.existsSync !== "function") {
console.error("[usageDb] fs module not available in this environment");
return [];
}
if (!LOG_FILE) {
console.error("[usageDb] LOG_FILE path not defined");
return [];
}
if (!fs.existsSync(LOG_FILE)) {
console.log(`[usageDb] Log file does not exist: ${LOG_FILE}`);
return [];
}
try {
const content = fs.readFileSync(LOG_FILE, "utf-8");
const lines = content.trim().split("\n");
return lines.slice(-limit).reverse();
} catch (error) {
console.error("[usageDb] Failed to read log.txt:", error.message);
console.error("[usageDb] LOG_FILE path:", LOG_FILE);
return [];
}
}
/**
* Calculate cost for a usage entry
* @param {string} provider - Provider ID
* @param {string} model - Model ID
* @param {object} tokens - Token counts
* @returns {number} Cost in dollars
*/
async function calculateCost(provider, model, tokens) {
if (!tokens || !provider || !model) return 0;
try {
const { getPricingForModel } = await import("@/lib/localDb.js");
const pricing = await getPricingForModel(provider, model);
if (!pricing) return 0;
let cost = 0;
// Input tokens (non-cached)
const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0;
const nonCachedInput = Math.max(0, inputTokens - cachedTokens);
cost += (nonCachedInput * (pricing.input / 1000000));
// Cached tokens
if (cachedTokens > 0) {
const cachedRate = pricing.cached || pricing.input; // Fallback to input rate
cost += (cachedTokens * (cachedRate / 1000000));
}
// Output tokens
const outputTokens = tokens.completion_tokens || tokens.output_tokens || 0;
cost += (outputTokens * (pricing.output / 1000000));
// Reasoning tokens
const reasoningTokens = tokens.reasoning_tokens || 0;
if (reasoningTokens > 0) {
const reasoningRate = pricing.reasoning || pricing.output; // Fallback to output rate
cost += (reasoningTokens * (reasoningRate / 1000000));
}
// Cache creation tokens
const cacheCreationTokens = tokens.cache_creation_input_tokens || 0;
if (cacheCreationTokens > 0) {
const cacheCreationRate = pricing.cache_creation || pricing.input; // Fallback to input rate
cost += (cacheCreationTokens * (cacheCreationRate / 1000000));
}
return cost;
} catch (error) {
console.error("Error calculating cost:", error);
return 0;
}
}
const PERIOD_MS = { "24h": 86400000, "7d": 604800000, "30d": 2592000000, "60d": 5184000000 };
/**
* Get aggregated usage stats
* @param {"24h"|"7d"|"30d"|"60d"|"all"} period - Time period to filter
*/
export async function getUsageStats(period = "all") {
const db = await getUsageDb();
const history = db.data.history || [];
const dailySummary = db.data.dailySummary || {};
const { getProviderConnections, getApiKeys, getProviderNodes } = await import("@/lib/localDb.js");
let allConnections = [];
try { allConnections = await getProviderConnections(); } catch {}
const connectionMap = {};
for (const conn of allConnections) {
connectionMap[conn.id] = conn.name || conn.email || conn.id;
}
const providerNodeNameMap = {};
try {
const nodes = await getProviderNodes();
for (const node of nodes) {
if (node.id && node.name) providerNodeNameMap[node.id] = node.name;
}
} catch {}
let allApiKeys = [];
try { allApiKeys = await getApiKeys(); } catch {}
const apiKeyMap = {};
for (const key of allApiKeys) {
apiKeyMap[key.key] = { name: key.name, id: key.id, createdAt: key.createdAt };
}
// Recent requests (always from live history)
const seen = new Set();
const recentRequests = [...history]
.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp))
.map((e) => {
const t = e.tokens || {};
return {
timestamp: e.timestamp, model: e.model, provider: e.provider || "",
promptTokens: t.prompt_tokens || t.input_tokens || 0,
completionTokens: t.completion_tokens || t.output_tokens || 0,
status: e.status || "ok",
};
})
.filter((e) => {
if (e.promptTokens === 0 && e.completionTokens === 0) return false;
const minute = e.timestamp ? e.timestamp.slice(0, 16) : "";
const key = `${e.model}|${e.provider}|${e.promptTokens}|${e.completionTokens}|${minute}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
})
.slice(0, 20);
// totalRequests: calculated from period-filtered data (not lifetime)
const stats = {
totalRequests: 0,
totalPromptTokens: 0, totalCompletionTokens: 0, totalCost: 0,
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
last10Minutes: [],
pending: pendingRequests,
activeRequests: [],
recentRequests,
errorProvider: (Date.now() - lastErrorProvider.ts < 10000) ? lastErrorProvider.provider : "",
};
// Active requests from pending
for (const [connectionId, models] of Object.entries(pendingRequests.byAccount)) {
for (const [modelKey, count] of Object.entries(models)) {
if (count > 0) {
const accountName = connectionMap[connectionId] || `Account ${connectionId.slice(0, 8)}...`;
const match = modelKey.match(/^(.*) \((.*)\)$/);
stats.activeRequests.push({
model: match ? match[1] : modelKey,
provider: match ? match[2] : "unknown",
account: accountName, count,
});
}
}
}
// last10Minutes — always from live history
const now = new Date();
const currentMinuteStart = new Date(Math.floor(now.getTime() / 60000) * 60000);
const tenMinutesAgo = new Date(currentMinuteStart.getTime() - 9 * 60 * 1000);
const bucketMap = {};
for (let i = 0; i < 10; i++) {
const bucketKey = currentMinuteStart.getTime() - (9 - i) * 60 * 1000;
bucketMap[bucketKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
stats.last10Minutes.push(bucketMap[bucketKey]);
}
for (const entry of history) {
const entryTime = new Date(entry.timestamp);
if (entryTime >= tenMinutesAgo && entryTime <= now) {
const entryMinuteStart = Math.floor(entryTime.getTime() / 60000) * 60000;
if (bucketMap[entryMinuteStart]) {
const pt = entry.tokens?.prompt_tokens || 0;
const ct = entry.tokens?.completion_tokens || 0;
bucketMap[entryMinuteStart].requests++;
bucketMap[entryMinuteStart].promptTokens += pt;
bucketMap[entryMinuteStart].completionTokens += ct;
bucketMap[entryMinuteStart].cost += entry.cost || 0;
}
}
}
// Determine if we use dailySummary (7d/30d/60d/all) or live history (24h)
const useDailySummary = period !== "24h";
if (useDailySummary) {
// Collect relevant date keys
const periodDays = { "7d": 7, "30d": 30, "60d": 60 };
const maxDays = periodDays[period] || null; // null = all
const today = new Date();
const dateKeys = Object.keys(dailySummary).filter((dateKey) => {
if (!maxDays) return true;
const parts = dateKey.split("-");
const d = new Date(Number(parts[0]), Number(parts[1]) - 1, Number(parts[2]));
const diffDays = Math.floor((today.getTime() - d.getTime()) / 86400000);
return diffDays < maxDays;
});
for (const dateKey of dateKeys) {
const day = dailySummary[dateKey];
stats.totalPromptTokens += day.promptTokens || 0;
stats.totalCompletionTokens += day.completionTokens || 0;
stats.totalCost += day.cost || 0;
// Merge byProvider
for (const [prov, pData] of Object.entries(day.byProvider || {})) {
if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
stats.byProvider[prov].requests += pData.requests || 0;
stats.byProvider[prov].promptTokens += pData.promptTokens || 0;
stats.byProvider[prov].completionTokens += pData.completionTokens || 0;
stats.byProvider[prov].cost += pData.cost || 0;
}
// Merge byModel (dailySummary key: "model|provider" → stats key: "model (provider)")
for (const [mk, mData] of Object.entries(day.byModel || {})) {
const rawModel = mData.rawModel || mk.split("|")[0];
const provider = mData.provider || mk.split("|")[1] || "";
const statsKey = provider ? `${rawModel} (${provider})` : rawModel;
const providerDisplayName = providerNodeNameMap[provider] || provider;
if (!stats.byModel[statsKey]) {
stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey };
}
stats.byModel[statsKey].requests += mData.requests || 0;
stats.byModel[statsKey].promptTokens += mData.promptTokens || 0;
stats.byModel[statsKey].completionTokens += mData.completionTokens || 0;
stats.byModel[statsKey].cost += mData.cost || 0;
if (dateKey > (stats.byModel[statsKey].lastUsed || "")) stats.byModel[statsKey].lastUsed = dateKey;
}
// Merge byAccount
for (const [connId, aData] of Object.entries(day.byAccount || {})) {
const accountName = connectionMap[connId] || `Account ${connId.slice(0, 8)}...`;
const rawModel = aData.rawModel || "";
const provider = aData.provider || "";
const providerDisplayName = providerNodeNameMap[provider] || provider;
const accountKey = `${rawModel} (${provider} - ${accountName})`;
if (!stats.byAccount[accountKey]) {
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey };
}
stats.byAccount[accountKey].requests += aData.requests || 0;
stats.byAccount[accountKey].promptTokens += aData.promptTokens || 0;
stats.byAccount[accountKey].completionTokens += aData.completionTokens || 0;
stats.byAccount[accountKey].cost += aData.cost || 0;
if (dateKey > (stats.byAccount[accountKey].lastUsed || "")) stats.byAccount[accountKey].lastUsed = dateKey;
}
// Merge byApiKey
for (const [akKey, akData] of Object.entries(day.byApiKey || {})) {
const rawModel = akData.rawModel || "";
const provider = akData.provider || "";
const providerDisplayName = providerNodeNameMap[provider] || provider;
const apiKeyVal = akData.apiKey;
const keyInfo = apiKeyVal ? apiKeyMap[apiKeyVal] : null;
const keyName = keyInfo?.name || (apiKeyVal ? apiKeyVal.slice(0, 8) + "..." : "Local (No API Key)");
const apiKeyKey = apiKeyVal || "local-no-key";
if (!stats.byApiKey[akKey]) {
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKey: apiKeyVal, keyName, apiKeyKey, lastUsed: dateKey };
}
stats.byApiKey[akKey].requests += akData.requests || 0;
stats.byApiKey[akKey].promptTokens += akData.promptTokens || 0;
stats.byApiKey[akKey].completionTokens += akData.completionTokens || 0;
stats.byApiKey[akKey].cost += akData.cost || 0;
if (dateKey > (stats.byApiKey[akKey].lastUsed || "")) stats.byApiKey[akKey].lastUsed = dateKey;
}
// Merge byEndpoint
for (const [epKey, epData] of Object.entries(day.byEndpoint || {})) {
const endpoint = epData.endpoint || epKey.split("|")[0] || "Unknown";
const rawModel = epData.rawModel || "";
const provider = epData.provider || "";
const providerDisplayName = providerNodeNameMap[provider] || provider;
if (!stats.byEndpoint[epKey]) {
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey };
}
stats.byEndpoint[epKey].requests += epData.requests || 0;
stats.byEndpoint[epKey].promptTokens += epData.promptTokens || 0;
stats.byEndpoint[epKey].completionTokens += epData.completionTokens || 0;
stats.byEndpoint[epKey].cost += epData.cost || 0;
if (dateKey > (stats.byEndpoint[epKey].lastUsed || "")) stats.byEndpoint[epKey].lastUsed = dateKey;
}
}
// Overlay lastUsed with precise ISO timestamps from live history (dailySummary only has YYYY-MM-DD)
const overlayCutoff = maxDays ? Date.now() - maxDays * 86400000 : 0;
for (const entry of history) {
const ts = entry.timestamp;
if (!ts || new Date(ts).getTime() < overlayCutoff) continue;
const modelKey = entry.provider ? `${entry.model} (${entry.provider})` : entry.model;
if (stats.byModel[modelKey] && new Date(ts) > new Date(stats.byModel[modelKey].lastUsed)) {
stats.byModel[modelKey].lastUsed = ts;
}
if (entry.connectionId) {
const accountName = connectionMap[entry.connectionId] || `Account ${entry.connectionId.slice(0, 8)}...`;
const accountKey = `${entry.model} (${entry.provider} - ${accountName})`;
if (stats.byAccount[accountKey] && new Date(ts) > new Date(stats.byAccount[accountKey].lastUsed)) {
stats.byAccount[accountKey].lastUsed = ts;
}
}
const apiKeyKey = (entry.apiKey && typeof entry.apiKey === "string")
? `${entry.apiKey}|${entry.model}|${entry.provider || "unknown"}`
: "local-no-key";
if (stats.byApiKey[apiKeyKey] && new Date(ts) > new Date(stats.byApiKey[apiKeyKey].lastUsed)) {
stats.byApiKey[apiKeyKey].lastUsed = ts;
}
const endpoint = entry.endpoint || "Unknown";
const endpointKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`;
if (stats.byEndpoint[endpointKey] && new Date(ts) > new Date(stats.byEndpoint[endpointKey].lastUsed)) {
stats.byEndpoint[endpointKey].lastUsed = ts;
}
}
} else {
// 24h: use live history (original logic)
const cutoff = Date.now() - PERIOD_MS["24h"];
const filtered = history.filter((e) => new Date(e.timestamp).getTime() >= cutoff);
for (const entry of filtered) {
const promptTokens = entry.tokens?.prompt_tokens || 0;
const completionTokens = entry.tokens?.completion_tokens || 0;
const entryCost = entry.cost || 0;
const providerDisplayName = providerNodeNameMap[entry.provider] || entry.provider;
stats.totalPromptTokens += promptTokens;
stats.totalCompletionTokens += completionTokens;
stats.totalCost += entryCost;
// byProvider
if (!stats.byProvider[entry.provider]) stats.byProvider[entry.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
stats.byProvider[entry.provider].requests++;
stats.byProvider[entry.provider].promptTokens += promptTokens;
stats.byProvider[entry.provider].completionTokens += completionTokens;
stats.byProvider[entry.provider].cost += entryCost;
// byModel
const modelKey = entry.provider ? `${entry.model} (${entry.provider})` : entry.model;
if (!stats.byModel[modelKey]) {
stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: entry.model, provider: providerDisplayName, lastUsed: entry.timestamp };
}
stats.byModel[modelKey].requests++;
stats.byModel[modelKey].promptTokens += promptTokens;
stats.byModel[modelKey].completionTokens += completionTokens;
stats.byModel[modelKey].cost += entryCost;
if (new Date(entry.timestamp) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = entry.timestamp;
// byAccount
if (entry.connectionId) {
const accountName = connectionMap[entry.connectionId] || `Account ${entry.connectionId.slice(0, 8)}...`;
const accountKey = `${entry.model} (${entry.provider} - ${accountName})`;
if (!stats.byAccount[accountKey]) {
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: entry.model, provider: providerDisplayName, connectionId: entry.connectionId, accountName, lastUsed: entry.timestamp };
}
stats.byAccount[accountKey].requests++;
stats.byAccount[accountKey].promptTokens += promptTokens;
stats.byAccount[accountKey].completionTokens += completionTokens;
stats.byAccount[accountKey].cost += entryCost;
if (new Date(entry.timestamp) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = entry.timestamp;
}
// byApiKey
if (entry.apiKey && typeof entry.apiKey === "string") {
const keyInfo = apiKeyMap[entry.apiKey];
const keyName = keyInfo?.name || entry.apiKey.slice(0, 8) + "...";
const apiKeyModelKey = `${entry.apiKey}|${entry.model}|${entry.provider || "unknown"}`;
if (!stats.byApiKey[apiKeyModelKey]) {
stats.byApiKey[apiKeyModelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: entry.model, provider: providerDisplayName, apiKey: entry.apiKey, keyName, apiKeyKey: entry.apiKey, lastUsed: entry.timestamp };
}
const ake = stats.byApiKey[apiKeyModelKey];
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
if (new Date(entry.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = entry.timestamp;
} else {
if (!stats.byApiKey["local-no-key"]) {
stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: entry.model, provider: providerDisplayName, apiKey: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: entry.timestamp };
}
const ake = stats.byApiKey["local-no-key"];
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
if (new Date(entry.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = entry.timestamp;
}
// byEndpoint
const endpoint = entry.endpoint || "Unknown";
const endpointModelKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`;
if (!stats.byEndpoint[endpointModelKey]) {
stats.byEndpoint[endpointModelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel: entry.model, provider: providerDisplayName, lastUsed: entry.timestamp };
}
const epe = stats.byEndpoint[endpointModelKey];
epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cost += entryCost;
if (new Date(entry.timestamp) > new Date(epe.lastUsed)) epe.lastUsed = entry.timestamp;
}
}
// Calculate totalRequests from period-filtered data (not lifetime)
stats.totalRequests = Object.values(stats.byProvider).reduce((sum, p) => sum + (p.requests || 0), 0);
return stats;
}
/**
* Get time-series chart data for a given period
* @param {"24h"|"7d"|"30d"|"60d"} period
* @returns {Promise<Array<{label: string, tokens: number, cost: number}>>}
*/
export async function getChartData(period = "7d") {
const db = await getUsageDb();
const history = db.data.history || [];
const dailySummary = db.data.dailySummary || {};
const now = Date.now();
// 24h: bucket by hour from live history
if (period === "24h") {
const bucketCount = 24;
const bucketMs = 3600000;
const labelFn = (ts) => new Date(ts).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false });
const startTime = now - bucketCount * bucketMs;
const buckets = Array.from({ length: bucketCount }, (_, i) => {
const ts = startTime + i * bucketMs;
return { label: labelFn(ts), tokens: 0, cost: 0 };
});
for (const entry of history) {
const entryTime = new Date(entry.timestamp).getTime();
if (entryTime < startTime || entryTime > now) continue;
const idx = Math.min(Math.floor((entryTime - startTime) / bucketMs), bucketCount - 1);
buckets[idx].tokens += (entry.tokens?.prompt_tokens || 0) + (entry.tokens?.completion_tokens || 0);
buckets[idx].cost += entry.cost || 0;
}
return buckets;
}
// 7d/30d/60d: bucket by day from dailySummary (local dates)
const bucketCount = period === "7d" ? 7 : period === "30d" ? 30 : 60;
const today = new Date();
const labelFn = (d) => d.toLocaleDateString("en-US", { month: "short", day: "numeric" });
const buckets = Array.from({ length: bucketCount }, (_, i) => {
const d = new Date(today);
d.setDate(d.getDate() - (bucketCount - 1 - i));
const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
const dayData = dailySummary[dateKey];
return {
label: labelFn(d),
tokens: dayData ? (dayData.promptTokens || 0) + (dayData.completionTokens || 0) : 0,
cost: dayData ? (dayData.cost || 0) : 0,
};
});
return buckets;
}
// Re-export request details functions from new SQLite-based module
export { saveRequestDetail, getRequestDetails, getRequestDetailById } from "./requestDetailsDb.js";
// Shim → re-export from new SQLite-based DB layer (src/lib/db/)
export {
statsEmitter, trackPendingRequest, getActiveRequests,
saveRequestUsage, getUsageHistory, getUsageStats, getChartData,
appendRequestLog, getRecentLogs,
saveRequestDetail, getRequestDetails, getRequestDetailById,
} from "@/lib/db/index.js";

49
src/mitm/dbReader.js Normal file
View file

@ -0,0 +1,49 @@
// CJS reader for MITM standalone process. Reads SQLite mitmAlias scope.
// Falls back to legacy db.json or db.json.migrated if SQLite unavailable.
const fs = require("fs");
const path = require("path");
const { DATA_DIR } = require("./paths");
const DB_FILE = path.join(DATA_DIR, "db", "data.sqlite");
const LEGACY_JSON = path.join(DATA_DIR, "db.json");
const LEGACY_MIGRATED = path.join(DATA_DIR, "db.json.migrated");
let sqliteDb = null;
let sqliteFailed = false;
function trySqlite() {
if (sqliteDb) return sqliteDb;
if (sqliteFailed) return null;
try {
if (!fs.existsSync(DB_FILE)) return null;
const Database = require("better-sqlite3");
sqliteDb = new Database(DB_FILE, { readonly: true, fileMustExist: true });
return sqliteDb;
} catch {
sqliteFailed = true;
return null;
}
}
function readLegacyJson() {
for (const file of [LEGACY_JSON, LEGACY_MIGRATED]) {
if (!fs.existsSync(file)) continue;
try { return JSON.parse(fs.readFileSync(file, "utf-8")); } catch {}
}
return null;
}
function getMitmAlias(toolName) {
const db = trySqlite();
if (db) {
try {
const row = db.prepare(`SELECT value FROM kv WHERE scope = 'mitmAlias' AND key = ?`).get(toolName);
if (row) return JSON.parse(row.value);
} catch {}
}
// Fallback to legacy JSON
const legacy = readLegacyJson();
return legacy?.mitmAlias?.[toolName] || null;
}
module.exports = { getMitmAlias };

2
src/mitm/dev

@ -1 +1 @@
Subproject commit b3b7473d6df221586f9056190bd98389d3b5c0d7
Subproject commit ea459d42d0963147c987752078d9de53001779ab

2
src/mitm/handlers/base.js
View file

@ -1,6 +1,6 @@
const { log, err } = require("../logger");
const DEFAULT_LOCAL_ROUTER = "http://localhost:20128";
const DEFAULT_LOCAL_ROUTER = "http://127.0.0.1:20128";
const ROUTER_BASE = String(process.env.MITM_ROUTER_BASE || DEFAULT_LOCAL_ROUTER)
.trim()
.replace(/\/+$/, "") || DEFAULT_LOCAL_ROUTER;

2
src/mitm/manager.js
View file

@ -16,7 +16,7 @@ const { isCertExpired } = require("./cert/rootCA");
const { DATA_DIR, MITM_DIR } = require("./paths");
const { log, err } = require("./logger");
const DEFAULT_MITM_ROUTER_BASE = "http://localhost:20128";
const DEFAULT_MITM_ROUTER_BASE = "http://127.0.0.1:20128";
function shellQuoteSingle(str) {
if (str == null || str === "") return "''";

7
src/mitm/server.js
View file

@ -8,8 +8,7 @@ const { log, err, dumpRequest, createResponseDumper } = require("./logger");
const { TARGET_HOSTS, URL_PATTERNS, MODEL_SYNONYMS, getToolForHost } = require("./config");
const { DATA_DIR, MITM_DIR } = require("./paths");
const { getCertForDomain } = require("./cert/generate");
const DB_FILE = path.join(DATA_DIR, "db.json");
const { getMitmAlias } = require("./dbReader");
const LOCAL_PORT = 443;
const IS_WIN = process.platform === "win32";
const ENABLE_FILE_LOG = true;
@ -108,9 +107,7 @@ function extractModel(url, body) {
function getMappedModel(tool, model) {
if (!model) return null;
try {
if (!fs.existsSync(DB_FILE)) return null;
const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
const aliases = db.mitmAlias?.[tool];
const aliases = getMitmAlias(tool);
if (!aliases) return null;
// Normalize via synonym map (e.g., gemini-default → gemini-3-flash)
const lookup = MODEL_SYNONYMS?.[tool]?.[model] || model;

2
src/shared/components/RequestLogger.js
View file

@ -114,7 +114,7 @@ export default function RequestLogger() {
</div>
</Card>
<div className="text-[10px] text-text-muted italic">
Logs are saved to log.txt in the application data directory.
Logs are loaded from the request history database.
</div>
</div>
);

2
src/shared/components/Sidebar.js
View file

@ -51,7 +51,7 @@ export default function Sidebar({ onClose }) {
const { copied, copy } = useCopyToClipboard(2000);
const INSTALL_CMD = UPDATER_CONFIG.installCmd;
const STATUS_URL = `http://localhost:${UPDATER_CONFIG.statusPort}/update/status`;
const STATUS_URL = `http://127.0.0.1:${UPDATER_CONFIG.statusPort}/update/status`;
useEffect(() => {
fetch("/api/settings")

5
src/shared/constants/config.js
View file

@ -2,7 +2,7 @@ import pkg from "../../../package.json" with { type: "json" };
// App configuration
export const APP_CONFIG = {
name: "9Router proxy",
name: "9Router Proxy",
description: "AI Infrastructure Management",
version: pkg.version,
};
@ -56,6 +56,9 @@ export const CONSOLE_LOG_CONFIG = {
pollIntervalMs: 1000,
};
// Client-side store TTL: how long fetched data stays fresh before re-fetching
export const CLIENT_STORE_TTL_MS = 60000;
// Provider API endpoints (for display only)
export const PROVIDER_ENDPOINTS = {
openrouter: "https://openrouter.ai/api/v1/chat/completions",

2
src/shared/services/cloudSyncScheduler.js
View file

@ -4,7 +4,7 @@ import { isCloudEnabled } from "@/lib/localDb";
const INTERNAL_BASE_URL =
process.env.BASE_URL ||
process.env.NEXT_PUBLIC_BASE_URL ||
"http://localhost:20128";
"http://127.0.0.1:20128";
/**
* Cloud sync scheduler

13
src/store/providerStore.js
View file

@ -1,13 +1,15 @@
"use client";
import { create } from "zustand";
import { CLIENT_STORE_TTL_MS } from "@/shared/constants/config";
const useProviderStore = create((set, get) => ({
providers: [],
loading: false,
error: null,
lastFetched: 0,
setProviders: (providers) => set({ providers }),
setProviders: (providers) => set({ providers, lastFetched: Date.now() }),
addProvider: (provider) =>
set((state) => ({ providers: [provider, ...state.providers] })),
@ -24,17 +26,22 @@ const useProviderStore = create((set, get) => ({
providers: state.providers.filter((p) => p._id !== id),
})),
invalidate: () => set({ lastFetched: 0 }),
setLoading: (loading) => set({ loading }),
setError: (error) => set({ error }),
fetchProviders: async () => {
// Skips network when cache is fresh (< CLIENT_STORE_TTL_MS). Pass {force:true} to override.
fetchProviders: async ({ force = false } = {}) => {
const { lastFetched, providers } = get();
if (!force && providers.length > 0 && Date.now() - lastFetched < CLIENT_STORE_TTL_MS) return;
set({ loading: true, error: null });
try {
const response = await fetch("/api/providers");
const data = await response.json();
if (response.ok) {
set({ providers: data.providers, loading: false });
set({ providers: data.connections || data.providers || [], loading: false, lastFetched: Date.now() });
} else {
set({ error: data.error, loading: false });
}

51
src/store/settingsStore.js Normal file
View file

@ -0,0 +1,51 @@
"use client";
import { create } from "zustand";
import { CLIENT_STORE_TTL_MS } from "@/shared/constants/config";
const useSettingsStore = create((set, get) => ({
settings: null,
loading: false,
error: null,
lastFetched: 0,
invalidate: () => set({ lastFetched: 0 }),
// Skips network when cache is fresh; pass {force:true} to override
fetchSettings: async ({ force = false } = {}) => {
const { lastFetched, settings } = get();
if (!force && settings && Date.now() - lastFetched < CLIENT_STORE_TTL_MS) return settings;
set({ loading: true, error: null });
try {
const res = await fetch("/api/settings");
const data = await res.json();
if (res.ok) {
set({ settings: data, loading: false, lastFetched: Date.now() });
return data;
}
set({ error: data.error, loading: false });
} catch (e) {
set({ error: "Failed to fetch settings", loading: false });
}
return null;
},
// PATCH server + merge into local cache (no extra fetch needed)
patchSettings: async (patch) => {
try {
const res = await fetch("/api/settings", {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(patch),
});
if (!res.ok) return null;
const updated = await res.json();
set({ settings: updated, lastFetched: Date.now() });
return updated;
} catch {
return null;
}
},
}));
export default useSettingsStore;

174
tests/unit/db-benchmark.test.js Normal file
View file

@ -0,0 +1,174 @@
// Benchmark: SQLite vs lowdb on equivalent workloads.
// Run: cd app/tests && npm test -- db-benchmark
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, it, beforeAll, afterAll, vi } from "vitest";
const N_ITEMS = 500;
const N_QUERIES = 200;
const originalDataDir = process.env.DATA_DIR;
let tempSqlite, tempLowdb;
let sqliteDb, lowDb;
function fmt(ms) { return `${ms.toFixed(2)}ms`; }
async function bench(label, fn) {
// warmup
await fn();
const t0 = performance.now();
await fn();
const dt = performance.now() - t0;
console.log(` ${label.padEnd(40)} ${fmt(dt)}`);
return dt;
}
beforeAll(async () => {
// SQLite setup
tempSqlite = fs.mkdtempSync(path.join(os.tmpdir(), "9router-bench-sqlite-"));
process.env.DATA_DIR = tempSqlite;
vi.resetModules();
sqliteDb = await import("@/lib/db/index.js");
await sqliteDb.initDb();
// Lowdb setup — direct lowdb usage (mimics legacy behavior)
tempLowdb = fs.mkdtempSync(path.join(os.tmpdir(), "9router-bench-lowdb-"));
const { Low } = await import("lowdb");
const { JSONFile } = await import("lowdb/node");
const dbFile = path.join(tempLowdb, "db.json");
fs.writeFileSync(dbFile, JSON.stringify({ providerConnections: [], usageHistory: [] }));
lowDb = new Low(new JSONFile(dbFile), { providerConnections: [], usageHistory: [] });
await lowDb.read();
});
afterAll(() => {
if (tempSqlite) fs.rmSync(tempSqlite, { recursive: true, force: true });
if (tempLowdb) fs.rmSync(tempLowdb, { recursive: true, force: true });
if (originalDataDir === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = originalDataDir;
});
describe("DB Benchmark — SQLite vs Lowdb", () => {
it(`INSERT ${N_ITEMS} provider connections`, async () => {
console.log(`\n[INSERT ${N_ITEMS}]`);
const sqliteTime = await bench("SQLite createProviderConnection", async () => {
for (let i = 0; i < N_ITEMS; i++) {
await sqliteDb.createProviderConnection({
provider: `bench-p${i % 5}`, authType: "apikey",
name: `name-${i}`, apiKey: `k-${i}`,
});
}
});
const lowdbTime = await bench("Lowdb push + write", async () => {
for (let i = 0; i < N_ITEMS; i++) {
lowDb.data.providerConnections.push({
id: `id-${i}`, provider: `bench-p${i % 5}`, authType: "apikey",
name: `name-${i}`, apiKey: `k-${i}`, priority: i + 1, isActive: true,
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
});
await lowDb.write();
}
});
const speedup = (lowdbTime / sqliteTime).toFixed(2);
console.log(` → SQLite is ${speedup}x faster`);
}, 60000);
it(`READ ${N_QUERIES} filtered queries`, async () => {
console.log(`\n[READ ${N_QUERIES} filtered queries]`);
const sqliteTime = await bench("SQLite getProviderConnections(filter)", async () => {
for (let i = 0; i < N_QUERIES; i++) {
await sqliteDb.getProviderConnections({ provider: `bench-p${i % 5}` });
}
});
const lowdbTime = await bench("Lowdb read + filter", async () => {
for (let i = 0; i < N_QUERIES; i++) {
await lowDb.read();
lowDb.data.providerConnections.filter((c) => c.provider === `bench-p${i % 5}`);
}
});
const speedup = (lowdbTime / sqliteTime).toFixed(2);
console.log(` → SQLite is ${speedup}x faster`);
}, 60000);
it(`READ ${N_QUERIES} by id (point lookup)`, async () => {
console.log(`\n[READ ${N_QUERIES} by id]`);
const sqliteAll = await sqliteDb.getProviderConnections();
const ids = sqliteAll.slice(0, N_QUERIES).map((c) => c.id);
const sqliteTime = await bench("SQLite getProviderConnectionById", async () => {
for (const id of ids) await sqliteDb.getProviderConnectionById(id);
});
const lowdbIds = lowDb.data.providerConnections.slice(0, N_QUERIES).map((c) => c.id);
const lowdbTime = await bench("Lowdb find by id", async () => {
for (const id of lowdbIds) {
await lowDb.read();
lowDb.data.providerConnections.find((c) => c.id === id);
}
});
const speedup = (lowdbTime / sqliteTime).toFixed(2);
console.log(` → SQLite is ${speedup}x faster`);
}, 60000);
it(`saveRequestUsage ${N_ITEMS} entries`, async () => {
console.log(`\n[saveRequestUsage ${N_ITEMS}]`);
const sqliteTime = await bench("SQLite saveRequestUsage", async () => {
for (let i = 0; i < N_ITEMS; i++) {
await sqliteDb.saveRequestUsage({
provider: "openai", model: `m-${i % 10}`, connectionId: `c-${i % 5}`,
tokens: { prompt_tokens: 100 + i, completion_tokens: 50 + i },
endpoint: "/v1/chat/completions", status: "ok",
});
}
});
const lowdbTime = await bench("Lowdb push history + write", async () => {
lowDb.data.usageHistory = [];
for (let i = 0; i < N_ITEMS; i++) {
lowDb.data.usageHistory.push({
timestamp: new Date().toISOString(), provider: "openai", model: `m-${i % 10}`,
connectionId: `c-${i % 5}`, tokens: { prompt_tokens: 100 + i, completion_tokens: 50 + i },
endpoint: "/v1/chat/completions", status: "ok", cost: 0,
});
await lowDb.write();
}
});
const speedup = (lowdbTime / sqliteTime).toFixed(2);
console.log(` → SQLite is ${speedup}x faster`);
}, 120000);
it(`getUsageStats(24h) repeat 50x`, async () => {
console.log(`\n[getUsageStats(24h) x 50]`);
const sqliteTime = await bench("SQLite getUsageStats(24h)", async () => {
for (let i = 0; i < 50; i++) await sqliteDb.getUsageStats("24h");
});
const lowdbTime = await bench("Lowdb read + aggregate", async () => {
for (let i = 0; i < 50; i++) {
await lowDb.read();
const cutoff = Date.now() - 86400000;
const hist = lowDb.data.usageHistory.filter((h) => new Date(h.timestamp).getTime() >= cutoff);
const stats = { byProvider: {}, byModel: {} };
for (const e of hist) {
if (!stats.byProvider[e.provider]) stats.byProvider[e.provider] = { requests: 0 };
stats.byProvider[e.provider].requests++;
}
}
});
const speedup = (lowdbTime / sqliteTime).toFixed(2);
console.log(` → SQLite is ${speedup}x faster`);
}, 60000);
});

171
tests/unit/db-concurrent.test.js Normal file
View file

@ -0,0 +1,171 @@
// Concurrency stress test — simulate many parallel saveRequestUsage / saveRequestDetail
// to verify atomic counter, no data loss, no race conditions.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
const originalDataDir = process.env.DATA_DIR;
let tempDir;
let db;
beforeAll(async () => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-concurrent-"));
process.env.DATA_DIR = tempDir;
vi.resetModules();
db = await import("@/lib/db/index.js");
await db.initDb();
});
afterAll(() => {
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
if (originalDataDir === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = originalDataDir;
});
describe("DB Concurrency — atomic safety", () => {
it("100 parallel saveRequestUsage → no count loss", async () => {
const N = 100;
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.saveRequestUsage({
provider: "openai", model: "gpt-4", connectionId: "c1",
tokens: { prompt_tokens: 10, completion_tokens: 5 },
endpoint: "/v1/chat", status: "ok",
}));
}
await Promise.all(promises);
const stats = await db.getUsageStats("24h");
expect(stats.totalRequests).toBe(N);
expect(stats.byProvider.openai.requests).toBe(N);
expect(stats.byProvider.openai.promptTokens).toBe(N * 10);
const hist = await db.getUsageHistory({ provider: "openai" });
expect(hist.length).toBe(N);
});
it("200 parallel saveRequestDetail → all flushed", async () => {
await db.updateSettings({ enableObservability: true, observabilityBatchSize: 10 });
const N = 200;
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.saveRequestDetail({
id: `det-${i}`, provider: "openai", model: "gpt-4",
connectionId: "c1", status: "ok",
tokens: { prompt_tokens: 1 }, request: { i }, response: { ok: true },
}));
}
await Promise.all(promises);
// Wait for any timer-based flush
await new Promise((r) => setTimeout(r, 6000));
const list = await db.getRequestDetails({ provider: "openai", pageSize: 500 });
expect(list.pagination.totalItems).toBeGreaterThanOrEqual(N);
}, 15000);
it("mixed concurrent: usage + details + connections + aliases", async () => {
const ops = [];
for (let i = 0; i < 50; i++) {
ops.push(db.saveRequestUsage({
provider: "anthropic", model: `m-${i % 3}`, connectionId: "c2",
tokens: { prompt_tokens: 20 }, status: "ok",
}));
ops.push(db.setModelAlias(`a-${i}`, `target-${i}`));
ops.push(db.disableModels("openai", [`d-${i}`]));
}
await Promise.all(ops);
const aliases = await db.getModelAliases();
expect(Object.keys(aliases).filter((k) => k.startsWith("a-")).length).toBe(50);
const disabled = await db.getDisabledByProvider("openai");
expect(disabled.length).toBeGreaterThanOrEqual(50);
const stats = await db.getUsageStats("24h");
expect(stats.byProvider.anthropic.requests).toBe(50);
}, 30000);
it("updateSettings parallel → no merge loss", async () => {
const N = 50;
await db.updateSettings({ counter: 0 });
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.updateSettings({ [`field${i}`]: `v${i}` }));
}
await Promise.all(promises);
const s = await db.getSettings();
for (let i = 0; i < N; i++) {
expect(s[`field${i}`]).toBe(`v${i}`); // all updates preserved
}
});
it("OAuth refresh race: parallel updateProviderConnection on same id", async () => {
const conn = await db.createProviderConnection({
provider: "oauth-test", authType: "oauth", email: "x@y.com",
accessToken: "initial", refreshToken: "rt-initial",
});
// 20 parallel updates each with a unique field
const N = 20;
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.updateProviderConnection(conn.id, { [`marker${i}`]: i }));
}
await Promise.all(promises);
const after = await db.getProviderConnectionById(conn.id);
for (let i = 0; i < N; i++) {
expect(after[`marker${i}`]).toBe(i); // no field lost
}
expect(after.refreshToken).toBe("rt-initial"); // base preserved
});
it("addCustomModel race: parallel duplicate adds → only 1 inserted", async () => {
const N = 30;
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.addCustomModel({ providerAlias: "racep", id: "racemodel", type: "llm", name: "r" }));
}
const results = await Promise.all(promises);
const trueCount = results.filter((r) => r === true).length;
expect(trueCount).toBe(1); // exactly one wins
const all = await db.getCustomModels();
expect(all.filter((m) => m.providerAlias === "racep" && m.id === "racemodel").length).toBe(1);
});
it("updatePricing race: parallel adds different models → all merged", async () => {
const N = 30;
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.updatePricing({ "race-prov": { [`m${i}`]: { input: i, output: i * 2 } } }));
}
await Promise.all(promises);
const p = await db.getPricing();
for (let i = 0; i < N; i++) {
expect(p["race-prov"][`m${i}`]).toEqual({ input: i, output: i * 2 });
}
});
it("daily summary aggregates correctly under parallel writes", async () => {
const N = 50;
const promises = [];
for (let i = 0; i < N; i++) {
promises.push(db.saveRequestUsage({
provider: "google", model: "gemini-pro", connectionId: "cG",
tokens: { prompt_tokens: 100, completion_tokens: 50 },
status: "ok",
}));
}
await Promise.all(promises);
const stats = await db.getUsageStats("7d");
const g = stats.byProvider.google;
expect(g).toBeDefined();
expect(g.requests).toBe(N);
expect(g.promptTokens).toBe(N * 100);
expect(g.completionTokens).toBe(N * 50);
});
});

100
tests/unit/db-migration-chain.test.js Normal file
View file

@ -0,0 +1,100 @@
// Verify schema migration chain runs correctly across versions.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
let tempDir;
const originalDataDir = process.env.DATA_DIR;
beforeEach(() => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-mig-"));
process.env.DATA_DIR = tempDir;
// Reset global singleton so each test gets fresh adapter pointed at tempDir
delete global._dbAdapter;
vi.resetModules();
});
afterEach(() => {
// Close adapter to release file handles before rm
try { global._dbAdapter?.instance?.close?.(); } catch {}
delete global._dbAdapter;
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
if (originalDataDir === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = originalDataDir;
});
describe("Schema migrations", () => {
it("fresh DB → applies migrations & stamps schemaVersion", async () => {
const { getAdapter } = await import("@/lib/db/driver.js");
const { latestVersion } = await import("@/lib/db/migrations/index.js");
const db = await getAdapter();
const row = db.get(`SELECT value FROM _meta WHERE key='schemaVersion'`);
expect(parseInt(row.value, 10)).toBe(latestVersion());
const tables = db.all(`SELECT name FROM sqlite_master WHERE type='table'`).map(t => t.name);
expect(tables).toEqual(expect.arrayContaining([
"_meta", "settings", "providerConnections", "providerNodes",
"proxyPools", "apiKeys", "combos", "kv", "usageHistory", "usageDaily", "requestDetails",
]));
});
it("existing DB at older schemaVersion → re-applies pending migrations on restart", async () => {
// 1st boot
const { getAdapter } = await import("@/lib/db/driver.js");
const db = await getAdapter();
db.run(`INSERT INTO settings(id, data) VALUES(1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data`, ['{"foo":"bar"}']);
db.run(`UPDATE _meta SET value = '0' WHERE key = 'schemaVersion'`);
db.close?.();
// 2nd boot: full reset to simulate process restart
delete global._dbAdapter;
vi.resetModules();
const { getAdapter: getAdapter2 } = await import("@/lib/db/driver.js");
const { latestVersion } = await import("@/lib/db/migrations/index.js");
const db2 = await getAdapter2();
const row = db2.get(`SELECT value FROM _meta WHERE key='schemaVersion'`);
expect(parseInt(row.value, 10)).toBe(latestVersion());
const settings = db2.get(`SELECT data FROM settings WHERE id=1`);
expect(JSON.parse(settings.data)).toEqual({ foo: "bar" });
});
it("fresh DB + legacy db.json → imports data automatically", async () => {
// Simulate user upgrading: place legacy JSON in DATA_DIR before first boot
const legacy = {
settings: { foo: "legacy-value" },
apiKeys: [{ id: "k1", key: "abc", name: "test", createdAt: new Date().toISOString() }],
modelAliases: { "gpt-4": "gpt-4-turbo" },
};
fs.writeFileSync(path.join(tempDir, "db.json"), JSON.stringify(legacy));
const { getAdapter } = await import("@/lib/db/driver.js");
const db = await getAdapter();
const settings = db.get(`SELECT data FROM settings WHERE id=1`);
expect(JSON.parse(settings.data)).toEqual({ foo: "legacy-value" });
const keys = db.all(`SELECT * FROM apiKeys`);
expect(keys).toHaveLength(1);
expect(keys[0].key).toBe("abc");
const aliases = db.all(`SELECT * FROM kv WHERE scope='modelAliases'`);
expect(aliases).toHaveLength(1);
});
it("auto-sync re-creates missing index when DB lacks it", async () => {
const { getAdapter } = await import("@/lib/db/driver.js");
const db = await getAdapter();
db.exec(`DROP INDEX IF EXISTS idx_pn_type`);
expect(db.all(`PRAGMA index_list(providerNodes)`).map(i => i.name)).not.toContain("idx_pn_type");
db.close?.();
delete global._dbAdapter;
vi.resetModules();
const { getAdapter: getAdapter2 } = await import("@/lib/db/driver.js");
const db2 = await getAdapter2();
const idx = db2.all(`PRAGMA index_list(providerNodes)`).map(i => i.name);
expect(idx).toContain("idx_pn_type");
});
});

274
tests/unit/db-sqlite-vs-lowdb.test.js Normal file
View file

@ -0,0 +1,274 @@
// Compare new SQLite-backed DB layer vs legacy lowdb behavior.
// Verifies: same public API signatures + equivalent results for core operations.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
const originalDataDir = process.env.DATA_DIR;
let tempDir;
let sqliteDb;
beforeAll(async () => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-db-compare-"));
process.env.DATA_DIR = tempDir;
vi.resetModules();
sqliteDb = await import("@/lib/db/index.js");
await sqliteDb.initDb();
});
afterAll(() => {
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
if (originalDataDir === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = originalDataDir;
});
describe("DB SQLite layer — public API parity", () => {
it("settings: get → defaults; update → merge", async () => {
const s = await sqliteDb.getSettings();
expect(s).toBeDefined();
expect(s.cloudEnabled).toBe(false);
expect(s.requireLogin).toBe(true);
const updated = await sqliteDb.updateSettings({ cloudEnabled: true, customField: "x" });
expect(updated.cloudEnabled).toBe(true);
expect(updated.customField).toBe("x");
expect(updated.requireLogin).toBe(true); // default preserved
const re = await sqliteDb.getSettings();
expect(re.cloudEnabled).toBe(true);
expect(re.customField).toBe("x");
});
it("isCloudEnabled reflects settings", async () => {
await sqliteDb.updateSettings({ cloudEnabled: true });
expect(await sqliteDb.isCloudEnabled()).toBe(true);
await sqliteDb.updateSettings({ cloudEnabled: false });
expect(await sqliteDb.isCloudEnabled()).toBe(false);
});
it("apiKeys: create/get/validate/delete", async () => {
const k = await sqliteDb.createApiKey("test-key", "machine-abc");
expect(k.id).toBeDefined();
expect(k.key).toMatch(/^sk-/);
expect(k.machineId).toBe("machine-abc");
expect(k.isActive).toBe(true);
const all = await sqliteDb.getApiKeys();
expect(all.find((x) => x.id === k.id)).toBeDefined();
expect(await sqliteDb.validateApiKey(k.key)).toBeTruthy();
expect(await sqliteDb.validateApiKey("invalid")).toBeFalsy();
const deleted = await sqliteDb.deleteApiKey(k.id);
expect(deleted).toBe(true);
expect(await sqliteDb.getApiKeyById(k.id)).toBeNull();
});
it("providerConnections: CRUD + reorder by priority", async () => {
const c1 = await sqliteDb.createProviderConnection({ provider: "test", authType: "apikey", name: "a", apiKey: "k1" });
const c2 = await sqliteDb.createProviderConnection({ provider: "test", authType: "apikey", name: "b", apiKey: "k2" });
const c3 = await sqliteDb.createProviderConnection({ provider: "test", authType: "apikey", name: "c", apiKey: "k3" });
const list = await sqliteDb.getProviderConnections({ provider: "test" });
expect(list).toHaveLength(3);
expect(list[0].priority).toBe(1);
expect(list[1].priority).toBe(2);
expect(list[2].priority).toBe(3);
// Update priority and reorder
await sqliteDb.updateProviderConnection(c3.id, { priority: 1 });
const reordered = await sqliteDb.getProviderConnections({ provider: "test" });
expect(reordered[0].name).toBe("c");
// Delete reorders remaining
await sqliteDb.deleteProviderConnection(c1.id);
const after = await sqliteDb.getProviderConnections({ provider: "test" });
expect(after).toHaveLength(2);
expect(after.every((c) => [1, 2].includes(c.priority))).toBe(true);
});
it("providerConnections: optional fields persisted via JSON column", async () => {
const c = await sqliteDb.createProviderConnection({
provider: "p2", authType: "oauth", email: "x@y.com",
accessToken: "tok", refreshToken: "rtok", expiresAt: 12345,
providerSpecificData: { foo: "bar" },
});
const back = await sqliteDb.getProviderConnectionById(c.id);
expect(back.accessToken).toBe("tok");
expect(back.refreshToken).toBe("rtok");
expect(back.expiresAt).toBe(12345);
expect(back.providerSpecificData).toEqual({ foo: "bar" });
});
it("providerNodes: CRUD", async () => {
const n = await sqliteDb.createProviderNode({ type: "openai", name: "Test", baseUrl: "https://api.test", apiType: "openai" });
expect(n.id).toBeDefined();
expect(n.baseUrl).toBe("https://api.test");
const all = await sqliteDb.getProviderNodes({ type: "openai" });
expect(all.find((x) => x.id === n.id)).toBeDefined();
await sqliteDb.updateProviderNode(n.id, { name: "Test2" });
const updated = await sqliteDb.getProviderNodeById(n.id);
expect(updated.name).toBe("Test2");
await sqliteDb.deleteProviderNode(n.id);
expect(await sqliteDb.getProviderNodeById(n.id)).toBeNull();
});
it("proxyPools: CRUD with sort by updatedAt desc", async () => {
const p1 = await sqliteDb.createProxyPool({ name: "p1", proxyUrl: "http://a", type: "http" });
await new Promise((r) => setTimeout(r, 10));
const p2 = await sqliteDb.createProxyPool({ name: "p2", proxyUrl: "http://b", type: "http" });
const list = await sqliteDb.getProxyPools();
expect(list[0].id).toBe(p2.id); // newest first
await sqliteDb.deleteProxyPool(p1.id);
await sqliteDb.deleteProxyPool(p2.id);
});
it("combos: CRUD", async () => {
const c = await sqliteDb.createCombo({ name: "combo1", models: ["m1", "m2"], kind: "fallback" });
expect(c.id).toBeDefined();
expect(c.models).toEqual(["m1", "m2"]);
const byName = await sqliteDb.getComboByName("combo1");
expect(byName.id).toBe(c.id);
await sqliteDb.updateCombo(c.id, { models: ["m3"] });
const updated = await sqliteDb.getComboById(c.id);
expect(updated.models).toEqual(["m3"]);
expect(await sqliteDb.deleteCombo(c.id)).toBe(true);
});
it("modelAliases: KV ops", async () => {
await sqliteDb.setModelAlias("alias1", "real-model-1");
await sqliteDb.setModelAlias("alias2", "real-model-2");
const all = await sqliteDb.getModelAliases();
expect(all.alias1).toBe("real-model-1");
expect(all.alias2).toBe("real-model-2");
await sqliteDb.deleteModelAlias("alias1");
expect((await sqliteDb.getModelAliases()).alias1).toBeUndefined();
});
it("customModels: add/list/delete with dedupe", async () => {
const ok1 = await sqliteDb.addCustomModel({ providerAlias: "p1", id: "m1", type: "llm", name: "Model 1" });
const dup = await sqliteDb.addCustomModel({ providerAlias: "p1", id: "m1", type: "llm" });
expect(ok1).toBe(true);
expect(dup).toBe(false);
const list = await sqliteDb.getCustomModels();
expect(list.find((m) => m.id === "m1")).toBeDefined();
await sqliteDb.deleteCustomModel({ providerAlias: "p1", id: "m1" });
const after = await sqliteDb.getCustomModels();
expect(after.find((m) => m.id === "m1")).toBeUndefined();
});
it("mitmAlias: get/set per tool", async () => {
await sqliteDb.setMitmAliasAll("cursor", { "gpt-5": "claude-3" });
const a = await sqliteDb.getMitmAlias("cursor");
expect(a["gpt-5"]).toBe("claude-3");
const all = await sqliteDb.getMitmAlias();
expect(all.cursor).toEqual({ "gpt-5": "claude-3" });
});
it("disabledModels: add/remove per provider", async () => {
await sqliteDb.disableModels("openai", ["gpt-3", "gpt-4"]);
expect(await sqliteDb.getDisabledByProvider("openai")).toEqual(expect.arrayContaining(["gpt-3", "gpt-4"]));
await sqliteDb.enableModels("openai", ["gpt-3"]);
expect(await sqliteDb.getDisabledByProvider("openai")).toEqual(["gpt-4"]);
await sqliteDb.enableModels("openai", []);
expect(await sqliteDb.getDisabledByProvider("openai")).toEqual([]);
});
it("usage: saveRequestUsage + getUsageHistory + getUsageStats", async () => {
await sqliteDb.saveRequestUsage({
provider: "openai", model: "gpt-4", connectionId: "c1",
tokens: { prompt_tokens: 100, completion_tokens: 50 },
endpoint: "/v1/chat/completions", status: "ok",
});
await sqliteDb.saveRequestUsage({
provider: "openai", model: "gpt-4", connectionId: "c1",
tokens: { prompt_tokens: 200, completion_tokens: 100 },
endpoint: "/v1/chat/completions", status: "ok",
});
const hist = await sqliteDb.getUsageHistory({ provider: "openai" });
expect(hist.length).toBeGreaterThanOrEqual(2);
expect(hist[0].tokens.prompt_tokens).toBeDefined();
const stats = await sqliteDb.getUsageStats("24h");
expect(stats.totalRequests).toBeGreaterThanOrEqual(2);
expect(stats.byProvider.openai).toBeDefined();
expect(stats.byProvider.openai.requests).toBeGreaterThanOrEqual(2);
expect(stats.byProvider.openai.promptTokens).toBeGreaterThanOrEqual(300);
});
it("usage: pending tracking in-memory", () => {
sqliteDb.trackPendingRequest("gpt-4", "openai", "c1", true);
expect(global._pendingRequests.byModel["gpt-4 (openai)"]).toBe(1);
sqliteDb.trackPendingRequest("gpt-4", "openai", "c1", false);
expect(global._pendingRequests.byModel["gpt-4 (openai)"]).toBeUndefined();
});
it("requestDetails: save → query with paging", async () => {
// Enable observability first
await sqliteDb.updateSettings({ enableObservability: true, observabilityBatchSize: 1 });
await sqliteDb.saveRequestDetail({
id: "d1", provider: "openai", model: "gpt-4", connectionId: "c1",
status: "ok", tokens: { prompt_tokens: 10 },
request: { method: "POST" }, response: { status: 200 },
});
// Wait for buffer flush
await new Promise((r) => setTimeout(r, 200));
const got = await sqliteDb.getRequestDetailById("d1");
expect(got).toBeDefined();
expect(got.id).toBe("d1");
const list = await sqliteDb.getRequestDetails({ provider: "openai" });
expect(list.details.length).toBeGreaterThanOrEqual(1);
expect(list.pagination.totalItems).toBeGreaterThanOrEqual(1);
});
it("exportDb / importDb roundtrip", async () => {
const exported = await sqliteDb.exportDb();
expect(exported.settings).toBeDefined();
expect(Array.isArray(exported.providerConnections)).toBe(true);
expect(typeof exported.modelAliases).toBe("object");
// Add marker, export, import a different payload, verify reset
await sqliteDb.setModelAlias("marker", "before");
const snap = await sqliteDb.exportDb();
await sqliteDb.setModelAlias("marker", "after");
expect((await sqliteDb.getModelAliases()).marker).toBe("after");
await sqliteDb.importDb(snap);
expect((await sqliteDb.getModelAliases()).marker).toBe("before");
});
it("pricing: user pricing merged with constants", async () => {
await sqliteDb.updatePricing({ openai: { "gpt-test": { input: 1, output: 2 } } });
const p = await sqliteDb.getPricing();
expect(p.openai["gpt-test"]).toEqual({ input: 1, output: 2 });
const single = await sqliteDb.getPricingForModel("openai", "gpt-test");
expect(single).toEqual({ input: 1, output: 2 });
await sqliteDb.resetPricing("openai", "gpt-test");
expect((await sqliteDb.getPricing()).openai?.["gpt-test"]).toBeUndefined();
});
it("getChartData: 24h buckets", async () => {
const data = await sqliteDb.getChartData("24h");
expect(data).toHaveLength(24);
expect(data[0]).toHaveProperty("label");
expect(data[0]).toHaveProperty("tokens");
expect(data[0]).toHaveProperty("cost");
});
it("getChartData: 7d buckets", async () => {
const data = await sqliteDb.getChartData("7d");
expect(data).toHaveLength(7);
});
});