diff --git a/.gitignore b/.gitignore index becf95e..e8af6e8 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ yarn-error.log* *.pem CLAUDE.md .serena +.local # Temp files /tmp diff --git a/apps/desktop/forge.env.d.ts b/apps/desktop/forge.env.d.ts index 9700e0a..b58e205 100644 --- a/apps/desktop/forge.env.d.ts +++ b/apps/desktop/forge.env.d.ts @@ -1 +1,6 @@ /// + +declare module "*?url" { + const url: string; + export default url; +} diff --git a/apps/desktop/package.json b/apps/desktop/package.json index f253c17..bc4bb79 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -59,6 +59,7 @@ "@dnd-kit/utilities": "^3.2.2", "@hookform/resolvers": "^5.0.1", "@libsql/client": "^0.15.9", + "@openrouter/ai-sdk-provider": "^0.7.2", "@radix-ui/react-accordion": "^1.2.10", "@radix-ui/react-alert-dialog": "^1.1.13", "@radix-ui/react-aspect-ratio": "^1.1.6", @@ -95,6 +96,7 @@ "@types/split2": "^4.2.3", "@types/uuid": "^10.0.0", "ai": "^4.3.16", + "ansi-colors": "^4.1.3", "async-mutex": "^0.5.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/apps/desktop/src/db/config.ts b/apps/desktop/src/db/config.ts index 189ef03..be41aae 100644 --- a/apps/desktop/src/db/config.ts +++ b/apps/desktop/src/db/config.ts @@ -17,6 +17,8 @@ export const db = drizzle(`file:${dbPath}`, { // Initialize database with migrations let isInitialized = false; +import { logger } from "../main/logger"; + export async function initializeDatabase() { if (isInitialized) { return; @@ -35,10 +37,10 @@ export async function initializeDatabase() { migrationsPath = path.join(process.resourcesPath, "migrations"); } - console.log("Attempting to run migrations from:", migrationsPath); - console.log("__dirname:", __dirname); - console.log("process.cwd():", process.cwd()); - console.log("isDev:", isDev); + logger.db.debug("Attempting to run migrations from:", migrationsPath); + logger.db.debug("__dirname:", __dirname); + logger.db.debug("process.cwd():", process.cwd()); + logger.db.debug("isDev:", isDev); // Check if the migrations path exists if (!fs.existsSync(migrationsPath)) { @@ -55,11 +57,13 @@ export async function initializeDatabase() { migrationsFolder: migrationsPath, }); - console.log("Database initialized and migrations completed successfully"); + logger.db.info( + "Database initialized and migrations completed successfully", + ); isInitialized = true; } catch (error) { - console.error("FATAL: Error initializing database:", error); - console.error( + logger.db.error("FATAL: Error initializing database:", error); + logger.db.error( "Application cannot continue without a working database. Exiting...", ); diff --git a/apps/desktop/src/db/downloaded-models.ts b/apps/desktop/src/db/downloaded-models.ts index 5e750f5..e149b5d 100644 --- a/apps/desktop/src/db/downloaded-models.ts +++ b/apps/desktop/src/db/downloaded-models.ts @@ -75,7 +75,7 @@ export async function deleteDownloadedModel(id: string) { return result[0] || null; } -// Get downloaded models as a record (for backward compatibility) +// Get downloaded models as a record export async function getDownloadedModelsRecord(): Promise< Record > { diff --git a/apps/desktop/src/db/migrate.ts b/apps/desktop/src/db/migrate.ts index ef8e5d6..e96aca7 100644 --- a/apps/desktop/src/db/migrate.ts +++ b/apps/desktop/src/db/migrate.ts @@ -1,13 +1,14 @@ import { migrate } from "drizzle-orm/libsql/migrator"; import { db } from "./config"; +import { logger } from "../main/logger"; export async function runMigrations() { try { // Run migrations await migrate(db, { migrationsFolder: "./src/db/migrations" }); - console.log("Migrations completed successfully"); + logger.db.info("Migrations completed successfully"); } catch (error) { - console.error("Error running migrations:", error); + logger.db.error("Error running migrations:", error); throw error; } } diff --git a/apps/desktop/public/audio-recorder-worklet.js b/apps/desktop/src/hooks/audio-recorder-worklet.ts similarity index 95% rename from apps/desktop/public/audio-recorder-worklet.js rename to apps/desktop/src/hooks/audio-recorder-worklet.ts index b36a9a3..4202452 100644 --- a/apps/desktop/public/audio-recorder-worklet.js +++ b/apps/desktop/src/hooks/audio-recorder-worklet.ts @@ -1,3 +1,5 @@ +// AudioWorklet processor source code +export const audioRecorderWorkletSource = ` // AudioWorklet processor for real-time audio capture // This runs in the audio rendering thread for low-latency processing /* eslint-env worker */ @@ -60,3 +62,4 @@ class AudioRecorderProcessor extends AudioWorkletProcessor { // Register the processor registerProcessor('audio-recorder-processor', AudioRecorderProcessor); +`; diff --git a/apps/desktop/src/hooks/useRecording.ts b/apps/desktop/src/hooks/useRecording.ts index e9eda54..958a584 100644 --- a/apps/desktop/src/hooks/useRecording.ts +++ b/apps/desktop/src/hooks/useRecording.ts @@ -1,6 +1,7 @@ import { useState, useEffect, useRef, useCallback } from "react"; import { MicVAD } from "@ricky0123/vad-web"; import { Mutex } from "async-mutex"; +import { audioRecorderWorkletSource } from "./audio-recorder-worklet"; export interface UseRecordingParams { onAudioChunk: ( @@ -72,19 +73,6 @@ export const useRecording = ({ "Hook: Internal: Stopping recording and sending final chunk...", ); - // Send final audio chunk before cleanup - try { - // Access the sendAudioChunk function from the current recording session - // We need to store this reference when starting recording - const sendFinalChunk = (window as any).currentSendAudioChunk; - if (sendFinalChunk) { - await sendFinalChunk(true); // Send final chunk - console.log("Hook: Final audio chunk sent."); - } - } catch (error) { - console.error("Hook: Error sending final audio chunk:", error); - } - // Cleanup all resources cleanupMediaResources(vadRef.current, streamRef.current); @@ -148,8 +136,13 @@ export const useRecording = ({ let chunkTimer: NodeJS.Timeout | null = null; let pendingAudioChunks: Float32Array[] = []; - // Load AudioWorklet module - await audioContext.audioWorklet.addModule("/audio-recorder-worklet.js"); + // Load AudioWorklet module using blob URL + const blob = new Blob([audioRecorderWorkletSource], { + type: "application/javascript", + }); + const audioWorkletUrl = URL.createObjectURL(blob); + await audioContext.audioWorklet.addModule(audioWorkletUrl); + URL.revokeObjectURL(audioWorkletUrl); // Clean up blob URL console.log("Hook: AudioWorklet module loaded successfully"); source = audioContext.createMediaStreamSource(localStream); diff --git a/apps/desktop/src/main/core/app-manager.ts b/apps/desktop/src/main/core/app-manager.ts new file mode 100644 index 0000000..49f40e7 --- /dev/null +++ b/apps/desktop/src/main/core/app-manager.ts @@ -0,0 +1,194 @@ +import { + app, + systemPreferences, + BrowserWindow, + globalShortcut, +} from "electron"; +import { initializeDatabase } from "../../db/config"; +import { logger } from "../logger"; +import { WindowManager } from "./window-manager"; +import { setupApplicationMenu } from "../menu"; +import { ServiceManager } from "../managers/service-manager"; +import { createIPCHandler } from "electron-trpc-experimental/main"; +import { router } from "../../trpc/router"; +import { EventHandlers } from "./event-handlers"; + +export class AppManager { + private windowManager: WindowManager; + private serviceManager: ServiceManager; + + constructor() { + this.windowManager = new WindowManager(); + this.serviceManager = ServiceManager.createInstance(); + this.windowManager.setMainWindowCreatedCallback( + this.onMainWindowCreated.bind(this), + ); + } + + async initialize(): Promise { + try { + await this.initializeDatabase(); + await this.requestPermissions(); + await this.serviceManager.initialize(this.windowManager); + this.exposeGlobalServices(); + await this.setupWindows(); + await this.setupMenu(); + + // Setup event handlers + const eventHandlers = new EventHandlers(this); + eventHandlers.setupEventHandlers(); + + // Schedule auto-update check after startup + this.scheduleAutoUpdateCheck(); + + logger.main.info("Application initialized successfully"); + } catch (error) { + logger.main.error("Error initializing app:", error); + throw error; + } + } + + private async initializeDatabase(): Promise { + await initializeDatabase(); + logger.db.info( + "Database initialized and migrations completed successfully", + ); + } + + private async requestPermissions(): Promise { + if (process.platform === "darwin") { + const accessibilityEnabled = + systemPreferences.isTrustedAccessibilityClient(false); + if (!accessibilityEnabled) { + logger.main.debug( + "Please enable accessibility permissions in System Preferences > Security & Privacy > Privacy > Accessibility", + ); + } + } + + const microphoneEnabled = + systemPreferences.getMediaAccessStatus("microphone"); + logger.main.info("Microphone access status:", { + status: microphoneEnabled, + }); + + if (microphoneEnabled !== "granted") { + await systemPreferences.askForMediaAccess("microphone"); + } + } + + private async setupWindows(): Promise { + this.windowManager.createWidgetWindow(); + this.setupTRPCHandler(); + + if (process.platform === "darwin" && app.dock) { + app.dock.show(); + } + } + + private setupTRPCHandler(): Promise { + const windows = this.windowManager + .getAllWindows() + .filter((w): w is BrowserWindow => w !== null); + createIPCHandler({ router, windows }); + return Promise.resolve(); + } + + updateTRPCHandler(): void { + const windows = this.windowManager + .getAllWindows() + .filter((w): w is BrowserWindow => w !== null); + createIPCHandler({ router, windows }); + } + + private async setupMenu(): Promise { + setupApplicationMenu( + () => this.windowManager.createOrShowMainWindow(), + () => { + const autoUpdaterService = this.serviceManager.getAutoUpdaterService(); + if (autoUpdaterService) { + autoUpdaterService.checkForUpdates(true); + } + }, + () => this.windowManager.openAllDevTools(), + ); + } + + private exposeGlobalServices(): void { + // Make services available globally for tRPC (temporary solution) + const transcriptionService = this.serviceManager.getTranscriptionService(); + const autoUpdaterService = this.serviceManager.getAutoUpdaterService(); + const settingsService = this.serviceManager.getSettingsService(); + const swiftBridge = this.serviceManager.getSwiftIOBridge(); + + (globalThis as any).modelManagerService = + this.serviceManager.getModelManagerService(); + (globalThis as any).transcriptionService = transcriptionService; + (globalThis as any).settingsService = settingsService; + (globalThis as any).logger = logger; + (globalThis as any).autoUpdaterService = autoUpdaterService; + (globalThis as any).swiftBridge = swiftBridge; + } + + getWindowManager(): WindowManager { + return this.windowManager; + } + + getServiceManager(): ServiceManager { + return this.serviceManager; + } + + getTranscriptionService(): any { + return this.serviceManager.getTranscriptionService(); + } + + getSwiftIOBridge(): any { + return this.serviceManager.getSwiftIOBridge(); + } + + getAutoUpdaterService(): any { + return this.serviceManager.getAutoUpdaterService(); + } + + private scheduleAutoUpdateCheck(): void { + // Check for updates on startup (after a brief delay) + setTimeout(() => { + try { + const autoUpdaterService = this.serviceManager.getAutoUpdaterService(); + autoUpdaterService.checkForUpdatesAndNotify(); + } catch (error) { + logger.main.warn("Auto-update check failed during startup", { + error: error instanceof Error ? error.message : String(error), + }); + } + }, 5000); // Wait 5 seconds after startup + } + + private onMainWindowCreated(window: BrowserWindow): void { + this.updateTRPCHandler(); + } + + async cleanup(): Promise { + globalShortcut.unregisterAll(); + await this.serviceManager.cleanup(); + if (this.windowManager) { + this.windowManager.cleanup(); + } + } + + handleActivate(): void { + const allWindows = this.windowManager.getAllWindows(); + + if (allWindows.every((w) => !w || w.isDestroyed())) { + this.windowManager.createWidgetWindow(); + } else { + const widgetWindow = this.windowManager.getWidgetWindow(); + if (!widgetWindow || widgetWindow.isDestroyed()) { + this.windowManager.createWidgetWindow(); + } else { + widgetWindow.show(); + } + this.windowManager.createOrShowMainWindow(); + } + } +} diff --git a/apps/desktop/src/main/core/event-handlers.ts b/apps/desktop/src/main/core/event-handlers.ts new file mode 100644 index 0000000..d58a530 --- /dev/null +++ b/apps/desktop/src/main/core/event-handlers.ts @@ -0,0 +1,63 @@ +import { HelperEvent } from "@amical/types"; +import { AppManager } from "./app-manager"; +import { logger } from "../logger"; + +export class EventHandlers { + private appManager: AppManager; + + constructor(appManager: AppManager) { + this.appManager = appManager; + } + + setupEventHandlers(): void { + this.setupSwiftBridgeEventHandlers(); + // Note: Audio IPC handlers are now managed by RecordingService + } + + private setupSwiftBridgeEventHandlers(): void { + try { + const swiftBridge = this.appManager.getSwiftIOBridge(); + const windowManager = this.appManager.getWindowManager(); + + swiftBridge.on("helperEvent", (event: HelperEvent) => { + logger.swift.debug("Received helperEvent from SwiftIOBridge", { + event, + }); + + switch (event.type) { + case "flagsChanged": { + const payload = event.payload; + if (payload?.fnKeyPressed !== undefined) { + logger.swift.info("Setting recording state", { + state: payload.fnKeyPressed, + }); + const widgetWindow = windowManager.getWidgetWindow(); + if (widgetWindow) { + widgetWindow.webContents.send( + "recording-state-changed", + payload.fnKeyPressed, + ); + } + } + break; + } + case "keyDown": + case "keyUp": + break; + default: + break; + } + }); + + swiftBridge.on("error", (error: Error) => { + logger.main.error("SwiftIOBridge error:", error); + }); + + swiftBridge.on("close", (code: number | null) => { + logger.swift.warn("Swift helper process closed", { code }); + }); + } catch (error) { + logger.main.warn("Swift bridge not available for event handlers"); + } + } +} diff --git a/apps/desktop/src/main/core/window-manager.ts b/apps/desktop/src/main/core/window-manager.ts new file mode 100644 index 0000000..6ea2db7 --- /dev/null +++ b/apps/desktop/src/main/core/window-manager.ts @@ -0,0 +1,195 @@ +import { BrowserWindow, screen, systemPreferences } from "electron"; +import path from "node:path"; +import { logger } from "../logger"; + +declare const MAIN_WINDOW_VITE_DEV_SERVER_URL: string; +declare const MAIN_WINDOW_VITE_NAME: string; +declare const WIDGET_WINDOW_VITE_NAME: string; + +export class WindowManager { + private mainWindow: BrowserWindow | null = null; + private widgetWindow: BrowserWindow | null = null; + private currentWindowDisplayId: number | null = null; + private activeSpaceChangeSubscriptionId: number | null = null; + private onMainWindowCreated?: (window: BrowserWindow) => void; + + createOrShowMainWindow(): void { + if (this.mainWindow && !this.mainWindow.isDestroyed()) { + this.mainWindow.show(); + this.mainWindow.focus(); + return; + } + + this.mainWindow = new BrowserWindow({ + width: 1200, + height: 800, + frame: false, + titleBarStyle: "hidden", + trafficLightPosition: { x: 20, y: 16 }, + useContentSize: true, + webPreferences: { + preload: path.join(__dirname, "preload.js"), + nodeIntegration: false, + contextIsolation: true, + }, + }); + + if (MAIN_WINDOW_VITE_DEV_SERVER_URL) { + this.mainWindow.loadURL(MAIN_WINDOW_VITE_DEV_SERVER_URL); + } else { + this.mainWindow.loadFile( + path.join(__dirname, `../renderer/${MAIN_WINDOW_VITE_NAME}/index.html`), + ); + } + + this.mainWindow.on("closed", () => { + this.mainWindow = null; + }); + + if (this.onMainWindowCreated) { + this.onMainWindowCreated(this.mainWindow); + } + } + + createWidgetWindow(): void { + const mainScreen = screen.getPrimaryDisplay(); + const { width, height } = mainScreen.workAreaSize; + + this.widgetWindow = new BrowserWindow({ + width, + height, + frame: false, + transparent: true, + alwaysOnTop: true, + resizable: false, + maximizable: false, + skipTaskbar: true, + focusable: false, + hasShadow: false, + webPreferences: { + preload: path.join(__dirname, "preload.js"), + nodeIntegration: false, + contextIsolation: true, + }, + }); + + this.currentWindowDisplayId = mainScreen.id; + this.widgetWindow.setIgnoreMouseEvents(true, { forward: true }); + + if (MAIN_WINDOW_VITE_DEV_SERVER_URL) { + const devUrl = new URL(MAIN_WINDOW_VITE_DEV_SERVER_URL); + devUrl.pathname = "widget.html"; + this.widgetWindow.loadURL(devUrl.toString()); + } else { + this.widgetWindow.loadFile( + path.join( + __dirname, + `../renderer/${WIDGET_WINDOW_VITE_NAME}/widget.html`, + ), + ); + } + + if (process.platform === "darwin") { + this.widgetWindow.setAlwaysOnTop(true, "floating", 1); + this.widgetWindow.setVisibleOnAllWorkspaces(true, { + visibleOnFullScreen: true, + }); + this.widgetWindow.setHiddenInMissionControl(true); + this.setupDisplayChangeNotifications(); + } + } + + private setupDisplayChangeNotifications(): void { + if (process.platform !== "darwin") return; + + try { + this.activeSpaceChangeSubscriptionId = + systemPreferences.subscribeWorkspaceNotification( + "NSWorkspaceActiveDisplayDidChangeNotification", + () => { + if (this.widgetWindow && !this.widgetWindow.isDestroyed()) { + try { + const cursorPoint = screen.getCursorScreenPoint(); + const displayForCursor = + screen.getDisplayNearestPoint(cursorPoint); + if (this.currentWindowDisplayId !== displayForCursor.id) { + logger.main.info("Moving floating window to display", { + displayId: displayForCursor.id, + }); + this.widgetWindow.setBounds(displayForCursor.workArea); + this.currentWindowDisplayId = displayForCursor.id; + } + } catch (error) { + logger.main.warn("Error handling display change:", error); + } + } + }, + ); + + if ( + this.activeSpaceChangeSubscriptionId !== undefined && + this.activeSpaceChangeSubscriptionId >= 0 + ) { + logger.main.info( + "Successfully subscribed to display change notifications", + ); + } else { + logger.main.error( + "Failed to subscribe to display change notifications", + ); + } + } catch (error) { + logger.main.error( + "Error during subscription to display notifications:", + error, + ); + this.activeSpaceChangeSubscriptionId = null; + } + } + + getMainWindow(): BrowserWindow | null { + return this.mainWindow; + } + + getWidgetWindow(): BrowserWindow | null { + return this.widgetWindow; + } + + getAllWindows(): (BrowserWindow | null)[] { + return [this.mainWindow, this.widgetWindow]; + } + + setMainWindowCreatedCallback( + callback: (window: BrowserWindow) => void, + ): void { + this.onMainWindowCreated = callback; + } + + openAllDevTools(): void { + const windows = this.getAllWindows().filter( + (window): window is BrowserWindow => + window !== null && !window.isDestroyed(), + ); + + windows.forEach((window) => { + if (window.webContents && !window.webContents.isDevToolsOpened()) { + window.webContents.openDevTools(); + } + }); + + logger.main.info(`Opened dev tools for ${windows.length} windows`); + } + + cleanup(): void { + if ( + process.platform === "darwin" && + this.activeSpaceChangeSubscriptionId !== null + ) { + systemPreferences.unsubscribeWorkspaceNotification( + this.activeSpaceChangeSubscriptionId, + ); + logger.main.info("Unsubscribed from display change notifications"); + this.activeSpaceChangeSubscriptionId = null; + } + } +} diff --git a/apps/desktop/src/main/logger.ts b/apps/desktop/src/main/logger.ts index 35414e5..23a1b7f 100644 --- a/apps/desktop/src/main/logger.ts +++ b/apps/desktop/src/main/logger.ts @@ -4,6 +4,7 @@ dotenv.config(); import log from "electron-log"; import { app } from "electron"; import path from "node:path"; +import colors from "ansi-colors"; // Configure electron-log immediately when module is imported const isDev = process.env.NODE_ENV === "development" || !app.isPackaged; @@ -35,9 +36,53 @@ log.transports.file.resolvePathFn = () => logPath; // Configure console transport for better development experience if (isDev) { - log.transports.console.format = - "[{y}-{m}-{d} {h}:{i}:{s}.{ms}] [{level}] [{scope}] {text}"; - log.transports.console.useStyles = true; + // Color functions for different scopes using ansi-colors + const scopeColorFunctions: Record string> = { + main: colors.blue.bold, + audio: colors.green.bold, + transcription: colors.magenta.bold, + swift: colors.yellow.bold, + pipeline: colors.red.bold, + widget: colors.cyan.bold, + mainWindow: colors.greenBright.bold, + renderer: colors.gray.bold, + ui: colors.magentaBright.bold, + db: colors.cyanBright.bold, + network: colors.yellowBright.bold, + updater: colors.blueBright.bold, + ipc: colors.green.bold, + default: colors.gray.bold, + }; + + // Color functions for different log levels + const levelColorFunctions: Record string> = { + error: colors.red.bold, + warn: colors.yellow.bold, + info: colors.blue, + verbose: colors.cyan, + debug: colors.gray, + silly: colors.magenta, + }; + + // Override console transport with custom colored output + log.transports.console.format = "{text}"; // Minimal formatting - just pass through the text + log.transports.console.writeFn = (info) => { + const { message } = info; + const scope = message.scope || "default"; + const level = message.level; + + // Get color functions + const scopeColorFn = + scopeColorFunctions[scope] || scopeColorFunctions.default; + const levelColorFn = levelColorFunctions[level] || ((text: string) => text); + + const timestamp = `${message.date.getFullYear()}-${String(message.date.getMonth() + 1).padStart(2, "0")}-${String(message.date.getDate()).padStart(2, "0")} ${String(message.date.getHours()).padStart(2, "0")}:${String(message.date.getMinutes()).padStart(2, "0")}:${String(message.date.getSeconds()).padStart(2, "0")}.${String(message.date.getMilliseconds()).padStart(3, "0")}`; + + // Let console.log handle message serialization naturally + const prefix = `${colors.dim(`[${timestamp}]`)} ${levelColorFn(`[${level}]`)} ${scopeColorFn(`[${scope}]`)}`; + + console.log(prefix, ...message.data); + }; } else { log.transports.console.format = "[{y}-{m}-{d} {h}:{i}:{s}.{ms}] [{level}] [{scope}] {text}"; @@ -115,11 +160,14 @@ export const logger = { renderer: createLoggerForScope("renderer"), network: createLoggerForScope("network"), audio: createLoggerForScope("audio"), - ai: createLoggerForScope("ai"), + pipeline: createLoggerForScope("pipeline"), swift: createLoggerForScope("swift"), ui: createLoggerForScope("ui"), db: createLoggerForScope("db"), updater: createLoggerForScope("updater"), + transcription: createLoggerForScope("transcription"), + widget: createLoggerForScope("widget"), + mainWindow: createLoggerForScope("mainWindow"), }; // Log startup information diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 73a7b72..8565da4 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -1,804 +1,19 @@ -// Load .env file FIRST before any other imports import dotenv from "dotenv"; dotenv.config(); -import { - app, - BrowserWindow, - systemPreferences, - globalShortcut, - ipcMain, - screen, - clipboard, -} from "electron"; -import path from "node:path"; -import fsPromises from "node:fs/promises"; // For reading the audio file (async) +import { app } from "electron"; import started from "electron-squirrel-startup"; -import { initializeDatabase } from "../db/config"; -import { HelperEvent, KeyEventPayload } from "@amical/types"; -import { logger, logError, logPerformance } from "./logger"; -import { AudioCapture } from "../modules/audio/audio-capture"; -import { setupApplicationMenu } from "./menu"; -import { AiService } from "../modules/ai/ai-service"; -import { SwiftIOBridge } from "./swift-io-bridge"; // Added import -import { DownloadedModel } from "../constants/models"; -import { ModelManagerService } from "../modules/models/model-manager"; -import { LocalWhisperClient } from "../modules/ai/local-whisper-client"; -import { - TranscriptionSession, - ChunkData, -} from "../modules/transcription/transcription-session"; -import { ContextualTranscriptionManager } from "../modules/transcription/contextual-transcription-manager"; -import { SettingsService } from "../modules/settings"; -import { createIPCHandler } from "electron-trpc-experimental/main"; -import { router } from "../trpc/router"; -import { AutoUpdaterService } from "./services/auto-updater"; +import { AppManager } from "./core/app-manager"; -// Handle creating/removing shortcuts on Windows when installing/uninstalling. if (started) { app.quit(); } -declare const MAIN_WINDOW_VITE_DEV_SERVER_URL: string; -declare const MAIN_WINDOW_VITE_NAME: string; -declare const WIDGET_WINDOW_VITE_NAME: string; +const appManager = new AppManager(); -let mainWindow: BrowserWindow | null = null; -let floatingButtonWindow: BrowserWindow | null = null; -let audioCapture: AudioCapture | null = null; -let aiService: AiService | null = null; -let swiftIOBridgeClientInstance: SwiftIOBridge | null = null; -let modelManagerService: ModelManagerService | null = null; -let localWhisperClient: LocalWhisperClient | null = null; -let currentWindowDisplayId: number | null = null; // For tracking current display -let activeSpaceChangeSubscriptionId: number | null = null; // For display change notifications - -// New chunk-based transcription variables -let contextualTranscriptionManager: ContextualTranscriptionManager | null = - null; -const activeTranscriptionSessions: Map = - new Map(); -let autoUpdaterService: AutoUpdaterService | null = null; - -// Store is imported from '../lib/store' and is database-backed - -// Function to create the local transcription client -const createTranscriptionClient = () => { - logger.ai.info("Using local Whisper inference"); - if (!localWhisperClient) { - throw new Error("Local Whisper client not initialized"); - } - return localWhisperClient; -}; - -// Formatter Configuration - Now handled by tRPC settings router - -const requestPermissions = async () => { - try { - // Request accessibility permissions - if (process.platform === "darwin") { - const accessibilityEnabled = - systemPreferences.isTrustedAccessibilityClient(false); - if (!accessibilityEnabled) { - // On macOS, we need to use a different approach for accessibility permissions - // The user will need to grant accessibility permissions through System Preferences - console.log( - "Please enable accessibility permissions in System Preferences > Security & Privacy > Privacy > Accessibility", - ); - } - } - - // Request microphone permissions - const microphoneEnabled = - systemPreferences.getMediaAccessStatus("microphone"); - logger.main.info("Microphone access status:", { - status: microphoneEnabled, - }); - if (microphoneEnabled !== "granted") { - await systemPreferences.askForMediaAccess("microphone"); - } - } catch (error) { - logError( - error instanceof Error ? error : new Error(String(error)), - "requesting permissions", - ); - } -}; - -const createOrShowMainWindow = () => { - if (mainWindow && !mainWindow.isDestroyed()) { - mainWindow.show(); - mainWindow.focus(); - return; - } - mainWindow = new BrowserWindow({ - width: 1200, - height: 800, - frame: false, - titleBarStyle: "hidden", - trafficLightPosition: { x: 20, y: 16 }, - useContentSize: true, - webPreferences: { - preload: path.join(__dirname, "preload.js"), - nodeIntegration: false, - contextIsolation: true, - }, - }); - if (MAIN_WINDOW_VITE_DEV_SERVER_URL) { - mainWindow.loadURL(MAIN_WINDOW_VITE_DEV_SERVER_URL); - } else { - mainWindow.loadFile( - path.join(__dirname, `../renderer/${MAIN_WINDOW_VITE_NAME}/index.html`), - ); - } - mainWindow.on("closed", () => { - mainWindow = null; - if (autoUpdaterService) { - autoUpdaterService.setMainWindow(null); - } - }); - - // Update tRPC handler to include the main window - createIPCHandler({ - router, - windows: [mainWindow, floatingButtonWindow].filter( - Boolean, - ) as BrowserWindow[], - }); - - // Set main window reference for auto-updater - if (autoUpdaterService) { - autoUpdaterService.setMainWindow(mainWindow); - } -}; - -const createFloatingButtonWindow = () => { - const mainScreen = screen.getPrimaryDisplay(); - const { width, height } = mainScreen.workAreaSize; - - floatingButtonWindow = new BrowserWindow({ - width, - height, - frame: false, - transparent: true, - alwaysOnTop: true, - resizable: false, - maximizable: false, - skipTaskbar: true, - focusable: false, - hasShadow: false, - webPreferences: { - preload: path.join(__dirname, "preload.js"), - nodeIntegration: false, - contextIsolation: true, - }, - }); - currentWindowDisplayId = mainScreen.id; // Initialize with the primary display's ID - - floatingButtonWindow.setIgnoreMouseEvents(true, { forward: true }); - - if (MAIN_WINDOW_VITE_DEV_SERVER_URL) { - const devUrl = new URL(MAIN_WINDOW_VITE_DEV_SERVER_URL); - devUrl.pathname = "widget.html"; - floatingButtonWindow.loadURL(devUrl.toString()); - } else { - floatingButtonWindow.loadFile( - path.join( - __dirname, - `../renderer/${WIDGET_WINDOW_VITE_NAME}/widget.html`, - ), - ); - } - - // Set a higher level for macOS to stay on top of fullscreen apps - if (process.platform === "darwin") { - floatingButtonWindow.setAlwaysOnTop(true, "floating", 1); - floatingButtonWindow.setVisibleOnAllWorkspaces(true, { - visibleOnFullScreen: true, - }); - floatingButtonWindow.setHiddenInMissionControl(true); - } - - // floatingButtonWindow.webContents.openDevTools({ mode: 'detach' }); // For debugging the button -}; - -// This method will be called when Electron has finished -// initialization and is ready to create browser windows. -// Some APIs can only be used after this event occurs. -app.on("ready", async () => { - // Initialize database and run migrations first - try { - await initializeDatabase(); - logger.db.info( - "Database initialized and migrations completed successfully", - ); - } catch (error) { - logError( - error instanceof Error ? error : new Error(String(error)), - "initializing database", - ); - // You might want to handle this error differently, perhaps showing a dialog to the user - } - - await requestPermissions(); - createFloatingButtonWindow(); - - // Setup tRPC IPC handler - createIPCHandler({ - router, - windows: [floatingButtonWindow!], - }); - - if (process.platform === "darwin" && app.dock) { - app.dock.show(); - } - - audioCapture = new AudioCapture(); - - // Initialize Model Manager Service - modelManagerService = new ModelManagerService(); - await modelManagerService.initialize(); - - // Initialize Local Whisper Client - localWhisperClient = new LocalWhisperClient(modelManagerService); - - // Make services available globally for tRPC - (globalThis as any).modelManagerService = modelManagerService; - (globalThis as any).localWhisperClient = localWhisperClient; - (globalThis as any).aiService = aiService; - (globalThis as any).logger = logger; - - // Initialize Contextual Transcription Manager - contextualTranscriptionManager = new ContextualTranscriptionManager( - modelManagerService, - ); - - // Initialize Auto-Updater Service - autoUpdaterService = new AutoUpdaterService(); - - // Make auto-updater service available globally for tRPC - (globalThis as any).autoUpdaterService = autoUpdaterService; - - // Check for updates on startup (after a brief delay) - setTimeout(() => { - if (autoUpdaterService) { - autoUpdaterService.checkForUpdatesAndNotify(); - } - }, 5000); // Wait 5 seconds after startup - - // Initialize AI service with the appropriate client based on configuration - try { - const transcriptionClient = createTranscriptionClient(); - aiService = new AiService(transcriptionClient); - - // Load and configure formatter - try { - const settingsService = SettingsService.getInstance(); - const formatterConfig = await settingsService.getFormatterConfig(); - if (formatterConfig) { - aiService.configureFormatter(formatterConfig); - logger.ai.info("Formatter configured", { - provider: formatterConfig.provider, - enabled: formatterConfig.enabled, - }); - } - } catch (formatterError) { - logger.ai.warn("Failed to load formatter configuration:", formatterError); - } - - logger.ai.info("AI Service initialized", { - client: "Local Whisper", - }); - } catch (error) { - logError( - error instanceof Error ? error : new Error(String(error)), - "initializing AI Service", - ); - logger.ai.warn("Transcription will not work until configuration is fixed"); - aiService = null; - } - - audioCapture.on("recording-finished", async (filePath: string) => { - // Ensure AI service is available and up-to-date - if (!aiService) { - try { - const transcriptionClient = createTranscriptionClient(); - aiService = new AiService(transcriptionClient); - - // Load and configure formatter - try { - const settingsService = SettingsService.getInstance(); - const formatterConfig = await settingsService.getFormatterConfig(); - if (formatterConfig) { - aiService.configureFormatter(formatterConfig); - logger.ai.info("Formatter reconfigured", { - provider: formatterConfig.provider, - enabled: formatterConfig.enabled, - }); - } - } catch (formatterError) { - logger.ai.warn( - "Failed to reload formatter configuration:", - formatterError, - ); - } - - logger.ai.info("AI Service reinitialized", { - client: "Local Whisper", - }); - } catch (error) { - logError( - error instanceof Error ? error : new Error(String(error)), - "reinitializing AI Service", - ); - } - } - - logger.audio.info("Recording finished", { filePath }); - if (aiService) { - try { - const startTime = Date.now(); - const audioBuffer = await fsPromises.readFile(filePath); - logger.audio.info("Audio file read", { - size: audioBuffer.length, - sizeKB: Math.round(audioBuffer.length / 1024), - }); - - const transcription = await aiService.transcribeAudio(audioBuffer); - logPerformance("audio transcription", startTime, { - audioSizeKB: Math.round(audioBuffer.length / 1024), - transcriptionLength: transcription?.length || 0, - }); - logger.ai.info("Transcription completed", { - resultLength: transcription?.length || 0, - hasResult: !!transcription, - }); - - // Save transcription to database - if ( - transcription && - typeof transcription === "string" && - transcription.trim().length > 0 - ) { - try { - const { createTranscription } = await import( - "../db/transcriptions.js" - ); - const savedTranscription = await createTranscription({ - text: transcription, - timestamp: new Date(), - audioFile: filePath, - language: "en", // Default to English, could be made configurable - }); - logger.db.info("Transcription saved to database", { - transcriptionId: savedTranscription.id, - textLength: transcription.length, - audioFile: filePath, - }); - } catch (dbError) { - logError( - dbError instanceof Error ? dbError : new Error(String(dbError)), - "saving transcription to database", - ); - } - } - - // Copy transcription to clipboard - if (transcription && typeof transcription === "string") { - logger.main.info("Transcription pasted to active application"); - // Attempt to paste into the active application - swiftIOBridgeClientInstance!.call("pasteText", { - transcript: transcription, - }); - } else { - logger.main.warn( - "Transcription result was empty or not a string, not copying", - ); - } - - // Optionally, delete the audio file after processing - // await fs.unlink(filePath); - // console.log(`Main: Deleted audio file: ${filePath}`); - } catch (error) { - logError( - error instanceof Error ? error : new Error(String(error)), - "transcription or file handling", - ); - } - } else { - logger.ai.warn("AI Service not available, cannot transcribe audio"); - } - }); - - audioCapture.on("recording-error", (error: Error) => { - console.error("Main: Received recording error from AudioCapture:", error); - }); - - // Handle individual audio chunks for real-time transcription - audioCapture.on("chunk-ready", async (chunkData: ChunkData) => { - logger.audio.info("Received chunk for transcription", { - sessionId: chunkData.sessionId, - chunkId: chunkData.chunkId, - audioDataSize: chunkData.audioData.length, - isFinalChunk: chunkData.isFinalChunk, - }); - - try { - // Get or create transcription session for this recording session - let transcriptionSession = activeTranscriptionSessions.get( - chunkData.sessionId, - ); - - if (!transcriptionSession) { - // Create new transcription session - const transcriptionClient = - contextualTranscriptionManager!.createDefaultClient(); - - transcriptionSession = new TranscriptionSession( - chunkData.sessionId, - transcriptionClient, - ); - activeTranscriptionSessions.set( - chunkData.sessionId, - transcriptionSession, - ); - - // Set up session event handlers - transcriptionSession.on("chunk-completed", (result) => { - logger.ai.info("Chunk transcription completed", { - sessionId: chunkData.sessionId, - chunkId: result.chunkId, - textLength: result.text.length, - processingTimeMs: result.processingTimeMs, - }); - }); - - transcriptionSession.on("session-completed", async (sessionResult) => { - logger.ai.info("Transcription session completed", { - sessionId: sessionResult.sessionId, - finalTextLength: sessionResult.finalText.length, - totalChunks: sessionResult.chunkResults.length, - totalProcessingTimeMs: sessionResult.totalProcessingTimeMs, - }); - - // Save chunk-based transcription to database - if ( - sessionResult.finalText && - sessionResult.finalText.trim().length > 0 - ) { - try { - const { createTranscription } = await import( - "../db/transcriptions.js" - ); - const savedTranscription = await createTranscription({ - text: sessionResult.finalText, - timestamp: new Date(), - audioFile: null, // Chunk-based transcriptions don't have a single audio file - language: "en", // Default to English, could be made configurable - }); - logger.db.info("Chunk-based transcription saved to database", { - transcriptionId: savedTranscription.id, - sessionId: sessionResult.sessionId, - textLength: sessionResult.finalText.length, - totalChunks: sessionResult.chunkResults.length, - }); - } catch (dbError) { - logError( - dbError instanceof Error ? dbError : new Error(String(dbError)), - "saving chunk-based transcription to database", - ); - } - - // Paste the final result to active application - logger.main.info( - "Final transcription pasted to active application", - { - textLength: sessionResult.finalText.length, - sessionId: sessionResult.sessionId, - }, - ); - swiftIOBridgeClientInstance!.call("pasteText", { - transcript: sessionResult.finalText, - }); - } else { - logger.main.warn("Final transcription was empty, not pasting"); - } - - // Clean up completed session - activeTranscriptionSessions.delete(chunkData.sessionId); - }); - - transcriptionSession.on("chunk-error", (errorInfo) => { - logger.ai.error("Chunk transcription error", { - sessionId: chunkData.sessionId, - chunkId: errorInfo.chunkId, - error: errorInfo.error, - }); - // Continue processing other chunks even if one fails - }); - - logger.ai.info("Created new transcription session", { - sessionId: chunkData.sessionId, - }); - } - - // Add chunk to session for processing - transcriptionSession.addChunk(chunkData); - } catch (error) { - logger.ai.error("Error handling chunk-ready event", { - sessionId: chunkData.sessionId, - chunkId: chunkData.chunkId, - error: error instanceof Error ? error.message : String(error), - }); - } - }); - - // Handle audio data chunks from renderer - ipcMain.handle( - "audio-data-chunk", - (event, chunk: ArrayBuffer, isFinalChunk: boolean) => { - if (chunk instanceof ArrayBuffer) { - console.log( - `Main: IPC received audio-data-chunk (ArrayBuffer) of size: ${chunk.byteLength} bytes. isFinalChunk: ${isFinalChunk}`, - ); - const buffer = Buffer.from(chunk); - if (buffer.length === 0) { - console.warn("Main: Received an empty audio chunk after conversion."); - } - // The AudioCapture class will now need to handle buffering and the isFinalChunk flag - audioCapture?.handleAudioChunk(buffer, isFinalChunk); - } else { - console.error( - "Main: Received audio chunk, but it is not an ArrayBuffer. Type:", - typeof chunk, - ); - throw new Error("Invalid audio chunk type received."); - } - }, - ); - - ipcMain.handle("recording-starting", async () => { - console.log("Main: Received recording-starting event."); - - // Preload the transcription model for fast processing - try { - if (contextualTranscriptionManager) { - if (!contextualTranscriptionManager.isModelLoaded()) { - logger.ai.info( - "Preloading transcription model for recording session", - ); - await contextualTranscriptionManager.preloadModel(); - logger.ai.info("Transcription model preloaded successfully"); - } else { - logger.ai.info("Transcription model already loaded"); - } - } - } catch (error) { - logger.ai.error("Error preloading transcription model", { - error: error instanceof Error ? error.message : String(error), - }); - } - - // Get accessibility context when recording starts - try { - //const accessibilityContext = await swiftIOBridgeClientInstance!.call('getAccessibilityContext', { editableOnly: true }); - //console.log('Main: Accessibility context captured:', JSON.stringify(accessibilityContext, null, 2)); - } catch (error) { - console.error("Main: Error getting accessibility context:", error); - } - - await swiftIOBridgeClientInstance!.call("muteSystemAudio", {}); - }); - - ipcMain.handle("recording-stopping", async () => { - console.log("Main: Received recording-stopping event."); - await swiftIOBridgeClientInstance!.call("restoreSystemAudio", {}); - }); - - // Initialize the SwiftIOBridgeClient - swiftIOBridgeClientInstance = new SwiftIOBridge(); - - swiftIOBridgeClientInstance.on("helperEvent", (event: HelperEvent) => { - logger.swift.debug("Received helperEvent from SwiftIOBridge", { event }); - - switch (event.type) { - case "flagsChanged": { - const payload = event.payload; - logger.swift.debug("Received flagsChanged event", { - fnKeyPressed: payload?.fnKeyPressed, - }); - // Use flagsChanged for more reliable Fn key state tracking - if (payload?.fnKeyPressed !== undefined) { - logger.swift.info("Setting recording state", { - state: payload.fnKeyPressed, - }); - floatingButtonWindow!.webContents.send( - "recording-state-changed", - payload.fnKeyPressed, - ); - } - break; - } - case "keyDown": { - const payload = event.payload; - // console.log(`Main: Received keyDown for key: ${payload?.key}.`); - // Keep keyDown handling as fallback, but flagsChanged should be primary - if (payload?.key?.toLowerCase() === "fn") { - // console.log('Main: Fn keyDown detected (fallback)'); - // Don't send recording-state-changed here as flagsChanged should handle it - } - break; - } - case "keyUp": { - const payload = event.payload; - // console.log(`Main: Received keyUp for key: ${payload?.key}.`); - // Keep keyUp handling as fallback, but flagsChanged should be primary - if (payload?.key?.toLowerCase() === "fn") { - // console.log('Main: Fn keyUp detected (fallback)'); - // Don't send recording-state-changed here as flagsChanged should handle it - } - break; - } - default: - // Optionally log or handle other event types if necessary - // console.log('Main: Unhandled helperEvent type:', (event as any).type); - break; - } - }); - - swiftIOBridgeClientInstance.on("error", (error) => { - logError( - error instanceof Error ? error : new Error(String(error)), - "SwiftIOBridge error", - ); - // Potentially notify the user or attempt to restart - }); - - swiftIOBridgeClientInstance.on("close", (code) => { - logger.swift.warn("Swift helper process closed", { code }); - // Handle unexpected close, maybe attempt restart - }); - - setupApplicationMenu(createOrShowMainWindow, () => { - if (autoUpdaterService) { - autoUpdaterService.checkForUpdates(true); - } - }); - - if (process.platform === "darwin") { - try { - console.log("Main: Setting up display change notifications"); - - activeSpaceChangeSubscriptionId = - systemPreferences.subscribeWorkspaceNotification( - "NSWorkspaceActiveDisplayDidChangeNotification", - () => { - if (floatingButtonWindow && !floatingButtonWindow.isDestroyed()) { - try { - const cursorPoint = screen.getCursorScreenPoint(); - const displayForCursor = - screen.getDisplayNearestPoint(cursorPoint); - if (currentWindowDisplayId !== displayForCursor.id) { - console.log( - `[Main Process] Moving floating window to display ID: ${displayForCursor.id}`, - ); - floatingButtonWindow.setBounds(displayForCursor.workArea); - currentWindowDisplayId = displayForCursor.id; - } - } catch (error) { - console.warn( - "[Main Process] Error handling display change:", - error, - ); - } - } - }, - ); - - if ( - activeSpaceChangeSubscriptionId !== undefined && - activeSpaceChangeSubscriptionId >= 0 - ) { - console.log( - `Main: Successfully subscribed to display change notifications`, - ); - } else { - console.error( - "Main: Failed to subscribe to display change notifications", - ); - } - } catch (e) { - console.error( - "Main: Error during subscription to display notifications:", - e, - ); - activeSpaceChangeSubscriptionId = null; - } - } else { - console.log("Main: Display change tracking is a macOS-only feature"); - } -}); - -// Clean up intervals and subscriptions -app.on("will-quit", () => { - // globalShortcut.unregisterAll(); - globalShortcut.unregisterAll(); - if (swiftIOBridgeClientInstance) { - console.log("Main: Stopping Swift helper..."); - swiftIOBridgeClientInstance.stopHelper(); - } - if (modelManagerService) { - console.log("Main: Cleaning up model downloads..."); - modelManagerService.cleanup(); - } - if (contextualTranscriptionManager) { - console.log("Main: Cleaning up transcription models..."); - contextualTranscriptionManager.dispose(); - } - if ( - process.platform === "darwin" && - activeSpaceChangeSubscriptionId !== null - ) { - systemPreferences.unsubscribeWorkspaceNotification( - activeSpaceChangeSubscriptionId, - ); - console.log("Main: Unsubscribed from display change notifications"); - activeSpaceChangeSubscriptionId = null; - } -}); - -// Quit when all windows are closed, except on macOS. There, it's common -// for applications and their menu bar to stay active until the user quits -// explicitly with Cmd + Q. +app.whenReady().then(() => appManager.initialize()); +app.on("will-quit", () => appManager.cleanup()); app.on("window-all-closed", () => { - if (process.platform !== "darwin") { - app.quit(); - } + if (process.platform !== "darwin") app.quit(); }); - -app.on("activate", () => { - // On OS X it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (BrowserWindow.getAllWindows().length === 0) { - // If no windows are open, create both FAB and main window - createFloatingButtonWindow(); - } else { - // If there are windows, ensure FAB is visible. - if (!floatingButtonWindow || floatingButtonWindow.isDestroyed()) { - createFloatingButtonWindow(); - } else { - floatingButtonWindow.show(); - } - - // Always show/create the main window when dock icon is clicked - createOrShowMainWindow(); - } -}); - -// In this file you can include the rest of your app's specific main process -// code. You can also put them in separate files and import them here. - -// Function to log the accessibility tree (added) -async function logAccessibilityTree() { - if ( - swiftIOBridgeClientInstance && - swiftIOBridgeClientInstance.isHelperRunning() - ) { - try { - // console.log('Main: Requesting full accessibility tree...'); - // Call with empty params for the whole tree, as per schema for GetAccessibilityTreeDetailsParams - const result = await swiftIOBridgeClientInstance.call( - "getAccessibilityTreeDetails", - {}, - ); - // Using JSON.stringify to see the whole structure since it's 'any' for now - // console.log('Main: Accessibility tree received:', JSON.stringify(result, null, 2)); - } catch (error) { - console.error("Main: Error calling getAccessibilityTreeDetails:", error); - } - } else { - console.warn( - "Main: SwiftIOBridge not ready or helper not running, cannot log accessibility tree.", - ); - } -} +app.on("activate", () => appManager.handleActivate()); diff --git a/apps/desktop/src/main/managers/service-manager.ts b/apps/desktop/src/main/managers/service-manager.ts new file mode 100644 index 0000000..f2c769f --- /dev/null +++ b/apps/desktop/src/main/managers/service-manager.ts @@ -0,0 +1,208 @@ +import { logger } from "../logger"; +import { ModelManagerService } from "../../services/model-manager"; +import { TranscriptionService } from "../../services/transcription-service"; +import { SettingsService } from "../../services/settings-service"; +import { SwiftIOBridge } from "../../services/platform/swift-bridge-service"; +import { AutoUpdaterService } from "../services/auto-updater"; +import { WindowManager } from "../core/window-manager"; +import { RecordingService } from "../../services/recording-service"; + +/** + * Manages service initialization and lifecycle + */ +export class ServiceManager { + private static instance: ServiceManager | null = null; + private isInitialized = false; + + private modelManagerService: ModelManagerService | null = null; + private transcriptionService: TranscriptionService | null = null; + private settingsService: SettingsService | null = null; + + private swiftIOBridge: SwiftIOBridge | null = null; + private autoUpdaterService: AutoUpdaterService | null = null; + private recordingService: RecordingService | null = null; + + async initialize(windowManager: WindowManager): Promise { + if (this.isInitialized) { + logger.main.warn( + "ServiceManager is already initialized, skipping initialization", + ); + return; + } + + try { + this.initializeSettingsService(); + await this.initializeModelServices(); + this.initializePlatformServices(); + await this.initializeAIServices(); + this.initializeRecordingService(); + this.initializeAutoUpdater(windowManager); + + this.isInitialized = true; + logger.main.info("Services initialized successfully"); + } catch (error) { + logger.main.error("Failed to initialize services:", error); + // Don't throw here - allow app to start even if some services fail + } + } + + private initializeSettingsService(): void { + this.settingsService = new SettingsService(); + logger.main.info("Settings service initialized"); + } + + private async initializeModelServices(): Promise { + // Initialize Model Manager Service + this.modelManagerService = new ModelManagerService(); + await this.modelManagerService.initialize(); + } + + private async initializeAIServices(): Promise { + try { + if (!this.modelManagerService) { + throw new Error("Model manager service not initialized"); + } + + this.transcriptionService = new TranscriptionService( + this.modelManagerService, + ); + + // Load and configure formatter + try { + if (!this.settingsService) { + throw new Error("SettingsService not initialized"); + } + const formatterConfig = await this.settingsService.getFormatterConfig(); + if (formatterConfig) { + this.transcriptionService.configureFormatter(formatterConfig); + logger.transcription.info("Formatter configured", { + provider: formatterConfig.provider, + enabled: formatterConfig.enabled, + }); + } + } catch (formatterError) { + logger.transcription.warn( + "Failed to load formatter configuration:", + formatterError, + ); + } + + logger.transcription.info("Transcription Service initialized", { + client: "Pipeline with Whisper", + }); + } catch (error) { + logger.transcription.error( + "Error initializing Transcription Service:", + error, + ); + logger.transcription.warn( + "Transcription will not work until configuration is fixed", + ); + this.transcriptionService = null; + } + } + + private initializePlatformServices(): void { + // Initialize Swift bridge for macOS integration + if (process.platform === "darwin") { + this.swiftIOBridge = new SwiftIOBridge(); + } + } + + private initializeRecordingService(): void { + this.recordingService = new RecordingService(this); + logger.main.info("Recording service initialized"); + } + + private initializeAutoUpdater(windowManager: WindowManager): void { + this.autoUpdaterService = new AutoUpdaterService(windowManager); + } + + // Getters for other managers to access services + getModelManagerService(): ModelManagerService { + if (!this.isInitialized) { + throw new Error( + "ServiceManager not initialized. Call initialize() first.", + ); + } + if (!this.modelManagerService) { + throw new Error("ModelManagerService failed to initialize"); + } + return this.modelManagerService; + } + + getTranscriptionService(): TranscriptionService { + if (!this.isInitialized) { + throw new Error( + "ServiceManager not initialized. Call initialize() first.", + ); + } + if (!this.transcriptionService) { + throw new Error("TranscriptionService failed to initialize"); + } + return this.transcriptionService; + } + + getSettingsService(): SettingsService { + if (!this.isInitialized) { + throw new Error( + "ServiceManager not initialized. Call initialize() first.", + ); + } + if (!this.settingsService) { + throw new Error("SettingsService failed to initialize"); + } + return this.settingsService; + } + + getSwiftIOBridge(): SwiftIOBridge { + if (!this.isInitialized) { + throw new Error( + "ServiceManager not initialized. Call initialize() first.", + ); + } + if (!this.swiftIOBridge) { + throw new Error("SwiftIOBridge not available on this platform"); + } + return this.swiftIOBridge; + } + + getAutoUpdaterService(): AutoUpdaterService { + if (!this.isInitialized) { + throw new Error( + "ServiceManager not initialized. Call initialize() first.", + ); + } + if (!this.autoUpdaterService) { + throw new Error("AutoUpdaterService failed to initialize"); + } + return this.autoUpdaterService; + } + + async cleanup(): Promise { + if (this.recordingService) { + logger.main.info("Cleaning up recording service..."); + await this.recordingService.cleanup(); + } + if (this.modelManagerService) { + logger.main.info("Cleaning up model downloads..."); + this.modelManagerService.cleanup(); + } + + if (this.swiftIOBridge) { + logger.main.info("Stopping Swift helper..."); + this.swiftIOBridge.stopHelper(); + } + } + + static getInstance(): ServiceManager | null { + return ServiceManager.instance; + } + + static createInstance(): ServiceManager { + if (!ServiceManager.instance) { + ServiceManager.instance = new ServiceManager(); + } + return ServiceManager.instance; + } +} diff --git a/apps/desktop/src/main/menu.ts b/apps/desktop/src/main/menu.ts index efee625..c164a75 100644 --- a/apps/desktop/src/main/menu.ts +++ b/apps/desktop/src/main/menu.ts @@ -6,6 +6,7 @@ import { app, Menu, MenuItemConstructorOptions, BrowserWindow } from "electron"; export const setupApplicationMenu = ( createOrShowSettingsWindow: () => void, checkForUpdates?: () => void, + openAllDevTools?: () => void, ) => { const menuTemplate: MenuItemConstructorOptions[] = [ // { role: 'appMenu' } for macOS @@ -97,6 +98,15 @@ export const setupApplicationMenu = ( { role: "reload" as const }, { role: "forceReload" as const }, { role: "toggleDevTools" as const }, + ...(openAllDevTools + ? [ + { + label: "Open All Dev Tools", + accelerator: "CmdOrCtrl+Shift+I", + click: () => openAllDevTools(), + } as MenuItemConstructorOptions, + ] + : []), { type: "separator" as const }, { role: "resetZoom" as const }, { role: "zoomIn" as const }, diff --git a/apps/desktop/src/main/preload.ts b/apps/desktop/src/main/preload.ts index e90684a..e10297a 100644 --- a/apps/desktop/src/main/preload.ts +++ b/apps/desktop/src/main/preload.ts @@ -2,11 +2,9 @@ // https://www.electronjs.org/docs/latest/tutorial/process-model#preload-scripts import { contextBridge, ipcRenderer, IpcRendererEvent } from "electron"; -import log from "electron-log/renderer"; import { exposeElectronTRPC } from "electron-trpc-experimental/preload"; import type { ElectronAPI } from "../types/electron-api"; -import type { FormatterConfig } from "../modules/formatter"; -import type { Transcription, NewTranscription } from "../db/schema"; +import type { FormatterConfig } from "../types/formatter"; interface ShortcutData { shortcut: string; @@ -94,7 +92,6 @@ const api: ElectronAPI = { // Transcription Database API (moved to tRPC) - // Vocabulary Database API on: (channel: string, callback: (...args: any[]) => void) => { const handler = (_event: IpcRendererEvent, ...args: any[]) => callback(...args); @@ -125,13 +122,26 @@ const api: ElectronAPI = { } }, - // Logging API for renderer process + // Logging API for renderer process - sends to main process via IPC log: { - info: (...args: any[]) => log.info(...args), - warn: (...args: any[]) => log.warn(...args), - error: (...args: any[]) => log.error(...args), - debug: (...args: any[]) => log.debug(...args), - scope: (name: string) => log.scope(name), + info: (...args: any[]) => + ipcRenderer.invoke("log-message", "info", "renderer", ...args), + warn: (...args: any[]) => + ipcRenderer.invoke("log-message", "warn", "renderer", ...args), + error: (...args: any[]) => + ipcRenderer.invoke("log-message", "error", "renderer", ...args), + debug: (...args: any[]) => + ipcRenderer.invoke("log-message", "debug", "renderer", ...args), + scope: (name: string) => ({ + info: (...args: any[]) => + ipcRenderer.invoke("log-message", "info", name, ...args), + warn: (...args: any[]) => + ipcRenderer.invoke("log-message", "warn", name, ...args), + error: (...args: any[]) => + ipcRenderer.invoke("log-message", "error", name, ...args), + debug: (...args: any[]) => + ipcRenderer.invoke("log-message", "debug", name, ...args), + }), }, }; diff --git a/apps/desktop/src/main/services/auto-updater.ts b/apps/desktop/src/main/services/auto-updater.ts index 8f5c8da..e774133 100644 --- a/apps/desktop/src/main/services/auto-updater.ts +++ b/apps/desktop/src/main/services/auto-updater.ts @@ -2,13 +2,13 @@ import { autoUpdater } from "electron-updater"; import { app, dialog, BrowserWindow } from "electron"; import { EventEmitter } from "events"; import { logger } from "../logger"; +import { WindowManager } from "../core/window-manager"; export class AutoUpdaterService extends EventEmitter { private checkingForUpdate = false; private updateAvailable = false; - private mainWindow: BrowserWindow | null = null; - constructor() { + constructor(private windowManager: WindowManager) { super(); // Only set up auto-updater in production @@ -19,10 +19,6 @@ export class AutoUpdaterService extends EventEmitter { } } - setMainWindow(window: BrowserWindow | null) { - this.mainWindow = window; - } - private setupAutoUpdater() { // Configure updater autoUpdater.autoDownload = false; // Don't auto-download, ask user first @@ -62,7 +58,8 @@ export class AutoUpdaterService extends EventEmitter { this.checkingForUpdate = false; // Show error dialog only if user manually checked for updates - if (this.mainWindow && !this.mainWindow.isDestroyed()) { + const mainWindow = this.windowManager.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { dialog.showErrorBox( "Update Error", `Error checking for updates: ${err.message}`, @@ -89,11 +86,12 @@ export class AutoUpdaterService extends EventEmitter { } private async showUpdateDialog(info: any) { - if (!this.mainWindow || this.mainWindow.isDestroyed()) { + const mainWindow = this.windowManager.getMainWindow(); + if (!mainWindow || mainWindow.isDestroyed()) { return; } - const result = await dialog.showMessageBox(this.mainWindow, { + const result = await dialog.showMessageBox(mainWindow, { type: "info", title: "Update Available", message: `A new version (${info.version}) is available.`, @@ -113,11 +111,12 @@ export class AutoUpdaterService extends EventEmitter { } private async showInstallDialog(info: any) { - if (!this.mainWindow || this.mainWindow.isDestroyed()) { + const mainWindow = this.windowManager.getMainWindow(); + if (!mainWindow || mainWindow.isDestroyed()) { return; } - const result = await dialog.showMessageBox(this.mainWindow, { + const result = await dialog.showMessageBox(mainWindow, { type: "info", title: "Update Ready", message: `Update ${info.version} has been downloaded.`, @@ -140,13 +139,16 @@ export class AutoUpdaterService extends EventEmitter { // Skip in development if (process.env.NODE_ENV === "development" || !app.isPackaged) { logger.updater.info("Skipping update check in development mode"); - if (userInitiated && this.mainWindow && !this.mainWindow.isDestroyed()) { - dialog.showMessageBox(this.mainWindow, { - type: "info", - title: "Development Mode", - message: "Update checking is disabled in development mode.", - buttons: ["OK"], - }); + if (userInitiated) { + const mainWindow = this.windowManager.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + dialog.showMessageBox(mainWindow, { + type: "info", + title: "Development Mode", + message: "Update checking is disabled in development mode.", + buttons: ["OK"], + }); + } } return; } @@ -164,11 +166,14 @@ export class AutoUpdaterService extends EventEmitter { error: error instanceof Error ? error.message : String(error), }); - if (userInitiated && this.mainWindow && !this.mainWindow.isDestroyed()) { - dialog.showErrorBox( - "Update Check Failed", - "Failed to check for updates. Please try again later.", - ); + if (userInitiated) { + const mainWindow = this.windowManager.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + dialog.showErrorBox( + "Update Check Failed", + "Failed to check for updates. Please try again later.", + ); + } } } } diff --git a/apps/desktop/src/modules/ai/ai-service.ts b/apps/desktop/src/modules/ai/ai-service.ts deleted file mode 100644 index 874e707..0000000 --- a/apps/desktop/src/modules/ai/ai-service.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { TranscriptionClient } from "./transcription-client"; -import { FormatterService } from "../formatter"; - -export class AiService { - private transcriptionClient: TranscriptionClient; - private formatterService: FormatterService; - - constructor(transcriptionClient: TranscriptionClient) { - this.transcriptionClient = transcriptionClient; - this.formatterService = new FormatterService(); - } - - async transcribeAudio(audioData: Buffer): Promise { - if (!this.transcriptionClient) { - throw new Error("Transcription client is not initialized."); - } - - // Step 1: Transcribe audio - const transcribedText = - await this.transcriptionClient.transcribe(audioData); - - // Step 2: Format the transcribed text if formatter is enabled - const formattedText = - await this.formatterService.formatText(transcribedText); - - return formattedText; - } - - /** - * Set formatter configuration - */ - configureFormatter(config: any): void { - this.formatterService.configure(config); - } - - /** - * Get formatter service instance - */ - getFormatterService(): FormatterService { - return this.formatterService; - } - - // Future methods for other AI functionalities can be added here - // e.g., text summarization, sentiment analysis, etc. -} diff --git a/apps/desktop/src/modules/ai/local-whisper-client.ts b/apps/desktop/src/modules/ai/local-whisper-client.ts deleted file mode 100644 index 0c12d3e..0000000 --- a/apps/desktop/src/modules/ai/local-whisper-client.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { TranscriptionClient } from "./transcription-client"; -import * as fs from "fs"; -import { logger } from "../../main/logger"; -import { ModelManagerService } from "../models/model-manager"; - -export class LocalWhisperClient implements TranscriptionClient { - private modelManager: ModelManagerService; - private selectedModelId: string | null = null; - private whisperInstance: any = null; // Will be imported from smart-whisper - - constructor(modelManager: ModelManagerService, selectedModelId?: string) { - this.modelManager = modelManager; - this.selectedModelId = selectedModelId || null; - } - - private async initializeWhisper(): Promise { - if (this.whisperInstance) { - return; // Already initialized - } - - const modelPath = await this.getBestAvailableModel(); - if (!modelPath) { - throw new Error( - "No Whisper models available. Please download a model first.", - ); - } - - try { - const { Whisper } = await import("smart-whisper"); - this.whisperInstance = new Whisper(modelPath, { gpu: true }); - logger.ai.info("Smart-whisper initialized", { modelPath }); - } catch (error) { - logger.ai.error("Failed to initialize smart-whisper", { - error: error instanceof Error ? error.message : String(error), - modelPath, - }); - throw new Error(`Failed to initialize smart-whisper: ${error}`); - } - } - - async transcribe(audioData: Buffer): Promise { - try { - await this.initializeWhisper(); - - // Convert audio buffer to the format expected by smart-whisper - const audioFloat32Array = await this.convertAudioBuffer(audioData); - - logger.ai.info("Starting smart-whisper transcription", { - audioDataSize: audioData.length, - convertedSize: audioFloat32Array.length, - }); - - // Transcribe using smart-whisper - const { result } = await this.whisperInstance.transcribe( - audioFloat32Array, - { - language: "auto", - }, - ); - - const transcription = await result; - - logger.ai.info("Smart-whisper transcription completed", { - resultLength: transcription.length, - }); - - return transcription; - } catch (error) { - logger.ai.error("Smart-whisper transcription failed", { - error: error instanceof Error ? error.message : String(error), - }); - throw new Error(`Transcription failed: ${error}`); - } - } - - private async convertAudioBuffer(audioData: Buffer): Promise { - // Smart-whisper expects Float32Array with 16kHz mono audio - // This is a simplified conversion - you may need more sophisticated audio processing - try { - // For now, assume the audio data is already in the correct format - // In a real implementation, you'd use an audio processing library like node-wav - // to properly decode and resample the audio - - // Convert buffer to Float32Array (simplified) - const float32Array = new Float32Array(audioData.length / 4); - for (let i = 0; i < float32Array.length; i++) { - // Read 32-bit float from buffer (little-endian) - float32Array[i] = audioData.readFloatLE(i * 4); - } - - return float32Array; - } catch (error) { - logger.ai.warn("Audio conversion failed, trying alternative method", { - error: error instanceof Error ? error.message : String(error), - }); - - // Fallback: convert as if it's PCM data - const samples = new Float32Array(audioData.length / 2); - for (let i = 0; i < samples.length; i++) { - // Convert 16-bit signed PCM to float (-1 to 1) - const sample = audioData.readInt16LE(i * 2); - samples[i] = sample / 32768.0; - } - - return samples; - } - } - - private async getBestAvailableModel(): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - - // If a specific model is selected and available, use it - if (this.selectedModelId && downloadedModels[this.selectedModelId]) { - const model = downloadedModels[this.selectedModelId]; - if (fs.existsSync(model.localPath)) { - return model.localPath; - } - } - - // Otherwise, find the best available model (prioritize by quality) - const preferredOrder = [ - "whisper-large-v1", - "whisper-medium", - "whisper-small", - "whisper-base", - "whisper-tiny", - ]; - - for (const modelId of preferredOrder) { - const model = downloadedModels[modelId]; - if (model && fs.existsSync(model.localPath)) { - return model.localPath; - } - } - - return null; - } - - // Set the model to use for transcription - async setSelectedModel(modelId: string): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - if (!downloadedModels[modelId]) { - throw new Error(`Model not downloaded: ${modelId}`); - } - - // If we're changing models, free the current instance - if (this.selectedModelId !== modelId && this.whisperInstance) { - this.freeWhisperInstance(); - } - - this.selectedModelId = modelId; - logger.ai.info("Selected model for transcription", { modelId }); - } - - // Get the currently selected model - getSelectedModel(): string | null { - return this.selectedModelId; - } - - // Check if whisper is available - async isAvailable(): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - return Object.keys(downloadedModels).some((modelId) => - fs.existsSync(downloadedModels[modelId].localPath), - ); - } - - // Get available models - async getAvailableModels(): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - return Object.keys(downloadedModels).filter((modelId) => - fs.existsSync(downloadedModels[modelId].localPath), - ); - } - - // Free resources - async dispose(): Promise { - await this.freeWhisperInstance(); - } - - private async freeWhisperInstance(): Promise { - if (this.whisperInstance) { - try { - await this.whisperInstance.free(); - logger.ai.info("Smart-whisper instance freed"); - } catch (error) { - logger.ai.warn("Error freeing smart-whisper instance", { - error: error instanceof Error ? error.message : String(error), - }); - } finally { - this.whisperInstance = null; - } - } - } -} diff --git a/apps/desktop/src/modules/ai/transcription-client.ts b/apps/desktop/src/modules/ai/transcription-client.ts deleted file mode 100644 index 71eccac..0000000 --- a/apps/desktop/src/modules/ai/transcription-client.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface TranscriptionClient { - transcribe(audioData: Buffer): Promise; -} diff --git a/apps/desktop/src/modules/audio/audio-capture.ts b/apps/desktop/src/modules/audio/audio-capture.ts deleted file mode 100644 index 5399c6e..0000000 --- a/apps/desktop/src/modules/audio/audio-capture.ts +++ /dev/null @@ -1,256 +0,0 @@ -import fs, { statSync } from "node:fs"; // Import statSync -import path from "node:path"; -import { app } from "electron"; // To get a writable path like appData -import { EventEmitter } from "node:events"; - -export class AudioCapture extends EventEmitter { - private currentRecordingPath: string | null = null; - private writableStream: fs.WriteStream | null = null; - private chunkCounter: number = 0; - private sessionId: string | null = null; - - constructor() { - super(); - // Ensure the recordings directory exists - const recordingsDir = path.join(app.getPath("userData"), "recordings"); - if (!fs.existsSync(recordingsDir)) { - fs.mkdirSync(recordingsDir, { recursive: true }); - } - } - - public isCurrentlyRecording(): boolean { - return this.writableStream !== null; - } - - private finalizeRecording(): void { - if (!this.writableStream) { - console.warn( - "AudioCapture: finalizeRecording called but no writableStream active. This might indicate a prior error or premature call.", - ); - return; - } - - console.log( - "AudioCapture: finalizeRecording() called, ending writable stream.", - ); - const streamToClose = this.writableStream; - const recordingPathToFinalize = this.currentRecordingPath; - - this.writableStream = null; // Prevent new writes and signal "not recording" - - streamToClose.end(() => { - console.log( - `AudioCapture: Writable stream .end() callback for: ${recordingPathToFinalize}`, - ); - if (recordingPathToFinalize) { - try { - const stats = statSync(recordingPathToFinalize); - console.log( - `AudioCapture: File size of ${recordingPathToFinalize} is ${stats.size} bytes before emitting 'recording-finished'.`, - ); - if (stats.size === 0) { - console.warn( - `AudioCapture: File ${recordingPathToFinalize} is empty. Transcription will likely fail.`, - ); - } - this.emit("recording-finished", recordingPathToFinalize); - } catch (error: any) { - console.error( - `AudioCapture: Error getting file stats for ${recordingPathToFinalize}:`, - error, - ); - this.emit( - "recording-error", - new Error( - `Failed to get stats for ${recordingPathToFinalize}: ${error.message}`, - ), - ); - } - // Only nullify currentRecordingPath if it matches the one being finalized. - if (this.currentRecordingPath === recordingPathToFinalize) { - this.currentRecordingPath = null; - this.sessionId = null; - this.chunkCounter = 0; - } - } - }); - - // The 'finish' event on streamToClose is mostly for logging here. - streamToClose.on("finish", () => { - console.log( - `AudioCapture: Writable stream 'finish' event for the recording at ${recordingPathToFinalize}.`, - ); - // Clean up path if still relevant, though .end() callback should handle primary cleanup. - if (this.currentRecordingPath === recordingPathToFinalize) { - this.currentRecordingPath = null; - this.sessionId = null; - this.chunkCounter = 0; - } - }); - // Note: The 'error' handler for streamToClose was set up when it was created. - // That handler is responsible for nulling writableStream and currentRecordingPath if an error occurs on *that* stream instance. - } - - public handleAudioChunk(chunk: Buffer, isFinalChunk: boolean = false): void { - if (!this.writableStream) { - // No active stream, this could be the start of a new recording - if (chunk.length > 0) { - // First non-empty chunk: Start a new recording - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - this.sessionId = `session-${timestamp}`; - this.chunkCounter = 0; - this.currentRecordingPath = path.join( - app.getPath("userData"), - "recordings", - `recording-${timestamp}.webm`, - ); - - const newStream = fs.createWriteStream(this.currentRecordingPath); - const recordingPathForThisStream = this.currentRecordingPath; // Capture path for this specific stream instance - console.log( - `AudioCapture: New recording started by first chunk. Saving to: ${recordingPathForThisStream}`, - ); - - newStream.on("error", (err) => { - console.error( - `AudioCapture: Error on writable stream for ${recordingPathForThisStream}:`, - err, - ); - this.emit("recording-error", err); - - // If the currently active stream in the class is the one that errored, nullify it. - if (this.writableStream === newStream) { - this.writableStream = null; - } - // If the current recording path in the class is for the stream that errored, nullify it. - if (this.currentRecordingPath === recordingPathForThisStream) { - this.currentRecordingPath = null; - } - // Ensure the stream is closed/destroyed to release resources - if (!newStream.destroyed) { - newStream.end(); - } - }); - - this.writableStream = newStream; // Assign to class property after setup - - // Write the first chunk - this.writableStream.write(chunk, (writeError) => { - if (writeError) { - console.error( - `AudioCapture: Error writing initial audio chunk to ${recordingPathForThisStream}:`, - writeError, - ); - this.emit("recording-error", writeError); - // If this write fails, the stream is likely compromised. Clean up. - if (this.writableStream === newStream) { - // Check if it's still our current stream - this.writableStream = null; - } - if (this.currentRecordingPath === recordingPathForThisStream) { - // Check if it's still our current path - this.currentRecordingPath = null; - } - if (!newStream.destroyed) { - newStream.end(); // Attempt to close the problematic stream - } - return; // Don't proceed to final chunk logic if initial write fails - } - - // Emit chunk-ready event for immediate transcription - this.chunkCounter++; - console.log( - `AudioCapture: Emitting chunk-ready for chunk ${this.chunkCounter}`, - ); - this.emit("chunk-ready", { - sessionId: this.sessionId, - chunkId: this.chunkCounter, - audioData: chunk, - isFinalChunk: isFinalChunk, - }); - - // If this very first chunk is also the final chunk - if (isFinalChunk) { - console.log( - "AudioCapture: First chunk is also the final chunk. Finalizing immediately.", - ); - this.finalizeRecording(); - } - }); - } else { - // Empty chunk and no stream - if (isFinalChunk) { - console.log( - "AudioCapture: Received an empty final chunk, but no recording was active. No action taken.", - ); - } else { - console.warn( - "AudioCapture: Received an empty non-final chunk, but no recording was active. Ignoring.", - ); - } - } - } else { - // WritableStream exists, so we are actively recording - const activeStream = this.writableStream; // Capture current stream for this operation scope - const activePath = this.currentRecordingPath; - - if (chunk.length > 0) { - // console.log(`AudioCapture: Writing audio chunk of size: ${chunk.length} bytes to ${activePath}. isFinalChunk: ${isFinalChunk}`); - activeStream.write(chunk, (writeError) => { - if (writeError) { - console.error( - `AudioCapture: Error writing subsequent audio chunk to ${activePath}:`, - writeError, - ); - this.emit("recording-error", writeError); - // The stream's main 'error' handler should manage cleanup if the stream itself errors. - // If only this write fails, but stream doesn't emit 'error', we might need to intervene. - // However, a write error often leads to a stream error. - // For safety, if this write fails, we consider the stream potentially compromised for further writes. - // The 'error' handler on `activeStream` should ideally handle this. - // If `isFinalChunk` was true, `finalizeRecording` won't be called due to return/error. - // Consider calling finalizeRecording or a similar cleanup if write error on final chunk. - // For now, relying on the stream's 'error' event for full cleanup. - } else { - // Emit chunk-ready event for immediate transcription - this.chunkCounter++; - console.log( - `AudioCapture: Emitting chunk-ready for chunk ${this.chunkCounter}`, - ); - this.emit("chunk-ready", { - sessionId: this.sessionId, - chunkId: this.chunkCounter, - audioData: chunk, - isFinalChunk: isFinalChunk, - }); - - if (isFinalChunk) { - console.log( - "AudioCapture: Final chunk written successfully. Finalizing recording.", - ); - this.finalizeRecording(); - } - } - }); - } else { - // Empty chunk during active recording - console.warn( - `AudioCapture: Received empty audio chunk while recording to ${activePath}. Not writing to file.`, - ); - if (isFinalChunk) { - console.log( - "AudioCapture: Empty final chunk received during active recording. Finalizing recording.", - ); - // Still emit the final chunk event even if empty - this.emit("chunk-ready", { - sessionId: this.sessionId, - chunkId: this.chunkCounter, // Don't increment for empty chunks - audioData: chunk, - isFinalChunk: true, - }); - this.finalizeRecording(); - } - } - } - } -} diff --git a/apps/desktop/src/modules/formatter/formatter-client.ts b/apps/desktop/src/modules/formatter/formatter-client.ts deleted file mode 100644 index c9f2d33..0000000 --- a/apps/desktop/src/modules/formatter/formatter-client.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Abstract base class for text formatting clients - */ -export abstract class FormatterClient { - abstract formatText(text: string): Promise; -} - -/** - * Configuration interface for formatter clients - */ -export interface FormatterConfig { - provider: "openrouter"; - model: string; - apiKey: string; - enabled: boolean; -} diff --git a/apps/desktop/src/modules/formatter/formatter-service.ts b/apps/desktop/src/modules/formatter/formatter-service.ts deleted file mode 100644 index c52523d..0000000 --- a/apps/desktop/src/modules/formatter/formatter-service.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { FormatterClient, FormatterConfig } from "./formatter-client"; -import { OpenRouterFormatterClient } from "./openrouter-formatter-client"; - -/** - * Main formatter service that manages different formatting providers - */ -export class FormatterService { - private client: FormatterClient | null = null; - private config: FormatterConfig | null = null; - - /** - * Configure the formatter service with the given configuration - */ - configure(config: FormatterConfig): void { - this.config = config; - - if (!config.enabled) { - this.client = null; - return; - } - - switch (config.provider) { - case "openrouter": - this.client = new OpenRouterFormatterClient( - config.apiKey, - config.model, - ); - break; - default: - throw new Error(`Unsupported formatter provider: ${config.provider}`); - } - } - - /** - * Format the given text using the configured formatter - * Returns the original text if formatter is not configured or disabled - */ - async formatText(text: string): Promise { - if (!this.client || !this.config?.enabled) { - return text; - } - - try { - return await this.client.formatText(text); - } catch (error) { - console.error("Error in formatter service:", error); - // Return original text if formatting fails - return text; - } - } - - /** - * Check if the formatter is configured and enabled - */ - isEnabled(): boolean { - return this.config?.enabled === true && this.client !== null; - } - - /** - * Get the current configuration - */ - getConfiguration(): FormatterConfig | null { - return this.config; - } -} diff --git a/apps/desktop/src/modules/formatter/index.ts b/apps/desktop/src/modules/formatter/index.ts deleted file mode 100644 index 60ea6b8..0000000 --- a/apps/desktop/src/modules/formatter/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { FormatterService } from "./formatter-service"; -export { FormatterClient, FormatterConfig } from "./formatter-client"; -export { OpenRouterFormatterClient } from "./openrouter-formatter-client"; diff --git a/apps/desktop/src/modules/formatter/openrouter-formatter-client.ts b/apps/desktop/src/modules/formatter/openrouter-formatter-client.ts deleted file mode 100644 index 540001f..0000000 --- a/apps/desktop/src/modules/formatter/openrouter-formatter-client.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createOpenAI } from "@ai-sdk/openai"; -import { generateText } from "ai"; -import { FormatterClient } from "./formatter-client"; - -/** - * OpenRouter-based text formatter client - */ -export class OpenRouterFormatterClient extends FormatterClient { - private provider: any; - private model: string; - - constructor(apiKey: string, model: string) { - super(); - - // Configure OpenRouter provider - this.provider = createOpenAI({ - baseURL: "https://openrouter.ai/api/v1", - apiKey: apiKey, - }); - - this.model = model; - } - - async formatText(text: string): Promise { - try { - const { text: formattedText } = await generateText({ - model: this.provider(this.model), - messages: [ - { - role: "system", - content: `You are a professional text formatter. Your task is to clean up and improve the formatting of transcribed text while preserving the original meaning and content. - -Please: -1. Fix obvious transcription errors and typos -2. Add proper punctuation where missing -3. Organize the text into proper paragraphs -4. Capitalize proper nouns and sentence beginnings -5. Remove unnecessary filler words (um, uh, etc.) but keep natural speech patterns -6. Maintain the speaker's original tone and style - -Return only the formatted text without any explanations or additional commentary.`, - }, - { - role: "user", - content: `Please format this transcribed text:\n\n${text}`, - }, - ], - temperature: 0.1, // Low temperature for consistent formatting - maxTokens: 2000, - }); - - return formattedText; - } catch (error) { - console.error("Error formatting text with OpenRouter:", error); - // Return original text if formatting fails - return text; - } - } -} diff --git a/apps/desktop/src/modules/settings/index.ts b/apps/desktop/src/modules/settings/index.ts deleted file mode 100644 index a7cc78a..0000000 --- a/apps/desktop/src/modules/settings/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { SettingsService } from "./settings-service"; diff --git a/apps/desktop/src/modules/transcription/contextual-local-whisper-client.ts b/apps/desktop/src/modules/transcription/contextual-local-whisper-client.ts deleted file mode 100644 index a37f545..0000000 --- a/apps/desktop/src/modules/transcription/contextual-local-whisper-client.ts +++ /dev/null @@ -1,387 +0,0 @@ -import { ContextualTranscriptionClient } from "./transcription-session"; -import * as fs from "fs"; -import { logger } from "../../main/logger"; -import { ModelManagerService } from "../models/model-manager"; -import { TranscribeFormat, TranscribeParams, Whisper } from "smart-whisper"; - -export class ContextualLocalWhisperClient - implements ContextualTranscriptionClient -{ - private modelManager: ModelManagerService; - private selectedModelId: string | null = null; - private whisperInstance: Whisper | null = null; // Will be imported from smart-whisper - private lastUsedTimestamp: number = 0; - private cleanupTimer: NodeJS.Timeout | null = null; - private readonly MODEL_CLEANUP_DELAY_MS = 30000; // 30 seconds after last use (configurable) - - constructor(modelManager: ModelManagerService, selectedModelId?: string) { - this.modelManager = modelManager; - this.selectedModelId = selectedModelId || null; - } - - private async initializeWhisper(): Promise { - if (this.whisperInstance) { - return; // Already initialized - } - - const modelPath = await this.getBestAvailableModel(); - if (!modelPath) { - throw new Error( - "No Whisper models available. Please download a model first.", - ); - } - - try { - //! esure gpu is used if available - this.whisperInstance = new Whisper(modelPath, { gpu: true }); - logger.ai.info( - "Smart-whisper instance created for contextual transcription", - { modelPath }, - ); - // Actually load the model into memory - await this.whisperInstance.load(); - logger.ai.info( - "Smart-whisper model loaded into memory for contextual transcription", - { - modelPath, - }, - ); - } catch (error) { - logger.ai.error( - "Failed to initialize and load smart-whisper for contextual transcription", - { - error: error instanceof Error ? error.message : String(error), - modelPath, - }, - ); - throw new Error(`Failed to initialize and load smart-whisper: ${error}`); - } - } - - async transcribeWithContext( - audioData: Buffer, - previousContext: string, - ): Promise { - try { - await this.initializeWhisper(); - this.updateLastUsedTimestamp(); // Update timestamp when model is used - - // Convert audio buffer to the format expected by smart-whisper - const audioFloat32Array = await this.convertAudioBuffer(audioData); - - // Prepare initial prompt with context for better continuity - let prompt = ""; - if (previousContext && previousContext.trim().length > 0) { - // Use last ~50 words as context/prompt - const contextWords = previousContext.trim().split(/\s+/); - const maxWords = 50; - prompt = - contextWords.length > maxWords - ? contextWords.slice(-maxWords).join(" ") - : previousContext.trim(); - } - - const modelInfo = await this.getCurrentModelInfo(); - logger.ai.info("Starting smart-whisper contextual transcription", { - audioDataSize: audioData.length, - convertedSize: audioFloat32Array.length, - hasContext: prompt.length > 0, - contextLength: prompt.length, - modelId: modelInfo.modelId, - modelPath: modelInfo.modelPath, - }); - - // Transcribe using smart-whisper with initial prompt for context - const transcriptionOptions: Partial> = - { - language: "auto", - }; - - // Add initial prompt if we have context - if (prompt) { - transcriptionOptions.initial_prompt = prompt; - } - - const { result } = await this.whisperInstance!.transcribe( - audioFloat32Array, - transcriptionOptions, - ); - const transcription = await result; - - // Extract text from the result object - const transcriptionText = transcription.reduce( - (acc, curr) => acc + curr.text, - "", - ); - - logger.ai.info("Smart-whisper contextual transcription completed", { - resultLength: transcriptionText.length, - hadContext: prompt.length > 0, - resultType: typeof result, - modelId: modelInfo.modelId, - modelPath: modelInfo.modelPath, - }); - - return transcriptionText; - } catch (error) { - logger.ai.error("Smart-whisper contextual transcription failed", { - error: error instanceof Error ? error.message : String(error), - }); - throw new Error(`Contextual transcription failed: ${error}`); - } - } - - private async convertAudioBuffer(audioData: Buffer): Promise { - // Smart-whisper expects Float32Array with 16kHz mono audio - // Now we're receiving raw Float32Array data from Web Audio API - - logger.ai.info("Converting audio buffer", { - bufferLength: audioData.length, - expectedFloat32Length: audioData.length / 4, - }); - - try { - // The audioData should now be raw Float32Array from Web Audio API (16kHz, mono) - // Check if buffer length is divisible by 4 (Float32 = 4 bytes) - if (audioData.length % 4 !== 0) { - logger.ai.warn( - "Audio buffer length not divisible by 4, may not be Float32Array", - { - length: audioData.length, - remainder: audioData.length % 4, - }, - ); - } - - // Convert buffer back to Float32Array - const float32Array = new Float32Array( - audioData.buffer, - audioData.byteOffset, - audioData.length / 4, - ); - - logger.ai.info("Successfully converted audio buffer", { - sampleCount: float32Array.length, - sampleRate: "16kHz (assumed)", - format: "Float32Array", - }); - - return float32Array; - } catch (error) { - logger.ai.error("Audio conversion failed", { - error: error instanceof Error ? error.message : String(error), - }); - - // Fallback: try to interpret as different formats - try { - // Try as 16-bit PCM - const samples = new Float32Array(audioData.length / 2); - for (let i = 0; i < samples.length; i++) { - const sample = audioData.readInt16LE(i * 2); - samples[i] = sample / 32768.0; - } - - logger.ai.info("Fallback: converted as 16-bit PCM", { - sampleCount: samples.length, - }); - return samples; - } catch (fallbackError) { - logger.ai.error("All audio conversion methods failed", { - originalError: error instanceof Error ? error.message : String(error), - fallbackError: - fallbackError instanceof Error - ? fallbackError.message - : String(fallbackError), - }); - - // Return empty array as last resort - return new Float32Array(0); - } - } - } - - private async getBestAvailableModel(): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - - // If a specific model is selected and available, use it - if (this.selectedModelId && downloadedModels[this.selectedModelId]) { - const model = downloadedModels[this.selectedModelId]; - if (fs.existsSync(model.localPath)) { - return model.localPath; - } - } - - // Otherwise, find the best available model (prioritize by quality) - const preferredOrder = [ - "whisper-large-v1", - "whisper-medium", - "whisper-small", - "whisper-base", - "whisper-tiny", - ]; - - for (const modelId of preferredOrder) { - const model = downloadedModels[modelId]; - if (model && fs.existsSync(model.localPath)) { - return model.localPath; - } - } - - return null; - } - - // Set the model to use for transcription - async setSelectedModel(modelId: string): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - if (!downloadedModels[modelId]) { - throw new Error(`Model not downloaded: ${modelId}`); - } - - // If we're changing models, free the current instance - if (this.selectedModelId !== modelId && this.whisperInstance) { - this.freeWhisperInstance(); - } - - this.selectedModelId = modelId; - logger.ai.info("Selected model for contextual transcription", { modelId }); - } - - // Get the currently selected model - getSelectedModel(): string | null { - return this.selectedModelId; - } - - // Check if whisper is available - async isAvailable(): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - return Object.keys(downloadedModels).some((modelId) => - fs.existsSync(downloadedModels[modelId].localPath), - ); - } - - // Get available models - async getAvailableModels(): Promise { - const downloadedModels = await this.modelManager.getDownloadedModels(); - return Object.keys(downloadedModels).filter((modelId) => - fs.existsSync(downloadedModels[modelId].localPath), - ); - } - - // Get current model information for logging - async getCurrentModelInfo(): Promise<{ - modelId: string | null; - modelPath: string | null; - }> { - const downloadedModels = await this.modelManager.getDownloadedModels(); - - // If a specific model is selected and available, use it - if (this.selectedModelId && downloadedModels[this.selectedModelId]) { - const model = downloadedModels[this.selectedModelId]; - if (fs.existsSync(model.localPath)) { - return { - modelId: this.selectedModelId, - modelPath: model.localPath, - }; - } - } - - // Otherwise, find the best available model (same logic as getBestAvailableModel) - const preferredOrder = [ - "whisper-large-v1", - "whisper-medium", - "whisper-small", - "whisper-base", - "whisper-tiny", - ]; - - for (const modelId of preferredOrder) { - const model = downloadedModels[modelId]; - if (model && fs.existsSync(model.localPath)) { - return { - modelId: modelId, - modelPath: model.localPath, - }; - } - } - - return { modelId: null, modelPath: null }; - } - - // Public method to preload the model - async loadModel(): Promise { - await this.initializeWhisper(); - this.updateLastUsedTimestamp(); - logger.ai.info("Model preloaded successfully", { - modelLoaded: this.isModelLoaded(), - cleanupDelayMs: this.MODEL_CLEANUP_DELAY_MS, - }); - } - - // Public method to free the model - async freeModel(): Promise { - this.clearCleanupTimer(); - await this.freeWhisperInstance(); - logger.ai.info("Model freed manually"); - } - - // Check if model is currently loaded - isModelLoaded(): boolean { - return this.whisperInstance !== null; - } - - // Free resources - async dispose(): Promise { - this.clearCleanupTimer(); - await this.freeWhisperInstance(); - } - - private async freeWhisperInstance(): Promise { - if (this.whisperInstance) { - try { - await this.whisperInstance.free(); - logger.ai.info("Smart-whisper contextual instance freed"); - } catch (error) { - logger.ai.warn("Error freeing smart-whisper contextual instance", { - error: error instanceof Error ? error.message : String(error), - }); - } finally { - this.whisperInstance = null; - } - } - } - - private updateLastUsedTimestamp(): void { - this.lastUsedTimestamp = Date.now(); - this.scheduleCleanup(); - } - - private scheduleCleanup(): void { - this.clearCleanupTimer(); - - this.cleanupTimer = setTimeout(async () => { - const timeSinceLastUse = Date.now() - this.lastUsedTimestamp; - - if (timeSinceLastUse >= this.MODEL_CLEANUP_DELAY_MS) { - logger.ai.info("Auto-freeing model after inactivity", { - inactiveTimeMs: timeSinceLastUse, - thresholdMs: this.MODEL_CLEANUP_DELAY_MS, - }); - await this.freeWhisperInstance(); - } else { - // Reschedule if model was used recently - const remainingTime = this.MODEL_CLEANUP_DELAY_MS - timeSinceLastUse; - this.cleanupTimer = setTimeout( - () => this.scheduleCleanup(), - remainingTime, - ); - } - }, this.MODEL_CLEANUP_DELAY_MS); - } - - private clearCleanupTimer(): void { - if (this.cleanupTimer) { - clearTimeout(this.cleanupTimer); - this.cleanupTimer = null; - } - } -} diff --git a/apps/desktop/src/modules/transcription/contextual-transcription-manager.ts b/apps/desktop/src/modules/transcription/contextual-transcription-manager.ts deleted file mode 100644 index a7477b0..0000000 --- a/apps/desktop/src/modules/transcription/contextual-transcription-manager.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { ContextualTranscriptionClient } from "./transcription-session"; -import { ContextualLocalWhisperClient } from "./contextual-local-whisper-client"; -import { ModelManagerService } from "../models/model-manager"; -import { createScopedLogger } from "../../main/logger"; - -export class ContextualTranscriptionManager { - private logger = createScopedLogger("contextual-transcription-manager"); - private defaultClient: ContextualLocalWhisperClient | null = null; - - constructor(private modelManagerService: ModelManagerService | null = null) {} - - createTranscriptionClient( - provider: "local", - options: { modelId?: string } = {}, - ): ContextualTranscriptionClient { - switch (provider) { - case "local": - if (!this.modelManagerService) { - throw new Error( - "ModelManagerService is required for local transcription client", - ); - } - this.logger.info( - "Creating local Whisper contextual transcription client", - { - selectedModelId: options.modelId, - }, - ); - return new ContextualLocalWhisperClient( - this.modelManagerService, - options.modelId, - ); - - default: - throw new Error(`Unknown transcription provider: ${provider}`); - } - } - - // Get the default provider based on configuration - getDefaultProvider(): "local" { - return "local"; - } - - // Create default client with current configuration - createDefaultClient(): ContextualTranscriptionClient { - if (!this.defaultClient) { - this.defaultClient = this.createTranscriptionClient( - "local", - ) as ContextualLocalWhisperClient; - } - return this.defaultClient; - } - - // Preload the model for faster transcription - async preloadModel(): Promise { - const client = this.createDefaultClient() as ContextualLocalWhisperClient; - await client.loadModel(); - this.logger.info("Model preloaded for contextual transcription"); - } - - // Free the model to save memory - async freeModel(): Promise { - if (this.defaultClient) { - await this.defaultClient.freeModel(); - this.logger.info("Model freed for contextual transcription"); - } - } - - // Check if model is loaded - isModelLoaded(): boolean { - return this.defaultClient ? this.defaultClient.isModelLoaded() : false; - } - - // Cleanup resources - async dispose(): Promise { - if (this.defaultClient) { - await this.defaultClient.dispose(); - this.defaultClient = null; - } - } -} diff --git a/apps/desktop/src/modules/transcription/transcription-session.ts b/apps/desktop/src/modules/transcription/transcription-session.ts deleted file mode 100644 index a8e738f..0000000 --- a/apps/desktop/src/modules/transcription/transcription-session.ts +++ /dev/null @@ -1,298 +0,0 @@ -import { EventEmitter } from "node:events"; -import { createScopedLogger } from "../../main/logger"; - -export interface ChunkData { - sessionId: string; - chunkId: number; - audioData: Buffer; - isFinalChunk: boolean; -} - -export interface ChunkResult { - chunkId: number; - text: string; - processingTimeMs: number; - startTime: number; - endTime: number; - modelInfo?: { - modelId: string | null; - modelPath: string | null; - }; -} - -export interface ContextualTranscriptionClient { - transcribeWithContext( - audioData: Buffer, - previousContext: string, - ): Promise; - getCurrentModelInfo?: () => Promise<{ - modelId: string | null; - modelPath: string | null; - }>; -} - -export class TranscriptionSession extends EventEmitter { - private logger = createScopedLogger("transcription-session"); - private sessionId: string; - private transcriptionClient: ContextualTranscriptionClient; - - private chunkQueue: ChunkData[] = []; - private results: ChunkResult[] = []; - private accumulatedText: string = ""; - private isProcessing: boolean = false; - private expectedChunkId: number = 1; - private isComplete: boolean = false; - private sessionStartTime: number; - - constructor( - sessionId: string, - transcriptionClient: ContextualTranscriptionClient, - ) { - super(); - this.sessionId = sessionId; - this.transcriptionClient = transcriptionClient; - this.sessionStartTime = Date.now(); - - this.logger.info("TranscriptionSession created", { - sessionId, - sessionStartTime: this.sessionStartTime, - sessionStartTimeISO: new Date(this.sessionStartTime).toISOString(), - }); - } - - public addChunk(chunkData: ChunkData): void { - if (chunkData.sessionId !== this.sessionId) { - this.logger.warn("Received chunk for different session", { - expected: this.sessionId, - received: chunkData.sessionId, - }); - return; - } - - if (this.isComplete) { - this.logger.warn("Session already complete, ignoring chunk", { - sessionId: this.sessionId, - chunkId: chunkData.chunkId, - }); - return; - } - - this.logger.info("Adding chunk to queue", { - sessionId: this.sessionId, - chunkId: chunkData.chunkId, - isFinalChunk: chunkData.isFinalChunk, - audioDataSize: chunkData.audioData.length, - }); - - this.chunkQueue.push(chunkData); - this.processNextChunk(); - } - - private async processNextChunk(): Promise { - if (this.isProcessing || this.chunkQueue.length === 0) { - return; - } - - // Find the next expected chunk in sequence - const nextChunkIndex = this.chunkQueue.findIndex( - (chunk) => chunk.chunkId === this.expectedChunkId, - ); - - if (nextChunkIndex === -1) { - this.logger.debug("Next expected chunk not yet available", { - expectedChunkId: this.expectedChunkId, - availableChunks: this.chunkQueue.map((c) => c.chunkId), - }); - return; - } - - const chunk = this.chunkQueue.splice(nextChunkIndex, 1)[0]; - this.isProcessing = true; - - try { - await this.transcribeChunk(chunk); - } catch (error) { - this.logger.error("Error processing chunk", { - sessionId: this.sessionId, - chunkId: chunk.chunkId, - error: error instanceof Error ? error.message : String(error), - }); - this.emit("chunk-error", { chunkId: chunk.chunkId, error }); - } finally { - this.isProcessing = false; - this.expectedChunkId++; - - // Check if this was the final chunk - if (chunk.isFinalChunk) { - this.completeSession(); - } else { - // Process next chunk if available - this.processNextChunk(); - } - } - } - - private async transcribeChunk(chunk: ChunkData): Promise { - const startTime = Date.now(); - const modelInfo = this.transcriptionClient.getCurrentModelInfo - ? await this.transcriptionClient.getCurrentModelInfo() - : { modelId: null, modelPath: null }; - - this.logger.info("Starting transcription for chunk", { - sessionId: this.sessionId, - chunkId: chunk.chunkId, - audioDataSize: chunk.audioData.length, - contextLength: this.accumulatedText.length, - startTime, - startTimeISO: new Date(startTime).toISOString(), - modelId: modelInfo.modelId, - modelPath: modelInfo.modelPath, - }); - - // Skip transcription for empty chunks (but still process them for completion) - if (chunk.audioData.length === 0) { - const endTime = Date.now(); - const processingTimeMs = endTime - startTime; - - this.logger.info("Skipping transcription for empty chunk", { - sessionId: this.sessionId, - chunkId: chunk.chunkId, - startTime, - endTime, - processingTimeMs, - startTimeISO: new Date(startTime).toISOString(), - endTimeISO: new Date(endTime).toISOString(), - modelId: modelInfo.modelId, - modelPath: modelInfo.modelPath, - }); - - const result: ChunkResult = { - chunkId: chunk.chunkId, - text: "", - processingTimeMs, - startTime, - endTime, - modelInfo, - }; - - this.results.push(result); - this.emit("chunk-completed", result); - return; - } - - const transcriptionText = - await this.transcriptionClient.transcribeWithContext( - chunk.audioData, - this.accumulatedText, - ); - - console.error("transcriptionText result ", transcriptionText); - - const endTime = Date.now(); - const processingTimeMs = endTime - startTime; - - const result: ChunkResult = { - chunkId: chunk.chunkId, - text: transcriptionText, - processingTimeMs, - startTime, - endTime, - modelInfo, - }; - - // Accumulate the transcription text for context - this.accumulatedText += - (this.accumulatedText ? " " : "") + transcriptionText; - - this.results.push(result); - - this.logger.error("Chunk transcription completed", { - sessionId: this.sessionId, - chunkId: chunk.chunkId, - textLength: transcriptionText.length, - processingTimeMs, - startTime, - endTime, - startTimeISO: new Date(startTime).toISOString(), - endTimeISO: new Date(endTime).toISOString(), - accumulatedTextLength: this.accumulatedText.length, - modelId: modelInfo.modelId, - modelPath: modelInfo.modelPath, - }); - - this.emit("chunk-completed", result); - } - - private completeSession(): void { - this.isComplete = true; - - const sessionEndTime = Date.now(); - const totalSessionTimeMs = sessionEndTime - this.sessionStartTime; - const totalProcessingTime = this.results.reduce( - (sum, result) => sum + result.processingTimeMs, - 0, - ); - - // Get model info from the last successful chunk result - const lastChunkWithModel = this.results.find((r) => r.modelInfo); - const sessionModelInfo = lastChunkWithModel?.modelInfo || { - modelId: null, - modelPath: null, - }; - - this.logger.error("Transcription session completed", { - sessionId: this.sessionId, - totalChunks: this.results.length, - finalTextLength: this.accumulatedText.length, - sessionStartTime: this.sessionStartTime, - sessionEndTime, - sessionStartTimeISO: new Date(this.sessionStartTime).toISOString(), - sessionEndTimeISO: new Date(sessionEndTime).toISOString(), - totalSessionTimeMs, - totalProcessingTimeMs: totalProcessingTime, - averageProcessingTimePerChunkMs: - this.results.length > 0 - ? Math.round(totalProcessingTime / this.results.length) - : 0, - processingEfficiency: - totalSessionTimeMs > 0 - ? Math.round((totalProcessingTime / totalSessionTimeMs) * 100) - : 0, - modelId: sessionModelInfo.modelId, - modelPath: sessionModelInfo.modelPath, - chunkTimings: this.results.map((r) => ({ - chunkId: r.chunkId, - processingTimeMs: r.processingTimeMs, - startTime: r.startTime, - endTime: r.endTime, - textLength: r.text.length, - })), - }); - - this.emit("session-completed", { - sessionId: this.sessionId, - finalText: this.accumulatedText, - chunkResults: this.results, - totalProcessingTimeMs: totalProcessingTime, - totalSessionTimeMs, - sessionStartTime: this.sessionStartTime, - sessionEndTime, - }); - } - - public getSessionId(): string { - return this.sessionId; - } - - public getAccumulatedText(): string { - return this.accumulatedText; - } - - public getResults(): ChunkResult[] { - return [...this.results]; - } - - public isSessionComplete(): boolean { - return this.isComplete; - } -} diff --git a/apps/desktop/src/pipeline/core/context.ts b/apps/desktop/src/pipeline/core/context.ts new file mode 100644 index 0000000..757b4b6 --- /dev/null +++ b/apps/desktop/src/pipeline/core/context.ts @@ -0,0 +1,46 @@ +/** + * Simple context management for the pipeline - no over-engineering + * Based on ARCHITECTURE.md specifications + */ + +export interface PipelineContext { + sessionId: string; + sharedData: SharedPipelineData; + metadata: Map; +} + +import { GetAccessibilityContextResult } from "@amical/types"; + +export interface SharedPipelineData { + vocabulary: Map; + userPreferences: { + language: string; + formattingStyle: "formal" | "casual" | "technical"; + }; + audioMetadata: { + source: "microphone" | "file" | "stream"; + duration?: number; + }; + accessibilityContext: GetAccessibilityContextResult | null; +} + +/** + * Create a default context for pipeline execution + */ +export function createDefaultContext(sessionId: string): PipelineContext { + return { + sessionId, + sharedData: { + vocabulary: new Map(), + userPreferences: { + language: "en", + formattingStyle: "formal", + }, + audioMetadata: { + source: "microphone", + }, + accessibilityContext: null, // Will be populated async by TranscriptionService + }, + metadata: new Map(), + }; +} diff --git a/apps/desktop/src/pipeline/core/pipeline-types.ts b/apps/desktop/src/pipeline/core/pipeline-types.ts new file mode 100644 index 0000000..27cd9a8 --- /dev/null +++ b/apps/desktop/src/pipeline/core/pipeline-types.ts @@ -0,0 +1,75 @@ +/** + * Core pipeline types - Simple interfaces without over-engineering + */ + +// Re-export context types from dedicated file +import { PipelineContext } from "./context"; +import { GetAccessibilityContextResult } from "@amical/types"; +export { PipelineContext, SharedPipelineData } from "./context"; + +// Transcription input parameters +export interface TranscribeParams { + audioData: Buffer; + context: { + vocabulary?: Map; + accessibilityContext?: GetAccessibilityContextResult | null; + previousChunk?: string; + aggregatedTranscription?: string; + }; +} + +// Formatting input parameters +export interface FormatParams { + text: string; + context: { + style?: string; + vocabulary?: Map; + accessibilityContext?: GetAccessibilityContextResult | null; + previousChunk?: string; + aggregatedTranscription?: string; + }; +} + +// Transcription provider interface +export interface TranscriptionProvider { + readonly name: string; + transcribe(params: TranscribeParams): Promise; +} + +// Formatting provider interface +export interface FormattingProvider { + readonly name: string; + format(params: FormatParams): Promise; +} + +// Pipeline execution result +export interface PipelineResult { + transcription: string; + sessionId: string; + metadata: { + duration?: number; + provider: string; + formatted: boolean; + }; +} + +// Streaming context for pipeline processing +export interface StreamingPipelineContext extends PipelineContext { + sessionId: string; + isPartial: boolean; + isFinal: boolean; + accumulatedTranscription?: string[]; // Store all partial results +} + +// Session data for streaming transcription +export interface StreamingSession { + context: StreamingPipelineContext; + transcriptionResults: string[]; // Accumulate all transcription chunks +} + +// Simple pipeline configuration +export interface PipelineConfig { + transcriptionProvider: TranscriptionProvider; + formattingProvider?: FormattingProvider; + saveToDatabase: boolean; +} diff --git a/apps/desktop/src/pipeline/index.ts b/apps/desktop/src/pipeline/index.ts new file mode 100644 index 0000000..1405151 --- /dev/null +++ b/apps/desktop/src/pipeline/index.ts @@ -0,0 +1,24 @@ +/** + * Pipeline module exports + */ + +// Core types +export type { + TranscriptionProvider, + FormattingProvider, + PipelineResult, + PipelineConfig, + StreamingPipelineContext, + StreamingSession, +} from "./core/pipeline-types"; + +// Context management +export { createDefaultContext } from "./core/context"; +export type { PipelineContext, SharedPipelineData } from "./core/context"; + +// Main service +export { TranscriptionService } from "../services/transcription-service"; + +// Providers (if needed externally) +export { WhisperProvider } from "./providers/transcription/whisper-provider"; +export { OpenRouterProvider } from "./providers/formatting/openrouter-formatter"; diff --git a/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts b/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts new file mode 100644 index 0000000..e3198ac --- /dev/null +++ b/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts @@ -0,0 +1,98 @@ +import { FormatParams } from "../../core/pipeline-types"; +import { GetAccessibilityContextResult, ApplicationInfo } from "@amical/types"; + +export function constructFormatterPrompt(context: FormatParams["context"]): { + systemPrompt: string; +} { + const { accessibilityContext } = context; + + // Build enhanced system prompt with context information + let systemPrompt = `You are a professional text formatter. Your task is to clean up and improve the formatting of transcribed text while preserving the original meaning and content. + +Please: +1. Fix obvious transcription errors and typos +2. Add proper punctuation where missing +3. Organize the text into proper paragraphs, with sufficient line breaks, etc. +4. Capitalize proper nouns and sentence beginnings +5. Remove unnecessary filler words (um, uh, etc.) but keep natural speech patterns +6. Maintain the speaker's original tone and style +7. If the text is empty, return an empty string +8. For formatting of emails make sure to use the correct email format`; + + // Build context information + const contextXml = buildContextXml(accessibilityContext); + + if (contextXml) { + systemPrompt += `\n\n${contextXml}`; + systemPrompt += `\n\nUse this context to better understand the environment where the text will be used and adjust formatting accordingly.`; + } + + systemPrompt += `\n\nReturn only the formatted text without any explanations or additional commentary.`; + + return { systemPrompt }; +} + +function buildContextXml( + accessibilityContext: GetAccessibilityContextResult | null | undefined, +): string | null { + if (!accessibilityContext?.context) return null; + + const contextParts: string[] = [""]; + + // Add application info + const appXml = buildApplicationXml(accessibilityContext.context.application); + if (appXml) contextParts.push(appXml); + + // Add URL info + const urlXml = buildUrlXml( + accessibilityContext.context.windowInfo?.url || undefined, + ); + if (urlXml) contextParts.push(urlXml); + + contextParts.push(""); + + // Only return context if we have actual content + return contextParts.length > 2 ? contextParts.join("\n") : null; +} + +function buildApplicationXml(application: ApplicationInfo): string | null { + if (!application?.name) return null; + + const appParts = [" ", ` ${application.name}`]; + + if (application.bundleIdentifier) { + appParts.push(` ${application.bundleIdentifier}`); + } + + appParts.push(" "); + return appParts.join("\n"); +} + +function buildUrlXml(url: string | undefined): string | null { + if (!url) return null; + + const domain = extractDomain(url); + if (!domain) return null; + + return [" ", ` ${domain}`, " "].join("\n"); +} + +function extractDomain(url: string): string | null { + try { + // Try standard URL parsing first + const parsedUrl = new URL(url); + return parsedUrl.hostname; + } catch { + // Handle URLs without protocol or malformed URLs + // Remove any leading slashes + const cleanUrl = url.replace(/^\/+/, ""); + + // Extract domain from patterns like "domain.com/path" or just "domain.com" + const match = cleanUrl.match(/^([^\/\s?#]+)/); + if (match && match[1].includes(".")) { + return match[1]; + } + + return null; + } +} diff --git a/apps/desktop/src/pipeline/providers/formatting/openrouter-formatter.ts b/apps/desktop/src/pipeline/providers/formatting/openrouter-formatter.ts new file mode 100644 index 0000000..e5da141 --- /dev/null +++ b/apps/desktop/src/pipeline/providers/formatting/openrouter-formatter.ts @@ -0,0 +1,62 @@ +import { FormattingProvider, FormatParams } from "../../core/pipeline-types"; +import { logger } from "../../../main/logger"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { constructFormatterPrompt } from "./formatter-prompt"; + +import { generateText } from "ai"; + +export class OpenRouterProvider implements FormattingProvider { + readonly name = "openrouter"; + + private provider: any; + private model: string; + + constructor(apiKey: string, model: string) { + // Configure OpenRouter provider + this.provider = createOpenRouter({ + apiKey: apiKey, + }); + + this.model = model; + } + + async format(params: FormatParams): Promise { + try { + // Extract parameters from the new structure + const { text, context } = params; + + // Construct the formatter prompt using the extracted function + const { systemPrompt } = constructFormatterPrompt(context); + + // Build user prompt with context + const userPrompt = text; + + const { text: formattedText } = await generateText({ + model: this.provider(this.model), + messages: [ + { + role: "system", + content: systemPrompt, + }, + { + role: "user", + content: userPrompt, + }, + ], + temperature: 0.1, // Low temperature for consistent formatting + maxTokens: 2000, + }); + + logger.pipeline.debug("Formatting completed", { + original: text, + formatted: formattedText, + }); + + return formattedText; + } catch (error) { + logger.pipeline.error("Formatting failed:", error); + // Return original text if formatting fails - simple fallback + return params.text; + } + } +} diff --git a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts new file mode 100644 index 0000000..e025e90 --- /dev/null +++ b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts @@ -0,0 +1,167 @@ +import { + TranscriptionProvider, + TranscribeParams, +} from "../../core/pipeline-types"; +import { logger } from "../../../main/logger"; +import { ModelManagerService } from "../../../services/model-manager"; +import { Whisper } from "smart-whisper"; + +export class WhisperProvider implements TranscriptionProvider { + readonly name = "whisper-local"; + + private modelManager: ModelManagerService; + private whisperInstance: Whisper | null = null; + + constructor(modelManager: ModelManagerService) { + this.modelManager = modelManager; + } + + async transcribe(params: TranscribeParams): Promise { + try { + await this.initializeWhisper(); + + // Extract parameters from the new structure + const { audioData, context } = params; + const { vocabulary, previousChunk, aggregatedTranscription } = context; + + // Convert audio buffer to the format expected by smart-whisper + const audioFloat32Array = await this.convertAudioBuffer(audioData); + + logger.transcription.debug( + `Starting transcription, audio size: ${audioData.length}`, + previousChunk + ? `Previous chunk: ${previousChunk.substring(0, 50)}...` + : "No previous chunk", + aggregatedTranscription + ? `Aggregated length: ${aggregatedTranscription.length}` + : "No aggregated transcription", + ); + + // Transcribe using smart-whisper + if (!this.whisperInstance) { + throw new Error("Whisper instance is not initialized"); + } + + // Generate initial prompt from vocabulary and recent context + const initialPrompt = this.generateInitialPrompt( + vocabulary, + aggregatedTranscription, + ); + + const { result } = await this.whisperInstance.transcribe( + audioFloat32Array, + { + language: "auto", + initial_prompt: initialPrompt, + }, + ); + + const transcription = await result; + + // Combine all transcription segments into a single string + const text = transcription + .map((segment) => segment.text) + .join(" ") + .trim(); + + logger.transcription.debug( + `Transcription completed, length: ${text.length}`, + ); + + return text; + } catch (error) { + logger.transcription.error("Transcription failed:", error); + throw new Error(`Transcription failed: ${error}`); + } + } + + private generateInitialPrompt( + vocabulary?: Map, + aggregatedTranscription?: string, + ): string { + const promptParts: string[] = []; + + // Add vocabulary terms if available + if (vocabulary && vocabulary.size > 0) { + // Extract vocabulary keys (the actual terms) and join with commas + const vocabularyTerms = Array.from(vocabulary.keys()); + const vocabularyText = vocabularyTerms.join(", "); + promptParts.push(vocabularyText); + } + + // Add last 8 words from aggregated transcription if available + if (aggregatedTranscription && aggregatedTranscription.trim().length > 0) { + const words = aggregatedTranscription.trim().split(/\s+/); + const lastWords = words.slice(-8).join(" "); + if (lastWords.length > 0) { + promptParts.push(lastWords); + } + } + + // Combine parts with a separator, or return empty string if no context + const prompt = promptParts.join(". "); + + logger.transcription.debug(`Generated initial prompt: "${prompt}"`); + + return prompt; + } + + private async initializeWhisper(): Promise { + if (this.whisperInstance) { + return; // Already initialized + } + + const modelPath = await this.modelManager.getBestAvailableModelPath(); + if (!modelPath) { + throw new Error( + "No Whisper models available. Please download a model first.", + ); + } + + try { + const { Whisper } = await import("smart-whisper"); + this.whisperInstance = new Whisper(modelPath, { gpu: true }); + logger.transcription.info(`Initialized with model: ${modelPath}`); + } catch (error) { + logger.transcription.error(`Failed to initialize:`, error); + throw new Error(`Failed to initialize smart-whisper: ${error}`); + } + } + + private async convertAudioBuffer(audioData: Buffer): Promise { + try { + // Convert buffer to Float32Array (simplified) + const float32Array = new Float32Array(audioData.length / 4); + for (let i = 0; i < float32Array.length; i++) { + float32Array[i] = audioData.readFloatLE(i * 4); + } + return float32Array; + } catch (error) { + logger.transcription.warn( + "Audio conversion failed, trying alternative method", + ); + + // Fallback: convert as if it's PCM data + const samples = new Float32Array(audioData.length / 2); + for (let i = 0; i < samples.length; i++) { + const sample = audioData.readInt16LE(i * 2); + samples[i] = sample / 32768.0; + } + return samples; + } + } + + // Simple cleanup method + async dispose(): Promise { + if (this.whisperInstance) { + try { + await this.whisperInstance.free(); + logger.transcription.debug("Instance freed"); + } catch (error) { + logger.transcription.warn("Error freeing instance:", error); + } finally { + this.whisperInstance = null; + } + } + } +} diff --git a/apps/desktop/src/renderer/main/index.tsx b/apps/desktop/src/renderer/main/index.tsx index 8b1b4df..0d7f596 100644 --- a/apps/desktop/src/renderer/main/index.tsx +++ b/apps/desktop/src/renderer/main/index.tsx @@ -1,30 +1,4 @@ -/** - * This file will automatically be loaded by vite and run in the "renderer" context. - * To learn more about the differences between the "main" and the "renderer" context in - * Electron, visit: - * - * https://electronjs.org/docs/tutorial/process-model - * - * By default, Node.js integration in this file is disabled. When enabling Node.js integration - * in a renderer process, please be aware of potential security implications. You can read - * more about security risks here: - * - * https://electronjs.org/docs/tutorial/security - * - * To enable Node.js integration in this file, open up `main.ts` and enable the `nodeIntegration` - * flag: - * - * ``` - * // Create the browser window. - * mainWindow = new BrowserWindow({ - * width: 800, - * height: 600, - * webPreferences: { - * nodeIntegration: true - * } - * }); - * ``` - */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import React, { useState, useEffect } from "react"; import { createRoot } from "react-dom/client"; @@ -46,6 +20,48 @@ import "@/styles/globals.css"; import { SiteHeader } from "@/components/site-header"; import { api } from "@/trpc/react"; +// Extend Console interface to include original methods +declare global { + interface Console { + original: { + log: (...args: any[]) => void; + info: (...args: any[]) => void; + warn: (...args: any[]) => void; + error: (...args: any[]) => void; + debug: (...args: any[]) => void; + }; + } +} + +// Main window scoped logger setup +const mainWindowLogger = window.electronAPI.log.scope("mainWindow"); + +// Proxy console methods to use BOTH original console AND main window logger +const originalConsole = { ...console }; +console.log = (...args: any[]) => { + originalConsole.log(...args); // Show in dev console + mainWindowLogger.info(...args); // Send via IPC +}; +console.info = (...args: any[]) => { + originalConsole.info(...args); + mainWindowLogger.info(...args); +}; +console.warn = (...args: any[]) => { + originalConsole.warn(...args); + mainWindowLogger.warn(...args); +}; +console.error = (...args: any[]) => { + originalConsole.error(...args); + mainWindowLogger.error(...args); +}; +console.debug = (...args: any[]) => { + originalConsole.debug(...args); + mainWindowLogger.debug(...args); +}; + +// Keep original methods available if needed +console.original = originalConsole; + // import { Waveform } from '../components/Waveform'; // Waveform might not be needed if hook is removed // import { useRecording } from '../hooks/useRecording'; // Remove hook import diff --git a/apps/desktop/src/renderer/main/pages/models/components/ModelsManager.tsx b/apps/desktop/src/renderer/main/pages/models/components/ModelsManager.tsx index 66a39d5..0e53f62 100644 --- a/apps/desktop/src/renderer/main/pages/models/components/ModelsManager.tsx +++ b/apps/desktop/src/renderer/main/pages/models/components/ModelsManager.tsx @@ -38,8 +38,8 @@ export const ModelsManager: React.FC = () => { const availableModelsQuery = api.models.getAvailableModels.useQuery(); const downloadedModelsQuery = api.models.getDownloadedModels.useQuery(); const activeDownloadsQuery = api.models.getActiveDownloads.useQuery(); - const isLocalWhisperAvailableQuery = - api.models.isLocalWhisperAvailable.useQuery(); + const isTranscriptionAvailableQuery = + api.models.isTranscriptionAvailable.useQuery(); const selectedModelQuery = api.models.getSelectedModel.useQuery(); const utils = api.useUtils(); @@ -243,13 +243,13 @@ export const ModelsManager: React.FC = () => { const loading = availableModelsQuery.isLoading || downloadedModelsQuery.isLoading || - isLocalWhisperAvailableQuery.isLoading || + isTranscriptionAvailableQuery.isLoading || selectedModelQuery.isLoading; // Data from queries const availableModels = availableModelsQuery.data || []; const downloadedModels = downloadedModelsQuery.data || {}; - const isLocalWhisperAvailable = isLocalWhisperAvailableQuery.data || false; + const isTranscriptionAvailable = isTranscriptionAvailableQuery.data || false; const selectedModel = selectedModelQuery.data; if (loading) { @@ -298,7 +298,7 @@ export const ModelsManager: React.FC = () => {