diff --git a/apps/desktop/src/hooks/useAudioCapture.ts b/apps/desktop/src/hooks/useAudioCapture.ts index 766ee4c..54913c8 100644 --- a/apps/desktop/src/hooks/useAudioCapture.ts +++ b/apps/desktop/src/hooks/useAudioCapture.ts @@ -1,6 +1,7 @@ import { useRef, useEffect, useState, useCallback } from "react"; import audioWorkletUrl from "@/assets/audio-recorder-processor.js?url"; import { api } from "@/trpc/react"; +import { Mutex } from "async-mutex"; // Audio configuration const FRAME_SIZE = 512; // 32ms at 16kHz @@ -28,6 +29,7 @@ export const useAudioCapture = ({ const sourceRef = useRef(null); const workletNodeRef = useRef(null); const streamRef = useRef(null); + const mutexRef = useRef(new Mutex()); // Subscribe to voice detection updates via tRPC api.recording.voiceDetectionUpdates.useSubscription(undefined, { @@ -41,117 +43,126 @@ export const useAudioCapture = ({ }); const startCapture = useCallback(async () => { - try { - console.log("AudioCapture: Starting audio capture"); + await mutexRef.current.runExclusive(async () => { + try { + console.log("AudioCapture: Starting audio capture"); - // Get microphone stream - streamRef.current = await navigator.mediaDevices.getUserMedia({ - audio: { - channelCount: 1, - sampleRate: SAMPLE_RATE, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }); + // Get microphone stream + streamRef.current = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + sampleRate: SAMPLE_RATE, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); - // Create audio context - audioContextRef.current = new AudioContext({ sampleRate: SAMPLE_RATE }); + // Create audio context + audioContextRef.current = new AudioContext({ sampleRate: SAMPLE_RATE }); - // Load audio worklet - await audioContextRef.current.audioWorklet.addModule(audioWorkletUrl); + // Load audio worklet + await audioContextRef.current.audioWorklet.addModule(audioWorkletUrl); - // Create nodes - sourceRef.current = audioContextRef.current.createMediaStreamSource( - streamRef.current, - ); - workletNodeRef.current = new AudioWorkletNode( - audioContextRef.current, - "audio-recorder-processor", - ); + // Create nodes + sourceRef.current = audioContextRef.current.createMediaStreamSource( + streamRef.current, + ); + workletNodeRef.current = new AudioWorkletNode( + audioContextRef.current, + "audio-recorder-processor", + ); - // Handle audio frames from worklet - workletNodeRef.current.port.onmessage = async (event) => { - if (event.data.type === "audioFrame") { - const frame = event.data.frame; - console.log("AudioCapture: Received frame", { - frameLength: frame.length, - isFinal: event.data.isFinal, - }); - const isFinal = event.data.isFinal || false; + // Handle audio frames from worklet + workletNodeRef.current.port.onmessage = async (event) => { + if (event.data.type === "audioFrame") { + const frame = event.data.frame; + console.debug("AudioCapture: Received frame", { + frameLength: frame.length, + isFinal: event.data.isFinal, + }); + const isFinal = event.data.isFinal || false; - // Convert to ArrayBuffer for IPC - const arrayBuffer = frame.buffer.slice( - frame.byteOffset, - frame.byteOffset + frame.byteLength, - ); + // Convert to ArrayBuffer for IPC + const arrayBuffer = frame.buffer.slice( + frame.byteOffset, + frame.byteOffset + frame.byteLength, + ); - // Send to main process for VAD processing - // Main process will update voice detection state - await onAudioChunk(arrayBuffer, 0, isFinal); // Speech probability will come from main + // Send to main process for VAD processing + // Main process will update voice detection state + await onAudioChunk(arrayBuffer, 0, isFinal); // Speech probability will come from main + } + }; - console.log( - `AudioCapture: Sent frame: ${frame.length} samples, isFinal: ${isFinal}`, - ); - } - }; + // Connect audio graph + sourceRef.current.connect(workletNodeRef.current); - // Connect audio graph - sourceRef.current.connect(workletNodeRef.current); - - console.log("AudioCapture: Audio capture started"); - } catch (error) { - console.error("AudioCapture: Failed to start capture:", error); - throw error; - } + console.log("AudioCapture: Audio capture started"); + } catch (error) { + console.error("AudioCapture: Failed to start capture:", error); + throw error; + } + }); }, [onAudioChunk]); - const stopCapture = useCallback(() => { - console.log("AudioCapture: Stopping audio capture"); + const stopCapture = useCallback(async () => { + await mutexRef.current.runExclusive(async () => { + try { + console.log("AudioCapture: Stopping audio capture"); - // Send flush command to worklet before disconnecting - if (workletNodeRef.current) { - workletNodeRef.current.port.postMessage({ type: "flush" }); - console.log("AudioCapture: Sent flush command to worklet"); - } + // Send flush command to worklet before disconnecting + if (workletNodeRef.current) { + workletNodeRef.current.port.postMessage({ type: "flush" }); + console.log("AudioCapture: Sent flush command to worklet"); + } - // Disconnect nodes - if (sourceRef.current && workletNodeRef.current) { - sourceRef.current.disconnect(workletNodeRef.current); - } + // Disconnect nodes + if (sourceRef.current && workletNodeRef.current) { + sourceRef.current.disconnect(workletNodeRef.current); + } - // Close audio context - if (audioContextRef.current && audioContextRef.current.state !== "closed") { - audioContextRef.current.close(); - } + // Close audio context + if ( + audioContextRef.current && + audioContextRef.current.state !== "closed" + ) { + await audioContextRef.current.close(); + } - // Stop media stream - if (streamRef.current) { - streamRef.current.getTracks().forEach((track) => track.stop()); - } + // Stop media stream + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } - // Clear refs - audioContextRef.current = null; - sourceRef.current = null; - workletNodeRef.current = null; - streamRef.current = null; + // Clear refs + audioContextRef.current = null; + sourceRef.current = null; + workletNodeRef.current = null; + streamRef.current = null; - setVoiceDetected(false); - console.log("AudioCapture: Audio capture stopped"); + console.log("AudioCapture: Audio capture stopped"); + } catch (error) { + console.error("AudioCapture: Error during stop:", error); + throw error; + } + }); }, []); // Start/stop based on enabled state useEffect(() => { - if (enabled) { - startCapture().catch((error) => { - console.error("AudioCapture: Failed to start:", error); - }); - } else { - stopCapture(); + if (!enabled) { + return; } + startCapture().catch((error) => { + console.error("AudioCapture: Failed to start:", error); + }); + return () => { - stopCapture(); + stopCapture().catch((error) => { + console.error("AudioCapture: Failed to stop:", error); + }); }; }, [enabled, startCapture, stopCapture]); diff --git a/apps/desktop/src/hooks/useRecording.ts b/apps/desktop/src/hooks/useRecording.ts index b45b00d..0d29d7a 100644 --- a/apps/desktop/src/hooks/useRecording.ts +++ b/apps/desktop/src/hooks/useRecording.ts @@ -1,53 +1,70 @@ -import { useCallback } from "react"; -import { useRecordingState } from "./useRecordingState"; +import { useCallback, useState } from "react"; import { useAudioCapture } from "./useAudioCapture"; +import { api } from "@/trpc/react"; import type { RecordingState } from "@/types/recording"; +import type { RecordingMode } from "@/main/managers/recording-manager"; -export interface UseRecordingParams { - onAudioFrame: ( - audioBuffer: ArrayBuffer, - speechProbability: number, - isFinal: boolean, - ) => Promise | void; - onRecordingStartCallback?: () => Promise | void; - onRecordingStopCallback?: () => Promise | void; +export interface RecordingStatus { + state: RecordingState; + mode: RecordingMode; } export interface UseRecordingOutput { - recordingState: RecordingState; + recordingStatus: RecordingStatus; voiceDetected: boolean; startRecording: () => Promise; stopRecording: () => Promise; } -export const useRecording = ({ - onAudioFrame, - onRecordingStartCallback, - onRecordingStopCallback, -}: UseRecordingParams): UseRecordingOutput => { - // Manage recording state via tRPC - const { - recordingState, - startRecording: startRecordingMutation, - stopRecording: stopRecordingMutation, - } = useRecordingState(); +export const useRecording = (): UseRecordingOutput => { + const [recordingStatus, setRecordingStatus] = useState({ + state: "idle", + mode: "idle", + }); - // Create handler for audio chunks - just pass through + const startRecordingMutation = api.recording.signalStart.useMutation(); + const stopRecordingMutation = api.recording.signalStop.useMutation(); + + // Subscribe to recording state updates via tRPC + api.recording.stateUpdates.useSubscription(undefined, { + onData: (update) => { + setRecordingStatus(update); + }, + onError: (error) => { + console.error("Error subscribing to recording state updates", error); + }, + }); + + // Handle audio frames by sending them to the main process const handleAudioChunk = useCallback( async ( arrayBuffer: ArrayBuffer, speechProbability: number, isFinalChunk: boolean, ) => { - // Direct pass-through - no aggregation needed - await onAudioFrame(arrayBuffer, speechProbability, isFinalChunk); + // Convert ArrayBuffer to Float32Array + const float32Array = new Float32Array(arrayBuffer); + + // Send frame directly to main process + // TODO: We need to update the IPC to include speech detection info + await window.electronAPI.sendAudioChunk(float32Array, isFinalChunk); + console.debug(`Sent audio frame`, { + samples: float32Array.length, + speechProbability: speechProbability.toFixed(3), + isFinal: isFinalChunk, + }); + + if (isFinalChunk) { + console.log("Final frame sent to main process"); + } }, - [onAudioFrame], + [], ); // Manage audio capture when recording is active const isActive = - recordingState === "recording" || recordingState === "starting"; + recordingStatus.state === "recording" || + recordingStatus.state === "starting"; const { voiceDetected } = useAudioCapture({ onAudioChunk: handleAudioChunk, @@ -55,83 +72,18 @@ export const useRecording = ({ }); const startRecording = useCallback(async () => { - // Check if already recording - if (recordingState !== "idle" && recordingState !== "error") { - console.log(`Hook: Start denied. Current status: ${recordingState}`); - return; - } - - try { - // Request main process to start recording - await startRecordingMutation(); - - // Call start callback if provided - if (onRecordingStartCallback) { - await onRecordingStartCallback(); - console.log("Hook: onRecordingStartCallback executed."); - } - - console.log("Hook: Recording fully started"); - } catch (error) { - console.error("Hook: Error starting recording:", error); - - // Try to stop recording in main process - try { - await stopRecordingMutation(); - } catch (stopError) { - console.error("Hook: Failed to stop recording after error", stopError); - } - - // Call stop callback if start callback was called - if (onRecordingStopCallback) { - try { - await onRecordingStopCallback(); - } catch (e) { - console.error( - "Hook: Error in onRecordingStopCallback during start error:", - e, - ); - } - } - } - }, [ - recordingState, - startRecordingMutation, - onRecordingStartCallback, - onRecordingStopCallback, - stopRecordingMutation, - ]); + // Request main process to start recording + await startRecordingMutation.mutateAsync(); + console.log("Hook: Recording fully started"); + }, [startRecordingMutation]); const stopRecording = useCallback(async () => { - // Check if recording - if (recordingState !== "recording" && recordingState !== "starting") { - console.log(`Hook: Stop called but status is ${recordingState}.`); - return; - } - - try { - // Request main process to stop recording - await stopRecordingMutation(); - - // Call stop callback if provided - if (onRecordingStopCallback) { - await onRecordingStopCallback(); - console.log("Hook: onRecordingStopCallback executed."); - } - - console.log("Hook: Recording stopped"); - } catch (error) { - console.error("Hook: Error stopping recording:", error); - } - }, [ - recordingState, - stopRecordingMutation, - onRecordingStopCallback, - onAudioFrame, - ]); + await stopRecordingMutation.mutateAsync(); + console.log("Hook: Recording stopped"); + }, [stopRecordingMutation]); return { - recordingState, + recordingStatus, voiceDetected, startRecording, stopRecording, diff --git a/apps/desktop/src/hooks/useRecordingState.ts b/apps/desktop/src/hooks/useRecordingState.ts deleted file mode 100644 index ff386a3..0000000 --- a/apps/desktop/src/hooks/useRecordingState.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { useState } from "react"; -import { api } from "@/trpc/react"; -import type { RecordingState } from "@/types/recording"; - -export interface UseRecordingStateOutput { - recordingState: RecordingState; - startRecording: () => Promise; - stopRecording: () => Promise; -} - -export const useRecordingState = (): UseRecordingStateOutput => { - const [recordingState, setRecordingState] = useState("idle"); - - console.log("recordingState", recordingState); - - const startRecordingMutation = api.recording.start.useMutation(); - const stopRecordingMutation = api.recording.stop.useMutation(); - - // Subscribe to recording state updates via tRPC - api.recording.stateUpdates.useSubscription(undefined, { - onData: (state: RecordingState) => { - console.log("recordingStatus", state); - setRecordingState(state); - }, - onError: (error) => { - console.error("Error subscribing to recording state updates", error); - }, - }); - - const startRecording = async (): Promise => { - try { - await startRecordingMutation.mutateAsync(); - } catch (error) { - console.error("Failed to start recording via tRPC", error); - throw error; - } - }; - - const stopRecording = async (): Promise => { - try { - await stopRecordingMutation.mutateAsync(); - } catch (error) { - console.error("Failed to stop recording via tRPC", error); - throw error; - } - }; - - return { - recordingState, - startRecording, - stopRecording, - }; -}; diff --git a/apps/desktop/src/main/core/app-manager.ts b/apps/desktop/src/main/core/app-manager.ts index 881edb4..3e583e5 100644 --- a/apps/desktop/src/main/core/app-manager.ts +++ b/apps/desktop/src/main/core/app-manager.ts @@ -14,6 +14,7 @@ export class AppManager { constructor() { this.windowManager = new WindowManager(); this.serviceManager = ServiceManager.createInstance(); + this.serviceManager.setWindowManager(this.windowManager); } async initialize(): Promise { @@ -87,9 +88,38 @@ export class AppManager { "Onboarding completed, restarting app for permissions to take effect", ); - // Relaunch the app to ensure all permissions take effect - app.relaunch(); - app.quit(); + // In development, reload windows instead of relaunching + if (process.env.NODE_ENV === "development") { + logger.main.info( + "Development mode: Reloading windows instead of relaunching", + ); + + // Close onboarding window + const onboardingWindow = this.windowManager.getOnboardingWindow(); + if (onboardingWindow && !onboardingWindow.isDestroyed()) { + onboardingWindow.close(); + } + + // Give a short delay for permissions to register + setTimeout(async () => { + await this.setupWindows(); + + // Force reload all windows to pick up new permissions + const mainWindow = this.windowManager.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.reload(); + } + + const widgetWindow = this.windowManager.getWidgetWindow(); + if (widgetWindow && !widgetWindow.isDestroyed()) { + widgetWindow.reload(); + } + }, 1000); + } else { + // Production mode: relaunch the app + app.relaunch(); + app.quit(); + } } private async setupWindows(): Promise { diff --git a/apps/desktop/src/main/managers/recording-manager.ts b/apps/desktop/src/main/managers/recording-manager.ts index a8436fd..92727f4 100644 --- a/apps/desktop/src/main/managers/recording-manager.ts +++ b/apps/desktop/src/main/managers/recording-manager.ts @@ -6,7 +6,7 @@ import type { RecordingState } from "../../types/recording"; import { Mutex } from "async-mutex"; import type { ShortcutManager } from "../services/shortcut-manager"; -export type RecordingMode = "idle" | "ptt" | "handsfree"; +export type RecordingMode = "idle" | "ptt" | "hands-free"; /** * Manages recording state and coordinates audio recording across the application @@ -61,6 +61,18 @@ export class RecordingManager extends EventEmitter { this.broadcastStateChange(); } + private setMode(newMode: RecordingMode): void { + const oldMode = this.recordingMode; + this.recordingMode = newMode; + logger.audio.info("Recording mode changed", { + oldMode, + newMode, + }); + + // Broadcast mode change to all windows + this.broadcastModeChange(); + } + public getState(): RecordingState { return this.recordingState; } @@ -70,6 +82,11 @@ export class RecordingManager extends EventEmitter { this.emit("state-changed", this.getState()); } + private broadcastModeChange(): void { + // Emit event for internal listeners (tRPC subscription will pick this up) + this.emit("mode-changed", this.getRecordingMode()); + } + private setupIPCHandlers(): void { // Handle audio data chunks from renderer ipcMain.handle( @@ -84,7 +101,7 @@ export class RecordingManager extends EventEmitter { // Convert ArrayBuffer back to Float32Array const float32Array = new Float32Array(chunk); - logger.audio.info("Received audio chunk", { + logger.audio.debug("Received audio chunk", { samples: float32Array.length, isFinalChunk, }); @@ -107,9 +124,15 @@ export class RecordingManager extends EventEmitter { ); } - public async startRecording() { - console.error("startRecording"); + public async startRecording(mode: "ptt" | "hands-free") { await this.recordingMutex.runExclusive(async () => { + // if we were previously in ptt mode, we override + // priority is given to hands-free mode + // we don't need to check the other way around + if (mode === "hands-free") { + this.setMode("hands-free"); + } + // Check if already recording if (this.recordingState !== "idle") { logger.audio.warn("Cannot start recording - already in progress", { @@ -119,6 +142,7 @@ export class RecordingManager extends EventEmitter { } this.setState("starting"); + this.setMode(mode); // Create session ID const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); @@ -149,6 +173,7 @@ export class RecordingManager extends EventEmitter { } public async stopRecording() { + console.error("stopRecording called", this.recordingState); await this.recordingMutex.runExclusive(async () => { // Check if recording if (this.recordingState !== "recording") { @@ -160,6 +185,9 @@ export class RecordingManager extends EventEmitter { this.setState("stopping"); + // Reset recording mode when stopping + this.recordingMode = "idle"; + // Restore system audio try { const swiftBridge = this.serviceManager.getService("swiftIOBridge"); @@ -180,55 +208,30 @@ export class RecordingManager extends EventEmitter { }); } - public async toggleRecording() { - if (this.recordingState === "idle") { - await this.startRecording(); - } else if (this.recordingState === "recording") { - await this.stopRecording(); - } else { - logger.audio.warn("Cannot toggle recording in current state", { - currentState: this.recordingState, - }); - } - } - // PTT-specific methods public async startPTT() { - // Don't start PTT if already in hands-free mode - if (this.recordingMode === "handsfree") { - logger.audio.info("Ignoring PTT - already in hands-free mode"); - return; - } - - this.recordingMode = "ptt"; - await this.startRecording(); + this.startRecording("ptt"); } public async stopPTT() { - // Only stop if we're actually in PTT mode - if (this.recordingMode === "ptt") { - this.recordingMode = "idle"; - await this.stopRecording(); + if (this.recordingMode !== "ptt") { + return; } + this.stopRecording(); } // Hands-free mode toggle public async toggleHandsFree() { - if (this.recordingMode === "handsfree") { - this.recordingMode = "idle"; - await this.stopRecording(); - logger.audio.info("Hands-free mode disabled"); - } else { - // If in PTT mode, just switch to hands-free without restarting - if (this.recordingMode === "ptt") { - this.recordingMode = "handsfree"; - logger.audio.info("Switched from PTT to hands-free mode"); - } else { - this.recordingMode = "handsfree"; - await this.startRecording(); - logger.audio.info("Hands-free mode enabled"); - } + if (this.recordingState === "idle") { + this.startRecording("hands-free"); + return; } + if (this.recordingMode === "hands-free") { + this.stopRecording(); + return; + } + this.startRecording("hands-free"); + return; } // Get current mode @@ -245,7 +248,7 @@ export class RecordingManager extends EventEmitter { this.recordingState !== "recording" && this.recordingState !== "stopping" ) { - logger.audio.warn("Received audio chunk while not recording", { + logger.audio.error("Received audio chunk while not recording", { state: this.recordingState, isFinalChunk, }); @@ -281,7 +284,7 @@ export class RecordingManager extends EventEmitter { isFinal: isFinalChunk, }); - logger.audio.error("Processed audio chunk", { + logger.audio.debug("Processed audio chunk", { chunkSize: chunk.length, processingTimeMs: Date.now() - startTime, resultLength: transcriptionResult.length, diff --git a/apps/desktop/src/main/managers/service-manager.ts b/apps/desktop/src/main/managers/service-manager.ts index ac100a2..7d19490 100644 --- a/apps/desktop/src/main/managers/service-manager.ts +++ b/apps/desktop/src/main/managers/service-manager.ts @@ -7,10 +7,10 @@ import { AutoUpdaterService } from "../services/auto-updater"; import { RecordingManager } from "./recording-manager"; import { VADService } from "../../services/vad-service"; import { ShortcutManager } from "../services/shortcut-manager"; +import { WindowManager } from "../core/window-manager"; import { createIPCHandler } from "electron-trpc-experimental/main"; import { router } from "../../trpc/router"; import { createContext } from "../../trpc/context"; -import { BrowserWindow } from "electron"; /** * Service map for type-safe service access @@ -24,6 +24,7 @@ export interface ServiceMap { autoUpdaterService: AutoUpdaterService; recordingManager: RecordingManager; shortcutManager: ShortcutManager; + windowManager: WindowManager; } /** @@ -42,6 +43,7 @@ export class ServiceManager { private autoUpdaterService: AutoUpdaterService | null = null; private recordingManager: RecordingManager | null = null; private shortcutManager: ShortcutManager | null = null; + private windowManager: WindowManager | null = null; private trpcHandler: ReturnType | null = null; async initialize(): Promise { @@ -215,6 +217,7 @@ export class ServiceManager { autoUpdaterService: this.autoUpdaterService ?? undefined, recordingManager: this.recordingManager ?? undefined, shortcutManager: this.shortcutManager ?? undefined, + windowManager: this.windowManager ?? undefined, }; return services[serviceName] ?? null; @@ -255,4 +258,9 @@ export class ServiceManager { } return ServiceManager.instance; } + + setWindowManager(windowManager: WindowManager): void { + this.windowManager = windowManager; + logger.main.info("Window manager registered with ServiceManager"); + } } diff --git a/apps/desktop/src/main/services/shortcut-manager.ts b/apps/desktop/src/main/services/shortcut-manager.ts index a3920e6..4f3de0a 100644 --- a/apps/desktop/src/main/services/shortcut-manager.ts +++ b/apps/desktop/src/main/services/shortcut-manager.ts @@ -168,11 +168,8 @@ export class ShortcutManager extends EventEmitter { const pttKeys = this.shortcuts.pushToTalk.split("+"); const activeKeysList = this.getActiveKeys(); - // Check if PTT keys match active keys exactly - return ( - pttKeys.length === activeKeysList.length && - pttKeys.every((key) => activeKeysList.includes(key)) - ); + //! This should only be a subset match + return pttKeys.every((key) => activeKeysList.includes(key)); } private isToggleRecordingShortcutPressed(): boolean { diff --git a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts index f480ab5..70370ba 100644 --- a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts @@ -169,7 +169,7 @@ export class WhisperProvider implements TranscriptionProvider { return true; } - logger.transcription.error("Not transcribing", { + logger.transcription.debug("Not transcribing", { bufferDurationMs, silenceDurationMs, frameBufferLength: this.frameBuffer.length, diff --git a/apps/desktop/src/renderer/widget/pages/widget/components/FloatingButton.tsx b/apps/desktop/src/renderer/widget/pages/widget/components/FloatingButton.tsx index d17101e..49f1262 100644 --- a/apps/desktop/src/renderer/widget/pages/widget/components/FloatingButton.tsx +++ b/apps/desktop/src/renderer/widget/pages/widget/components/FloatingButton.tsx @@ -1,16 +1,60 @@ -import React, { useState, useCallback, useRef, useEffect } from "react"; +import React, { useState, useRef, useEffect } from "react"; +import { Square } from "lucide-react"; import { Waveform } from "@/components/Waveform"; import { useRecording } from "@/hooks/useRecording"; -import type { RecordingState } from "@/types/recording"; +import { api } from "@/trpc/react"; -const NUM_WAVEFORM_BARS = 8; // Fewer bars for a smaller button -const DEBOUNCE_DELAY = 100; // milliseconds; +const NUM_WAVEFORM_BARS = 6; // Fewer bars to make room for stop button +const DEBOUNCE_DELAY = 100; // milliseconds + +// Separate component for the stop button +const StopButton: React.FC<{ onClick: (e: React.MouseEvent) => void }> = ({ + onClick, +}) => ( + +); + +// Separate component for the processing indicator +const ProcessingIndicator: React.FC = () => ( +
+
+
+
+
+); + +// Separate component for the waveform visualization +const WaveformVisualization: React.FC<{ + isRecording: boolean; + voiceDetected: boolean; +}> = ({ isRecording, voiceDetected }) => ( + <> + {Array.from({ length: NUM_WAVEFORM_BARS }).map((_, index) => ( + + ))} + +); export const FloatingButton: React.FC = () => { const [isHovered, setIsHovered] = useState(false); - const fabRef = useRef(null); const leaveTimeoutRef = useRef(null); // Ref for debounce timeout + // tRPC mutation to control widget mouse events + const setIgnoreMouseEvents = api.widget.setIgnoreMouseEvents.useMutation(); + // Log component initialization useEffect(() => { console.log("FloatingButton component initialized"); @@ -19,147 +63,118 @@ export const FloatingButton: React.FC = () => { }; }, []); - const handleAudioFrame = useCallback( - async ( - audioBuffer: ArrayBuffer, - speechProbability: number, - isFinal: boolean, - ) => { - try { - // Convert ArrayBuffer to Float32Array - const float32Array = new Float32Array(audioBuffer); - - // Send frame directly to main process - // TODO: We need to update the IPC to include speech detection info - await window.electronAPI.sendAudioChunk(float32Array, isFinal); - console.debug(`Sent audio frame`, { - samples: float32Array.length, - speechProbability: speechProbability.toFixed(3), - isFinal, - }); - - if (isFinal) { - console.log("Final frame sent to main process"); - } - } catch (error) { - console.error("Error sending audio frame:", error); - } - }, - [], - ); - - const { recordingState, startRecording, stopRecording, voiceDetected } = - useRecording({ - onAudioFrame: handleAudioFrame, - }); + const { recordingStatus, stopRecording, voiceDetected, startRecording } = + useRecording(); const isRecording = - recordingState === "recording" || recordingState === "starting"; - const isAwaitingFinalChunk = recordingState === "stopping"; + recordingStatus.state === "recording" || + recordingStatus.state === "starting"; + const isStopping = recordingStatus.state === "stopping"; + const isHandsFreeMode = recordingStatus.mode === "hands-free"; - // Log recording status changes - useEffect(() => { - console.debug("Recording status changed", { recordingState }); - }, [recordingState]); + // Handler for widget click to start recording in hands-free mode + const handleButtonClick = async (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + console.log("FAB: Button clicked! Current status:", recordingStatus); - // Recording state is now managed centrally, no need for separate listener - - // This handler is for the button click. - // It now uses the toggleRecording from the hook. - const handleButtonClickToggleRecording = () => { - console.log("FAB: Invoking toggleRecording from hook."); - // The hook internally manages starting/stopping MediaRecorder and VAD. - // The hook also listens for global state changes from the main process. - }; - - // Function to send the FAB's size to Electron - const updateWindowSizeToFab = () => { - if (isHovered || isRecording) { - //window.electronAPI.resizeWindow(96, 32); + // Only start recording if not already recording + if (recordingStatus.state === "idle") { + await startRecording(); + console.log("FAB: Started hands-free recording"); } else { - //window.electronAPI.resizeWindow(48, 16); + console.log("FAB: Already recording, ignoring click"); } }; - // Update window size when recording or hover state changes - useEffect(() => { - console.debug("Widget state changed", { isHovered, isRecording }); - updateWindowSizeToFab(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [isRecording, isHovered]); + // Handler for stop button in hands-free mode + const handleStopClick = async (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); // Prevent triggering the main button click + console.log("FAB: Stopping hands-free recording"); + await stopRecording(); + }; // Debounced mouse leave handler - const handleMouseLeave = () => { + const handleMouseLeave = async () => { if (leaveTimeoutRef.current) { clearTimeout(leaveTimeoutRef.current); } - leaveTimeoutRef.current = setTimeout(() => { + leaveTimeoutRef.current = setTimeout(async () => { setIsHovered(false); + // Re-enable mouse event forwarding when not hovering + try { + await setIgnoreMouseEvents.mutateAsync({ ignore: true }); + console.debug("Re-enabled mouse event forwarding"); + } catch (error) { + console.error("Failed to re-enable mouse event forwarding:", error); + } }, DEBOUNCE_DELAY); }; // Mouse enter handler - clears any pending leave timeout - const handleMouseEnter = () => { + const handleMouseEnter = async () => { if (leaveTimeoutRef.current) { clearTimeout(leaveTimeoutRef.current); leaveTimeoutRef.current = null; } setIsHovered(true); + // Disable mouse event forwarding to make widget clickable + await setIgnoreMouseEvents.mutateAsync({ ignore: false }); + console.debug("Disabled mouse event forwarding for clicking"); }; - // Clear timeout on unmount - useEffect(() => { - return () => { - if (leaveTimeoutRef.current) { - clearTimeout(leaveTimeoutRef.current); - } - }; - }, []); + const expanded = isRecording || isStopping || isHovered; - const expanded = - recordingState === "recording" || - recordingState === "starting" || - recordingState === "stopping" || - isHovered; + // Function to render widget content based on state + const renderWidgetContent = () => { + if (!expanded) return null; + + // Show processing indicator when stopping + if (isStopping) { + return ; + } + + // Show waveform with stop button when in hands-free mode and recording + if (isHandsFreeMode && isRecording) { + return ( + <> + +
+ +
+ + ); + } + + // Show waveform visualization for all other states + return ( + + ); + }; return ( diff --git a/apps/desktop/src/services/transcription-service.ts b/apps/desktop/src/services/transcription-service.ts index b5ab2ac..d1ee6b9 100644 --- a/apps/desktop/src/services/transcription-service.ts +++ b/apps/desktop/src/services/transcription-service.ts @@ -272,7 +272,7 @@ export class TranscriptionService { }); } - logger.transcription.error("Processed frame", { + logger.transcription.debug("Processed frame", { sessionId, frameSize: audioChunk.length, hadTranscription: chunkTranscription.length > 0, diff --git a/apps/desktop/src/trpc/router.ts b/apps/desktop/src/trpc/router.ts index 72da159..63868b7 100644 --- a/apps/desktop/src/trpc/router.ts +++ b/apps/desktop/src/trpc/router.ts @@ -5,6 +5,7 @@ import { modelsRouter } from "./routers/models"; import { settingsRouter } from "./routers/settings"; import { updaterRouter } from "./routers/updater"; import { recordingRouter } from "./routers/recording"; +import { widgetRouter } from "./routers/widget"; import { createRouter, procedure } from "./trpc"; export const router = createRouter({ @@ -49,6 +50,9 @@ export const router = createRouter({ // Recording router recording: recordingRouter, + + // Widget router + widget: widgetRouter, }); export type AppRouter = typeof router; diff --git a/apps/desktop/src/trpc/routers/recording.ts b/apps/desktop/src/trpc/routers/recording.ts index 3712f46..449bec6 100644 --- a/apps/desktop/src/trpc/routers/recording.ts +++ b/apps/desktop/src/trpc/routers/recording.ts @@ -1,17 +1,24 @@ import { observable } from "@trpc/server/observable"; import { createRouter, procedure } from "../trpc"; +import { z } from "zod"; import type { RecordingState } from "../../types/recording"; +import type { RecordingMode } from "../../main/managers/recording-manager"; + +interface RecordingStateUpdate { + state: RecordingState; + mode: RecordingMode; +} export const recordingRouter = createRouter({ - start: procedure.mutation(async ({ ctx }) => { + signalStart: procedure.mutation(async ({ ctx }) => { const recordingManager = ctx.serviceManager.getService("recordingManager"); if (!recordingManager) { throw new Error("Recording manager not available"); } - return await recordingManager.startRecording(); + return await recordingManager.startRecording("hands-free"); }), - stop: procedure.mutation(async ({ ctx }) => { + signalStop: procedure.mutation(async ({ ctx }) => { const recordingManager = ctx.serviceManager.getService("recordingManager"); if (!recordingManager) { throw new Error("Recording manager not available"); @@ -26,7 +33,7 @@ export const recordingRouter = createRouter({ // TODO: Remove this workaround when electron-trpc is updated to handle native Symbol.asyncDispose // eslint-disable-next-line deprecation/deprecation stateUpdates: procedure.subscription(({ ctx }) => { - return observable((emit) => { + return observable((emit) => { const recordingManager = ctx.serviceManager.getService("recordingManager"); if (!recordingManager) { @@ -34,18 +41,33 @@ export const recordingRouter = createRouter({ } // Emit initial state - emit.next(recordingManager.getState()); + emit.next({ + state: recordingManager.getState(), + mode: recordingManager.getRecordingMode(), + }); // Set up listener for state changes const handleStateChange = (status: RecordingState) => { - emit.next(status); + emit.next({ + state: status, + mode: recordingManager.getRecordingMode(), + }); + }; + + const handleModeChange = (mode: RecordingMode) => { + emit.next({ + state: recordingManager.getState(), + mode, + }); }; recordingManager.on("state-changed", handleStateChange); + recordingManager.on("mode-changed", handleModeChange); // Cleanup function return () => { recordingManager.off("state-changed", handleStateChange); + recordingManager.off("mode-changed", handleModeChange); }; }); }), diff --git a/apps/desktop/src/trpc/routers/widget.ts b/apps/desktop/src/trpc/routers/widget.ts new file mode 100644 index 0000000..e41de20 --- /dev/null +++ b/apps/desktop/src/trpc/routers/widget.ts @@ -0,0 +1,26 @@ +import { createRouter, procedure } from "../trpc"; +import { z } from "zod"; +import { logger } from "@/main/logger"; + +export const widgetRouter = createRouter({ + setIgnoreMouseEvents: procedure + .input( + z.object({ + ignore: z.boolean(), + }), + ) + .mutation(async ({ ctx, input }) => { + const windowManager = ctx.serviceManager.getService("windowManager"); + if (!windowManager) { + logger.main.error("Window manager service not available"); + return false; + } + + const widgetWindow = windowManager.getWidgetWindow(); + widgetWindow!.setIgnoreMouseEvents(input.ignore, { + forward: true, + }); + logger.main.debug("Set widget ignore mouse events", input); + return true; + }), +});