feat: recording toggle via widget click

This commit is contained in:
haritabh-z01 2025-07-08 04:13:05 +05:30
parent f8095d8ac0
commit c82f86143a
13 changed files with 425 additions and 410 deletions

View file

@ -1,6 +1,7 @@
import { useRef, useEffect, useState, useCallback } from "react";
import audioWorkletUrl from "@/assets/audio-recorder-processor.js?url";
import { api } from "@/trpc/react";
import { Mutex } from "async-mutex";
// Audio configuration
const FRAME_SIZE = 512; // 32ms at 16kHz
@ -28,6 +29,7 @@ export const useAudioCapture = ({
const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
const workletNodeRef = useRef<AudioWorkletNode | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const mutexRef = useRef(new Mutex());
// Subscribe to voice detection updates via tRPC
api.recording.voiceDetectionUpdates.useSubscription(undefined, {
@ -41,117 +43,126 @@ export const useAudioCapture = ({
});
const startCapture = useCallback(async () => {
try {
console.log("AudioCapture: Starting audio capture");
await mutexRef.current.runExclusive(async () => {
try {
console.log("AudioCapture: Starting audio capture");
// Get microphone stream
streamRef.current = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: SAMPLE_RATE,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
// Get microphone stream
streamRef.current = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: SAMPLE_RATE,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
// Create audio context
audioContextRef.current = new AudioContext({ sampleRate: SAMPLE_RATE });
// Create audio context
audioContextRef.current = new AudioContext({ sampleRate: SAMPLE_RATE });
// Load audio worklet
await audioContextRef.current.audioWorklet.addModule(audioWorkletUrl);
// Load audio worklet
await audioContextRef.current.audioWorklet.addModule(audioWorkletUrl);
// Create nodes
sourceRef.current = audioContextRef.current.createMediaStreamSource(
streamRef.current,
);
workletNodeRef.current = new AudioWorkletNode(
audioContextRef.current,
"audio-recorder-processor",
);
// Create nodes
sourceRef.current = audioContextRef.current.createMediaStreamSource(
streamRef.current,
);
workletNodeRef.current = new AudioWorkletNode(
audioContextRef.current,
"audio-recorder-processor",
);
// Handle audio frames from worklet
workletNodeRef.current.port.onmessage = async (event) => {
if (event.data.type === "audioFrame") {
const frame = event.data.frame;
console.log("AudioCapture: Received frame", {
frameLength: frame.length,
isFinal: event.data.isFinal,
});
const isFinal = event.data.isFinal || false;
// Handle audio frames from worklet
workletNodeRef.current.port.onmessage = async (event) => {
if (event.data.type === "audioFrame") {
const frame = event.data.frame;
console.debug("AudioCapture: Received frame", {
frameLength: frame.length,
isFinal: event.data.isFinal,
});
const isFinal = event.data.isFinal || false;
// Convert to ArrayBuffer for IPC
const arrayBuffer = frame.buffer.slice(
frame.byteOffset,
frame.byteOffset + frame.byteLength,
);
// Convert to ArrayBuffer for IPC
const arrayBuffer = frame.buffer.slice(
frame.byteOffset,
frame.byteOffset + frame.byteLength,
);
// Send to main process for VAD processing
// Main process will update voice detection state
await onAudioChunk(arrayBuffer, 0, isFinal); // Speech probability will come from main
// Send to main process for VAD processing
// Main process will update voice detection state
await onAudioChunk(arrayBuffer, 0, isFinal); // Speech probability will come from main
}
};
console.log(
`AudioCapture: Sent frame: ${frame.length} samples, isFinal: ${isFinal}`,
);
}
};
// Connect audio graph
sourceRef.current.connect(workletNodeRef.current);
// Connect audio graph
sourceRef.current.connect(workletNodeRef.current);
console.log("AudioCapture: Audio capture started");
} catch (error) {
console.error("AudioCapture: Failed to start capture:", error);
throw error;
}
console.log("AudioCapture: Audio capture started");
} catch (error) {
console.error("AudioCapture: Failed to start capture:", error);
throw error;
}
});
}, [onAudioChunk]);
const stopCapture = useCallback(() => {
console.log("AudioCapture: Stopping audio capture");
const stopCapture = useCallback(async () => {
await mutexRef.current.runExclusive(async () => {
try {
console.log("AudioCapture: Stopping audio capture");
// Send flush command to worklet before disconnecting
if (workletNodeRef.current) {
workletNodeRef.current.port.postMessage({ type: "flush" });
console.log("AudioCapture: Sent flush command to worklet");
}
// Send flush command to worklet before disconnecting
if (workletNodeRef.current) {
workletNodeRef.current.port.postMessage({ type: "flush" });
console.log("AudioCapture: Sent flush command to worklet");
}
// Disconnect nodes
if (sourceRef.current && workletNodeRef.current) {
sourceRef.current.disconnect(workletNodeRef.current);
}
// Disconnect nodes
if (sourceRef.current && workletNodeRef.current) {
sourceRef.current.disconnect(workletNodeRef.current);
}
// Close audio context
if (audioContextRef.current && audioContextRef.current.state !== "closed") {
audioContextRef.current.close();
}
// Close audio context
if (
audioContextRef.current &&
audioContextRef.current.state !== "closed"
) {
await audioContextRef.current.close();
}
// Stop media stream
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
}
// Stop media stream
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
}
// Clear refs
audioContextRef.current = null;
sourceRef.current = null;
workletNodeRef.current = null;
streamRef.current = null;
// Clear refs
audioContextRef.current = null;
sourceRef.current = null;
workletNodeRef.current = null;
streamRef.current = null;
setVoiceDetected(false);
console.log("AudioCapture: Audio capture stopped");
console.log("AudioCapture: Audio capture stopped");
} catch (error) {
console.error("AudioCapture: Error during stop:", error);
throw error;
}
});
}, []);
// Start/stop based on enabled state
useEffect(() => {
if (enabled) {
startCapture().catch((error) => {
console.error("AudioCapture: Failed to start:", error);
});
} else {
stopCapture();
if (!enabled) {
return;
}
startCapture().catch((error) => {
console.error("AudioCapture: Failed to start:", error);
});
return () => {
stopCapture();
stopCapture().catch((error) => {
console.error("AudioCapture: Failed to stop:", error);
});
};
}, [enabled, startCapture, stopCapture]);

View file

@ -1,53 +1,70 @@
import { useCallback } from "react";
import { useRecordingState } from "./useRecordingState";
import { useCallback, useState } from "react";
import { useAudioCapture } from "./useAudioCapture";
import { api } from "@/trpc/react";
import type { RecordingState } from "@/types/recording";
import type { RecordingMode } from "@/main/managers/recording-manager";
export interface UseRecordingParams {
onAudioFrame: (
audioBuffer: ArrayBuffer,
speechProbability: number,
isFinal: boolean,
) => Promise<void> | void;
onRecordingStartCallback?: () => Promise<void> | void;
onRecordingStopCallback?: () => Promise<void> | void;
export interface RecordingStatus {
state: RecordingState;
mode: RecordingMode;
}
export interface UseRecordingOutput {
recordingState: RecordingState;
recordingStatus: RecordingStatus;
voiceDetected: boolean;
startRecording: () => Promise<void>;
stopRecording: () => Promise<void>;
}
export const useRecording = ({
onAudioFrame,
onRecordingStartCallback,
onRecordingStopCallback,
}: UseRecordingParams): UseRecordingOutput => {
// Manage recording state via tRPC
const {
recordingState,
startRecording: startRecordingMutation,
stopRecording: stopRecordingMutation,
} = useRecordingState();
export const useRecording = (): UseRecordingOutput => {
const [recordingStatus, setRecordingStatus] = useState<RecordingStatus>({
state: "idle",
mode: "idle",
});
// Create handler for audio chunks - just pass through
const startRecordingMutation = api.recording.signalStart.useMutation();
const stopRecordingMutation = api.recording.signalStop.useMutation();
// Subscribe to recording state updates via tRPC
api.recording.stateUpdates.useSubscription(undefined, {
onData: (update) => {
setRecordingStatus(update);
},
onError: (error) => {
console.error("Error subscribing to recording state updates", error);
},
});
// Handle audio frames by sending them to the main process
const handleAudioChunk = useCallback(
async (
arrayBuffer: ArrayBuffer,
speechProbability: number,
isFinalChunk: boolean,
) => {
// Direct pass-through - no aggregation needed
await onAudioFrame(arrayBuffer, speechProbability, isFinalChunk);
// Convert ArrayBuffer to Float32Array
const float32Array = new Float32Array(arrayBuffer);
// Send frame directly to main process
// TODO: We need to update the IPC to include speech detection info
await window.electronAPI.sendAudioChunk(float32Array, isFinalChunk);
console.debug(`Sent audio frame`, {
samples: float32Array.length,
speechProbability: speechProbability.toFixed(3),
isFinal: isFinalChunk,
});
if (isFinalChunk) {
console.log("Final frame sent to main process");
}
},
[onAudioFrame],
[],
);
// Manage audio capture when recording is active
const isActive =
recordingState === "recording" || recordingState === "starting";
recordingStatus.state === "recording" ||
recordingStatus.state === "starting";
const { voiceDetected } = useAudioCapture({
onAudioChunk: handleAudioChunk,
@ -55,83 +72,18 @@ export const useRecording = ({
});
const startRecording = useCallback(async () => {
// Check if already recording
if (recordingState !== "idle" && recordingState !== "error") {
console.log(`Hook: Start denied. Current status: ${recordingState}`);
return;
}
try {
// Request main process to start recording
await startRecordingMutation();
// Call start callback if provided
if (onRecordingStartCallback) {
await onRecordingStartCallback();
console.log("Hook: onRecordingStartCallback executed.");
}
console.log("Hook: Recording fully started");
} catch (error) {
console.error("Hook: Error starting recording:", error);
// Try to stop recording in main process
try {
await stopRecordingMutation();
} catch (stopError) {
console.error("Hook: Failed to stop recording after error", stopError);
}
// Call stop callback if start callback was called
if (onRecordingStopCallback) {
try {
await onRecordingStopCallback();
} catch (e) {
console.error(
"Hook: Error in onRecordingStopCallback during start error:",
e,
);
}
}
}
}, [
recordingState,
startRecordingMutation,
onRecordingStartCallback,
onRecordingStopCallback,
stopRecordingMutation,
]);
// Request main process to start recording
await startRecordingMutation.mutateAsync();
console.log("Hook: Recording fully started");
}, [startRecordingMutation]);
const stopRecording = useCallback(async () => {
// Check if recording
if (recordingState !== "recording" && recordingState !== "starting") {
console.log(`Hook: Stop called but status is ${recordingState}.`);
return;
}
try {
// Request main process to stop recording
await stopRecordingMutation();
// Call stop callback if provided
if (onRecordingStopCallback) {
await onRecordingStopCallback();
console.log("Hook: onRecordingStopCallback executed.");
}
console.log("Hook: Recording stopped");
} catch (error) {
console.error("Hook: Error stopping recording:", error);
}
}, [
recordingState,
stopRecordingMutation,
onRecordingStopCallback,
onAudioFrame,
]);
await stopRecordingMutation.mutateAsync();
console.log("Hook: Recording stopped");
}, [stopRecordingMutation]);
return {
recordingState,
recordingStatus,
voiceDetected,
startRecording,
stopRecording,

View file

@ -1,53 +0,0 @@
import { useState } from "react";
import { api } from "@/trpc/react";
import type { RecordingState } from "@/types/recording";
export interface UseRecordingStateOutput {
recordingState: RecordingState;
startRecording: () => Promise<void>;
stopRecording: () => Promise<void>;
}
export const useRecordingState = (): UseRecordingStateOutput => {
const [recordingState, setRecordingState] = useState<RecordingState>("idle");
console.log("recordingState", recordingState);
const startRecordingMutation = api.recording.start.useMutation();
const stopRecordingMutation = api.recording.stop.useMutation();
// Subscribe to recording state updates via tRPC
api.recording.stateUpdates.useSubscription(undefined, {
onData: (state: RecordingState) => {
console.log("recordingStatus", state);
setRecordingState(state);
},
onError: (error) => {
console.error("Error subscribing to recording state updates", error);
},
});
const startRecording = async (): Promise<void> => {
try {
await startRecordingMutation.mutateAsync();
} catch (error) {
console.error("Failed to start recording via tRPC", error);
throw error;
}
};
const stopRecording = async (): Promise<void> => {
try {
await stopRecordingMutation.mutateAsync();
} catch (error) {
console.error("Failed to stop recording via tRPC", error);
throw error;
}
};
return {
recordingState,
startRecording,
stopRecording,
};
};

View file

@ -14,6 +14,7 @@ export class AppManager {
constructor() {
this.windowManager = new WindowManager();
this.serviceManager = ServiceManager.createInstance();
this.serviceManager.setWindowManager(this.windowManager);
}
async initialize(): Promise<void> {
@ -87,9 +88,38 @@ export class AppManager {
"Onboarding completed, restarting app for permissions to take effect",
);
// Relaunch the app to ensure all permissions take effect
app.relaunch();
app.quit();
// In development, reload windows instead of relaunching
if (process.env.NODE_ENV === "development") {
logger.main.info(
"Development mode: Reloading windows instead of relaunching",
);
// Close onboarding window
const onboardingWindow = this.windowManager.getOnboardingWindow();
if (onboardingWindow && !onboardingWindow.isDestroyed()) {
onboardingWindow.close();
}
// Give a short delay for permissions to register
setTimeout(async () => {
await this.setupWindows();
// Force reload all windows to pick up new permissions
const mainWindow = this.windowManager.getMainWindow();
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.reload();
}
const widgetWindow = this.windowManager.getWidgetWindow();
if (widgetWindow && !widgetWindow.isDestroyed()) {
widgetWindow.reload();
}
}, 1000);
} else {
// Production mode: relaunch the app
app.relaunch();
app.quit();
}
}
private async setupWindows(): Promise<void> {

View file

@ -6,7 +6,7 @@ import type { RecordingState } from "../../types/recording";
import { Mutex } from "async-mutex";
import type { ShortcutManager } from "../services/shortcut-manager";
export type RecordingMode = "idle" | "ptt" | "handsfree";
export type RecordingMode = "idle" | "ptt" | "hands-free";
/**
* Manages recording state and coordinates audio recording across the application
@ -61,6 +61,18 @@ export class RecordingManager extends EventEmitter {
this.broadcastStateChange();
}
private setMode(newMode: RecordingMode): void {
const oldMode = this.recordingMode;
this.recordingMode = newMode;
logger.audio.info("Recording mode changed", {
oldMode,
newMode,
});
// Broadcast mode change to all windows
this.broadcastModeChange();
}
public getState(): RecordingState {
return this.recordingState;
}
@ -70,6 +82,11 @@ export class RecordingManager extends EventEmitter {
this.emit("state-changed", this.getState());
}
private broadcastModeChange(): void {
// Emit event for internal listeners (tRPC subscription will pick this up)
this.emit("mode-changed", this.getRecordingMode());
}
private setupIPCHandlers(): void {
// Handle audio data chunks from renderer
ipcMain.handle(
@ -84,7 +101,7 @@ export class RecordingManager extends EventEmitter {
// Convert ArrayBuffer back to Float32Array
const float32Array = new Float32Array(chunk);
logger.audio.info("Received audio chunk", {
logger.audio.debug("Received audio chunk", {
samples: float32Array.length,
isFinalChunk,
});
@ -107,9 +124,15 @@ export class RecordingManager extends EventEmitter {
);
}
public async startRecording() {
console.error("startRecording");
public async startRecording(mode: "ptt" | "hands-free") {
await this.recordingMutex.runExclusive(async () => {
// if we were previously in ptt mode, we override
// priority is given to hands-free mode
// we don't need to check the other way around
if (mode === "hands-free") {
this.setMode("hands-free");
}
// Check if already recording
if (this.recordingState !== "idle") {
logger.audio.warn("Cannot start recording - already in progress", {
@ -119,6 +142,7 @@ export class RecordingManager extends EventEmitter {
}
this.setState("starting");
this.setMode(mode);
// Create session ID
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
@ -149,6 +173,7 @@ export class RecordingManager extends EventEmitter {
}
public async stopRecording() {
console.error("stopRecording called", this.recordingState);
await this.recordingMutex.runExclusive(async () => {
// Check if recording
if (this.recordingState !== "recording") {
@ -160,6 +185,9 @@ export class RecordingManager extends EventEmitter {
this.setState("stopping");
// Reset recording mode when stopping
this.recordingMode = "idle";
// Restore system audio
try {
const swiftBridge = this.serviceManager.getService("swiftIOBridge");
@ -180,55 +208,30 @@ export class RecordingManager extends EventEmitter {
});
}
public async toggleRecording() {
if (this.recordingState === "idle") {
await this.startRecording();
} else if (this.recordingState === "recording") {
await this.stopRecording();
} else {
logger.audio.warn("Cannot toggle recording in current state", {
currentState: this.recordingState,
});
}
}
// PTT-specific methods
public async startPTT() {
// Don't start PTT if already in hands-free mode
if (this.recordingMode === "handsfree") {
logger.audio.info("Ignoring PTT - already in hands-free mode");
return;
}
this.recordingMode = "ptt";
await this.startRecording();
this.startRecording("ptt");
}
public async stopPTT() {
// Only stop if we're actually in PTT mode
if (this.recordingMode === "ptt") {
this.recordingMode = "idle";
await this.stopRecording();
if (this.recordingMode !== "ptt") {
return;
}
this.stopRecording();
}
// Hands-free mode toggle
public async toggleHandsFree() {
if (this.recordingMode === "handsfree") {
this.recordingMode = "idle";
await this.stopRecording();
logger.audio.info("Hands-free mode disabled");
} else {
// If in PTT mode, just switch to hands-free without restarting
if (this.recordingMode === "ptt") {
this.recordingMode = "handsfree";
logger.audio.info("Switched from PTT to hands-free mode");
} else {
this.recordingMode = "handsfree";
await this.startRecording();
logger.audio.info("Hands-free mode enabled");
}
if (this.recordingState === "idle") {
this.startRecording("hands-free");
return;
}
if (this.recordingMode === "hands-free") {
this.stopRecording();
return;
}
this.startRecording("hands-free");
return;
}
// Get current mode
@ -245,7 +248,7 @@ export class RecordingManager extends EventEmitter {
this.recordingState !== "recording" &&
this.recordingState !== "stopping"
) {
logger.audio.warn("Received audio chunk while not recording", {
logger.audio.error("Received audio chunk while not recording", {
state: this.recordingState,
isFinalChunk,
});
@ -281,7 +284,7 @@ export class RecordingManager extends EventEmitter {
isFinal: isFinalChunk,
});
logger.audio.error("Processed audio chunk", {
logger.audio.debug("Processed audio chunk", {
chunkSize: chunk.length,
processingTimeMs: Date.now() - startTime,
resultLength: transcriptionResult.length,

View file

@ -7,10 +7,10 @@ import { AutoUpdaterService } from "../services/auto-updater";
import { RecordingManager } from "./recording-manager";
import { VADService } from "../../services/vad-service";
import { ShortcutManager } from "../services/shortcut-manager";
import { WindowManager } from "../core/window-manager";
import { createIPCHandler } from "electron-trpc-experimental/main";
import { router } from "../../trpc/router";
import { createContext } from "../../trpc/context";
import { BrowserWindow } from "electron";
/**
* Service map for type-safe service access
@ -24,6 +24,7 @@ export interface ServiceMap {
autoUpdaterService: AutoUpdaterService;
recordingManager: RecordingManager;
shortcutManager: ShortcutManager;
windowManager: WindowManager;
}
/**
@ -42,6 +43,7 @@ export class ServiceManager {
private autoUpdaterService: AutoUpdaterService | null = null;
private recordingManager: RecordingManager | null = null;
private shortcutManager: ShortcutManager | null = null;
private windowManager: WindowManager | null = null;
private trpcHandler: ReturnType<typeof createIPCHandler> | null = null;
async initialize(): Promise<void> {
@ -215,6 +217,7 @@ export class ServiceManager {
autoUpdaterService: this.autoUpdaterService ?? undefined,
recordingManager: this.recordingManager ?? undefined,
shortcutManager: this.shortcutManager ?? undefined,
windowManager: this.windowManager ?? undefined,
};
return services[serviceName] ?? null;
@ -255,4 +258,9 @@ export class ServiceManager {
}
return ServiceManager.instance;
}
setWindowManager(windowManager: WindowManager): void {
this.windowManager = windowManager;
logger.main.info("Window manager registered with ServiceManager");
}
}

View file

@ -168,11 +168,8 @@ export class ShortcutManager extends EventEmitter {
const pttKeys = this.shortcuts.pushToTalk.split("+");
const activeKeysList = this.getActiveKeys();
// Check if PTT keys match active keys exactly
return (
pttKeys.length === activeKeysList.length &&
pttKeys.every((key) => activeKeysList.includes(key))
);
//! This should only be a subset match
return pttKeys.every((key) => activeKeysList.includes(key));
}
private isToggleRecordingShortcutPressed(): boolean {

View file

@ -169,7 +169,7 @@ export class WhisperProvider implements TranscriptionProvider {
return true;
}
logger.transcription.error("Not transcribing", {
logger.transcription.debug("Not transcribing", {
bufferDurationMs,
silenceDurationMs,
frameBufferLength: this.frameBuffer.length,

View file

@ -1,16 +1,60 @@
import React, { useState, useCallback, useRef, useEffect } from "react";
import React, { useState, useRef, useEffect } from "react";
import { Square } from "lucide-react";
import { Waveform } from "@/components/Waveform";
import { useRecording } from "@/hooks/useRecording";
import type { RecordingState } from "@/types/recording";
import { api } from "@/trpc/react";
const NUM_WAVEFORM_BARS = 8; // Fewer bars for a smaller button
const DEBOUNCE_DELAY = 100; // milliseconds;
const NUM_WAVEFORM_BARS = 6; // Fewer bars to make room for stop button
const DEBOUNCE_DELAY = 100; // milliseconds
// Separate component for the stop button
const StopButton: React.FC<{ onClick: (e: React.MouseEvent) => void }> = ({
onClick,
}) => (
<button
onClick={onClick}
className="flex items-center justify-center w-[20px] h-[20px] bg-red-500 hover:bg-red-600 rounded transition-colors"
aria-label="Stop recording"
>
<Square className="w-[12px] h-[12px] text-white fill-white" />
</button>
);
// Separate component for the processing indicator
const ProcessingIndicator: React.FC = () => (
<div className="flex gap-[4px] items-center justify-center">
<div className="w-[4px] h-[4px] bg-blue-500 rounded-full animate-bounce [animation-delay:-0.3s]" />
<div className="w-[4px] h-[4px] bg-blue-500 rounded-full animate-bounce [animation-delay:-0.15s]" />
<div className="w-[4px] h-[4px] bg-blue-500 rounded-full animate-bounce" />
</div>
);
// Separate component for the waveform visualization
const WaveformVisualization: React.FC<{
isRecording: boolean;
voiceDetected: boolean;
}> = ({ isRecording, voiceDetected }) => (
<>
{Array.from({ length: NUM_WAVEFORM_BARS }).map((_, index) => (
<Waveform
key={index}
index={index}
isRecording={isRecording}
voiceDetected={voiceDetected}
baseHeight={100}
silentHeight={20}
/>
))}
</>
);
export const FloatingButton: React.FC = () => {
const [isHovered, setIsHovered] = useState(false);
const fabRef = useRef<HTMLButtonElement>(null);
const leaveTimeoutRef = useRef<NodeJS.Timeout | null>(null); // Ref for debounce timeout
// tRPC mutation to control widget mouse events
const setIgnoreMouseEvents = api.widget.setIgnoreMouseEvents.useMutation();
// Log component initialization
useEffect(() => {
console.log("FloatingButton component initialized");
@ -19,147 +63,118 @@ export const FloatingButton: React.FC = () => {
};
}, []);
const handleAudioFrame = useCallback(
async (
audioBuffer: ArrayBuffer,
speechProbability: number,
isFinal: boolean,
) => {
try {
// Convert ArrayBuffer to Float32Array
const float32Array = new Float32Array(audioBuffer);
// Send frame directly to main process
// TODO: We need to update the IPC to include speech detection info
await window.electronAPI.sendAudioChunk(float32Array, isFinal);
console.debug(`Sent audio frame`, {
samples: float32Array.length,
speechProbability: speechProbability.toFixed(3),
isFinal,
});
if (isFinal) {
console.log("Final frame sent to main process");
}
} catch (error) {
console.error("Error sending audio frame:", error);
}
},
[],
);
const { recordingState, startRecording, stopRecording, voiceDetected } =
useRecording({
onAudioFrame: handleAudioFrame,
});
const { recordingStatus, stopRecording, voiceDetected, startRecording } =
useRecording();
const isRecording =
recordingState === "recording" || recordingState === "starting";
const isAwaitingFinalChunk = recordingState === "stopping";
recordingStatus.state === "recording" ||
recordingStatus.state === "starting";
const isStopping = recordingStatus.state === "stopping";
const isHandsFreeMode = recordingStatus.mode === "hands-free";
// Log recording status changes
useEffect(() => {
console.debug("Recording status changed", { recordingState });
}, [recordingState]);
// Handler for widget click to start recording in hands-free mode
const handleButtonClick = async (e: React.MouseEvent) => {
e.preventDefault();
e.stopPropagation();
console.log("FAB: Button clicked! Current status:", recordingStatus);
// Recording state is now managed centrally, no need for separate listener
// This handler is for the button click.
// It now uses the toggleRecording from the hook.
const handleButtonClickToggleRecording = () => {
console.log("FAB: Invoking toggleRecording from hook.");
// The hook internally manages starting/stopping MediaRecorder and VAD.
// The hook also listens for global state changes from the main process.
};
// Function to send the FAB's size to Electron
const updateWindowSizeToFab = () => {
if (isHovered || isRecording) {
//window.electronAPI.resizeWindow(96, 32);
// Only start recording if not already recording
if (recordingStatus.state === "idle") {
await startRecording();
console.log("FAB: Started hands-free recording");
} else {
//window.electronAPI.resizeWindow(48, 16);
console.log("FAB: Already recording, ignoring click");
}
};
// Update window size when recording or hover state changes
useEffect(() => {
console.debug("Widget state changed", { isHovered, isRecording });
updateWindowSizeToFab();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isRecording, isHovered]);
// Handler for stop button in hands-free mode
const handleStopClick = async (e: React.MouseEvent) => {
e.preventDefault();
e.stopPropagation(); // Prevent triggering the main button click
console.log("FAB: Stopping hands-free recording");
await stopRecording();
};
// Debounced mouse leave handler
const handleMouseLeave = () => {
const handleMouseLeave = async () => {
if (leaveTimeoutRef.current) {
clearTimeout(leaveTimeoutRef.current);
}
leaveTimeoutRef.current = setTimeout(() => {
leaveTimeoutRef.current = setTimeout(async () => {
setIsHovered(false);
// Re-enable mouse event forwarding when not hovering
try {
await setIgnoreMouseEvents.mutateAsync({ ignore: true });
console.debug("Re-enabled mouse event forwarding");
} catch (error) {
console.error("Failed to re-enable mouse event forwarding:", error);
}
}, DEBOUNCE_DELAY);
};
// Mouse enter handler - clears any pending leave timeout
const handleMouseEnter = () => {
const handleMouseEnter = async () => {
if (leaveTimeoutRef.current) {
clearTimeout(leaveTimeoutRef.current);
leaveTimeoutRef.current = null;
}
setIsHovered(true);
// Disable mouse event forwarding to make widget clickable
await setIgnoreMouseEvents.mutateAsync({ ignore: false });
console.debug("Disabled mouse event forwarding for clicking");
};
// Clear timeout on unmount
useEffect(() => {
return () => {
if (leaveTimeoutRef.current) {
clearTimeout(leaveTimeoutRef.current);
}
};
}, []);
const expanded = isRecording || isStopping || isHovered;
const expanded =
recordingState === "recording" ||
recordingState === "starting" ||
recordingState === "stopping" ||
isHovered;
// Function to render widget content based on state
const renderWidgetContent = () => {
if (!expanded) return null;
// Show processing indicator when stopping
if (isStopping) {
return <ProcessingIndicator />;
}
// Show waveform with stop button when in hands-free mode and recording
if (isHandsFreeMode && isRecording) {
return (
<>
<WaveformVisualization
isRecording={isRecording}
voiceDetected={voiceDetected}
/>
<div className="ml-[4px]">
<StopButton onClick={handleStopClick} />
</div>
</>
);
}
// Show waveform visualization for all other states
return (
<WaveformVisualization
isRecording={isRecording}
voiceDetected={voiceDetected}
/>
);
};
return (
<button
role="button"
ref={fabRef}
// onClick={handleButtonClickToggleRecording} // Removed onClick to disable manual toggle
onClick={handleButtonClick}
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseLeave}
className={`
transition-all duration-200 ease-in-out
${expanded ? "h-[32px] w-[96px]" : "h-[16px] w-[48px]"}
rounded-full border-2 border-text-muted bg-black/10 border-muted-foreground
mb-2
${expanded ? "h-[32px] w-[112px]" : "h-[16px] w-[48px]"}
rounded-full border-2 border-text-muted bg-black/50 border-muted-foreground
mb-2 cursor-pointer select-none
`}
style={{ pointerEvents: "auto" }}
>
{expanded && (
<div className="flex gap-[2px] items-end h-[40%] justify-center w-full">
{recordingState === "stopping" ? (
// Show processing indicator when stopping
<div className="flex gap-[4px] items-center justify-center">
<div className="w-[4px] h-[4px] bg-blue-500 rounded-full animate-bounce [animation-delay:-0.3s]"></div>
<div className="w-[4px] h-[4px] bg-blue-500 rounded-full animate-bounce [animation-delay:-0.15s]"></div>
<div className="w-[4px] h-[4px] bg-blue-500 rounded-full animate-bounce"></div>
</div>
) : (
// Show waveform for other states
Array.from({ length: NUM_WAVEFORM_BARS }).map((_, index) => (
<Waveform
key={index}
index={index}
isRecording={
recordingState === "recording" ||
recordingState === "starting"
}
voiceDetected={voiceDetected} // Use local state for VAD
baseHeight={100} // Percentage of its container (the 40% height div)
silentHeight={20} // Percentage
/>
))
)}
{renderWidgetContent()}
</div>
)}
</button>

View file

@ -272,7 +272,7 @@ export class TranscriptionService {
});
}
logger.transcription.error("Processed frame", {
logger.transcription.debug("Processed frame", {
sessionId,
frameSize: audioChunk.length,
hadTranscription: chunkTranscription.length > 0,

View file

@ -5,6 +5,7 @@ import { modelsRouter } from "./routers/models";
import { settingsRouter } from "./routers/settings";
import { updaterRouter } from "./routers/updater";
import { recordingRouter } from "./routers/recording";
import { widgetRouter } from "./routers/widget";
import { createRouter, procedure } from "./trpc";
export const router = createRouter({
@ -49,6 +50,9 @@ export const router = createRouter({
// Recording router
recording: recordingRouter,
// Widget router
widget: widgetRouter,
});
export type AppRouter = typeof router;

View file

@ -1,17 +1,24 @@
import { observable } from "@trpc/server/observable";
import { createRouter, procedure } from "../trpc";
import { z } from "zod";
import type { RecordingState } from "../../types/recording";
import type { RecordingMode } from "../../main/managers/recording-manager";
interface RecordingStateUpdate {
state: RecordingState;
mode: RecordingMode;
}
export const recordingRouter = createRouter({
start: procedure.mutation(async ({ ctx }) => {
signalStart: procedure.mutation(async ({ ctx }) => {
const recordingManager = ctx.serviceManager.getService("recordingManager");
if (!recordingManager) {
throw new Error("Recording manager not available");
}
return await recordingManager.startRecording();
return await recordingManager.startRecording("hands-free");
}),
stop: procedure.mutation(async ({ ctx }) => {
signalStop: procedure.mutation(async ({ ctx }) => {
const recordingManager = ctx.serviceManager.getService("recordingManager");
if (!recordingManager) {
throw new Error("Recording manager not available");
@ -26,7 +33,7 @@ export const recordingRouter = createRouter({
// TODO: Remove this workaround when electron-trpc is updated to handle native Symbol.asyncDispose
// eslint-disable-next-line deprecation/deprecation
stateUpdates: procedure.subscription(({ ctx }) => {
return observable<RecordingState>((emit) => {
return observable<RecordingStateUpdate>((emit) => {
const recordingManager =
ctx.serviceManager.getService("recordingManager");
if (!recordingManager) {
@ -34,18 +41,33 @@ export const recordingRouter = createRouter({
}
// Emit initial state
emit.next(recordingManager.getState());
emit.next({
state: recordingManager.getState(),
mode: recordingManager.getRecordingMode(),
});
// Set up listener for state changes
const handleStateChange = (status: RecordingState) => {
emit.next(status);
emit.next({
state: status,
mode: recordingManager.getRecordingMode(),
});
};
const handleModeChange = (mode: RecordingMode) => {
emit.next({
state: recordingManager.getState(),
mode,
});
};
recordingManager.on("state-changed", handleStateChange);
recordingManager.on("mode-changed", handleModeChange);
// Cleanup function
return () => {
recordingManager.off("state-changed", handleStateChange);
recordingManager.off("mode-changed", handleModeChange);
};
});
}),

View file

@ -0,0 +1,26 @@
import { createRouter, procedure } from "../trpc";
import { z } from "zod";
import { logger } from "@/main/logger";
export const widgetRouter = createRouter({
setIgnoreMouseEvents: procedure
.input(
z.object({
ignore: z.boolean(),
}),
)
.mutation(async ({ ctx, input }) => {
const windowManager = ctx.serviceManager.getService("windowManager");
if (!windowManager) {
logger.main.error("Window manager service not available");
return false;
}
const widgetWindow = windowManager.getWidgetWindow();
widgetWindow!.setIgnoreMouseEvents(input.ignore, {
forward: true,
});
logger.main.debug("Set widget ignore mouse events", input);
return true;
}),
});