diff --git a/packages/native-helpers/swift-helper/Sources/SwiftHelper/AudioService.swift b/packages/native-helpers/swift-helper/Sources/SwiftHelper/AudioService.swift new file mode 100644 index 0000000..5e7b36f --- /dev/null +++ b/packages/native-helpers/swift-helper/Sources/SwiftHelper/AudioService.swift @@ -0,0 +1,106 @@ +import AVFoundation +import Foundation + +class AudioService: NSObject, AVAudioPlayerDelegate { + private var audioPlayer: AVAudioPlayer? + private var audioCompletionHandler: (() -> Void)? + private var preloadedAudio: [String: Data] = [:] + private let dateFormatter: DateFormatter + + override init() { + self.dateFormatter = DateFormatter() + self.dateFormatter.dateFormat = "yyyy-MM-dd HH:mm:ss.SSS" + super.init() + preloadSounds() + } + + private func preloadSounds() { + // Preload audio files at startup for faster playback + preloadedAudio["rec-start"] = Data(PackageResources.rec_start_mp3) + logToStderr("[AudioService] Preloaded rec-start.mp3 (\(preloadedAudio["rec-start"]?.count ?? 0) bytes)") + + preloadedAudio["rec-stop"] = Data(PackageResources.rec_stop_mp3) + logToStderr("[AudioService] Preloaded rec-stop.mp3 (\(preloadedAudio["rec-stop"]?.count ?? 0) bytes)") + + logToStderr("[AudioService] Audio files preloaded at startup") + } + + func playSound(named soundName: String, completion: (() -> Void)? = nil) { + logToStderr("[AudioService] playSound called with soundName: \(soundName)") + + // Stop any currently playing sound + if audioPlayer?.isPlaying == true { + logToStderr( + "[AudioService] Sound '\(audioPlayer?.url?.lastPathComponent ?? "previous")' is playing. Stopping it." + ) + audioPlayer?.delegate = nil + audioPlayer?.stop() + } + audioPlayer = nil + audioCompletionHandler = nil + + audioCompletionHandler = completion + + // Use preloaded audio data (fast) or fall back to loading from resources + let soundData: Data + if let preloaded = preloadedAudio[soundName] { + logToStderr("[AudioService] Using preloaded audio for \(soundName).mp3 (\(preloaded.count) bytes)") + soundData = preloaded + } else { + logToStderr("[AudioService] Audio not preloaded, loading from PackageResources: \(soundName)") + switch soundName { + case "rec-start": + soundData = Data(PackageResources.rec_start_mp3) + case "rec-stop": + soundData = Data(PackageResources.rec_stop_mp3) + default: + logToStderr("[AudioService] Error: Unknown sound name '\(soundName)'. Completion will not be called.") + audioCompletionHandler = nil + return + } + } + + do { + audioPlayer = try AVAudioPlayer(data: soundData) + audioPlayer?.delegate = self + + if audioPlayer?.play() == true { + logToStderr("[AudioService] Playing sound: \(soundName).mp3. Delegate will handle completion.") + } else { + logToStderr( + "[AudioService] Failed to start playing sound: \(soundName).mp3. Completion will not be called." + ) + audioCompletionHandler = nil + } + } catch { + logToStderr( + "[AudioService] Error initializing AVAudioPlayer for \(soundName).mp3: \(error.localizedDescription). Completion will not be called." + ) + audioCompletionHandler = nil + } + } + + // MARK: - AVAudioPlayerDelegate + + func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + logToStderr( + "[AudioService] Sound playback finished (player URL: \(player.url?.lastPathComponent ?? "unknown"), successfully: \(flag))." + ) + + let handlerToCall = audioCompletionHandler + audioCompletionHandler = nil + + if flag { + logToStderr("[AudioService] Sound finished successfully. Executing completion handler.") + handlerToCall?() + } else { + logToStderr("[AudioService] Sound did not finish successfully. Not executing completion handler.") + } + } + + private func logToStderr(_ message: String) { + let timestamp = dateFormatter.string(from: Date()) + let logMessage = "[\(timestamp)] \(message)\n" + FileHandle.standardError.write(logMessage.data(using: .utf8)!) + } +} diff --git a/packages/native-helpers/swift-helper/Sources/SwiftHelper/RpcHandler.swift b/packages/native-helpers/swift-helper/Sources/SwiftHelper/RpcHandler.swift index f4b2b2c..eb0ae49 100644 --- a/packages/native-helpers/swift-helper/Sources/SwiftHelper/RpcHandler.swift +++ b/packages/native-helpers/swift-helper/Sources/SwiftHelper/RpcHandler.swift @@ -1,18 +1,17 @@ -import AVFoundation // ADDED import Foundation -class IOBridge: NSObject, AVAudioPlayerDelegate { +class IOBridge: NSObject { private let jsonEncoder: JSONEncoder private let jsonDecoder: JSONDecoder private let accessibilityService: AccessibilityService - private var audioPlayer: AVAudioPlayer? - private var audioCompletionHandler: (() -> Void)? + private let audioService: AudioService private let dateFormatter: DateFormatter init(jsonEncoder: JSONEncoder, jsonDecoder: JSONDecoder) { self.jsonEncoder = jsonEncoder self.jsonDecoder = jsonDecoder self.accessibilityService = AccessibilityService() + self.audioService = AudioService() // Audio preloaded here at startup self.dateFormatter = DateFormatter() self.dateFormatter.dateFormat = "yyyy-MM-dd HH:mm:ss.SSS" super.init() @@ -24,78 +23,6 @@ class IOBridge: NSObject, AVAudioPlayerDelegate { FileHandle.standardError.write(logMessage.data(using: .utf8)!) } - private func playSound(named soundName: String, completion: (() -> Void)? = nil) { - logToStderr("[IOBridge] playSound called with soundName: \(soundName)") - - if audioPlayer?.isPlaying == true { - logToStderr( - "[IOBridge] Sound '\(audioPlayer?.url?.lastPathComponent ?? "previous")' is playing. Stopping it before playing \(soundName)." - ) - audioPlayer?.delegate = nil - audioPlayer?.stop() - } - audioPlayer = nil - self.audioCompletionHandler = nil - - self.audioCompletionHandler = completion - - // Get the embedded audio data - let audioData: [UInt8] - do { - switch soundName { - case "rec-start": - logToStderr("[IOBridge] Attempting to load rec-start.mp3 from PackageResources") - audioData = PackageResources.rec_start_mp3 - logToStderr( - "[IOBridge] Successfully loaded rec-start.mp3, data size: \(audioData.count) bytes" - ) - case "rec-stop": - logToStderr("[IOBridge] Attempting to load rec-stop.mp3 from PackageResources") - audioData = PackageResources.rec_stop_mp3 - logToStderr( - "[IOBridge] Successfully loaded rec-stop.mp3, data size: \(audioData.count) bytes" - ) - default: - logToStderr( - "[IOBridge] Error: Unknown sound name '\(soundName)'. Completion will not be called." - ) - self.audioCompletionHandler = nil - return - } - } catch { - logToStderr( - "[IOBridge] Error loading embedded audio data for '\(soundName)': \(error.localizedDescription). Completion will not be called." - ) - self.audioCompletionHandler = nil - return - } - - do { - // Convert embedded data to Data object - let soundData = Data(audioData) - - // Initialize the audio player with the embedded data - audioPlayer = try AVAudioPlayer(data: soundData) - audioPlayer?.delegate = self - - if audioPlayer?.play() == true { - logToStderr( - "[IOBridge] Playing embedded sound: \(soundName).mp3. Delegate will handle completion." - ) - } else { - logToStderr( - "[IOBridge] Failed to start playing embedded sound: \(soundName).mp3 (audioPlayer.play() returned false or player is nil). Completion will not be called." - ) - self.audioCompletionHandler = nil - } - } catch { - logToStderr( - "[IOBridge] Error initializing AVAudioPlayer for embedded \(soundName).mp3: \(error.localizedDescription). Completion will not be called." - ) - self.audioCompletionHandler = nil - } - } - // Handles a single RPC Request func handleRpcRequest(_ request: RPCRequestSchema) { var rpcResponse: RPCResponseSchema @@ -157,7 +84,7 @@ class IOBridge: NSObject, AVAudioPlayerDelegate { case .muteSystemAudio: logToStderr("[IOBridge] Handling muteSystemAudio for ID: \(request.id)") - playSound(named: "rec-start") { [weak self] in + audioService.playSound(named: "rec-start") { [weak self] in guard let self = self else { let timestamp = DateFormatter().string(from: Date()) let logMessage = @@ -200,7 +127,7 @@ class IOBridge: NSObject, AVAudioPlayerDelegate { let success = accessibilityService.restoreSystemAudio() if success { // Play sound only if restore was successful - playSound(named: "rec-stop") + audioService.playSound(named: "rec-stop") } let resultPayload = RestoreSystemAudioResultSchema( message: success ? "Restore command sent" : "Failed to send restore command", @@ -398,22 +325,4 @@ class IOBridge: NSObject, AVAudioPlayerDelegate { } } - // MARK: - AVAudioPlayerDelegate - func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { - logToStderr( - "[IOBridge] Sound playback finished (player URL: \(player.url?.lastPathComponent ?? "unknown"), successfully: \(flag))." - ) - - let handlerToCall = audioCompletionHandler - audioCompletionHandler = nil - - if flag { - logToStderr("[IOBridge] Sound finished successfully. Executing completion handler.") - handlerToCall?() - } else { - logToStderr( - "[IOBridge] Sound did not finish successfully (e.g., stopped or error). Not executing completion handler." - ) - } - } } diff --git a/packages/native-helpers/windows-helper/src/Program.cs b/packages/native-helpers/windows-helper/src/Program.cs index 151685c..fe76f99 100644 --- a/packages/native-helpers/windows-helper/src/Program.cs +++ b/packages/native-helpers/windows-helper/src/Program.cs @@ -25,19 +25,20 @@ namespace WindowsHelper { // Initialize components shortcutMonitor = new ShortcutMonitor(); - rpcHandler = new RpcHandler(); + // Pass shortcutMonitor to RpcHandler for STA thread dispatch (audio operations) + rpcHandler = new RpcHandler(shortcutMonitor); // Set up event handlers shortcutMonitor.KeyEventOccurred += OnKeyEvent; // Start RPC processing in background task - var rpcTask = Task.Run(() => + var rpcTask = Task.Run(() => { LogToStderr("Starting RPC processing in background thread..."); rpcHandler.ProcessRpcRequests(cancellationTokenSource.Token); }, cancellationTokenSource.Token); - // Start shortcut monitoring (this will run the Windows message loop) + // Start shortcut monitoring (this will run the Windows message loop with STA support) LogToStderr("Starting shortcut monitoring in main thread..."); shortcutMonitor.Start(); diff --git a/packages/native-helpers/windows-helper/src/RpcHandler.cs b/packages/native-helpers/windows-helper/src/RpcHandler.cs index 7896c5f..617ddbe 100644 --- a/packages/native-helpers/windows-helper/src/RpcHandler.cs +++ b/packages/native-helpers/windows-helper/src/RpcHandler.cs @@ -13,16 +13,24 @@ namespace WindowsHelper private readonly JsonSerializerOptions jsonOptions; private readonly AccessibilityService accessibilityService; private readonly AudioService audioService; + private readonly ShortcutMonitor? shortcutMonitor; private Action? audioCompletionHandler; - public RpcHandler() + public RpcHandler(ShortcutMonitor? shortcutMonitor = null) { + this.shortcutMonitor = shortcutMonitor; + // Use the generated converter settings from the models jsonOptions = WindowsHelper.Models.Converter.Settings; - + accessibilityService = new AccessibilityService(); audioService = new AudioService(); audioService.SoundPlaybackCompleted += OnSoundPlaybackCompleted; + + if (shortcutMonitor != null) + { + LogToStderr("RpcHandler: STA thread dispatch enabled via ShortcutMonitor"); + } } public void ProcessRpcRequests(CancellationToken cancellationToken) @@ -253,10 +261,10 @@ namespace WindowsHelper private async Task HandleMuteSystemAudio(RpcRequest request) { LogToStderr($"Handling muteSystemAudio for ID: {request.Id}"); - + // Store the request ID for the completion handler var requestId = request.Id.ToString(); - + audioCompletionHandler = (id) => { LogToStderr($"rec-start.mp3 finished playing. Proceeding to mute system audio. ID: {id}"); @@ -274,9 +282,21 @@ namespace WindowsHelper audioCompletionHandler = null; }; - // Play sound and wait for completion - await audioService.PlaySound("rec-start", requestId); - + // Play sound on STA thread if available (faster due to COM/audio API preferences) + if (shortcutMonitor != null) + { + LogToStderr("Dispatching audio playback to STA thread"); + await shortcutMonitor.InvokeOnStaAsync(async () => + { + await audioService.PlaySound("rec-start", requestId); + return true; + }); + } + else + { + await audioService.PlaySound("rec-start", requestId); + } + // Return dummy response (real response sent after audio completion) return new RpcResponse { Id = request.Id.ToString() }; } @@ -284,14 +304,24 @@ namespace WindowsHelper private RpcResponse HandleRestoreSystemAudio(RpcRequest request) { LogToStderr($"Handling restoreSystemAudio for ID: {request.Id}"); - + var success = audioService.RestoreSystemAudio(); if (success) { - // Play sound asynchronously (don't wait) - _ = audioService.PlaySound("rec-stop", request.Id.ToString()); + // Play sound asynchronously on STA thread if available (don't wait) + if (shortcutMonitor != null) + { + shortcutMonitor.PostToSta(async () => + { + await audioService.PlaySound("rec-stop", request.Id.ToString()); + }); + } + else + { + _ = audioService.PlaySound("rec-stop", request.Id.ToString()); + } } - + return new RpcResponse { Id = request.Id.ToString(), diff --git a/packages/native-helpers/windows-helper/src/Services/AudioService.cs b/packages/native-helpers/windows-helper/src/Services/AudioService.cs index 5169794..37351a9 100644 --- a/packages/native-helpers/windows-helper/src/Services/AudioService.cs +++ b/packages/native-helpers/windows-helper/src/Services/AudioService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Reflection; using System.Runtime.InteropServices; @@ -14,7 +15,10 @@ namespace WindowsHelper.Services private MMDeviceEnumerator? deviceEnumerator; private float originalVolume = 1.0f; private bool originalMuteState = false; - + + // Preloaded audio data for faster playback + private readonly Dictionary preloadedAudio = new(); + public event EventHandler? SoundPlaybackCompleted; public AudioService() @@ -22,10 +26,41 @@ namespace WindowsHelper.Services try { deviceEnumerator = new MMDeviceEnumerator(); + + // Preload audio files at startup for faster playback + PreloadSound("rec-start"); + PreloadSound("rec-stop"); + LogToStderr("Audio files preloaded at startup"); } catch (Exception ex) { - LogToStderr($"Failed to initialize audio device enumerator: {ex.Message}"); + LogToStderr($"Failed to initialize audio service: {ex.Message}"); + } + } + + private void PreloadSound(string soundName) + { + try + { + var assembly = Assembly.GetExecutingAssembly(); + var resourceName = $"WindowsHelper.Resources.{soundName}.mp3"; + + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream != null) + { + using var ms = new MemoryStream(); + stream.CopyTo(ms); + preloadedAudio[soundName] = ms.ToArray(); + LogToStderr($"Preloaded {soundName}.mp3 ({preloadedAudio[soundName].Length} bytes)"); + } + else + { + LogToStderr($"Resource not found for preloading: {resourceName}"); + } + } + catch (Exception ex) + { + LogToStderr($"Failed to preload {soundName}: {ex.Message}"); } } @@ -34,7 +69,7 @@ namespace WindowsHelper.Services try { LogToStderr($"PlaySound called with soundName: {soundName}"); - + // Stop any currently playing sound if (waveOut != null && waveOut.PlaybackState == PlaybackState.Playing) { @@ -42,49 +77,49 @@ namespace WindowsHelper.Services waveOut.Dispose(); waveOut = null; } - - // Get embedded resource - var assembly = Assembly.GetExecutingAssembly(); - var resourceName = $"WindowsHelper.Resources.{soundName}.mp3"; - - using (var stream = assembly.GetManifestResourceStream(resourceName)) + + // Use preloaded audio data (fast) or fall back to loading from resources + byte[]? audioData; + if (!preloadedAudio.TryGetValue(soundName, out audioData)) { + LogToStderr($"Audio not preloaded, loading from resources: {soundName}"); + var assembly = Assembly.GetExecutingAssembly(); + var resourceName = $"WindowsHelper.Resources.{soundName}.mp3"; + + using var stream = assembly.GetManifestResourceStream(resourceName); if (stream == null) { LogToStderr($"Resource not found: {resourceName}"); return; } - - // Create memory stream from embedded resource - using (var memoryStream = new MemoryStream()) - { - await stream.CopyToAsync(memoryStream); - memoryStream.Position = 0; - - // Create audio file reader - using (var audioFile = new Mp3FileReader(memoryStream)) - { - waveOut = new WaveOutEvent(); - waveOut.Init(audioFile); - - // Set up completion handler - var completionSource = new TaskCompletionSource(); - waveOut.PlaybackStopped += (sender, args) => - { - LogToStderr($"Sound playback finished for {soundName}"); - completionSource.TrySetResult(true); - SoundPlaybackCompleted?.Invoke(this, requestId); - }; - - // Start playback - waveOut.Play(); - LogToStderr($"Playing embedded sound: {soundName}.mp3"); - - // Wait for completion - await completionSource.Task; - } - } + + using var ms = new MemoryStream(); + await stream.CopyToAsync(ms); + audioData = ms.ToArray(); } + + // Create memory stream from preloaded/loaded audio data + using var memoryStream = new MemoryStream(audioData); + using var audioFile = new Mp3FileReader(memoryStream); + + waveOut = new WaveOutEvent(); + waveOut.Init(audioFile); + + // Set up completion handler + var completionSource = new TaskCompletionSource(); + waveOut.PlaybackStopped += (sender, args) => + { + LogToStderr($"Sound playback finished for {soundName}"); + completionSource.TrySetResult(true); + SoundPlaybackCompleted?.Invoke(this, requestId); + }; + + // Start playback + waveOut.Play(); + LogToStderr($"Playing sound: {soundName}.mp3"); + + // Wait for completion + await completionSource.Task; } catch (Exception ex) { diff --git a/packages/native-helpers/windows-helper/src/ShortcutMonitor.cs b/packages/native-helpers/windows-helper/src/ShortcutMonitor.cs index 288bdeb..4af0e3c 100644 --- a/packages/native-helpers/windows-helper/src/ShortcutMonitor.cs +++ b/packages/native-helpers/windows-helper/src/ShortcutMonitor.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Tasks; using WindowsHelper.Models; namespace WindowsHelper @@ -15,6 +17,13 @@ namespace WindowsHelper private const int WM_SYSKEYDOWN = 0x0104; private const int WM_SYSKEYUP = 0x0105; + // For MsgWaitForMultipleObjects + private const uint QS_ALLINPUT = 0x04FF; + private const uint WAIT_OBJECT_0 = 0; + private const uint WAIT_TIMEOUT = 258; + private const uint INFINITE = 0xFFFFFFFF; + private const int PM_REMOVE = 0x0001; + private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam); [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)] @@ -32,6 +41,12 @@ namespace WindowsHelper [DllImport("user32.dll")] private static extern short GetAsyncKeyState(int vKey); + [DllImport("user32.dll")] + private static extern uint MsgWaitForMultipleObjects(uint nCount, IntPtr[] pHandles, bool bWaitAll, uint dwMilliseconds, uint dwWakeMask); + + [DllImport("user32.dll")] + private static extern bool PeekMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax, int wRemoveMsg); + [StructLayout(LayoutKind.Sequential)] private struct KBDLLHOOKSTRUCT { @@ -64,6 +79,10 @@ namespace WindowsHelper private Thread? messageLoopThread; private bool isRunning = false; + // STA thread work queue for dispatching work from other threads + private readonly ConcurrentQueue staWorkQueue = new(); + private readonly AutoResetEvent staWorkEvent = new(false); + // Track modifier key states internally to avoid GetAsyncKeyState issues // Track left and right separately to handle cases where both are pressed private bool leftShiftPressed = false; @@ -83,6 +102,65 @@ namespace WindowsHelper public event EventHandler? KeyEventOccurred; + /// + /// Invokes an async action on the STA thread and waits for completion. + /// Use this for audio/COM operations that require STA thread. + /// + public Task InvokeOnStaAsync(Func> asyncAction) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + staWorkQueue.Enqueue(async () => + { + try + { + var result = await asyncAction(); + tcs.SetResult(result); + } + catch (Exception ex) + { + tcs.SetException(ex); + } + }); + staWorkEvent.Set(); + + return tcs.Task; + } + + /// + /// Invokes a synchronous action on the STA thread and waits for completion. + /// Use this for audio/COM operations that require STA thread. + /// + public Task InvokeOnSta(Func action) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + staWorkQueue.Enqueue(() => + { + try + { + var result = action(); + tcs.SetResult(result); + } + catch (Exception ex) + { + tcs.SetException(ex); + } + }); + staWorkEvent.Set(); + + return tcs.Task; + } + + /// + /// Posts an action to the STA thread without waiting for completion. + /// + public void PostToSta(Action action) + { + staWorkQueue.Enqueue(action); + staWorkEvent.Set(); + } + public void Start() { if (isRunning) return; @@ -113,7 +191,7 @@ namespace WindowsHelper { // Keep a reference to the delegate to prevent GC hookProc = HookCallback; - + using (Process curProcess = Process.GetCurrentProcess()) using (ProcessModule? curModule = curProcess.MainModule) { @@ -131,13 +209,36 @@ namespace WindowsHelper } LogToStderr("Shortcut hook installed successfully"); + LogToStderr("STA thread ready for work dispatch"); - // Run Windows message loop - MSG msg; - while (isRunning && GetMessage(out msg, IntPtr.Zero, 0, 0) > 0) + // Run Windows message loop with support for STA work queue + var waitHandles = new IntPtr[] { staWorkEvent.SafeWaitHandle.DangerousGetHandle() }; + + while (isRunning) { - TranslateMessage(ref msg); - DispatchMessage(ref msg); + // Wait for either a Windows message or a work item + var waitResult = MsgWaitForMultipleObjects(1, waitHandles, false, INFINITE, QS_ALLINPUT); + + if (waitResult == WAIT_OBJECT_0) + { + // Work item signaled - process all queued work + ProcessStaWorkQueue(); + } + else if (waitResult == WAIT_OBJECT_0 + 1) + { + // Windows message available - process all pending messages + MSG msg; + while (PeekMessage(out msg, IntPtr.Zero, 0, 0, PM_REMOVE)) + { + if (msg.message == 0x0012) // WM_QUIT + { + isRunning = false; + break; + } + TranslateMessage(ref msg); + DispatchMessage(ref msg); + } + } } } catch (Exception ex) @@ -154,6 +255,21 @@ namespace WindowsHelper } } + private void ProcessStaWorkQueue() + { + while (staWorkQueue.TryDequeue(out var action)) + { + try + { + action(); + } + catch (Exception ex) + { + LogToStderr($"Error processing STA work item: {ex.Message}"); + } + } + } + private IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam) { if (nCode >= 0)