marketing-shibata50/amical
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: improve perf for muting/unmuting by caching the short clips in mem

Browse source
This commit is contained in:
haritabh-z01 2025-12-10 13:46:51 +05:30
parent a7fe8dbac0
commit 213ad6a703
6 changed files with 352 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

106
packages/native-helpers/swift-helper/Sources/SwiftHelper/AudioService.swift Normal file
View file

@ -0,0 +1,106 @@
import AVFoundation
import Foundation
class AudioService: NSObject, AVAudioPlayerDelegate {
private var audioPlayer: AVAudioPlayer?
private var audioCompletionHandler: (() -> Void)?
private var preloadedAudio: [String: Data] = [:]
private let dateFormatter: DateFormatter
override init() {
self.dateFormatter = DateFormatter()
self.dateFormatter.dateFormat = "yyyy-MM-dd HH:mm:ss.SSS"
super.init()
preloadSounds()
}
private func preloadSounds() {
// Preload audio files at startup for faster playback
preloadedAudio["rec-start"] = Data(PackageResources.rec_start_mp3)
logToStderr("[AudioService] Preloaded rec-start.mp3 (\(preloadedAudio["rec-start"]?.count ?? 0) bytes)")
preloadedAudio["rec-stop"] = Data(PackageResources.rec_stop_mp3)
logToStderr("[AudioService] Preloaded rec-stop.mp3 (\(preloadedAudio["rec-stop"]?.count ?? 0) bytes)")
logToStderr("[AudioService] Audio files preloaded at startup")
}
func playSound(named soundName: String, completion: (() -> Void)? = nil) {
logToStderr("[AudioService] playSound called with soundName: \(soundName)")
// Stop any currently playing sound
if audioPlayer?.isPlaying == true {
logToStderr(
"[AudioService] Sound '\(audioPlayer?.url?.lastPathComponent ?? "previous")' is playing. Stopping it."
)
audioPlayer?.delegate = nil
audioPlayer?.stop()
}
audioPlayer = nil
audioCompletionHandler = nil
audioCompletionHandler = completion
// Use preloaded audio data (fast) or fall back to loading from resources
let soundData: Data
if let preloaded = preloadedAudio[soundName] {
logToStderr("[AudioService] Using preloaded audio for \(soundName).mp3 (\(preloaded.count) bytes)")
soundData = preloaded
} else {
logToStderr("[AudioService] Audio not preloaded, loading from PackageResources: \(soundName)")
switch soundName {
case "rec-start":
soundData = Data(PackageResources.rec_start_mp3)
case "rec-stop":
soundData = Data(PackageResources.rec_stop_mp3)
default:
logToStderr("[AudioService] Error: Unknown sound name '\(soundName)'. Completion will not be called.")
audioCompletionHandler = nil
return
}
}
do {
audioPlayer = try AVAudioPlayer(data: soundData)
audioPlayer?.delegate = self
if audioPlayer?.play() == true {
logToStderr("[AudioService] Playing sound: \(soundName).mp3. Delegate will handle completion.")
} else {
logToStderr(
"[AudioService] Failed to start playing sound: \(soundName).mp3. Completion will not be called."
)
audioCompletionHandler = nil
}
} catch {
logToStderr(
"[AudioService] Error initializing AVAudioPlayer for \(soundName).mp3: \(error.localizedDescription). Completion will not be called."
)
audioCompletionHandler = nil
}
}
// MARK: - AVAudioPlayerDelegate
func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) {
logToStderr(
"[AudioService] Sound playback finished (player URL: \(player.url?.lastPathComponent ?? "unknown"), successfully: \(flag))."
)
let handlerToCall = audioCompletionHandler
audioCompletionHandler = nil
if flag {
logToStderr("[AudioService] Sound finished successfully. Executing completion handler.")
handlerToCall?()
} else {
logToStderr("[AudioService] Sound did not finish successfully. Not executing completion handler.")
}
}
private func logToStderr(_ message: String) {
let timestamp = dateFormatter.string(from: Date())
let logMessage = "[\(timestamp)] \(message)\n"
FileHandle.standardError.write(logMessage.data(using: .utf8)!)
}
}

101
packages/native-helpers/swift-helper/Sources/SwiftHelper/RpcHandler.swift
View file

@ -1,18 +1,17 @@
import AVFoundation // ADDED
import Foundation
class IOBridge: NSObject, AVAudioPlayerDelegate {
class IOBridge: NSObject {
private let jsonEncoder: JSONEncoder
private let jsonDecoder: JSONDecoder
private let accessibilityService: AccessibilityService
private var audioPlayer: AVAudioPlayer?
private var audioCompletionHandler: (() -> Void)?
private let audioService: AudioService
private let dateFormatter: DateFormatter
init(jsonEncoder: JSONEncoder, jsonDecoder: JSONDecoder) {
self.jsonEncoder = jsonEncoder
self.jsonDecoder = jsonDecoder
self.accessibilityService = AccessibilityService()
self.audioService = AudioService() // Audio preloaded here at startup
self.dateFormatter = DateFormatter()
self.dateFormatter.dateFormat = "yyyy-MM-dd HH:mm:ss.SSS"
super.init()
@ -24,78 +23,6 @@ class IOBridge: NSObject, AVAudioPlayerDelegate {
FileHandle.standardError.write(logMessage.data(using: .utf8)!)
}
private func playSound(named soundName: String, completion: (() -> Void)? = nil) {
logToStderr("[IOBridge] playSound called with soundName: \(soundName)")
if audioPlayer?.isPlaying == true {
logToStderr(
"[IOBridge] Sound '\(audioPlayer?.url?.lastPathComponent ?? "previous")' is playing. Stopping it before playing \(soundName)."
)
audioPlayer?.delegate = nil
audioPlayer?.stop()
}
audioPlayer = nil
self.audioCompletionHandler = nil
self.audioCompletionHandler = completion
// Get the embedded audio data
let audioData: [UInt8]
do {
switch soundName {
case "rec-start":
logToStderr("[IOBridge] Attempting to load rec-start.mp3 from PackageResources")
audioData = PackageResources.rec_start_mp3
logToStderr(
"[IOBridge] Successfully loaded rec-start.mp3, data size: \(audioData.count) bytes"
)
case "rec-stop":
logToStderr("[IOBridge] Attempting to load rec-stop.mp3 from PackageResources")
audioData = PackageResources.rec_stop_mp3
logToStderr(
"[IOBridge] Successfully loaded rec-stop.mp3, data size: \(audioData.count) bytes"
)
default:
logToStderr(
"[IOBridge] Error: Unknown sound name '\(soundName)'. Completion will not be called."
)
self.audioCompletionHandler = nil
return
}
} catch {
logToStderr(
"[IOBridge] Error loading embedded audio data for '\(soundName)': \(error.localizedDescription). Completion will not be called."
)
self.audioCompletionHandler = nil
return
}
do {
// Convert embedded data to Data object
let soundData = Data(audioData)
// Initialize the audio player with the embedded data
audioPlayer = try AVAudioPlayer(data: soundData)
audioPlayer?.delegate = self
if audioPlayer?.play() == true {
logToStderr(
"[IOBridge] Playing embedded sound: \(soundName).mp3. Delegate will handle completion."
)
} else {
logToStderr(
"[IOBridge] Failed to start playing embedded sound: \(soundName).mp3 (audioPlayer.play() returned false or player is nil). Completion will not be called."
)
self.audioCompletionHandler = nil
}
} catch {
logToStderr(
"[IOBridge] Error initializing AVAudioPlayer for embedded \(soundName).mp3: \(error.localizedDescription). Completion will not be called."
)
self.audioCompletionHandler = nil
}
}
// Handles a single RPC Request
func handleRpcRequest(_ request: RPCRequestSchema) {
var rpcResponse: RPCResponseSchema
@ -157,7 +84,7 @@ class IOBridge: NSObject, AVAudioPlayerDelegate {
case .muteSystemAudio:
logToStderr("[IOBridge] Handling muteSystemAudio for ID: \(request.id)")
playSound(named: "rec-start") { [weak self] in
audioService.playSound(named: "rec-start") { [weak self] in
guard let self = self else {
let timestamp = DateFormatter().string(from: Date())
let logMessage =
@ -200,7 +127,7 @@ class IOBridge: NSObject, AVAudioPlayerDelegate {
let success = accessibilityService.restoreSystemAudio()
if success { // Play sound only if restore was successful
playSound(named: "rec-stop")
audioService.playSound(named: "rec-stop")
}
let resultPayload = RestoreSystemAudioResultSchema(
message: success ? "Restore command sent" : "Failed to send restore command",
@ -398,22 +325,4 @@ class IOBridge: NSObject, AVAudioPlayerDelegate {
}
}
// MARK: - AVAudioPlayerDelegate
func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) {
logToStderr(
"[IOBridge] Sound playback finished (player URL: \(player.url?.lastPathComponent ?? "unknown"), successfully: \(flag))."
)
let handlerToCall = audioCompletionHandler
audioCompletionHandler = nil
if flag {
logToStderr("[IOBridge] Sound finished successfully. Executing completion handler.")
handlerToCall?()
} else {
logToStderr(
"[IOBridge] Sound did not finish successfully (e.g., stopped or error). Not executing completion handler."
)
}
}
}

7
packages/native-helpers/windows-helper/src/Program.cs
View file

@ -25,19 +25,20 @@ namespace WindowsHelper
{
// Initialize components
shortcutMonitor = new ShortcutMonitor();
rpcHandler = new RpcHandler();
// Pass shortcutMonitor to RpcHandler for STA thread dispatch (audio operations)
rpcHandler = new RpcHandler(shortcutMonitor);
// Set up event handlers
shortcutMonitor.KeyEventOccurred += OnKeyEvent;
// Start RPC processing in background task
var rpcTask = Task.Run(() =>
var rpcTask = Task.Run(() =>
{
LogToStderr("Starting RPC processing in background thread...");
rpcHandler.ProcessRpcRequests(cancellationTokenSource.Token);
}, cancellationTokenSource.Token);
// Start shortcut monitoring (this will run the Windows message loop)
// Start shortcut monitoring (this will run the Windows message loop with STA support)
LogToStderr("Starting shortcut monitoring in main thread...");
shortcutMonitor.Start();

52
packages/native-helpers/windows-helper/src/RpcHandler.cs
View file

@ -13,16 +13,24 @@ namespace WindowsHelper
private readonly JsonSerializerOptions jsonOptions;
private readonly AccessibilityService accessibilityService;
private readonly AudioService audioService;
private readonly ShortcutMonitor? shortcutMonitor;
private Action<string>? audioCompletionHandler;
public RpcHandler()
public RpcHandler(ShortcutMonitor? shortcutMonitor = null)
{
this.shortcutMonitor = shortcutMonitor;
// Use the generated converter settings from the models
jsonOptions = WindowsHelper.Models.Converter.Settings;
accessibilityService = new AccessibilityService();
audioService = new AudioService();
audioService.SoundPlaybackCompleted += OnSoundPlaybackCompleted;
if (shortcutMonitor != null)
{
LogToStderr("RpcHandler: STA thread dispatch enabled via ShortcutMonitor");
}
}
public void ProcessRpcRequests(CancellationToken cancellationToken)
@ -253,10 +261,10 @@ namespace WindowsHelper
private async Task<RpcResponse> HandleMuteSystemAudio(RpcRequest request)
{
LogToStderr($"Handling muteSystemAudio for ID: {request.Id}");
// Store the request ID for the completion handler
var requestId = request.Id.ToString();
audioCompletionHandler = (id) =>
{
LogToStderr($"rec-start.mp3 finished playing. Proceeding to mute system audio. ID: {id}");
@ -274,9 +282,21 @@ namespace WindowsHelper
audioCompletionHandler = null;
};
// Play sound and wait for completion
await audioService.PlaySound("rec-start", requestId);
// Play sound on STA thread if available (faster due to COM/audio API preferences)
if (shortcutMonitor != null)
{
LogToStderr("Dispatching audio playback to STA thread");
await shortcutMonitor.InvokeOnStaAsync(async () =>
{
await audioService.PlaySound("rec-start", requestId);
return true;
});
}
else
{
await audioService.PlaySound("rec-start", requestId);
}
// Return dummy response (real response sent after audio completion)
return new RpcResponse { Id = request.Id.ToString() };
}
@ -284,14 +304,24 @@ namespace WindowsHelper
private RpcResponse HandleRestoreSystemAudio(RpcRequest request)
{
LogToStderr($"Handling restoreSystemAudio for ID: {request.Id}");
var success = audioService.RestoreSystemAudio();
if (success)
{
// Play sound asynchronously (don't wait)
_ = audioService.PlaySound("rec-stop", request.Id.ToString());
// Play sound asynchronously on STA thread if available (don't wait)
if (shortcutMonitor != null)
{
shortcutMonitor.PostToSta(async () =>
{
await audioService.PlaySound("rec-stop", request.Id.ToString());
});
}
else
{
_ = audioService.PlaySound("rec-stop", request.Id.ToString());
}
}
return new RpcResponse
{
Id = request.Id.ToString(),

113
packages/native-helpers/windows-helper/src/Services/AudioService.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Runtime.InteropServices;
@ -14,7 +15,10 @@ namespace WindowsHelper.Services
private MMDeviceEnumerator? deviceEnumerator;
private float originalVolume = 1.0f;
private bool originalMuteState = false;
// Preloaded audio data for faster playback
private readonly Dictionary<string, byte[]> preloadedAudio = new();
public event EventHandler<string>? SoundPlaybackCompleted;
public AudioService()
@ -22,10 +26,41 @@ namespace WindowsHelper.Services
try
{
deviceEnumerator = new MMDeviceEnumerator();
// Preload audio files at startup for faster playback
PreloadSound("rec-start");
PreloadSound("rec-stop");
LogToStderr("Audio files preloaded at startup");
}
catch (Exception ex)
{
LogToStderr($"Failed to initialize audio device enumerator: {ex.Message}");
LogToStderr($"Failed to initialize audio service: {ex.Message}");
}
}
private void PreloadSound(string soundName)
{
try
{
var assembly = Assembly.GetExecutingAssembly();
var resourceName = $"WindowsHelper.Resources.{soundName}.mp3";
using var stream = assembly.GetManifestResourceStream(resourceName);
if (stream != null)
{
using var ms = new MemoryStream();
stream.CopyTo(ms);
preloadedAudio[soundName] = ms.ToArray();
LogToStderr($"Preloaded {soundName}.mp3 ({preloadedAudio[soundName].Length} bytes)");
}
else
{
LogToStderr($"Resource not found for preloading: {resourceName}");
}
}
catch (Exception ex)
{
LogToStderr($"Failed to preload {soundName}: {ex.Message}");
}
}
@ -34,7 +69,7 @@ namespace WindowsHelper.Services
try
{
LogToStderr($"PlaySound called with soundName: {soundName}");
// Stop any currently playing sound
if (waveOut != null && waveOut.PlaybackState == PlaybackState.Playing)
{
@ -42,49 +77,49 @@ namespace WindowsHelper.Services
waveOut.Dispose();
waveOut = null;
}
// Get embedded resource
var assembly = Assembly.GetExecutingAssembly();
var resourceName = $"WindowsHelper.Resources.{soundName}.mp3";
using (var stream = assembly.GetManifestResourceStream(resourceName))
// Use preloaded audio data (fast) or fall back to loading from resources
byte[]? audioData;
if (!preloadedAudio.TryGetValue(soundName, out audioData))
{
LogToStderr($"Audio not preloaded, loading from resources: {soundName}");
var assembly = Assembly.GetExecutingAssembly();
var resourceName = $"WindowsHelper.Resources.{soundName}.mp3";
using var stream = assembly.GetManifestResourceStream(resourceName);
if (stream == null)
{
LogToStderr($"Resource not found: {resourceName}");
return;
}
// Create memory stream from embedded resource
using (var memoryStream = new MemoryStream())
{
await stream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
// Create audio file reader
using (var audioFile = new Mp3FileReader(memoryStream))
{
waveOut = new WaveOutEvent();
waveOut.Init(audioFile);
// Set up completion handler
var completionSource = new TaskCompletionSource<bool>();
waveOut.PlaybackStopped += (sender, args) =>
{
LogToStderr($"Sound playback finished for {soundName}");
completionSource.TrySetResult(true);
SoundPlaybackCompleted?.Invoke(this, requestId);
};
// Start playback
waveOut.Play();
LogToStderr($"Playing embedded sound: {soundName}.mp3");
// Wait for completion
await completionSource.Task;
}
}
using var ms = new MemoryStream();
await stream.CopyToAsync(ms);
audioData = ms.ToArray();
}
// Create memory stream from preloaded/loaded audio data
using var memoryStream = new MemoryStream(audioData);
using var audioFile = new Mp3FileReader(memoryStream);
waveOut = new WaveOutEvent();
waveOut.Init(audioFile);
// Set up completion handler
var completionSource = new TaskCompletionSource<bool>();
waveOut.PlaybackStopped += (sender, args) =>
{
LogToStderr($"Sound playback finished for {soundName}");
completionSource.TrySetResult(true);
SoundPlaybackCompleted?.Invoke(this, requestId);
};
// Start playback
waveOut.Play();
LogToStderr($"Playing sound: {soundName}.mp3");
// Wait for completion
await completionSource.Task;
}
catch (Exception ex)
{

128
packages/native-helpers/windows-helper/src/ShortcutMonitor.cs
View file

@ -1,7 +1,9 @@
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using WindowsHelper.Models;
namespace WindowsHelper
@ -15,6 +17,13 @@ namespace WindowsHelper
private const int WM_SYSKEYDOWN = 0x0104;
private const int WM_SYSKEYUP = 0x0105;
// For MsgWaitForMultipleObjects
private const uint QS_ALLINPUT = 0x04FF;
private const uint WAIT_OBJECT_0 = 0;
private const uint WAIT_TIMEOUT = 258;
private const uint INFINITE = 0xFFFFFFFF;
private const int PM_REMOVE = 0x0001;
private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
[DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
@ -32,6 +41,12 @@ namespace WindowsHelper
[DllImport("user32.dll")]
private static extern short GetAsyncKeyState(int vKey);
[DllImport("user32.dll")]
private static extern uint MsgWaitForMultipleObjects(uint nCount, IntPtr[] pHandles, bool bWaitAll, uint dwMilliseconds, uint dwWakeMask);
[DllImport("user32.dll")]
private static extern bool PeekMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax, int wRemoveMsg);
[StructLayout(LayoutKind.Sequential)]
private struct KBDLLHOOKSTRUCT
{
@ -64,6 +79,10 @@ namespace WindowsHelper
private Thread? messageLoopThread;
private bool isRunning = false;
// STA thread work queue for dispatching work from other threads
private readonly ConcurrentQueue<Action> staWorkQueue = new();
private readonly AutoResetEvent staWorkEvent = new(false);
// Track modifier key states internally to avoid GetAsyncKeyState issues
// Track left and right separately to handle cases where both are pressed
private bool leftShiftPressed = false;
@ -83,6 +102,65 @@ namespace WindowsHelper
public event EventHandler<HelperEvent>? KeyEventOccurred;
/// <summary>
/// Invokes an async action on the STA thread and waits for completion.
/// Use this for audio/COM operations that require STA thread.
/// </summary>
public Task<T> InvokeOnStaAsync<T>(Func<Task<T>> asyncAction)
{
var tcs = new TaskCompletionSource<T>(TaskCreationOptions.RunContinuationsAsynchronously);
staWorkQueue.Enqueue(async () =>
{
try
{
var result = await asyncAction();
tcs.SetResult(result);
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
staWorkEvent.Set();
return tcs.Task;
}
/// <summary>
/// Invokes a synchronous action on the STA thread and waits for completion.
/// Use this for audio/COM operations that require STA thread.
/// </summary>
public Task<T> InvokeOnSta<T>(Func<T> action)
{
var tcs = new TaskCompletionSource<T>(TaskCreationOptions.RunContinuationsAsynchronously);
staWorkQueue.Enqueue(() =>
{
try
{
var result = action();
tcs.SetResult(result);
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
staWorkEvent.Set();
return tcs.Task;
}
/// <summary>
/// Posts an action to the STA thread without waiting for completion.
/// </summary>
public void PostToSta(Action action)
{
staWorkQueue.Enqueue(action);
staWorkEvent.Set();
}
public void Start()
{
if (isRunning) return;
@ -113,7 +191,7 @@ namespace WindowsHelper
{
// Keep a reference to the delegate to prevent GC
hookProc = HookCallback;
using (Process curProcess = Process.GetCurrentProcess())
using (ProcessModule? curModule = curProcess.MainModule)
{
@ -131,13 +209,36 @@ namespace WindowsHelper
}
LogToStderr("Shortcut hook installed successfully");
LogToStderr("STA thread ready for work dispatch");
// Run Windows message loop
MSG msg;
while (isRunning && GetMessage(out msg, IntPtr.Zero, 0, 0) > 0)
// Run Windows message loop with support for STA work queue
var waitHandles = new IntPtr[] { staWorkEvent.SafeWaitHandle.DangerousGetHandle() };
while (isRunning)
{
TranslateMessage(ref msg);
DispatchMessage(ref msg);
// Wait for either a Windows message or a work item
var waitResult = MsgWaitForMultipleObjects(1, waitHandles, false, INFINITE, QS_ALLINPUT);
if (waitResult == WAIT_OBJECT_0)
{
// Work item signaled - process all queued work
ProcessStaWorkQueue();
}
else if (waitResult == WAIT_OBJECT_0 + 1)
{
// Windows message available - process all pending messages
MSG msg;
while (PeekMessage(out msg, IntPtr.Zero, 0, 0, PM_REMOVE))
{
if (msg.message == 0x0012) // WM_QUIT
{
isRunning = false;
break;
}
TranslateMessage(ref msg);
DispatchMessage(ref msg);
}
}
}
}
catch (Exception ex)
@ -154,6 +255,21 @@ namespace WindowsHelper
}
}
private void ProcessStaWorkQueue()
{
while (staWorkQueue.TryDequeue(out var action))
{
try
{
action();
}
catch (Exception ex)
{
LogToStderr($"Error processing STA work item: {ex.Message}");
}
}
}
private IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam)
{
if (nCode >= 0)