diff --git a/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts b/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts index 443f768..5251067 100644 --- a/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts +++ b/apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts @@ -17,8 +17,11 @@ const BASE_INSTRUCTIONS = [ "Do not include any commentary, explanations, or text outside the XML tags", ]; +// Application type union +type AppType = "email" | "chat" | "notes" | "default"; + // Application type specific rules -const APPLICATION_TYPE_RULES: Record = { +const APPLICATION_TYPE_RULES: Record = { email: [ "Format with proper email structure (greeting, body paragraphs, closing)", "Preserve email metadata if present (From, To, Subject, Date)", @@ -41,7 +44,7 @@ const APPLICATION_TYPE_RULES: Record = { "Format action items and tasks clearly", "Preserve any existing formatting hints", ], - general: [ + default: [ "Apply standard formatting for general text", "Create logical paragraph breaks based on content flow", "Maintain consistent formatting throughout", @@ -50,7 +53,7 @@ const APPLICATION_TYPE_RULES: Record = { }; // Map bundle identifiers to application types -const BUNDLE_TO_TYPE: Record = { +const BUNDLE_TO_TYPE: Record = { "com.apple.mail": "email", "com.microsoft.Outlook": "email", "com.readdle.smartemail": "email", @@ -79,8 +82,8 @@ const BROWSER_BUNDLE_IDS = [ "com.vivaldi.Vivaldi", ]; -// URL patterns for web applications -const URL_PATTERNS: Record = { +// URL patterns for web applications (general has no patterns, falls through) +const URL_PATTERNS: Partial> = { email: [ /mail\.google\.com/, /outlook\.live\.com/, @@ -145,11 +148,11 @@ export function constructFormatterPrompt(context: FormatParams["context"]): { return { systemPrompt: parts.join("\n") }; } -function detectApplicationType( +export function detectApplicationType( accessibilityContext: GetAccessibilityContextResult | null | undefined, -): string { +): "email" | "chat" | "notes" | "default" { if (!accessibilityContext?.context?.application?.bundleIdentifier) { - return "general"; + return "default"; } const bundleId = accessibilityContext.context.application.bundleIdentifier; @@ -163,8 +166,11 @@ function detectApplicationType( // Try to detect type from URL const url = accessibilityContext.context.windowInfo.url.toLowerCase(); - for (const [type, patterns] of Object.entries(URL_PATTERNS)) { - if (patterns.some((pattern) => pattern.test(url))) { + for (const [type, patterns] of Object.entries(URL_PATTERNS) as [ + AppType, + RegExp[], + ][]) { + if (patterns?.some((pattern) => pattern.test(url))) { return type; } } @@ -176,12 +182,15 @@ function detectApplicationType( } // Check for partial matches - for (const [key, type] of Object.entries(BUNDLE_TO_TYPE)) { + for (const [key, type] of Object.entries(BUNDLE_TO_TYPE) as [ + string, + AppType, + ][]) { if (bundleId.includes(key) || key.includes(bundleId)) { return type; } } - // Default to general - return "general"; + // Default to default + return "default"; } diff --git a/apps/desktop/src/pipeline/providers/transcription/amical-cloud-provider.ts b/apps/desktop/src/pipeline/providers/transcription/amical-cloud-provider.ts index f07bc56..411e056 100644 --- a/apps/desktop/src/pipeline/providers/transcription/amical-cloud-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/amical-cloud-provider.ts @@ -5,6 +5,8 @@ import { import { logger } from "../../../main/logger"; import { AuthService } from "../../../services/auth-service"; import { getUserAgent } from "../../../utils/http-client"; +import { detectApplicationType } from "../formatting/formatter-prompt"; +import type { GetAccessibilityContextResult } from "@amical/types"; interface CloudTranscriptionResponse { success: boolean; @@ -27,6 +29,8 @@ export class AmicalCloudProvider implements TranscriptionProvider { private currentSilenceFrameCount = 0; private lastSpeechTimestamp = 0; private currentLanguage: string | undefined; + private currentAccessibilityContext: GetAccessibilityContextResult | null = + null; // Configuration private readonly FRAME_SIZE = 512; // 32ms at 16kHz @@ -58,6 +62,9 @@ export class AmicalCloudProvider implements TranscriptionProvider { // Store language for use in API call (undefined = auto-detect) this.currentLanguage = context.language; + // Store accessibility context for the API request + this.currentAccessibilityContext = context?.accessibilityContext ?? null; + // Check authentication if (!(await this.authService.isAuthenticated())) { throw new Error("Authentication required for cloud transcription"); @@ -163,6 +170,30 @@ export class AmicalCloudProvider implements TranscriptionProvider { body: JSON.stringify({ audioData: Array.from(audioData), language: this.currentLanguage, + sharedContext: this.currentAccessibilityContext + ? { + selectedText: + this.currentAccessibilityContext.context?.textSelection + ?.selectedText, + beforeText: + this.currentAccessibilityContext.context?.textSelection + ?.preSelectionText, + afterText: + this.currentAccessibilityContext.context?.textSelection + ?.postSelectionText, + appType: detectApplicationType( + this.currentAccessibilityContext, + ), + appBundleId: + this.currentAccessibilityContext.context?.application + ?.bundleIdentifier, + appName: + this.currentAccessibilityContext.context?.application?.name, + appUrl: + this.currentAccessibilityContext.context?.windowInfo?.url, + surroundingContext: "", // Empty for now, future enhancement + } + : undefined, }), });