feat: enable formatting via cloud models - pass context info

apps/desktop/src/pipeline/providers/formatting/formatter-prompt.ts

@@ -17,8 +17,11 @@ const BASE_INSTRUCTIONS = [
   "Do not include any commentary, explanations, or text outside the XML tags",
 ];
 
+// Application type union
+type AppType = "email" | "chat" | "notes" | "default";
+
 // Application type specific rules
-const APPLICATION_TYPE_RULES: Record<string, string[]> = {
+const APPLICATION_TYPE_RULES: Record<AppType, string[]> = {
   email: [
     "Format with proper email structure (greeting, body paragraphs, closing)",
     "Preserve email metadata if present (From, To, Subject, Date)",
@@ -41,7 +44,7 @@ const APPLICATION_TYPE_RULES: Record<string, string[]> = {
     "Format action items and tasks clearly",
     "Preserve any existing formatting hints",
   ],
-  general: [
+  default: [
     "Apply standard formatting for general text",
     "Create logical paragraph breaks based on content flow",
     "Maintain consistent formatting throughout",
@@ -50,7 +53,7 @@ const APPLICATION_TYPE_RULES: Record<string, string[]> = {
 };
 
 // Map bundle identifiers to application types
-const BUNDLE_TO_TYPE: Record<string, string> = {
+const BUNDLE_TO_TYPE: Record<string, AppType> = {
   "com.apple.mail": "email",
   "com.microsoft.Outlook": "email",
   "com.readdle.smartemail": "email",
@@ -79,8 +82,8 @@ const BROWSER_BUNDLE_IDS = [
   "com.vivaldi.Vivaldi",
 ];
 
-// URL patterns for web applications
-const URL_PATTERNS: Record<string, RegExp[]> = {
+// URL patterns for web applications (general has no patterns, falls through)
+const URL_PATTERNS: Partial<Record<AppType, RegExp[]>> = {
   email: [
     /mail\.google\.com/,
     /outlook\.live\.com/,
@@ -145,11 +148,11 @@ export function constructFormatterPrompt(context: FormatParams["context"]): {
   return { systemPrompt: parts.join("\n") };
 }
 
-function detectApplicationType(
+export function detectApplicationType(
   accessibilityContext: GetAccessibilityContextResult | null | undefined,
-): string {
+): "email" | "chat" | "notes" | "default" {
   if (!accessibilityContext?.context?.application?.bundleIdentifier) {
-    return "general";
+    return "default";
   }
 
   const bundleId = accessibilityContext.context.application.bundleIdentifier;
@@ -163,8 +166,11 @@ function detectApplicationType(
     // Try to detect type from URL
     const url = accessibilityContext.context.windowInfo.url.toLowerCase();
 
-    for (const [type, patterns] of Object.entries(URL_PATTERNS)) {
-      if (patterns.some((pattern) => pattern.test(url))) {
+    for (const [type, patterns] of Object.entries(URL_PATTERNS) as [
+      AppType,
+      RegExp[],
+    ][]) {
+      if (patterns?.some((pattern) => pattern.test(url))) {
         return type;
       }
     }
@@ -176,12 +182,15 @@ function detectApplicationType(
   }
 
   // Check for partial matches
-  for (const [key, type] of Object.entries(BUNDLE_TO_TYPE)) {
+  for (const [key, type] of Object.entries(BUNDLE_TO_TYPE) as [
+    string,
+    AppType,
+  ][]) {
     if (bundleId.includes(key) || key.includes(bundleId)) {
       return type;
     }
   }
 
-  // Default to general
-  return "general";
+  // Default to default
+  return "default";
 }

apps/desktop/src/pipeline/providers/transcription/amical-cloud-provider.ts

@@ -5,6 +5,8 @@ import {
 import { logger } from "../../../main/logger";
 import { AuthService } from "../../../services/auth-service";
 import { getUserAgent } from "../../../utils/http-client";
+import { detectApplicationType } from "../formatting/formatter-prompt";
+import type { GetAccessibilityContextResult } from "@amical/types";
 
 interface CloudTranscriptionResponse {
   success: boolean;
@@ -27,6 +29,8 @@ export class AmicalCloudProvider implements TranscriptionProvider {
   private currentSilenceFrameCount = 0;
   private lastSpeechTimestamp = 0;
   private currentLanguage: string | undefined;
+  private currentAccessibilityContext: GetAccessibilityContextResult | null =
+    null;
 
   // Configuration
   private readonly FRAME_SIZE = 512; // 32ms at 16kHz
@@ -58,6 +62,9 @@ export class AmicalCloudProvider implements TranscriptionProvider {
       // Store language for use in API call (undefined = auto-detect)
       this.currentLanguage = context.language;
 
+      // Store accessibility context for the API request
+      this.currentAccessibilityContext = context?.accessibilityContext ?? null;
+
       // Check authentication
       if (!(await this.authService.isAuthenticated())) {
         throw new Error("Authentication required for cloud transcription");
@@ -163,6 +170,30 @@ export class AmicalCloudProvider implements TranscriptionProvider {
         body: JSON.stringify({
           audioData: Array.from(audioData),
           language: this.currentLanguage,
+          sharedContext: this.currentAccessibilityContext
+            ? {
+                selectedText:
+                  this.currentAccessibilityContext.context?.textSelection
+                    ?.selectedText,
+                beforeText:
+                  this.currentAccessibilityContext.context?.textSelection
+                    ?.preSelectionText,
+                afterText:
+                  this.currentAccessibilityContext.context?.textSelection
+                    ?.postSelectionText,
+                appType: detectApplicationType(
+                  this.currentAccessibilityContext,
+                ),
+                appBundleId:
+                  this.currentAccessibilityContext.context?.application
+                    ?.bundleIdentifier,
+                appName:
+                  this.currentAccessibilityContext.context?.application?.name,
+                appUrl:
+                  this.currentAccessibilityContext.context?.windowInfo?.url,
+                surroundingContext: "", // Empty for now, future enhancement
+              }
+            : undefined,
         }),
       });