Add hidden CLI command for live terminal debugging (#1599)

* Add hidden terminal debug CLI command

* Expand orphan terminal debug metadata

* Remove stray CLIProcessRunner test target wiring

* Tighten debug terminal diagnostics handling

---------

Co-authored-by: Lawrence Chen <lawrencecchen@users.noreply.github.com>

Sources/GhosttyTerminalView.swift

@@ -2489,6 +2489,30 @@ final class GhosttyMetalLayer: CAMetalLayer {
     }
 }
 
+final class TerminalSurfaceRegistry {
+    static let shared = TerminalSurfaceRegistry()
+
+    private let lock = NSLock()
+    private let surfaces = NSHashTable<AnyObject>.weakObjects()
+
+    private init() {}
+
+    func register(_ surface: TerminalSurface) {
+        lock.lock()
+        defer { lock.unlock() }
+        surfaces.add(surface)
+    }
+
+    func allSurfaces() -> [TerminalSurface] {
+        lock.lock()
+        let objects = surfaces.allObjects.compactMap { $0 as? TerminalSurface }
+        lock.unlock()
+        return objects.sorted { lhs, rhs in
+            lhs.id.uuidString < rhs.id.uuidString
+        }
+    }
+}
+
 // MARK: - Terminal Surface (owns the ghostty_surface_t lifecycle)
 
 final class TerminalSurface: Identifiable, ObservableObject {
@@ -2538,6 +2562,11 @@ final class TerminalSurface: Identifiable, ObservableObject {
     private var lastPixelHeight: UInt32 = 0
     private var lastXScale: CGFloat = 0
     private var lastYScale: CGFloat = 0
+    private let debugMetadataLock = NSLock()
+    private let createdAt: Date = Date()
+    private var runtimeSurfaceCreatedAt: Date?
+    private var teardownRequestedAt: Date?
+    private var teardownRequestReason: String?
     private var pendingTextQueue: [Data] = []
     private var pendingTextBytes: Int = 0
     private let maxPendingTextBytes = 1_048_576
@@ -2623,6 +2652,7 @@ final class TerminalSurface: Identifiable, ObservableObject {
         self.hostedView = GhosttySurfaceScrollView(surfaceView: view)
         // Surface is created when attached to a view
         hostedView.attachSurface(self)
+        TerminalSurfaceRegistry.shared.register(self)
     }
 
 
@@ -2679,6 +2709,47 @@ final class TerminalSurface: Identifiable, ObservableObject {
         portalLifecycleState.rawValue
     }
 
+    private func withDebugMetadataLock<T>(_ body: () -> T) -> T {
+        debugMetadataLock.lock()
+        defer { debugMetadataLock.unlock() }
+        return body()
+    }
+
+    func debugCreatedAt() -> Date {
+        withDebugMetadataLock { createdAt }
+    }
+
+    func debugRuntimeSurfaceCreatedAt() -> Date? {
+        withDebugMetadataLock { runtimeSurfaceCreatedAt }
+    }
+
+    func debugTeardownRequest() -> (requestedAt: Date?, reason: String?) {
+        withDebugMetadataLock { (teardownRequestedAt, teardownRequestReason) }
+    }
+
+    func debugLastKnownWorkspaceId() -> UUID {
+        tabId
+    }
+
+    func debugSurfaceContextLabel() -> String {
+        cmuxSurfaceContextName(surfaceContext)
+    }
+
+    func debugInitialCommand() -> String? {
+        initialCommand
+    }
+
+    func debugPortalHostLease() -> (hostId: String?, inWindow: Bool?, area: CGFloat?) {
+        guard let activePortalHostLease else {
+            return (nil, nil, nil)
+        }
+        return (
+            hostId: String(describing: activePortalHostLease.hostId),
+            inWindow: activePortalHostLease.inWindow,
+            area: activePortalHostLease.area
+        )
+    }
+
     func canAcceptPortalBinding(expectedSurfaceId: UUID?, expectedGeneration: UInt64?) -> Bool {
         guard portalLifecycleState == .live else { return false }
         if let expectedSurfaceId, expectedSurfaceId != id {
@@ -2774,9 +2845,28 @@ final class TerminalSurface: Identifiable, ObservableObject {
 #endif
     }
 
+    private func recordTeardownRequest(reason: String) {
+        withDebugMetadataLock {
+            if teardownRequestedAt == nil {
+                teardownRequestedAt = Date()
+            }
+            if let existing = teardownRequestReason, !existing.isEmpty {
+                return
+            }
+            teardownRequestReason = reason
+        }
+    }
+
+    private func recordRuntimeSurfaceCreation() {
+        withDebugMetadataLock {
+            runtimeSurfaceCreatedAt = Date()
+        }
+    }
+
     func beginPortalCloseLifecycle(reason: String) {
         guard portalLifecycleState != .closed else { return }
         guard portalLifecycleState != .closing else { return }
+        recordTeardownRequest(reason: reason)
         portalLifecycleState = .closing
         portalLifecycleGeneration &+= 1
 #if DEBUG
@@ -2805,6 +2895,7 @@ final class TerminalSurface: Identifiable, ObservableObject {
     /// before deinit; deinit will skip the free if already torn down.
     @MainActor
     func teardownSurface() {
+        recordTeardownRequest(reason: "surface.teardown")
         markPortalLifecycleClosed(reason: "teardown")
 
         let callbackContext = surfaceCallbackContext
@@ -3198,6 +3289,7 @@ final class TerminalSurface: Identifiable, ObservableObject {
             return
         }
         guard let createdSurface = surface else { return }
+        recordRuntimeSurfaceCreation()
 
         // Session scrollback replay must be one-shot. Reusing it on a later runtime
         // surface recreation would inject stale restored output into a live shell.

Sources/TerminalController.swift

@@ -2090,6 +2090,8 @@ class TerminalController {
             return v2Result(id: id, self.v2SurfaceRefresh(params: params))
         case "surface.health":
             return v2Result(id: id, self.v2SurfaceHealth(params: params))
+        case "debug.terminals":
+            return v2Result(id: id, self.v2DebugTerminals(params: params))
         case "surface.send_text":
             return v2Result(id: id, self.v2SurfaceSendText(params: params))
         case "surface.send_key":
@@ -2432,6 +2434,7 @@ class TerminalController {
             "tab.action",
             "surface.refresh",
             "surface.health",
+            "debug.terminals",
             "surface.send_text",
             "surface.send_key",
             "surface.read_text",
@@ -4917,6 +4920,265 @@ class TerminalController {
         return .ok(payload)
     }
 
+    private func v2DebugTerminals(params _: [String: Any]) -> V2CallResult {
+        var payload: [String: Any]?
+
+        v2MainSync {
+            guard let app = AppDelegate.shared else { return }
+
+            struct MappedTerminalLocation {
+                let windowIndex: Int
+                let windowId: UUID
+                let window: NSWindow?
+                let workspaceIndex: Int
+                let workspaceSelected: Bool
+                let workspace: Workspace
+                let terminalPanel: TerminalPanel
+                let paneId: PaneID?
+                let paneIndex: Int?
+                let surfaceIndex: Int
+                let selectedInPane: Bool?
+                let bonsplitTabId: TabID?
+            }
+
+            func nonEmpty(_ raw: String?) -> String? {
+                guard let raw else { return nil }
+                let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
+                return trimmed.isEmpty ? nil : trimmed
+            }
+
+            func rectPayload(_ rect: CGRect) -> [String: Double] {
+                [
+                    "x": Double(rect.origin.x),
+                    "y": Double(rect.origin.y),
+                    "width": Double(rect.size.width),
+                    "height": Double(rect.size.height)
+                ]
+            }
+
+            func objectPointerString(_ object: AnyObject?) -> String {
+                guard let object else { return "nil" }
+                return String(describing: Unmanaged.passUnretained(object).toOpaque())
+            }
+
+            func ghosttyPointerString(_ surface: ghostty_surface_t?) -> String {
+                guard let surface else { return "nil" }
+                return String(describing: surface)
+            }
+
+            func className(_ object: AnyObject?) -> String? {
+                guard let object else { return nil }
+                return String(describing: type(of: object))
+            }
+
+            let iso8601Formatter = ISO8601DateFormatter()
+            let now = Date()
+
+            func iso8601String(_ date: Date?) -> String? {
+                guard let date else { return nil }
+                return iso8601Formatter.string(from: date)
+            }
+
+            func ageSeconds(since date: Date?) -> Double? {
+                guard let date else { return nil }
+                return (now.timeIntervalSince(date) * 1000).rounded() / 1000
+            }
+
+            @MainActor
+            func superviewClassChain(for view: NSView, limit: Int = 8) -> [String] {
+                var chain: [String] = [String(describing: type(of: view))]
+                var currentSuperview = view.superview
+                while chain.count < limit, let nextSuperview = currentSuperview {
+                    chain.append(String(describing: type(of: nextSuperview)))
+                    currentSuperview = nextSuperview.superview
+                }
+                if currentSuperview != nil {
+                    chain.append("...")
+                }
+                return chain
+            }
+
+            let windows = app.scriptableMainWindows()
+            let windowIndexById = Dictionary(
+                uniqueKeysWithValues: windows.enumerated().map { ($0.element.windowId, $0.offset) }
+            )
+
+            @MainActor
+            func resolvedWindowMetadata(for window: NSWindow?) -> (windowId: UUID?, windowIndex: Int?) {
+                guard let window else { return (nil, nil) }
+
+                if let match = windows.enumerated().first(where: { _, state in
+                    guard let stateWindow = state.window else { return false }
+                    return stateWindow === window || stateWindow.windowNumber == window.windowNumber
+                }) {
+                    return (match.element.windowId, match.offset)
+                }
+
+                guard let raw = window.identifier?.rawValue else { return (nil, nil) }
+                let prefix = "cmux.main."
+                guard raw.hasPrefix(prefix),
+                      let parsedWindowId = UUID(uuidString: String(raw.dropFirst(prefix.count))) else {
+                    return (nil, nil)
+                }
+                return (parsedWindowId, windowIndexById[parsedWindowId])
+            }
+
+            var mappedLocations: [ObjectIdentifier: MappedTerminalLocation] = [:]
+            for (windowIndex, state) in windows.enumerated() {
+                let tabManager = state.tabManager
+                for (workspaceIndex, workspace) in tabManager.tabs.enumerated() {
+                    let paneIndexById = Dictionary(
+                        uniqueKeysWithValues: workspace.bonsplitController.allPaneIds.enumerated().map {
+                            ($0.element.id, $0.offset)
+                        }
+                    )
+                    var selectedInPaneByPanelId: [UUID: Bool] = [:]
+                    for paneId in workspace.bonsplitController.allPaneIds {
+                        let selectedTab = workspace.bonsplitController.selectedTab(inPane: paneId)
+                        for tab in workspace.bonsplitController.tabs(inPane: paneId) {
+                            guard let panelId = workspace.panelIdFromSurfaceId(tab.id) else { continue }
+                            selectedInPaneByPanelId[panelId] = (tab.id == selectedTab?.id)
+                        }
+                    }
+
+                    for (surfaceIndex, panel) in orderedPanels(in: workspace).enumerated() {
+                        guard let terminalPanel = panel as? TerminalPanel else { continue }
+                        mappedLocations[ObjectIdentifier(terminalPanel.surface)] = MappedTerminalLocation(
+                            windowIndex: windowIndex,
+                            windowId: state.windowId,
+                            window: state.window,
+                            workspaceIndex: workspaceIndex,
+                            workspaceSelected: workspace.id == tabManager.selectedTabId,
+                            workspace: workspace,
+                            terminalPanel: terminalPanel,
+                            paneId: workspace.paneId(forPanelId: terminalPanel.id),
+                            paneIndex: workspace.paneId(forPanelId: terminalPanel.id).flatMap { paneIndexById[$0.id] },
+                            surfaceIndex: surfaceIndex,
+                            selectedInPane: selectedInPaneByPanelId[terminalPanel.id],
+                            bonsplitTabId: workspace.surfaceIdFromPanelId(terminalPanel.id)
+                        )
+                    }
+                }
+            }
+
+            let surfaces = TerminalSurfaceRegistry.shared.allSurfaces()
+            let terminals: [[String: Any]] = surfaces.enumerated().map { index, terminalSurface in
+                let mapped = mappedLocations[ObjectIdentifier(terminalSurface)]
+                let hostedView = terminalSurface.hostedView
+                let hostedWindow = mapped?.window ?? hostedView.window
+                let fallbackWindowMetadata = resolvedWindowMetadata(for: hostedWindow)
+                let resolvedWindowId = mapped?.windowId ?? fallbackWindowMetadata.windowId
+                let resolvedWindowIndex = mapped?.windowIndex ?? fallbackWindowMetadata.windowIndex
+                let workspace = mapped?.workspace
+                let panelId = mapped?.terminalPanel.id ?? terminalSurface.id
+                let portalState = hostedView.portalBindingGuardState()
+                let portalHostLease = terminalSurface.debugPortalHostLease()
+                let gitBranchState = workspace?.panelGitBranches[panelId]
+                let listeningPorts = (workspace?.surfaceListeningPorts[panelId] ?? []).sorted()
+                let title = workspace?.panelTitle(panelId: panelId)
+                let paneId = mapped?.paneId
+                let treeVisible = mapped?.bonsplitTabId != nil && paneId != nil
+                let ttyName = workspace?.surfaceTTYNames[panelId]
+                let currentDirectory = nonEmpty(workspace?.panelDirectories[panelId] ?? mapped?.terminalPanel.directory)
+                let teardownRequest = terminalSurface.debugTeardownRequest()
+                let lastKnownWorkspaceId = terminalSurface.debugLastKnownWorkspaceId()
+
+                var item: [String: Any] = [
+                    "index": index,
+                    "mapped": mapped != nil,
+                    "tree_visible": treeVisible,
+                    "window_index": v2OrNull(resolvedWindowIndex),
+                    "window_id": v2OrNull(resolvedWindowId?.uuidString),
+                    "window_ref": v2Ref(kind: .window, uuid: resolvedWindowId),
+                    "window_number": v2OrNull(hostedWindow?.windowNumber),
+                    "window_key": hostedWindow?.isKeyWindow ?? false,
+                    "window_main": hostedWindow?.isMainWindow ?? false,
+                    "window_visible": hostedWindow?.isVisible ?? false,
+                    "window_occluded": hostedWindow.map { !$0.occlusionState.contains(.visible) } ?? false,
+                    "window_identifier": v2OrNull(hostedWindow?.identifier?.rawValue),
+                    "window_title": v2OrNull(nonEmpty(hostedWindow?.title)),
+                    "window_class": v2OrNull(className(hostedWindow)),
+                    "window_delegate_class": v2OrNull(className(hostedWindow?.delegate as AnyObject?)),
+                    "window_controller_class": v2OrNull(className(hostedWindow?.windowController)),
+                    "window_level": v2OrNull(hostedWindow?.level.rawValue),
+                    "window_frame": hostedWindow.map { rectPayload($0.frame) } ?? NSNull(),
+                    "workspace_index": v2OrNull(mapped?.workspaceIndex),
+                    "workspace_id": v2OrNull(workspace?.id.uuidString),
+                    "workspace_ref": v2Ref(kind: .workspace, uuid: workspace?.id),
+                    "workspace_title": v2OrNull(workspace?.title),
+                    "workspace_selected": v2OrNull(mapped?.workspaceSelected),
+                    "pane_index": v2OrNull(mapped?.paneIndex),
+                    "pane_id": v2OrNull(paneId?.id.uuidString),
+                    "pane_ref": v2Ref(kind: .pane, uuid: paneId?.id),
+                    "surface_index": v2OrNull(mapped?.surfaceIndex),
+                    "surface_index_in_pane": v2OrNull(workspace?.indexInPane(forPanelId: panelId)),
+                    "surface_id": panelId.uuidString,
+                    "surface_ref": v2Ref(kind: .surface, uuid: panelId),
+                    "surface_title": v2OrNull(title),
+                    "surface_focused": v2OrNull(workspace.map { panelId == $0.focusedPanelId }),
+                    "surface_selected_in_pane": v2OrNull(mapped?.selectedInPane),
+                    "surface_pinned": v2OrNull(workspace.map { $0.isPanelPinned(panelId) }),
+                    "surface_context": terminalSurface.debugSurfaceContextLabel(),
+                    "surface_created_at": v2OrNull(iso8601String(terminalSurface.debugCreatedAt())),
+                    "surface_age_seconds": v2OrNull(ageSeconds(since: terminalSurface.debugCreatedAt())),
+                    "runtime_surface_created_at": v2OrNull(iso8601String(terminalSurface.debugRuntimeSurfaceCreatedAt())),
+                    "runtime_surface_age_seconds": v2OrNull(ageSeconds(since: terminalSurface.debugRuntimeSurfaceCreatedAt())),
+                    "bonsplit_tab_id": v2OrNull(mapped?.bonsplitTabId?.uuid.uuidString),
+                    "terminal_object_ptr": objectPointerString(terminalSurface),
+                    "ghostty_surface_ptr": ghosttyPointerString(terminalSurface.surface),
+                    "runtime_surface_ready": terminalSurface.surface != nil,
+                    "hosted_view_ptr": objectPointerString(hostedView),
+                    "hosted_view_class": className(hostedView) ?? "nil",
+                    "hosted_view_in_window": hostedView.window != nil,
+                    "hosted_view_has_superview": hostedView.superview != nil,
+                    "hosted_view_hidden": hostedView.isHidden,
+                    "hosted_view_hidden_or_ancestor_hidden": hostedView.isHiddenOrHasHiddenAncestor,
+                    "hosted_view_alpha": hostedView.alphaValue,
+                    "hosted_view_visible_in_ui": hostedView.debugPortalVisibleInUI,
+                    "hosted_view_superview_chain": superviewClassChain(for: hostedView),
+                    "surface_view_first_responder": hostedView.isSurfaceViewFirstResponder(),
+                    "hosted_view_frame": rectPayload(hostedView.frame),
+                    "hosted_view_bounds": rectPayload(hostedView.bounds),
+                    "hosted_view_frame_in_window": rectPayload(hostedView.debugPortalFrameInWindow),
+                    "portal_binding_state": portalState.state,
+                    "portal_binding_generation": v2OrNull(portalState.generation),
+                    "portal_host_id": v2OrNull(portalHostLease.hostId),
+                    "portal_host_in_window": v2OrNull(portalHostLease.inWindow),
+                    "portal_host_area": v2OrNull(portalHostLease.area.map(Double.init)),
+                    "tty": v2OrNull(ttyName),
+                    "current_directory": v2OrNull(currentDirectory),
+                    "requested_working_directory": v2OrNull(nonEmpty(terminalSurface.requestedWorkingDirectory)),
+                    "initial_command": v2OrNull(nonEmpty(terminalSurface.debugInitialCommand())),
+                    "git_branch": v2OrNull(nonEmpty(gitBranchState?.branch)),
+                    "git_dirty": v2OrNull(gitBranchState?.isDirty),
+                    "listening_ports": listeningPorts,
+                    "key_state_indicator": v2OrNull(nonEmpty(terminalSurface.currentKeyStateIndicatorText)),
+                    "last_known_workspace_id": lastKnownWorkspaceId.uuidString,
+                    "last_known_workspace_ref": v2Ref(kind: .workspace, uuid: lastKnownWorkspaceId),
+                    "teardown_requested": teardownRequest.requestedAt != nil,
+                    "teardown_requested_at": v2OrNull(iso8601String(teardownRequest.requestedAt)),
+                    "teardown_requested_age_seconds": v2OrNull(ageSeconds(since: teardownRequest.requestedAt)),
+                    "teardown_requested_reason": v2OrNull(nonEmpty(teardownRequest.reason))
+                ]
+
+                if title == nil, let fallbackTitle = mapped?.terminalPanel.displayTitle, !fallbackTitle.isEmpty {
+                    item["surface_title"] = fallbackTitle
+                }
+                return item
+            }
+
+            payload = [
+                "count": terminals.count,
+                "terminals": terminals
+            ]
+        }
+
+        guard let payload else {
+            return .err(code: "unavailable", message: "AppDelegate not available", data: nil)
+        }
+        return .ok(payload)
+    }
+
     private func v2SurfaceSendText(params: [String: Any]) -> V2CallResult {
         guard let tabManager = v2ResolveTabManager(params: params) else {
             return .err(code: "unavailable", message: "TabManager not available", data: nil)