From daa340fa87ee9d7ff38d653ffad00a11e7186958 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 24 Feb 2026 22:09:27 -0800 Subject: [PATCH] Harden cmux ssh for mixed-version remote sessions --- CLI/cmux.swift | 12 +++- Sources/Workspace.swift | 64 ++++++++++++++++++++-- daemon/remote/README.md | 7 ++- daemon/remote/cmd/cmuxd-remote/cli.go | 12 +++- daemon/remote/cmd/cmuxd-remote/cli_test.go | 14 +++++ docs/remote-daemon-spec.md | 8 ++- tests_v2/test_ssh_remote_cli_metadata.py | 11 +++- tests_v2/test_ssh_remote_cli_relay.py | 45 ++++++++++----- 8 files changed, 142 insertions(+), 31 deletions(-) diff --git a/CLI/cmux.swift b/CLI/cmux.swift index 1587b5ca..c4e6bcc2 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -2128,9 +2128,19 @@ struct CMUXCLI { parts.append("-tt") } if !hasSSHOptionKey(options.sshOptions, key: "RemoteCommand") { + var startupExports = [ + "export PATH=\"$HOME/.cmux/bin:$PATH\"", + ] + if options.remoteRelayPort > 0 { + // Pin this shell to the relay allocated for this workspace so parallel + // SSH sessions (including from different cmux versions) don't race on + // shared ~/.cmux/socket_addr. + startupExports.append("export CMUX_SOCKET_PATH=127.0.0.1:\(options.remoteRelayPort)") + } + startupExports.append("exec \"${SHELL:-/bin/zsh}\" -l") parts += [ "-o", - "RemoteCommand=export PATH=\"$HOME/.cmux/bin:$PATH\"; exec \"${SHELL:-/bin/zsh}\" -l", + "RemoteCommand=\(startupExports.joined(separator: "; "))", ] } parts.append(options.destination) diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index f995779b..164b64dc 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -494,6 +494,7 @@ private final class WorkspaceRemoteSessionController { guard let self, !self.isStopping else { return } guard self.reverseRelayProcess?.isRunning == true else { return } self.writeRemoteSocketAddrLocked() + self.writeRemoteRelayDaemonMappingLocked() } return true @@ -704,15 +705,46 @@ private final class WorkspaceRemoteSessionController { return try helloRemoteDaemonLocked(remotePath: remotePath) } - /// Creates `cmux` symlinks pointing to the daemon binary. + /// Installs a stable `cmux` wrapper on the remote and updates the default daemon target. + /// The wrapper resolves daemon path using relay-port metadata, allowing multiple local + /// cmux versions to coexist on the same remote host without clobbering each other. /// Tries `/usr/local/bin` first (already in PATH, no rc changes needed), falls back to /// `~/.cmux/bin`. Non-fatal: logs on failure but does not throw. private func createRemoteCLISymlinkLocked(daemonRemotePath: String) { let script = """ - mkdir -p "$HOME/.cmux/bin" - ln -sf "$HOME/\(daemonRemotePath)" "$HOME/.cmux/bin/cmux" - ln -sf "$HOME/\(daemonRemotePath)" /usr/local/bin/cmux 2>/dev/null \ - || sudo -n ln -sf "$HOME/\(daemonRemotePath)" /usr/local/bin/cmux 2>/dev/null \ + mkdir -p "$HOME/.cmux/bin" "$HOME/.cmux/relay" + ln -sf "$HOME/\(daemonRemotePath)" "$HOME/.cmux/bin/cmuxd-remote-current" + cat > "$HOME/.cmux/bin/cmux" <<'CMUX_REMOTE_WRAPPER' + #!/bin/sh + set -eu + + if [ -n "${CMUX_SOCKET_PATH:-}" ]; then + _cmux_port="${CMUX_SOCKET_PATH##*:}" + case "$_cmux_port" in + ''|*[!0-9]*) + ;; + *) + _cmux_map="$HOME/.cmux/relay/${_cmux_port}.daemon_path" + if [ -r "$_cmux_map" ]; then + _cmux_daemon="$(cat "$_cmux_map" 2>/dev/null || true)" + if [ -n "$_cmux_daemon" ] && [ -x "$_cmux_daemon" ]; then + exec "$_cmux_daemon" cli "$@" + fi + fi + ;; + esac + fi + + if [ -x "$HOME/.cmux/bin/cmuxd-remote-current" ]; then + exec "$HOME/.cmux/bin/cmuxd-remote-current" cli "$@" + fi + + echo "cmux: remote daemon not installed; reconnect from local cmux." >&2 + exit 127 + CMUX_REMOTE_WRAPPER + chmod 755 "$HOME/.cmux/bin/cmux" + ln -sf "$HOME/.cmux/bin/cmux" /usr/local/bin/cmux 2>/dev/null \ + || sudo -n ln -sf "$HOME/.cmux/bin/cmux" /usr/local/bin/cmux 2>/dev/null \ || true """ let command = "sh -lc \(Self.shellSingleQuoted(script))" @@ -747,6 +779,28 @@ private final class WorkspaceRemoteSessionController { } } + /// Writes relay-port -> daemon binary mapping used by the remote `cmux` wrapper. + /// This keeps CLI dispatch stable when multiple local cmux versions target the same host. + private func writeRemoteRelayDaemonMappingLocked() { + guard let relayPort = configuration.relayPort, relayPort > 0, + let daemonRemotePath, !daemonRemotePath.isEmpty else { return } + let script = """ + mkdir -p "$HOME/.cmux/relay" && \ + printf '%s' "$HOME/\(daemonRemotePath)" > "$HOME/.cmux/relay/\(relayPort).daemon_path" + """ + let command = "sh -lc \(Self.shellSingleQuoted(script))" + do { + let result = try sshExec(arguments: sshCommonArguments(batchMode: true) + [configuration.destination, command], timeout: 8) + if result.status != 0 { + NSLog("[cmux] warning: failed to write remote relay daemon mapping (exit %d): %@", + result.status, + Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) ?? "unknown error") + } + } catch { + NSLog("[cmux] warning: failed to write remote relay daemon mapping: %@", error.localizedDescription) + } + } + private func resolveRemotePlatformLocked() throws -> RemotePlatform { let script = "uname -s; uname -m" let command = "sh -lc \(Self.shellSingleQuoted(script))" diff --git a/daemon/remote/README.md b/daemon/remote/README.md index f84c75f8..64a94337 100644 --- a/daemon/remote/README.md +++ b/daemon/remote/README.md @@ -8,7 +8,7 @@ Go remote daemon for `cmux ssh` bootstrap, capability negotiation, and CLI relay 2. `cmuxd-remote serve --stdio` 3. `cmuxd-remote cli [args...]` — relay cmux commands to the local app over the reverse TCP forward -When invoked as `cmux` (via symlink created during bootstrap), the binary auto-dispatches to the `cli` subcommand. This is busybox-style argv[0] detection. +When invoked as `cmux` (via wrapper/symlink installed during bootstrap), the binary auto-dispatches to the `cli` subcommand. This is busybox-style argv[0] detection. ## RPC methods (newline-delimited JSON over stdio) @@ -17,7 +17,7 @@ When invoked as `cmux` (via symlink created during bootstrap), the binary auto-d ## CLI relay -The `cli` subcommand (or `cmux` symlink) connects to the local cmux app's socket through an SSH reverse TCP forward and relays commands. It supports both v1 text protocol and v2 JSON-RPC commands. +The `cli` subcommand (or `cmux` wrapper/symlink) connects to the local cmux app's socket through an SSH reverse TCP forward and relays commands. It supports both v1 text protocol and v2 JSON-RPC commands. Socket discovery order: 1. `--socket ` flag @@ -31,5 +31,6 @@ For TCP addresses, the CLI retries for up to 15 seconds on connection refused, r 1. `workspace.remote.configure` bootstraps this binary over SSH when missing. 2. Client sends `hello` before enabling remote port probing/forwarding. 3. Daemon status/capabilities are exposed in `workspace.remote.status -> remote.daemon`. -4. Bootstrap creates `~/.cmux/bin/cmux` symlink pointing to the daemon binary. +4. Bootstrap installs `~/.cmux/bin/cmux` wrapper and keeps a default daemon target (`~/.cmux/bin/cmuxd-remote-current`). 5. A background `ssh -N -R` process reverse-forwards a TCP port to the local cmux Unix socket. The relay address is written to `~/.cmux/socket_addr` on the remote. +6. Relay startup writes `~/.cmux/relay/.daemon_path` so the wrapper can route each shell to the correct daemon binary when multiple local cmux instances/versions coexist. diff --git a/daemon/remote/cmd/cmuxd-remote/cli.go b/daemon/remote/cmd/cmuxd-remote/cli.go index 77654e7d..fad8b4d9 100644 --- a/daemon/remote/cmd/cmuxd-remote/cli.go +++ b/daemon/remote/cmd/cmuxd-remote/cli.go @@ -93,6 +93,9 @@ func runCLI(args []string) int { i++ case "--json": jsonOutput = true + case "--help", "-h": + cliUsage() + return 0 default: remaining = append(remaining, args[i:]...) goto doneFlags @@ -104,6 +107,12 @@ doneFlags: cliUsage() return 2 } + cmdName := remaining[0] + cmdArgs := remaining[1:] + if cmdName == "help" { + cliUsage() + return 0 + } // refreshAddr is set when the address came from socket_addr file (not env/flag), // allowing retry loops to pick up updated relay ports. @@ -117,9 +126,6 @@ doneFlags: return 1 } - cmdName := remaining[0] - cmdArgs := remaining[1:] - // Special case: "rpc" passthrough if cmdName == "rpc" { return runRPC(socketPath, cmdArgs, jsonOutput, refreshAddr) diff --git a/daemon/remote/cmd/cmuxd-remote/cli_test.go b/daemon/remote/cmd/cmuxd-remote/cli_test.go index bfb876b1..924c4e00 100644 --- a/daemon/remote/cmd/cmuxd-remote/cli_test.go +++ b/daemon/remote/cmd/cmuxd-remote/cli_test.go @@ -381,6 +381,20 @@ func TestCLINoArgs(t *testing.T) { } } +func TestCLIHelpFlag(t *testing.T) { + code := runCLI([]string{"--help"}) + if code != 0 { + t.Fatalf("--help should return 0, got %d", code) + } +} + +func TestCLIHelpCommand(t *testing.T) { + code := runCLI([]string{"help"}) + if code != 0 { + t.Fatalf("help should return 0, got %d", code) + } +} + func TestFlagToParamKey(t *testing.T) { tests := []struct { input, expected string diff --git a/docs/remote-daemon-spec.md b/docs/remote-daemon-spec.md index a5e4ebf3..d28cb27a 100644 --- a/docs/remote-daemon-spec.md +++ b/docs/remote-daemon-spec.md @@ -33,15 +33,17 @@ This is a **living implementation spec** (also called an **execution spec**): a - `DONE` local app probes remote platform, builds/uploads `cmuxd-remote`, and runs `serve --stdio`. - `DONE` daemon `hello` handshake is enforced. - `DONE` bootstrap/probe failures surface actionable details. -- `DONE` bootstrap creates `~/.cmux/bin/cmux` symlink (also tries `/usr/local/bin/cmux`) so `cmux` is available in PATH on the remote. +- `DONE` bootstrap installs `~/.cmux/bin/cmux` wrapper (also tries `/usr/local/bin/cmux`) so `cmux` is available in PATH on the remote. ### 3.5 CLI Relay (Running cmux Commands From Remote) - `DONE` `cmuxd-remote` includes a table-driven CLI relay (`cli` subcommand) that maps CLI args to v1 text or v2 JSON-RPC messages. -- `DONE` busybox-style argv[0] detection: when invoked as `cmux` via symlink, auto-dispatches to CLI relay. +- `DONE` busybox-style argv[0] detection: when invoked as `cmux` via wrapper/symlink, auto-dispatches to CLI relay. - `DONE` background `ssh -N -R 127.0.0.1:PORT:/local/cmux.sock` process reverse-forwards a TCP port to the local cmux socket. Uses TCP instead of Unix socket forwarding because many servers have `AllowStreamLocalForwarding` disabled. - `DONE` relay process uses `ControlPath=none` (avoids ControlMaster multiplexing and inherited `RemoteForward` directives) and `ExitOnForwardFailure=no` (inherited forwards from user ssh config failing should not kill the relay). - `DONE` relay address written to `~/.cmux/socket_addr` on the remote with a 3s delay after the relay process starts, giving SSH time to establish the `-R` forward. - `DONE` Go CLI re-reads `~/.cmux/socket_addr` on each TCP retry to pick up updated relay ports when multiple workspaces overwrite the file. +- `DONE` `cmux ssh` startup exports session-local `CMUX_SOCKET_PATH=127.0.0.1:` so parallel sessions pin to their own relay instead of racing on shared socket_addr. +- `DONE` relay startup writes `~/.cmux/relay/.daemon_path`; remote `cmux` wrapper uses this to select the right daemon binary per session, including mixed local cmux versions. - `DONE` ephemeral port range (49152-65535) filtered from probe results to exclude relay ports from other workspaces. - `DONE` multi-workspace port conflict detection uses TCP connect check (`isLoopbackPortReachable`) so ports already forwarded by another workspace are silently skipped instead of flagged as conflicts. - `DONE` orphaned relay SSH processes from previous app sessions are cleaned up before starting a new relay. @@ -112,7 +114,7 @@ Recompute effective size on: | M-002 | Remote bootstrap/upload/start + hello handshake | DONE | Current `cmuxd-remote` is minimal (`hello`, `ping`) | | M-003 | Reconnect/disconnect UX + API + improved error surfacing | DONE | Includes retry count in surfaced errors | | M-004 | Docker e2e for bootstrap/reconnect shell niceties | DONE | Existing docker tests currently validate mirroring-era path | -| M-004b | CLI relay: run cmux commands from within SSH sessions | DONE | Reverse TCP forward + Go CLI relay + bootstrap symlink (PR #374) | +| M-004b | CLI relay: run cmux commands from within SSH sessions | DONE | Reverse TCP forward + Go CLI relay + bootstrap wrapper (PR #374) | | M-005 | Remove automatic remote port mirroring path | TODO | Delete probe/listen mirror loop from `WorkspaceRemoteSessionController` | | M-006 | Transport-scoped local proxy broker (SOCKS5 + CONNECT) | TODO | Local component in app/daemon layer | | M-007 | Remote proxy stream RPC in `cmuxd-remote` | TODO | Add `proxy.open/close` and multiplexed stream handling | diff --git a/tests_v2/test_ssh_remote_cli_metadata.py b/tests_v2/test_ssh_remote_cli_metadata.py index 5ff706de..2d0ad892 100644 --- a/tests_v2/test_ssh_remote_cli_metadata.py +++ b/tests_v2/test_ssh_remote_cli_metadata.py @@ -95,6 +95,9 @@ def main() -> int: workspace_id = str(row.get("id") or "") break _must(bool(workspace_id), f"cmux ssh output missing workspace_id: {payload}") + remote_relay_port = payload.get("remote_relay_port") + _must(remote_relay_port is not None, f"cmux ssh output missing remote_relay_port: {payload}") + remote_socket_addr = f"127.0.0.1:{int(remote_relay_port)}" ssh_command = str(payload.get("ssh_command") or "") _must(bool(ssh_command), f"cmux ssh output missing ssh_command: {payload}") _must( @@ -117,8 +120,12 @@ def main() -> int: _must("-o ControlPersist=600" in ssh_command, f"ssh command should keep master alive for reuse: {ssh_command!r}") _must("ControlPath=/tmp/cmux-ssh-" in ssh_command, f"ssh command should use shared control path template: {ssh_command!r}") _must( - "RemoteCommand=export PATH=\"$HOME/.cmux/bin:$PATH\"; exec \"${SHELL:-/bin/zsh}\" -l" in ssh_command, - f"cmux ssh should use -o RemoteCommand for PATH bootstrap (not positional command): {ssh_command!r}", + ( + f"RemoteCommand=export PATH=\"$HOME/.cmux/bin:$PATH\"; " + f"export CMUX_SOCKET_PATH={remote_socket_addr}; " + "exec \"${SHELL:-/bin/zsh}\" -l" + ) in ssh_command, + f"cmux ssh should use -o RemoteCommand for PATH/bootstrap env pinning (not positional command): {ssh_command!r}", ) listed_row = None diff --git a/tests_v2/test_ssh_remote_cli_relay.py b/tests_v2/test_ssh_remote_cli_relay.py index 89853d15..53e01a95 100644 --- a/tests_v2/test_ssh_remote_cli_relay.py +++ b/tests_v2/test_ssh_remote_cli_relay.py @@ -204,11 +204,20 @@ def main() -> int: workspace_id = str(row.get("id") or "") break _must(bool(workspace_id), f"cmux ssh output missing workspace_id: {payload}") + remote_relay_port = payload.get("remote_relay_port") + _must(remote_relay_port is not None, f"cmux ssh output missing remote_relay_port: {payload}") + remote_relay_port = int(remote_relay_port) + _must(49152 <= remote_relay_port <= 65535, f"remote_relay_port should be in ephemeral range: {remote_relay_port}") + remote_socket_addr = f"127.0.0.1:{remote_relay_port}" startup_cmd = str(payload.get("ssh_startup_command") or "") _must( 'PATH="$HOME/.cmux/bin:$PATH"' in startup_cmd, f"ssh startup command should prepend ~/.cmux/bin for remote cmux CLI: {startup_cmd!r}", ) + _must( + f"CMUX_SOCKET_PATH={remote_socket_addr}" in startup_cmd, + f"ssh startup command should pin CMUX_SOCKET_PATH to workspace relay: {startup_cmd!r}", + ) workspace_window_id = payload.get("window_id") current_params = {"window_id": workspace_window_id} if isinstance(workspace_window_id, str) and workspace_window_id else {} current = client._call("workspace.current", current_params) or {} @@ -218,12 +227,6 @@ def main() -> int: f"cmux ssh should focus created workspace: current={current_workspace_id!r} created={workspace_id!r}", ) - remote_relay_port = payload.get("remote_relay_port") - _must(remote_relay_port is not None, f"cmux ssh output missing remote_relay_port: {payload}") - remote_relay_port = int(remote_relay_port) - _must(49152 <= remote_relay_port <= 65535, f"remote_relay_port should be in ephemeral range: {remote_relay_port}") - remote_socket_addr = f"127.0.0.1:{remote_relay_port}" - # Wait for daemon to be ready first_status = _wait_for_remote_ready(client, workspace_id) first_remote = first_status.get("remote") or {} @@ -238,15 +241,24 @@ def main() -> int: f"expected no forwarded ports when none are eligible: {first_status}", ) - # Verify the cmux symlink exists on the remote - symlink_check = _ssh_run( - host, host_ssh_port, key_path, - "test -L \"$HOME/.cmux/bin/cmux\" && echo symlink-ok", - check=False, - ) + # Verify remote cmux wrapper + relay-specific daemon mapping were installed. + wrapper_check = None + wrapper_deadline = time.time() + 10.0 + while time.time() < wrapper_deadline: + wrapper_check = _ssh_run( + host, host_ssh_port, key_path, + f"test -x \"$HOME/.cmux/bin/cmux\" && test -f \"$HOME/.cmux/bin/cmux\" && " + f"map=\"$HOME/.cmux/relay/{remote_relay_port}.daemon_path\" && " + "daemon=\"$(cat \"$map\" 2>/dev/null || true)\" && " + "test -n \"$daemon\" && test -x \"$daemon\" && echo wrapper-ok", + check=False, + ) + if "wrapper-ok" in (wrapper_check.stdout or ""): + break + time.sleep(0.4) _must( - "symlink-ok" in symlink_check.stdout, - f"Expected cmux symlink at ~/.cmux/bin/cmux on remote: {symlink_check.stdout} {symlink_check.stderr}", + wrapper_check is not None and "wrapper-ok" in (wrapper_check.stdout or ""), + f"Expected remote cmux wrapper+relay mapping to exist: {wrapper_check.stdout if wrapper_check else ''} {wrapper_check.stderr if wrapper_check else ''}", ) # Start a second SSH workspace to the same destination and verify both @@ -282,6 +294,11 @@ def main() -> int: f"relay ports should differ per workspace: {remote_relay_port_2} vs {remote_relay_port}", ) remote_socket_addr_2 = f"127.0.0.1:{remote_relay_port_2}" + startup_cmd_2 = str(payload_2.get("ssh_startup_command") or "") + _must( + f"CMUX_SOCKET_PATH={remote_socket_addr_2}" in startup_cmd_2, + f"second ssh startup command should pin CMUX_SOCKET_PATH to second relay: {startup_cmd_2!r}", + ) _ = _wait_for_remote_ready(client, workspace_id_2) stability_deadline = time.time() + 8.0