Harden cmux ssh for mixed-version remote sessions

CLI/cmux.swift

@@ -2128,9 +2128,19 @@ struct CMUXCLI {
                 parts.append("-tt")
             }
             if !hasSSHOptionKey(options.sshOptions, key: "RemoteCommand") {
+                var startupExports = [
+                    "export PATH=\"$HOME/.cmux/bin:$PATH\"",
+                ]
+                if options.remoteRelayPort > 0 {
+                    // Pin this shell to the relay allocated for this workspace so parallel
+                    // SSH sessions (including from different cmux versions) don't race on
+                    // shared ~/.cmux/socket_addr.
+                    startupExports.append("export CMUX_SOCKET_PATH=127.0.0.1:\(options.remoteRelayPort)")
+                }
+                startupExports.append("exec \"${SHELL:-/bin/zsh}\" -l")
                 parts += [
                     "-o",
-                    "RemoteCommand=export PATH=\"$HOME/.cmux/bin:$PATH\"; exec \"${SHELL:-/bin/zsh}\" -l",
+                    "RemoteCommand=\(startupExports.joined(separator: "; "))",
                 ]
             }
             parts.append(options.destination)

Sources/Workspace.swift

@@ -494,6 +494,7 @@ private final class WorkspaceRemoteSessionController {
                 guard let self, !self.isStopping else { return }
                 guard self.reverseRelayProcess?.isRunning == true else { return }
                 self.writeRemoteSocketAddrLocked()
+                self.writeRemoteRelayDaemonMappingLocked()
             }
 
             return true
@@ -704,15 +705,46 @@ private final class WorkspaceRemoteSessionController {
         return try helloRemoteDaemonLocked(remotePath: remotePath)
     }
 
-    /// Creates `cmux` symlinks pointing to the daemon binary.
+    /// Installs a stable `cmux` wrapper on the remote and updates the default daemon target.
+    /// The wrapper resolves daemon path using relay-port metadata, allowing multiple local
+    /// cmux versions to coexist on the same remote host without clobbering each other.
     /// Tries `/usr/local/bin` first (already in PATH, no rc changes needed), falls back to
     /// `~/.cmux/bin`. Non-fatal: logs on failure but does not throw.
     private func createRemoteCLISymlinkLocked(daemonRemotePath: String) {
         let script = """
-        mkdir -p "$HOME/.cmux/bin"
+        mkdir -p "$HOME/.cmux/bin" "$HOME/.cmux/relay"
-        ln -sf "$HOME/\(daemonRemotePath)" "$HOME/.cmux/bin/cmux"
+        ln -sf "$HOME/\(daemonRemotePath)" "$HOME/.cmux/bin/cmuxd-remote-current"
-        ln -sf "$HOME/\(daemonRemotePath)" /usr/local/bin/cmux 2>/dev/null \
+        cat > "$HOME/.cmux/bin/cmux" <<'CMUX_REMOTE_WRAPPER'
-          || sudo -n ln -sf "$HOME/\(daemonRemotePath)" /usr/local/bin/cmux 2>/dev/null \
+        #!/bin/sh
+        set -eu
+
+        if [ -n "${CMUX_SOCKET_PATH:-}" ]; then
+          _cmux_port="${CMUX_SOCKET_PATH##*:}"
+          case "$_cmux_port" in
+            ''|*[!0-9]*)
+              ;;
+            *)
+              _cmux_map="$HOME/.cmux/relay/${_cmux_port}.daemon_path"
+              if [ -r "$_cmux_map" ]; then
+                _cmux_daemon="$(cat "$_cmux_map" 2>/dev/null || true)"
+                if [ -n "$_cmux_daemon" ] && [ -x "$_cmux_daemon" ]; then
+                  exec "$_cmux_daemon" cli "$@"
+                fi
+              fi
+              ;;
+          esac
+        fi
+
+        if [ -x "$HOME/.cmux/bin/cmuxd-remote-current" ]; then
+          exec "$HOME/.cmux/bin/cmuxd-remote-current" cli "$@"
+        fi
+
+        echo "cmux: remote daemon not installed; reconnect from local cmux." >&2
+        exit 127
+        CMUX_REMOTE_WRAPPER
+        chmod 755 "$HOME/.cmux/bin/cmux"
+        ln -sf "$HOME/.cmux/bin/cmux" /usr/local/bin/cmux 2>/dev/null \
+          || sudo -n ln -sf "$HOME/.cmux/bin/cmux" /usr/local/bin/cmux 2>/dev/null \
           || true
         """
         let command = "sh -lc \(Self.shellSingleQuoted(script))"
@@ -747,6 +779,28 @@ private final class WorkspaceRemoteSessionController {
         }
     }
 
+    /// Writes relay-port -> daemon binary mapping used by the remote `cmux` wrapper.
+    /// This keeps CLI dispatch stable when multiple local cmux versions target the same host.
+    private func writeRemoteRelayDaemonMappingLocked() {
+        guard let relayPort = configuration.relayPort, relayPort > 0,
+              let daemonRemotePath, !daemonRemotePath.isEmpty else { return }
+        let script = """
+        mkdir -p "$HOME/.cmux/relay" && \
+        printf '%s' "$HOME/\(daemonRemotePath)" > "$HOME/.cmux/relay/\(relayPort).daemon_path"
+        """
+        let command = "sh -lc \(Self.shellSingleQuoted(script))"
+        do {
+            let result = try sshExec(arguments: sshCommonArguments(batchMode: true) + [configuration.destination, command], timeout: 8)
+            if result.status != 0 {
+                NSLog("[cmux] warning: failed to write remote relay daemon mapping (exit %d): %@",
+                      result.status,
+                      Self.bestErrorLine(stderr: result.stderr, stdout: result.stdout) ?? "unknown error")
+            }
+        } catch {
+            NSLog("[cmux] warning: failed to write remote relay daemon mapping: %@", error.localizedDescription)
+        }
+    }
+
     private func resolveRemotePlatformLocked() throws -> RemotePlatform {
         let script = "uname -s; uname -m"
         let command = "sh -lc \(Self.shellSingleQuoted(script))"

daemon/remote/README.md

@@ -8,7 +8,7 @@ Go remote daemon for `cmux ssh` bootstrap, capability negotiation, and CLI relay
 2. `cmuxd-remote serve --stdio`
 3. `cmuxd-remote cli <command> [args...]` — relay cmux commands to the local app over the reverse TCP forward
 
-When invoked as `cmux` (via symlink created during bootstrap), the binary auto-dispatches to the `cli` subcommand. This is busybox-style argv[0] detection.
+When invoked as `cmux` (via wrapper/symlink installed during bootstrap), the binary auto-dispatches to the `cli` subcommand. This is busybox-style argv[0] detection.
 
 ## RPC methods (newline-delimited JSON over stdio)
 
@@ -17,7 +17,7 @@ When invoked as `cmux` (via symlink created during bootstrap), the binary auto-d
 
 ## CLI relay
 
-The `cli` subcommand (or `cmux` symlink) connects to the local cmux app's socket through an SSH reverse TCP forward and relays commands. It supports both v1 text protocol and v2 JSON-RPC commands.
+The `cli` subcommand (or `cmux` wrapper/symlink) connects to the local cmux app's socket through an SSH reverse TCP forward and relays commands. It supports both v1 text protocol and v2 JSON-RPC commands.
 
 Socket discovery order:
 1. `--socket <path>` flag
@@ -31,5 +31,6 @@ For TCP addresses, the CLI retries for up to 15 seconds on connection refused, r
 1. `workspace.remote.configure` bootstraps this binary over SSH when missing.
 2. Client sends `hello` before enabling remote port probing/forwarding.
 3. Daemon status/capabilities are exposed in `workspace.remote.status -> remote.daemon`.
-4. Bootstrap creates `~/.cmux/bin/cmux` symlink pointing to the daemon binary.
+4. Bootstrap installs `~/.cmux/bin/cmux` wrapper and keeps a default daemon target (`~/.cmux/bin/cmuxd-remote-current`).
 5. A background `ssh -N -R` process reverse-forwards a TCP port to the local cmux Unix socket. The relay address is written to `~/.cmux/socket_addr` on the remote.
+6. Relay startup writes `~/.cmux/relay/<port>.daemon_path` so the wrapper can route each shell to the correct daemon binary when multiple local cmux instances/versions coexist.

daemon/remote/cmd/cmuxd-remote/cli.go

@@ -93,6 +93,9 @@ func runCLI(args []string) int {
 			i++
 		case "--json":
 			jsonOutput = true
+		case "--help", "-h":
+			cliUsage()
+			return 0
 		default:
 			remaining = append(remaining, args[i:]...)
 			goto doneFlags
@@ -104,6 +107,12 @@ doneFlags:
 		cliUsage()
 		return 2
 	}
+	cmdName := remaining[0]
+	cmdArgs := remaining[1:]
+	if cmdName == "help" {
+		cliUsage()
+		return 0
+	}
 
 	// refreshAddr is set when the address came from socket_addr file (not env/flag),
 	// allowing retry loops to pick up updated relay ports.
@@ -117,9 +126,6 @@ doneFlags:
 		return 1
 	}
 
-	cmdName := remaining[0]
-	cmdArgs := remaining[1:]
-
 	// Special case: "rpc" passthrough
 	if cmdName == "rpc" {
 		return runRPC(socketPath, cmdArgs, jsonOutput, refreshAddr)

daemon/remote/cmd/cmuxd-remote/cli_test.go

@@ -381,6 +381,20 @@ func TestCLINoArgs(t *testing.T) {
 	}
 }
 
+func TestCLIHelpFlag(t *testing.T) {
+	code := runCLI([]string{"--help"})
+	if code != 0 {
+		t.Fatalf("--help should return 0, got %d", code)
+	}
+}
+
+func TestCLIHelpCommand(t *testing.T) {
+	code := runCLI([]string{"help"})
+	if code != 0 {
+		t.Fatalf("help should return 0, got %d", code)
+	}
+}
+
 func TestFlagToParamKey(t *testing.T) {
 	tests := []struct {
 		input, expected string

docs/remote-daemon-spec.md

@@ -33,15 +33,17 @@ This is a **living implementation spec** (also called an **execution spec**): a
 - `DONE` local app probes remote platform, builds/uploads `cmuxd-remote`, and runs `serve --stdio`.
 - `DONE` daemon `hello` handshake is enforced.
 - `DONE` bootstrap/probe failures surface actionable details.
-- `DONE` bootstrap creates `~/.cmux/bin/cmux` symlink (also tries `/usr/local/bin/cmux`) so `cmux` is available in PATH on the remote.
+- `DONE` bootstrap installs `~/.cmux/bin/cmux` wrapper (also tries `/usr/local/bin/cmux`) so `cmux` is available in PATH on the remote.
 
 ### 3.5 CLI Relay (Running cmux Commands From Remote)
 - `DONE` `cmuxd-remote` includes a table-driven CLI relay (`cli` subcommand) that maps CLI args to v1 text or v2 JSON-RPC messages.
-- `DONE` busybox-style argv[0] detection: when invoked as `cmux` via symlink, auto-dispatches to CLI relay.
+- `DONE` busybox-style argv[0] detection: when invoked as `cmux` via wrapper/symlink, auto-dispatches to CLI relay.
 - `DONE` background `ssh -N -R 127.0.0.1:PORT:/local/cmux.sock` process reverse-forwards a TCP port to the local cmux socket. Uses TCP instead of Unix socket forwarding because many servers have `AllowStreamLocalForwarding` disabled.
 - `DONE` relay process uses `ControlPath=none` (avoids ControlMaster multiplexing and inherited `RemoteForward` directives) and `ExitOnForwardFailure=no` (inherited forwards from user ssh config failing should not kill the relay).
 - `DONE` relay address written to `~/.cmux/socket_addr` on the remote with a 3s delay after the relay process starts, giving SSH time to establish the `-R` forward.
 - `DONE` Go CLI re-reads `~/.cmux/socket_addr` on each TCP retry to pick up updated relay ports when multiple workspaces overwrite the file.
+- `DONE` `cmux ssh` startup exports session-local `CMUX_SOCKET_PATH=127.0.0.1:<relay_port>` so parallel sessions pin to their own relay instead of racing on shared socket_addr.
+- `DONE` relay startup writes `~/.cmux/relay/<relay_port>.daemon_path`; remote `cmux` wrapper uses this to select the right daemon binary per session, including mixed local cmux versions.
 - `DONE` ephemeral port range (49152-65535) filtered from probe results to exclude relay ports from other workspaces.
 - `DONE` multi-workspace port conflict detection uses TCP connect check (`isLoopbackPortReachable`) so ports already forwarded by another workspace are silently skipped instead of flagged as conflicts.
 - `DONE` orphaned relay SSH processes from previous app sessions are cleaned up before starting a new relay.
@@ -112,7 +114,7 @@ Recompute effective size on:
 | M-002 | Remote bootstrap/upload/start + hello handshake | DONE | Current `cmuxd-remote` is minimal (`hello`, `ping`) |
 | M-003 | Reconnect/disconnect UX + API + improved error surfacing | DONE | Includes retry count in surfaced errors |
 | M-004 | Docker e2e for bootstrap/reconnect shell niceties | DONE | Existing docker tests currently validate mirroring-era path |
-| M-004b | CLI relay: run cmux commands from within SSH sessions | DONE | Reverse TCP forward + Go CLI relay + bootstrap symlink (PR #374) |
+| M-004b | CLI relay: run cmux commands from within SSH sessions | DONE | Reverse TCP forward + Go CLI relay + bootstrap wrapper (PR #374) |
 | M-005 | Remove automatic remote port mirroring path | TODO | Delete probe/listen mirror loop from `WorkspaceRemoteSessionController` |
 | M-006 | Transport-scoped local proxy broker (SOCKS5 + CONNECT) | TODO | Local component in app/daemon layer |
 | M-007 | Remote proxy stream RPC in `cmuxd-remote` | TODO | Add `proxy.open/close` and multiplexed stream handling |

tests_v2/test_ssh_remote_cli_metadata.py

@@ -95,6 +95,9 @@ def main() -> int:
                         workspace_id = str(row.get("id") or "")
                         break
             _must(bool(workspace_id), f"cmux ssh output missing workspace_id: {payload}")
+            remote_relay_port = payload.get("remote_relay_port")
+            _must(remote_relay_port is not None, f"cmux ssh output missing remote_relay_port: {payload}")
+            remote_socket_addr = f"127.0.0.1:{int(remote_relay_port)}"
             ssh_command = str(payload.get("ssh_command") or "")
             _must(bool(ssh_command), f"cmux ssh output missing ssh_command: {payload}")
             _must(
@@ -117,8 +120,12 @@ def main() -> int:
             _must("-o ControlPersist=600" in ssh_command, f"ssh command should keep master alive for reuse: {ssh_command!r}")
             _must("ControlPath=/tmp/cmux-ssh-" in ssh_command, f"ssh command should use shared control path template: {ssh_command!r}")
             _must(
-                "RemoteCommand=export PATH=\"$HOME/.cmux/bin:$PATH\"; exec \"${SHELL:-/bin/zsh}\" -l" in ssh_command,
+                (
-                f"cmux ssh should use -o RemoteCommand for PATH bootstrap (not positional command): {ssh_command!r}",
+                    f"RemoteCommand=export PATH=\"$HOME/.cmux/bin:$PATH\"; "
+                    f"export CMUX_SOCKET_PATH={remote_socket_addr}; "
+                    "exec \"${SHELL:-/bin/zsh}\" -l"
+                ) in ssh_command,
+                f"cmux ssh should use -o RemoteCommand for PATH/bootstrap env pinning (not positional command): {ssh_command!r}",
             )
 
             listed_row = None

tests_v2/test_ssh_remote_cli_relay.py

@@ -204,11 +204,20 @@ def main() -> int:
                         workspace_id = str(row.get("id") or "")
                         break
             _must(bool(workspace_id), f"cmux ssh output missing workspace_id: {payload}")
+            remote_relay_port = payload.get("remote_relay_port")
+            _must(remote_relay_port is not None, f"cmux ssh output missing remote_relay_port: {payload}")
+            remote_relay_port = int(remote_relay_port)
+            _must(49152 <= remote_relay_port <= 65535, f"remote_relay_port should be in ephemeral range: {remote_relay_port}")
+            remote_socket_addr = f"127.0.0.1:{remote_relay_port}"
             startup_cmd = str(payload.get("ssh_startup_command") or "")
             _must(
                 'PATH="$HOME/.cmux/bin:$PATH"' in startup_cmd,
                 f"ssh startup command should prepend ~/.cmux/bin for remote cmux CLI: {startup_cmd!r}",
             )
+            _must(
+                f"CMUX_SOCKET_PATH={remote_socket_addr}" in startup_cmd,
+                f"ssh startup command should pin CMUX_SOCKET_PATH to workspace relay: {startup_cmd!r}",
+            )
             workspace_window_id = payload.get("window_id")
             current_params = {"window_id": workspace_window_id} if isinstance(workspace_window_id, str) and workspace_window_id else {}
             current = client._call("workspace.current", current_params) or {}
@@ -218,12 +227,6 @@ def main() -> int:
                 f"cmux ssh should focus created workspace: current={current_workspace_id!r} created={workspace_id!r}",
             )
 
-            remote_relay_port = payload.get("remote_relay_port")
-            _must(remote_relay_port is not None, f"cmux ssh output missing remote_relay_port: {payload}")
-            remote_relay_port = int(remote_relay_port)
-            _must(49152 <= remote_relay_port <= 65535, f"remote_relay_port should be in ephemeral range: {remote_relay_port}")
-            remote_socket_addr = f"127.0.0.1:{remote_relay_port}"
-
             # Wait for daemon to be ready
             first_status = _wait_for_remote_ready(client, workspace_id)
             first_remote = first_status.get("remote") or {}
@@ -238,15 +241,24 @@ def main() -> int:
                 f"expected no forwarded ports when none are eligible: {first_status}",
             )
 
-            # Verify the cmux symlink exists on the remote
+            # Verify remote cmux wrapper + relay-specific daemon mapping were installed.
-            symlink_check = _ssh_run(
+            wrapper_check = None
-                host, host_ssh_port, key_path,
+            wrapper_deadline = time.time() + 10.0
-                "test -L \"$HOME/.cmux/bin/cmux\" && echo symlink-ok",
+            while time.time() < wrapper_deadline:
-                check=False,
+                wrapper_check = _ssh_run(
-            )
+                    host, host_ssh_port, key_path,
+                    f"test -x \"$HOME/.cmux/bin/cmux\" && test -f \"$HOME/.cmux/bin/cmux\" && "
+                    f"map=\"$HOME/.cmux/relay/{remote_relay_port}.daemon_path\" && "
+                    "daemon=\"$(cat \"$map\" 2>/dev/null || true)\" && "
+                    "test -n \"$daemon\" && test -x \"$daemon\" && echo wrapper-ok",
+                    check=False,
+                )
+                if "wrapper-ok" in (wrapper_check.stdout or ""):
+                    break
+                time.sleep(0.4)
             _must(
-                "symlink-ok" in symlink_check.stdout,
+                wrapper_check is not None and "wrapper-ok" in (wrapper_check.stdout or ""),
-                f"Expected cmux symlink at ~/.cmux/bin/cmux on remote: {symlink_check.stdout} {symlink_check.stderr}",
+                f"Expected remote cmux wrapper+relay mapping to exist: {wrapper_check.stdout if wrapper_check else ''} {wrapper_check.stderr if wrapper_check else ''}",
             )
 
             # Start a second SSH workspace to the same destination and verify both
@@ -282,6 +294,11 @@ def main() -> int:
                 f"relay ports should differ per workspace: {remote_relay_port_2} vs {remote_relay_port}",
             )
             remote_socket_addr_2 = f"127.0.0.1:{remote_relay_port_2}"
+            startup_cmd_2 = str(payload_2.get("ssh_startup_command") or "")
+            _must(
+                f"CMUX_SOCKET_PATH={remote_socket_addr_2}" in startup_cmd_2,
+                f"second ssh startup command should pin CMUX_SOCKET_PATH to second relay: {startup_cmd_2!r}",
+            )
             _ = _wait_for_remote_ready(client, workspace_id_2)
 
             stability_deadline = time.time() + 8.0