From ccd84bd5789b092275bd2f85f201fc4e945a4988 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:24:37 -0700 Subject: [PATCH] Fix nightly SSH remote daemon checksum mismatch (#2225) * Fix nightly SSH remote daemon checksum mismatch Each nightly build overwrites the shared cmuxd-remote-* assets on the nightly release, but older nightly DMGs have manifests with checksums from their build time. When a user's nightly is even one build behind, the downloaded binary doesn't match their embedded manifest. Two-layer fix: 1. CI: version nightly remote daemon asset names with the build number (e.g. cmuxd-remote-darwin-arm64-2362248028801) so each nightly's manifest points to immutable files. Unsuffixed "latest" copies are still uploaded for tooling compatibility. 2. Client: on checksum mismatch, fetch the live manifest from the release and verify against that. This handles users on older nightlies that predate the CI fix. Fixes https://github.com/manaflow-ai/cmux/issues/1745 * Fix unsuffixed checksums file to use generic filenames Regenerate cmuxd-remote-checksums.txt from the unsuffixed alias binaries so `shasum -c` works against the generic asset names. Also document that unsuffixed manifest intentionally keeps versioned downloadURLs and that aliases don't carry attestation. --------- Co-authored-by: Lawrence Chen --- .github/workflows/nightly.yml | 59 ++++++++++++------- Sources/Workspace.swift | 45 ++++++++++++-- scripts/build_remote_daemon_release_assets.sh | 35 ++++++++--- tests/test_remote_daemon_release_assets.sh | 43 ++++++++++++++ 4 files changed, 148 insertions(+), 34 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 92ef8ece..1f277a2a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -317,12 +317,17 @@ jobs: if: needs.decide.outputs.should_publish != 'true' || (steps.current_head_prebuild.outputs.still_current == 'true' && steps.current_head_postbuild.outputs.still_current == 'true') run: | set -euo pipefail + # Build with --asset-suffix so manifest download URLs point to + # immutable, build-specific asset names (e.g. cmuxd-remote-darwin-arm64-2362248028801). + # This prevents checksum mismatches when a newer nightly overwrites + # the shared "latest" assets on the release. ./scripts/build_remote_daemon_release_assets.sh \ --version "$NIGHTLY_REMOTE_DAEMON_VERSION" \ --release-tag "nightly" \ --repo "manaflow-ai/cmux" \ - --output-dir "remote-daemon-assets" - MANIFEST_JSON="$(python3 -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1], encoding="utf-8")), separators=(",",":")))' remote-daemon-assets/cmuxd-remote-manifest.json)" + --output-dir "remote-daemon-assets" \ + --asset-suffix "$NIGHTLY_BUILD" + MANIFEST_JSON="$(python3 -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1], encoding="utf-8")), separators=(",",":")))' "remote-daemon-assets/cmuxd-remote-manifest-${NIGHTLY_BUILD}.json")" APP_PLIST="build-universal/Build/Products/Release/cmux NIGHTLY.app/Contents/Info.plist" if [ ! -f "$APP_PLIST" ]; then echo "Missing nightly app Info.plist at $APP_PLIST" >&2 @@ -330,6 +335,30 @@ jobs: fi plutil -remove CMUXRemoteDaemonManifestJSON "$APP_PLIST" >/dev/null 2>&1 || true plutil -insert CMUXRemoteDaemonManifestJSON -string "$MANIFEST_JSON" "$APP_PLIST" + # Also create unsuffixed "latest" copies for the release page and + # any tooling that fetches the generic asset names. The manifest's + # downloadURLs still point to the versioned filenames (intentional: + # the live manifest is used by the client-side checksum fallback + # which only reads sha256, not downloadURL). The unsuffixed copies + # are convenience aliases and don't carry build-provenance + # attestation (attested versioned files are canonical). + for platform in darwin-arm64 darwin-amd64 linux-arm64 linux-amd64; do + cp "remote-daemon-assets/cmuxd-remote-${platform}-${NIGHTLY_BUILD}" \ + "remote-daemon-assets/cmuxd-remote-${platform}" + done + # Regenerate unsuffixed checksums with generic filenames so + # `shasum -c cmuxd-remote-checksums.txt` works against the aliases. + ( + cd remote-daemon-assets + shasum -a 256 \ + cmuxd-remote-darwin-arm64 \ + cmuxd-remote-darwin-amd64 \ + cmuxd-remote-linux-arm64 \ + cmuxd-remote-linux-amd64 \ + > cmuxd-remote-checksums.txt + ) + cp "remote-daemon-assets/cmuxd-remote-manifest-${NIGHTLY_BUILD}.json" \ + "remote-daemon-assets/cmuxd-remote-manifest.json" - name: Import signing cert if: needs.decide.outputs.should_publish != 'true' || (steps.current_head_prebuild.outputs.still_current == 'true' && steps.current_head_postbuild.outputs.still_current == 'true') @@ -479,12 +508,12 @@ jobs: uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 with: subject-path: | - remote-daemon-assets/cmuxd-remote-darwin-arm64 - remote-daemon-assets/cmuxd-remote-darwin-amd64 - remote-daemon-assets/cmuxd-remote-linux-arm64 - remote-daemon-assets/cmuxd-remote-linux-amd64 - remote-daemon-assets/cmuxd-remote-checksums.txt - remote-daemon-assets/cmuxd-remote-manifest.json + remote-daemon-assets/cmuxd-remote-darwin-arm64-${{ env.NIGHTLY_BUILD }} + remote-daemon-assets/cmuxd-remote-darwin-amd64-${{ env.NIGHTLY_BUILD }} + remote-daemon-assets/cmuxd-remote-linux-arm64-${{ env.NIGHTLY_BUILD }} + remote-daemon-assets/cmuxd-remote-linux-amd64-${{ env.NIGHTLY_BUILD }} + remote-daemon-assets/cmuxd-remote-checksums-${{ env.NIGHTLY_BUILD }}.txt + remote-daemon-assets/cmuxd-remote-manifest-${{ env.NIGHTLY_BUILD }}.json - name: Upload branch nightly artifacts if: needs.decide.outputs.should_publish != 'true' @@ -494,12 +523,7 @@ jobs: path: | cmux-nightly-macos*.dmg appcast.xml - remote-daemon-assets/cmuxd-remote-darwin-arm64 - remote-daemon-assets/cmuxd-remote-darwin-amd64 - remote-daemon-assets/cmuxd-remote-linux-arm64 - remote-daemon-assets/cmuxd-remote-linux-amd64 - remote-daemon-assets/cmuxd-remote-checksums.txt - remote-daemon-assets/cmuxd-remote-manifest.json + remote-daemon-assets/cmuxd-remote-* appcast-universal.xml if-no-files-found: error @@ -533,12 +557,7 @@ jobs: cmux-nightly-macos-${{ github.run_id }}*.dmg cmux-nightly-macos.dmg appcast.xml - remote-daemon-assets/cmuxd-remote-darwin-arm64 - remote-daemon-assets/cmuxd-remote-darwin-amd64 - remote-daemon-assets/cmuxd-remote-linux-arm64 - remote-daemon-assets/cmuxd-remote-linux-amd64 - remote-daemon-assets/cmuxd-remote-checksums.txt - remote-daemon-assets/cmuxd-remote-manifest.json + remote-daemon-assets/cmuxd-remote-* appcast-universal.xml overwrite_files: true diff --git a/Sources/Workspace.swift b/Sources/Workspace.swift index 77920681..ab1b9c37 100644 --- a/Sources/Workspace.swift +++ b/Sources/Workspace.swift @@ -4146,7 +4146,29 @@ final class WorkspaceRemoteSessionController { .appendingPathComponent("cmuxd-remote", isDirectory: false) } - private func downloadRemoteDaemonBinaryLocked(entry: WorkspaceRemoteDaemonManifest.Entry, version: String) throws -> URL { + /// Fetch the live manifest JSON from the release, returning nil on any failure. + private static func fetchRemoteManifestLocked(releaseURL: String, version: String) -> WorkspaceRemoteDaemonManifest? { + guard let manifestURL = URL(string: "\(releaseURL)/cmuxd-remote-manifest.json") else { return nil } + let request = NSMutableURLRequest(url: manifestURL) + request.timeoutInterval = 15 + request.setValue("cmux/\(version)", forHTTPHeaderField: "User-Agent") + let session = URLSession(configuration: .ephemeral) + let semaphore = DispatchSemaphore(value: 0) + var resultData: Data? + session.dataTask(with: request as URLRequest) { data, response, error in + defer { semaphore.signal() } + guard error == nil, + let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { return } + resultData = data + }.resume() + _ = semaphore.wait(timeout: .now() + 20.0) + session.finishTasksAndInvalidate() + guard let data = resultData else { return nil } + return try? JSONDecoder().decode(WorkspaceRemoteDaemonManifest.self, from: data) + } + + private func downloadRemoteDaemonBinaryLocked(entry: WorkspaceRemoteDaemonManifest.Entry, version: String, releaseURL: String? = nil) throws -> URL { guard let url = URL(string: entry.downloadURL) else { throw NSError(domain: "cmux.remote.daemon", code: 25, userInfo: [ NSLocalizedDescriptionKey: "remote daemon manifest has an invalid download URL", @@ -4193,10 +4215,21 @@ final class WorkspaceRemoteSessionController { } let downloadedSHA = try Self.sha256Hex(forFile: downloadedURL) - guard downloadedSHA == entry.sha256.lowercased() else { - throw NSError(domain: "cmux.remote.daemon", code: 28, userInfo: [ - NSLocalizedDescriptionKey: "remote daemon checksum mismatch for \(entry.assetName)", - ]) + if downloadedSHA != entry.sha256.lowercased() { + // The embedded manifest's checksum doesn't match the downloaded binary. + // This can happen when a newer nightly overwrites the shared release + // asset after this build's manifest was embedded. As a fallback, fetch + // the live manifest from the release and verify against that. + if let releaseURL, + let liveManifest = Self.fetchRemoteManifestLocked(releaseURL: releaseURL, version: version), + let liveEntry = liveManifest.entry(goOS: entry.goOS, goArch: entry.goArch), + downloadedSHA == liveEntry.sha256.lowercased() { + debugLog("remote.download.checksum-fallback: embedded manifest checksum stale, live manifest matched for \(entry.assetName)") + } else { + throw NSError(domain: "cmux.remote.daemon", code: 28, userInfo: [ + NSLocalizedDescriptionKey: "remote daemon checksum mismatch for \(entry.assetName)", + ]) + } } let tempURL = cacheURL.deletingLastPathComponent() @@ -4229,7 +4262,7 @@ final class WorkspaceRemoteSessionController { } try? FileManager.default.removeItem(at: cacheURL) } - let downloadedURL = try downloadRemoteDaemonBinaryLocked(entry: entry, version: manifest.appVersion) + let downloadedURL = try downloadRemoteDaemonBinaryLocked(entry: entry, version: manifest.appVersion, releaseURL: manifest.releaseURL) debugLog("remote.build.downloaded path=\(downloadedURL.path)") return downloadedURL } diff --git a/scripts/build_remote_daemon_release_assets.sh b/scripts/build_remote_daemon_release_assets.sh index 6765fb38..a4de3391 100755 --- a/scripts/build_remote_daemon_release_assets.sh +++ b/scripts/build_remote_daemon_release_assets.sh @@ -7,12 +7,17 @@ Usage: scripts/build_remote_daemon_release_assets.sh \ --version \ --release-tag \ --repo \ - --output-dir + --output-dir \ + [--asset-suffix ] Builds cmuxd-remote release assets for the supported remote platforms and emits: - cmuxd-remote-- - cmuxd-remote-checksums.txt - cmuxd-remote-manifest.json + cmuxd-remote--[-] + cmuxd-remote-checksums[-].txt + cmuxd-remote-manifest[-].json + +When --asset-suffix is provided, all output filenames and manifest download URLs +include the suffix, making each build's assets immutable (used by nightly builds +to avoid checksum mismatches when assets are overwritten by later builds). EOF } @@ -20,6 +25,7 @@ VERSION="" RELEASE_TAG="" REPO="" OUTPUT_DIR="" +ASSET_SUFFIX="" while [[ $# -gt 0 ]]; do case "$1" in @@ -39,6 +45,10 @@ while [[ $# -gt 0 ]]; do OUTPUT_DIR="${2:-}" shift 2 ;; + --asset-suffix) + ASSET_SUFFIX="${2:-}" + shift 2 + ;; -h|--help) usage exit 0 @@ -77,9 +87,14 @@ DAEMON_GO_BUILD_ARGS=( -ldflags "$DAEMON_GO_LDFLAGS" ) -CHECKSUMS_ASSET_NAME="cmuxd-remote-checksums.txt" +SUFFIX_TAG="" +if [[ -n "$ASSET_SUFFIX" ]]; then + SUFFIX_TAG="-${ASSET_SUFFIX}" +fi + +CHECKSUMS_ASSET_NAME="cmuxd-remote-checksums${SUFFIX_TAG}.txt" CHECKSUMS_PATH="${OUTPUT_DIR}/${CHECKSUMS_ASSET_NAME}" -MANIFEST_PATH="${OUTPUT_DIR}/cmuxd-remote-manifest.json" +MANIFEST_PATH="${OUTPUT_DIR}/cmuxd-remote-manifest${SUFFIX_TAG}.json" TARGETS=( "darwin arm64" @@ -95,18 +110,22 @@ trap 'rm -f "$ENTRIES_FILE"' EXIT for target in "${TARGETS[@]}"; do read -r GOOS GOARCH <<<"$target" - ASSET_NAME="cmuxd-remote-${GOOS}-${GOARCH}" + ASSET_NAME="cmuxd-remote-${GOOS}-${GOARCH}${SUFFIX_TAG}" OUTPUT_PATH="${OUTPUT_DIR}/${ASSET_NAME}" + # Build into a temp path first, then rename (the binary content is the same + # regardless of suffix, so we build once and move). + BUILD_PATH="${OUTPUT_DIR}/cmuxd-remote-${GOOS}-${GOARCH}.build" ( cd "$DAEMON_ROOT" GOOS="$GOOS" \ GOARCH="$GOARCH" \ CGO_ENABLED=0 \ go "${DAEMON_GO_BUILD_ARGS[@]}" \ - -o "$OUTPUT_PATH" \ + -o "$BUILD_PATH" \ ./cmd/cmuxd-remote ) + mv "$BUILD_PATH" "$OUTPUT_PATH" chmod 755 "$OUTPUT_PATH" SHA256="$(shasum -a 256 "$OUTPUT_PATH" | awk '{print $1}')" diff --git a/tests/test_remote_daemon_release_assets.sh b/tests/test_remote_daemon_release_assets.sh index 8495d835..2afb1d11 100755 --- a/tests/test_remote_daemon_release_assets.sh +++ b/tests/test_remote_daemon_release_assets.sh @@ -63,3 +63,46 @@ for entry in manifest["entries"]: print("PASS: remote daemon release assets include all targets and manifest entries") PY + +# ------------------------------------------------------------------ +# Test with --asset-suffix (nightly-style immutable asset names) +# ------------------------------------------------------------------ +SUFFIX_DIR="$(mktemp -d "${TMPDIR:-/tmp}/cmux-remote-assets-suffix-test.XXXXXX")" +trap 'rm -rf "$OUTPUT_DIR" "$SUFFIX_DIR"' EXIT + +"$ROOT_DIR/scripts/build_remote_daemon_release_assets.sh" \ + --version "0.62.0-nightly.123456" \ + --release-tag "nightly" \ + --repo "manaflow-ai/cmux" \ + --output-dir "$SUFFIX_DIR" \ + --asset-suffix "123456" >/dev/null + +for asset in \ + cmuxd-remote-darwin-arm64-123456 \ + cmuxd-remote-darwin-amd64-123456 \ + cmuxd-remote-linux-arm64-123456 \ + cmuxd-remote-linux-amd64-123456 \ + cmuxd-remote-checksums-123456.txt \ + cmuxd-remote-manifest-123456.json +do + if [[ ! -f "$SUFFIX_DIR/$asset" ]]; then + echo "FAIL: missing suffixed asset $asset" >&2 + exit 1 + fi +done + +python3 - <<'PY' "$SUFFIX_DIR/cmuxd-remote-manifest-123456.json" +import json +import sys +from pathlib import Path + +manifest = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + +for entry in manifest["entries"]: + if not entry["assetName"].endswith("-123456"): + raise SystemExit(f"FAIL: suffixed asset name missing suffix: {entry['assetName']}") + if not entry["downloadURL"].endswith("/" + entry["assetName"]): + raise SystemExit(f"FAIL: downloadURL mismatch for {entry['assetName']}") + +print("PASS: --asset-suffix produces correctly suffixed assets and manifest entries") +PY