From ae3bcd7eea6f3f20a0d9b4a7d1e5b9d078d1e799 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 27 Feb 2026 04:18:26 -0800 Subject: [PATCH] Set up full test suite in CI and Xcode Cloud (#447) * Set up full test suite in CI and Xcode Cloud Add build-ghosttykit.yml workflow to pre-build and publish GhosttyKit.xcframework as a GitHub release on manaflow-ai/ghostty, keyed by submodule SHA. Add ci_scripts/ci_post_clone.sh for Xcode Cloud to download the pre-built xcframework with retry logic. Create cmux-ci scheme that runs both cmuxTests and cmuxUITests. Switch the CI tests job from running a single UI test class to the full suite. * Run unit tests + single UI test class on self-hosted runner The self-hosted runner can't launch the full app for UI tests (no GUI session), so run all unit tests via cmux-unit scheme and keep the original UpdatePillUITests as a smoke test. Full UI test suite runs on Xcode Cloud which has proper macOS GUI support. * Handle expected test failures in unit tests step xcodebuild returns exit code 65 even for expected failures (XCTExpectFailure). Parse the summary line to only fail the CI job when there are unexpected failures. --- .github/workflows/build-ghosttykit.yml | 97 +++++++++++++++++++ .github/workflows/ci.yml | 24 ++++- .../xcshareddata/xcschemes/cmux-ci.xcscheme | 38 ++++++++ ci_scripts/ci_post_clone.sh | 48 +++++++++ tests/test_ci_self_hosted_guard.sh | 12 +-- 5 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/build-ghosttykit.yml create mode 100644 GhosttyTabs.xcodeproj/xcshareddata/xcschemes/cmux-ci.xcscheme create mode 100755 ci_scripts/ci_post_clone.sh diff --git a/.github/workflows/build-ghosttykit.yml b/.github/workflows/build-ghosttykit.yml new file mode 100644 index 00000000..0dc2871a --- /dev/null +++ b/.github/workflows/build-ghosttykit.yml @@ -0,0 +1,97 @@ +name: Build GhosttyKit + +on: + push: + branches: + - main + pull_request: + +jobs: + build-ghosttykit: + # Never run self-hosted jobs for fork pull requests. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + concurrency: + group: self-hosted-build + cancel-in-progress: false + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + submodules: recursive + + - name: Get ghostty SHA + id: ghostty-sha + run: | + SHA=$(git -C ghostty rev-parse HEAD) + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + echo "Ghostty SHA: $SHA" + + - name: Check if xcframework release already exists + id: check-release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="xcframework-${{ steps.ghostty-sha.outputs.sha }}" + if gh release view "$TAG" --repo manaflow-ai/ghostty >/dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "Release $TAG already exists, skipping build" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "Release $TAG not found, will build" + fi + + - name: Select Xcode + if: steps.check-release.outputs.exists == 'false' + run: | + set -euo pipefail + if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then + XCODE_DIR="/Applications/Xcode.app/Contents/Developer" + else + XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | head -n 1 || true)" + if [ -n "$XCODE_APP" ]; then + XCODE_DIR="$XCODE_APP/Contents/Developer" + else + echo "No Xcode.app found under /Applications" >&2 + exit 1 + fi + fi + echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV" + export DEVELOPER_DIR="$XCODE_DIR" + xcodebuild -version + + - name: Build GhosttyKit.xcframework + if: steps.check-release.outputs.exists == 'false' + run: | + set -euo pipefail + if ! command -v zig >/dev/null 2>&1; then + if command -v brew >/dev/null 2>&1; then + brew install zig + else + echo "zig is required to build GhosttyKit.xcframework. Install zig and retry." >&2 + exit 1 + fi + fi + cd ghostty && zig build -Demit-xcframework=true -Demit-macos-app=false -Doptimize=ReleaseFast + + - name: Package xcframework + if: steps.check-release.outputs.exists == 'false' + run: | + set -euo pipefail + rm -rf GhosttyKit.xcframework + cp -R ghostty/macos/GhosttyKit.xcframework GhosttyKit.xcframework + tar czf GhosttyKit.xcframework.tar.gz GhosttyKit.xcframework + + - name: Upload xcframework release + if: steps.check-release.outputs.exists == 'false' + env: + GH_TOKEN: ${{ secrets.GHOSTTY_RELEASE_TOKEN }} + run: | + set -euo pipefail + TAG="xcframework-${{ steps.ghostty-sha.outputs.sha }}" + gh release create "$TAG" \ + --repo manaflow-ai/ghostty \ + --title "GhosttyKit xcframework (${{ steps.ghostty-sha.outputs.sha }})" \ + --notes "Pre-built GhosttyKit.xcframework for commit ${{ steps.ghostty-sha.outputs.sha }}" \ + GhosttyKit.xcframework.tar.gz + echo "Published release $TAG" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ac36d11..da2d147c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: - name: Typecheck run: bun tsc --noEmit - ui-tests: + tests: # Never run self-hosted jobs for fork pull requests. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: self-hosted @@ -93,8 +93,28 @@ jobs: # Remove stale build cache to avoid incremental build errors rm -rf ~/Library/Developer/Xcode/DerivedData/GhosttyTabs-* + - name: Run unit tests + run: | + set -euo pipefail + # xcodebuild exits 65 even for expected failures (XCTExpectFailure). + # Capture output and fail only if there are unexpected failures. + set +e + OUTPUT=$(xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux-unit -configuration Debug \ + -destination "platform=macOS" test 2>&1) + EXIT_CODE=$? + set -e + echo "$OUTPUT" + if [ "$EXIT_CODE" -ne 0 ]; then + SUMMARY=$(echo "$OUTPUT" | grep "Executed.*tests.*with.*failures" | tail -1) + if echo "$SUMMARY" | grep -q "(0 unexpected)"; then + echo "All failures are expected, treating as pass" + else + echo "Unexpected test failures detected" + exit 1 + fi + fi + - name: Run UI tests run: | set -euo pipefail - # Run directly on the self-hosted macOS runner. xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug -destination "platform=macOS" -only-testing:cmuxUITests/UpdatePillUITests test diff --git a/GhosttyTabs.xcodeproj/xcshareddata/xcschemes/cmux-ci.xcscheme b/GhosttyTabs.xcodeproj/xcshareddata/xcschemes/cmux-ci.xcscheme new file mode 100644 index 00000000..415b3867 --- /dev/null +++ b/GhosttyTabs.xcodeproj/xcshareddata/xcschemes/cmux-ci.xcscheme @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ci_scripts/ci_post_clone.sh b/ci_scripts/ci_post_clone.sh new file mode 100755 index 00000000..986b22a8 --- /dev/null +++ b/ci_scripts/ci_post_clone.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -euo pipefail + +echo "=== ci_post_clone.sh ===" + +# Initialize submodules (needed for vendor/bonsplit SPM package) +echo "Initializing submodules..." +git submodule update --init --recursive + +# Get ghostty submodule SHA +GHOSTTY_SHA=$(git -C "$CI_PRIMARY_REPOSITORY_PATH/ghostty" rev-parse HEAD) +echo "Ghostty SHA: $GHOSTTY_SHA" + +# Download pre-built xcframework from manaflow-ai/ghostty releases +TAG="xcframework-$GHOSTTY_SHA" +URL="https://github.com/manaflow-ai/ghostty/releases/download/$TAG/GhosttyKit.xcframework.tar.gz" + +echo "Downloading xcframework from $URL" + +MAX_RETRIES=30 +RETRY_DELAY=20 + +for i in $(seq 1 $MAX_RETRIES); do + if curl -fSL -o "$CI_PRIMARY_REPOSITORY_PATH/GhosttyKit.xcframework.tar.gz" "$URL"; then + echo "Download succeeded on attempt $i" + break + fi + if [ "$i" -eq "$MAX_RETRIES" ]; then + echo "Failed to download xcframework after $MAX_RETRIES attempts" >&2 + exit 1 + fi + echo "Attempt $i/$MAX_RETRIES failed, retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY +done + +# Extract xcframework to project root +echo "Extracting xcframework..." +cd "$CI_PRIMARY_REPOSITORY_PATH" +tar xzf GhosttyKit.xcframework.tar.gz +rm GhosttyKit.xcframework.tar.gz +test -d GhosttyKit.xcframework +echo "GhosttyKit.xcframework extracted successfully" + +# Download Metal toolchain (required for shader compilation) +echo "Downloading Metal toolchain..." +xcodebuild -downloadComponent MetalToolchain + +echo "=== ci_post_clone.sh done ===" diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index f046141c..c63a3111 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -16,14 +16,14 @@ if ! grep -Fq "$EXPECTED_IF" "$WORKFLOW_FILE"; then fi if ! awk ' - /^ ui-tests:/ { in_ui_tests=1; next } - in_ui_tests && /^ [^[:space:]]/ { in_ui_tests=0 } - in_ui_tests && /runs-on: self-hosted/ { saw_self_hosted=1 } - in_ui_tests && /github.event.pull_request.head.repo.full_name == github.repository/ { saw_guard=1 } + /^ tests:/ { in_tests=1; next } + in_tests && /^ [^[:space:]]/ { in_tests=0 } + in_tests && /runs-on: self-hosted/ { saw_self_hosted=1 } + in_tests && /github.event.pull_request.head.repo.full_name == github.repository/ { saw_guard=1 } END { exit !(saw_self_hosted && saw_guard) } ' "$WORKFLOW_FILE"; then - echo "FAIL: ui-tests block must keep both self-hosted and fork guard" + echo "FAIL: tests block must keep both self-hosted and fork guard" exit 1 fi -echo "PASS: ui-tests self-hosted fork guard is present" +echo "PASS: tests self-hosted fork guard is present"