From cd0c3cfa93fc3809b980be11e4e2354e5f101e9f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 2 Mar 2026 22:16:38 -0800 Subject: [PATCH] Add E2E test workflow with video recording (#778) * Add E2E test workflow with video recording and issue posting New workflow_dispatch workflow (test-e2e.yml) that runs XCUITests on GitHub-hosted macos-15 runners, records the virtual display, uploads the video as an artifact, and posts results as an issue on cmux-dev-artifacts. Includes scripts/run-e2e.sh for convenient triggering from the terminal. * Print issue URL in workflow annotation and run-e2e.sh output - Capture gh issue create output URL, print as ::notice annotation - Search issues by run ID instead of grabbing most recent --- .github/workflows/test-e2e.yml | 272 +++++++++++++++++++++++++++++++++ scripts/run-e2e.sh | 101 ++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 .github/workflows/test-e2e.yml create mode 100755 scripts/run-e2e.sh diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 00000000..4d8483eb --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,272 @@ +name: E2E test with video recording + +on: + workflow_dispatch: + inputs: + ref: + description: Branch or SHA to test + required: false + default: "" + test_filter: + description: "Test class or class/method (e.g. UpdatePillUITests or UpdatePillUITests/testSomething)" + required: true + test_timeout: + description: "Per-test timeout in seconds" + required: false + default: "120" + record_video: + description: Record the virtual display during tests + required: false + default: true + type: boolean + +jobs: + e2e: + runs-on: macos-15 + env: + TEST_REF: ${{ inputs.ref || github.ref }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ inputs.ref || github.ref }} + submodules: recursive + + - name: Capture SHA + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Select Xcode + run: | + set -euo pipefail + if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then + XCODE_DIR="/Applications/Xcode.app/Contents/Developer" + else + XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | head -n 1 || true)" + if [ -n "$XCODE_APP" ]; then + XCODE_DIR="$XCODE_APP/Contents/Developer" + else + echo "No Xcode.app found under /Applications" >&2 + exit 1 + fi + fi + echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV" + export DEVELOPER_DIR="$XCODE_DIR" + xcodebuild -version + xcrun --sdk macosx --show-sdk-path + + - name: Download pre-built GhosttyKit.xcframework + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + GHOSTTY_SHA=$(git -C ghostty rev-parse HEAD) + TAG="xcframework-$GHOSTTY_SHA" + URL="https://github.com/manaflow-ai/ghostty/releases/download/$TAG/GhosttyKit.xcframework.tar.gz" + echo "Downloading xcframework for ghostty $GHOSTTY_SHA" + MAX_RETRIES=30 + RETRY_DELAY=20 + for i in $(seq 1 $MAX_RETRIES); do + if curl -fSL -o GhosttyKit.xcframework.tar.gz "$URL"; then + echo "Download succeeded on attempt $i" + break + fi + if [ "$i" -eq "$MAX_RETRIES" ]; then + echo "Failed to download xcframework after $MAX_RETRIES attempts" >&2 + exit 1 + fi + echo "Attempt $i/$MAX_RETRIES failed, retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + done + tar xzf GhosttyKit.xcframework.tar.gz + rm GhosttyKit.xcframework.tar.gz + test -d GhosttyKit.xcframework + + - name: Create virtual display + run: | + set -euo pipefail + echo "=== Display before ===" + system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)" + echo "" + clang -framework Foundation -framework CoreGraphics \ + -o /tmp/create-virtual-display scripts/create-virtual-display.m + /tmp/create-virtual-display & + VDISPLAY_PID=$! + echo "VDISPLAY_PID=$VDISPLAY_PID" >> "$GITHUB_ENV" + sleep 3 + echo "=== Display after ===" + system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)" + + - name: Grant TCC screen recording permission + continue-on-error: true + run: | + # Ephemeral CI runner, TCC database is writable + TCC_DB="$HOME/Library/Application Support/com.apple.TCC/TCC.db" + if [ -f "$TCC_DB" ]; then + sqlite3 "$TCC_DB" "INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version) VALUES ('kTCCServiceScreenCapture', '/usr/sbin/screencapture', 1, 2, 4, 1);" 2>/dev/null || true + fi + + - name: Start screen recording + if: ${{ inputs.record_video }} + run: | + screencapture -v -D 1 -x /tmp/test-recording.mov & + RECORD_PID=$! + echo "RECORD_PID=$RECORD_PID" >> "$GITHUB_ENV" + sleep 2 + echo "Recording started (PID $RECORD_PID)" + + - name: Clean DerivedData + run: rm -rf ~/Library/Developer/Xcode/DerivedData/GhosttyTabs-* + + - name: Resolve Swift packages + run: | + set -euo pipefail + SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" + rm -rf "$SOURCE_PACKAGES_DIR" + mkdir -p "$SOURCE_PACKAGES_DIR" + for attempt in 1 2 3; do + if xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux-unit -configuration Debug \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -resolvePackageDependencies; then + exit 0 + fi + if [ "$attempt" -eq 3 ]; then + echo "Failed to resolve Swift packages after 3 attempts" >&2 + exit 1 + fi + echo "Package resolution failed on attempt $attempt, retrying..." + sleep $((attempt * 5)) + done + + - name: Run UI tests + id: tests + env: + TEST_FILTER: ${{ inputs.test_filter }} + TEST_TIMEOUT: ${{ inputs.test_timeout || '120' }} + run: | + set -euo pipefail + SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" + + ONLY_TESTING="-only-testing:cmuxUITests/$TEST_FILTER" + + set +e + OUTPUT=$(xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -disableAutomaticPackageResolution \ + -destination "platform=macOS" \ + -maximum-test-execution-time-allowance "$TEST_TIMEOUT" \ + $ONLY_TESTING test 2>&1) + EXIT_CODE=$? + set -e + + echo "$OUTPUT" + + # Save summary for the issue + SUMMARY=$(echo "$OUTPUT" | grep -E "(Test Suite|Executed|FAIL|PASS)" | tail -20) + { + echo "test_summary<> "$GITHUB_OUTPUT" + + if [ "$EXIT_CODE" -eq 0 ]; then + echo "test_result=passed" >> "$GITHUB_OUTPUT" + else + echo "test_result=failed" >> "$GITHUB_OUTPUT" + # Save full output for the issue body + { + echo "test_output<> "$GITHUB_OUTPUT" + exit 1 + fi + + - name: Stop screen recording + if: ${{ always() && inputs.record_video && env.RECORD_PID != '' }} + run: | + kill -INT "$RECORD_PID" 2>/dev/null || true + # Wait for screencapture to finalize the .mov file + for i in $(seq 1 30); do + if ! kill -0 "$RECORD_PID" 2>/dev/null; then + echo "Recording stopped after ${i}s" + break + fi + sleep 1 + done + kill -9 "$RECORD_PID" 2>/dev/null || true + ls -lh /tmp/test-recording.mov 2>/dev/null || echo "No recording file found" + + - name: Upload recording artifact + if: ${{ always() && inputs.record_video }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-recording + path: /tmp/test-recording.mov + if-no-files-found: warn + + - name: Post results to cmux-dev-artifacts + if: always() + env: + GH_TOKEN: ${{ secrets.DEV_ARTIFACTS_TOKEN }} + TEST_RESULT: ${{ steps.tests.outputs.test_result || 'failed' }} + TEST_SUMMARY: ${{ steps.tests.outputs.test_summary }} + TEST_OUTPUT: ${{ steps.tests.outputs.test_output }} + TEST_FILTER: ${{ inputs.test_filter }} + COMMIT_SHA: ${{ steps.sha.outputs.sha }} + RUN_ID: ${{ github.run_id }} + RECORD_VIDEO: ${{ inputs.record_video }} + run: | + set -euo pipefail + + LABEL="$TEST_RESULT" + if [ "$TEST_RESULT" = "passed" ]; then + STATUS_EMOJI="PASSED" + else + STATUS_EMOJI="FAILED" + fi + + REF_DISPLAY="${{ inputs.ref || github.ref_name }}" + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID" + ARTIFACT_URL="$RUN_URL#artifacts" + + # Build issue body (no leading whitespace) + BODY="**Status:** $STATUS_EMOJI + **Ref:** \`$REF_DISPLAY\` + **SHA:** [\`${COMMIT_SHA:0:12}\`](https://github.com/${{ github.repository }}/commit/$COMMIT_SHA) + **Test:** \`$TEST_FILTER\` + **Workflow run:** $RUN_URL" + + if [ "$RECORD_VIDEO" = "true" ]; then + BODY="$BODY + **Recording:** [Download from artifacts]($ARTIFACT_URL)" + fi + + if [ -n "$TEST_OUTPUT" ]; then + BODY="$BODY + +
Test output (last 200 lines) + + \`\`\` + $TEST_OUTPUT + \`\`\` + +
" + fi + + if [ -n "$TEST_SUMMARY" ]; then + BODY="$BODY + + \`\`\` + $TEST_SUMMARY + \`\`\`" + fi + + ISSUE_URL=$(gh issue create \ + --repo manaflow-ai/cmux-dev-artifacts \ + --title "[$STATUS_EMOJI] $TEST_FILTER @ ${COMMIT_SHA:0:7} ($REF_DISPLAY)" \ + --body "$BODY" \ + --label "$LABEL") + + echo "Issue posted: $ISSUE_URL" + echo "::notice title=Test Result Issue::$ISSUE_URL" diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh new file mode 100755 index 00000000..4d26c416 --- /dev/null +++ b/scripts/run-e2e.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Trigger the test-e2e.yml workflow and optionally wait for results. +# +# Usage: +# ./scripts/run-e2e.sh UpdatePillUITests +# ./scripts/run-e2e.sh UpdatePillUITests --wait +# ./scripts/run-e2e.sh UpdatePillUITests/testFoo --ref my-branch +# ./scripts/run-e2e.sh UpdatePillUITests --no-video --timeout 300 +set -euo pipefail + +REPO="manaflow-ai/cmux" +WORKFLOW="test-e2e.yml" + +# Defaults +REF="" +WAIT=false +RECORD_VIDEO=true +TIMEOUT=120 + +usage() { + cat < [options] + +Arguments: + test_filter Test class or class/method (e.g. UpdatePillUITests) + +Options: + --ref Branch or SHA to test (default: current branch) + --wait Wait for the run to complete and print result + --no-video Disable video recording + --timeout Per-test timeout in seconds (default: 120) + -h, --help Show this help +EOF + exit 0 +} + +if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage +fi + +TEST_FILTER="$1" +shift + +while [ $# -gt 0 ]; do + case "$1" in + --ref) + REF="$2" + shift 2 + ;; + --wait) + WAIT=true + shift + ;; + --no-video) + RECORD_VIDEO=false + shift + ;; + --timeout) + TIMEOUT="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + usage + ;; + esac +done + +# Build workflow dispatch fields +FIELDS=(-f "test_filter=$TEST_FILTER" -f "record_video=$RECORD_VIDEO" -f "test_timeout=$TIMEOUT") +if [ -n "$REF" ]; then + FIELDS+=(-f "ref=$REF") +fi + +echo "Triggering $WORKFLOW with test_filter=$TEST_FILTER ref=${REF:-} video=$RECORD_VIDEO timeout=$TIMEOUT" +gh workflow run "$WORKFLOW" --repo "$REPO" "${FIELDS[@]}" + +# Wait a moment for the run to register +sleep 3 + +# Get the latest run ID +RUN_ID=$(gh run list --repo "$REPO" --workflow "$WORKFLOW" --limit 1 --json databaseId --jq '.[0].databaseId') +RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID" + +echo "Run: $RUN_URL" + +if [ "$WAIT" = true ]; then + echo "Waiting for run to complete..." + gh run watch --repo "$REPO" "$RUN_ID" --exit-status || true + + STATUS=$(gh run view --repo "$REPO" "$RUN_ID" --json conclusion --jq '.conclusion') + echo "" + echo "Result: $STATUS" + echo "Run: $RUN_URL" + + # Find the issue created for this run (search by run ID in body) + ISSUE_URL=$(gh search issues "$RUN_ID" --repo manaflow-ai/cmux-dev-artifacts --limit 1 --json url --jq '.[0].url' 2>/dev/null || true) + if [ -n "$ISSUE_URL" ]; then + echo "Issue: $ISSUE_URL" + fi +fi