Add E2E test workflow with video recording (#778)
* Add E2E test workflow with video recording and issue posting New workflow_dispatch workflow (test-e2e.yml) that runs XCUITests on GitHub-hosted macos-15 runners, records the virtual display, uploads the video as an artifact, and posts results as an issue on cmux-dev-artifacts. Includes scripts/run-e2e.sh for convenient triggering from the terminal. * Print issue URL in workflow annotation and run-e2e.sh output - Capture gh issue create output URL, print as ::notice annotation - Search issues by run ID instead of grabbing most recent
This commit is contained in:
Lawrence Chen
GitHub
parent
fe3e2d06d9
commit
cd0c3cfa93
2 changed files with 373 additions and 0 deletions
272
.github/workflows/test-e2e.yml
vendored
Normal file
272
.github/workflows/test-e2e.yml
vendored
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
name: E2E test with video recording
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Branch or SHA to test
|
||||
required: false
|
||||
default: ""
|
||||
test_filter:
|
||||
description: "Test class or class/method (e.g. UpdatePillUITests or UpdatePillUITests/testSomething)"
|
||||
required: true
|
||||
test_timeout:
|
||||
description: "Per-test timeout in seconds"
|
||||
required: false
|
||||
default: "120"
|
||||
record_video:
|
||||
description: Record the virtual display during tests
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
runs-on: macos-15
|
||||
env:
|
||||
TEST_REF: ${{ inputs.ref || github.ref }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
submodules: recursive
|
||||
|
||||
- name: Capture SHA
|
||||
id: sha
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Select Xcode
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then
|
||||
XCODE_DIR="/Applications/Xcode.app/Contents/Developer"
|
||||
else
|
||||
XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | head -n 1 || true)"
|
||||
if [ -n "$XCODE_APP" ]; then
|
||||
XCODE_DIR="$XCODE_APP/Contents/Developer"
|
||||
else
|
||||
echo "No Xcode.app found under /Applications" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV"
|
||||
export DEVELOPER_DIR="$XCODE_DIR"
|
||||
xcodebuild -version
|
||||
xcrun --sdk macosx --show-sdk-path
|
||||
|
||||
- name: Download pre-built GhosttyKit.xcframework
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
GHOSTTY_SHA=$(git -C ghostty rev-parse HEAD)
|
||||
TAG="xcframework-$GHOSTTY_SHA"
|
||||
URL="https://github.com/manaflow-ai/ghostty/releases/download/$TAG/GhosttyKit.xcframework.tar.gz"
|
||||
echo "Downloading xcframework for ghostty $GHOSTTY_SHA"
|
||||
MAX_RETRIES=30
|
||||
RETRY_DELAY=20
|
||||
for i in $(seq 1 $MAX_RETRIES); do
|
||||
if curl -fSL -o GhosttyKit.xcframework.tar.gz "$URL"; then
|
||||
echo "Download succeeded on attempt $i"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq "$MAX_RETRIES" ]; then
|
||||
echo "Failed to download xcframework after $MAX_RETRIES attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Attempt $i/$MAX_RETRIES failed, retrying in ${RETRY_DELAY}s..."
|
||||
sleep $RETRY_DELAY
|
||||
done
|
||||
tar xzf GhosttyKit.xcframework.tar.gz
|
||||
rm GhosttyKit.xcframework.tar.gz
|
||||
test -d GhosttyKit.xcframework
|
||||
|
||||
- name: Create virtual display
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "=== Display before ==="
|
||||
system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)"
|
||||
echo ""
|
||||
clang -framework Foundation -framework CoreGraphics \
|
||||
-o /tmp/create-virtual-display scripts/create-virtual-display.m
|
||||
/tmp/create-virtual-display &
|
||||
VDISPLAY_PID=$!
|
||||
echo "VDISPLAY_PID=$VDISPLAY_PID" >> "$GITHUB_ENV"
|
||||
sleep 3
|
||||
echo "=== Display after ==="
|
||||
system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)"
|
||||
|
||||
- name: Grant TCC screen recording permission
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# Ephemeral CI runner, TCC database is writable
|
||||
TCC_DB="$HOME/Library/Application Support/com.apple.TCC/TCC.db"
|
||||
if [ -f "$TCC_DB" ]; then
|
||||
sqlite3 "$TCC_DB" "INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version) VALUES ('kTCCServiceScreenCapture', '/usr/sbin/screencapture', 1, 2, 4, 1);" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Start screen recording
|
||||
if: ${{ inputs.record_video }}
|
||||
run: |
|
||||
screencapture -v -D 1 -x /tmp/test-recording.mov &
|
||||
RECORD_PID=$!
|
||||
echo "RECORD_PID=$RECORD_PID" >> "$GITHUB_ENV"
|
||||
sleep 2
|
||||
echo "Recording started (PID $RECORD_PID)"
|
||||
|
||||
- name: Clean DerivedData
|
||||
run: rm -rf ~/Library/Developer/Xcode/DerivedData/GhosttyTabs-*
|
||||
|
||||
- name: Resolve Swift packages
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
|
||||
rm -rf "$SOURCE_PACKAGES_DIR"
|
||||
mkdir -p "$SOURCE_PACKAGES_DIR"
|
||||
for attempt in 1 2 3; do
|
||||
if xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux-unit -configuration Debug \
|
||||
-clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
|
||||
-resolvePackageDependencies; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$attempt" -eq 3 ]; then
|
||||
echo "Failed to resolve Swift packages after 3 attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Package resolution failed on attempt $attempt, retrying..."
|
||||
sleep $((attempt * 5))
|
||||
done
|
||||
|
||||
- name: Run UI tests
|
||||
id: tests
|
||||
env:
|
||||
TEST_FILTER: ${{ inputs.test_filter }}
|
||||
TEST_TIMEOUT: ${{ inputs.test_timeout || '120' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
|
||||
|
||||
ONLY_TESTING="-only-testing:cmuxUITests/$TEST_FILTER"
|
||||
|
||||
set +e
|
||||
OUTPUT=$(xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \
|
||||
-clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
|
||||
-disableAutomaticPackageResolution \
|
||||
-destination "platform=macOS" \
|
||||
-maximum-test-execution-time-allowance "$TEST_TIMEOUT" \
|
||||
$ONLY_TESTING test 2>&1)
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo "$OUTPUT"
|
||||
|
||||
# Save summary for the issue
|
||||
SUMMARY=$(echo "$OUTPUT" | grep -E "(Test Suite|Executed|FAIL|PASS)" | tail -20)
|
||||
{
|
||||
echo "test_summary<<EOFSUM"
|
||||
echo "$SUMMARY"
|
||||
echo "EOFSUM"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$EXIT_CODE" -eq 0 ]; then
|
||||
echo "test_result=passed" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "test_result=failed" >> "$GITHUB_OUTPUT"
|
||||
# Save full output for the issue body
|
||||
{
|
||||
echo "test_output<<EOFOUT"
|
||||
echo "$OUTPUT" | tail -200
|
||||
echo "EOFOUT"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Stop screen recording
|
||||
if: ${{ always() && inputs.record_video && env.RECORD_PID != '' }}
|
||||
run: |
|
||||
kill -INT "$RECORD_PID" 2>/dev/null || true
|
||||
# Wait for screencapture to finalize the .mov file
|
||||
for i in $(seq 1 30); do
|
||||
if ! kill -0 "$RECORD_PID" 2>/dev/null; then
|
||||
echo "Recording stopped after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
kill -9 "$RECORD_PID" 2>/dev/null || true
|
||||
ls -lh /tmp/test-recording.mov 2>/dev/null || echo "No recording file found"
|
||||
|
||||
- name: Upload recording artifact
|
||||
if: ${{ always() && inputs.record_video }}
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: test-recording
|
||||
path: /tmp/test-recording.mov
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Post results to cmux-dev-artifacts
|
||||
if: always()
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.DEV_ARTIFACTS_TOKEN }}
|
||||
TEST_RESULT: ${{ steps.tests.outputs.test_result || 'failed' }}
|
||||
TEST_SUMMARY: ${{ steps.tests.outputs.test_summary }}
|
||||
TEST_OUTPUT: ${{ steps.tests.outputs.test_output }}
|
||||
TEST_FILTER: ${{ inputs.test_filter }}
|
||||
COMMIT_SHA: ${{ steps.sha.outputs.sha }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
RECORD_VIDEO: ${{ inputs.record_video }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
LABEL="$TEST_RESULT"
|
||||
if [ "$TEST_RESULT" = "passed" ]; then
|
||||
STATUS_EMOJI="PASSED"
|
||||
else
|
||||
STATUS_EMOJI="FAILED"
|
||||
fi
|
||||
|
||||
REF_DISPLAY="${{ inputs.ref || github.ref_name }}"
|
||||
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID"
|
||||
ARTIFACT_URL="$RUN_URL#artifacts"
|
||||
|
||||
# Build issue body (no leading whitespace)
|
||||
BODY="**Status:** $STATUS_EMOJI
|
||||
**Ref:** \`$REF_DISPLAY\`
|
||||
**SHA:** [\`${COMMIT_SHA:0:12}\`](https://github.com/${{ github.repository }}/commit/$COMMIT_SHA)
|
||||
**Test:** \`$TEST_FILTER\`
|
||||
**Workflow run:** $RUN_URL"
|
||||
|
||||
if [ "$RECORD_VIDEO" = "true" ]; then
|
||||
BODY="$BODY
|
||||
**Recording:** [Download from artifacts]($ARTIFACT_URL)"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_OUTPUT" ]; then
|
||||
BODY="$BODY
|
||||
|
||||
<details><summary>Test output (last 200 lines)</summary>
|
||||
|
||||
\`\`\`
|
||||
$TEST_OUTPUT
|
||||
\`\`\`
|
||||
|
||||
</details>"
|
||||
fi
|
||||
|
||||
if [ -n "$TEST_SUMMARY" ]; then
|
||||
BODY="$BODY
|
||||
|
||||
\`\`\`
|
||||
$TEST_SUMMARY
|
||||
\`\`\`"
|
||||
fi
|
||||
|
||||
ISSUE_URL=$(gh issue create \
|
||||
--repo manaflow-ai/cmux-dev-artifacts \
|
||||
--title "[$STATUS_EMOJI] $TEST_FILTER @ ${COMMIT_SHA:0:7} ($REF_DISPLAY)" \
|
||||
--body "$BODY" \
|
||||
--label "$LABEL")
|
||||
|
||||
echo "Issue posted: $ISSUE_URL"
|
||||
echo "::notice title=Test Result Issue::$ISSUE_URL"
|
||||
101
scripts/run-e2e.sh
Executable file
101
scripts/run-e2e.sh
Executable file
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env bash
|
||||
# Trigger the test-e2e.yml workflow and optionally wait for results.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/run-e2e.sh UpdatePillUITests
|
||||
# ./scripts/run-e2e.sh UpdatePillUITests --wait
|
||||
# ./scripts/run-e2e.sh UpdatePillUITests/testFoo --ref my-branch
|
||||
# ./scripts/run-e2e.sh UpdatePillUITests --no-video --timeout 300
|
||||
set -euo pipefail
|
||||
|
||||
REPO="manaflow-ai/cmux"
|
||||
WORKFLOW="test-e2e.yml"
|
||||
|
||||
# Defaults
|
||||
REF=""
|
||||
WAIT=false
|
||||
RECORD_VIDEO=true
|
||||
TIMEOUT=120
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") <test_filter> [options]
|
||||
|
||||
Arguments:
|
||||
test_filter Test class or class/method (e.g. UpdatePillUITests)
|
||||
|
||||
Options:
|
||||
--ref <ref> Branch or SHA to test (default: current branch)
|
||||
--wait Wait for the run to complete and print result
|
||||
--no-video Disable video recording
|
||||
--timeout <sec> Per-test timeout in seconds (default: 120)
|
||||
-h, --help Show this help
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
TEST_FILTER="$1"
|
||||
shift
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--ref)
|
||||
REF="$2"
|
||||
shift 2
|
||||
;;
|
||||
--wait)
|
||||
WAIT=true
|
||||
shift
|
||||
;;
|
||||
--no-video)
|
||||
RECORD_VIDEO=false
|
||||
shift
|
||||
;;
|
||||
--timeout)
|
||||
TIMEOUT="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build workflow dispatch fields
|
||||
FIELDS=(-f "test_filter=$TEST_FILTER" -f "record_video=$RECORD_VIDEO" -f "test_timeout=$TIMEOUT")
|
||||
if [ -n "$REF" ]; then
|
||||
FIELDS+=(-f "ref=$REF")
|
||||
fi
|
||||
|
||||
echo "Triggering $WORKFLOW with test_filter=$TEST_FILTER ref=${REF:-<default>} video=$RECORD_VIDEO timeout=$TIMEOUT"
|
||||
gh workflow run "$WORKFLOW" --repo "$REPO" "${FIELDS[@]}"
|
||||
|
||||
# Wait a moment for the run to register
|
||||
sleep 3
|
||||
|
||||
# Get the latest run ID
|
||||
RUN_ID=$(gh run list --repo "$REPO" --workflow "$WORKFLOW" --limit 1 --json databaseId --jq '.[0].databaseId')
|
||||
RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID"
|
||||
|
||||
echo "Run: $RUN_URL"
|
||||
|
||||
if [ "$WAIT" = true ]; then
|
||||
echo "Waiting for run to complete..."
|
||||
gh run watch --repo "$REPO" "$RUN_ID" --exit-status || true
|
||||
|
||||
STATUS=$(gh run view --repo "$REPO" "$RUN_ID" --json conclusion --jq '.conclusion')
|
||||
echo ""
|
||||
echo "Result: $STATUS"
|
||||
echo "Run: $RUN_URL"
|
||||
|
||||
# Find the issue created for this run (search by run ID in body)
|
||||
ISSUE_URL=$(gh search issues "$RUN_ID" --repo manaflow-ai/cmux-dev-artifacts --limit 1 --json url --jq '.[0].url' 2>/dev/null || true)
|
||||
if [ -n "$ISSUE_URL" ]; then
|
||||
echo "Issue: $ISSUE_URL"
|
||||
fi
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue