436 lines
17 KiB
YAML
436 lines
17 KiB
YAML
name: E2E test with video recording
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
ref:
|
|
description: Branch or SHA to test
|
|
required: false
|
|
default: ""
|
|
test_filter:
|
|
description: "Test class or class/method (e.g. UpdatePillUITests or UpdatePillUITests/testSomething)"
|
|
required: true
|
|
test_timeout:
|
|
description: "Per-test timeout in seconds"
|
|
required: false
|
|
default: "120"
|
|
record_video:
|
|
description: Record the virtual display during tests
|
|
required: false
|
|
default: true
|
|
type: boolean
|
|
runner:
|
|
description: "Runner OS (Depot runners for GUI activation support)"
|
|
required: false
|
|
default: "depot-macos-latest"
|
|
type: choice
|
|
options:
|
|
- depot-macos-latest
|
|
- depot-macos-14
|
|
|
|
jobs:
|
|
e2e:
|
|
runs-on: ${{ inputs.runner || 'depot-macos-latest' }}
|
|
timeout-minutes: 20
|
|
env:
|
|
TEST_REF: ${{ inputs.ref || github.ref }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
|
with:
|
|
ref: ${{ inputs.ref || github.ref }}
|
|
submodules: recursive
|
|
|
|
- name: Capture SHA
|
|
id: sha
|
|
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Select Xcode
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then
|
|
XCODE_DIR="/Applications/Xcode.app/Contents/Developer"
|
|
else
|
|
XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | head -n 1 || true)"
|
|
if [ -n "$XCODE_APP" ]; then
|
|
XCODE_DIR="$XCODE_APP/Contents/Developer"
|
|
else
|
|
echo "No Xcode.app found under /Applications" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV"
|
|
export DEVELOPER_DIR="$XCODE_DIR"
|
|
xcodebuild -version
|
|
xcrun --sdk macosx --show-sdk-path
|
|
|
|
- name: Download pre-built GhosttyKit.xcframework
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
GHOSTTY_SHA=$(git -C ghostty rev-parse HEAD)
|
|
TAG="xcframework-$GHOSTTY_SHA"
|
|
URL="https://github.com/manaflow-ai/ghostty/releases/download/$TAG/GhosttyKit.xcframework.tar.gz"
|
|
echo "Downloading xcframework for ghostty $GHOSTTY_SHA"
|
|
MAX_RETRIES=30
|
|
RETRY_DELAY=20
|
|
for i in $(seq 1 $MAX_RETRIES); do
|
|
if curl -fSL -o GhosttyKit.xcframework.tar.gz "$URL"; then
|
|
echo "Download succeeded on attempt $i"
|
|
break
|
|
fi
|
|
if [ "$i" -eq "$MAX_RETRIES" ]; then
|
|
echo "Failed to download xcframework after $MAX_RETRIES attempts" >&2
|
|
exit 1
|
|
fi
|
|
echo "Attempt $i/$MAX_RETRIES failed, retrying in ${RETRY_DELAY}s..."
|
|
sleep $RETRY_DELAY
|
|
done
|
|
tar xzf GhosttyKit.xcframework.tar.gz
|
|
rm GhosttyKit.xcframework.tar.gz
|
|
test -d GhosttyKit.xcframework
|
|
|
|
- name: Install zig
|
|
run: |
|
|
ZIG_REQUIRED="0.15.2"
|
|
if command -v zig >/dev/null 2>&1 && zig version 2>/dev/null | grep -q "^${ZIG_REQUIRED}"; then
|
|
echo "zig ${ZIG_REQUIRED} already installed"
|
|
else
|
|
echo "Installing zig ${ZIG_REQUIRED} from tarball"
|
|
curl -fSL "https://ziglang.org/download/${ZIG_REQUIRED}/zig-aarch64-macos-${ZIG_REQUIRED}.tar.xz" -o /tmp/zig.tar.xz
|
|
tar xf /tmp/zig.tar.xz -C /tmp
|
|
sudo mkdir -p /usr/local/bin /usr/local/lib
|
|
sudo cp -f /tmp/zig-aarch64-macos-${ZIG_REQUIRED}/zig /usr/local/bin/zig
|
|
sudo cp -rf /tmp/zig-aarch64-macos-${ZIG_REQUIRED}/lib /usr/local/lib/zig
|
|
export PATH="/usr/local/bin:$PATH"
|
|
zig version
|
|
fi
|
|
|
|
- name: Create virtual display
|
|
if: ${{ inputs.test_filter != 'DisplayResolutionRegressionUITests' }}
|
|
run: |
|
|
set -euo pipefail
|
|
echo "=== Display before ==="
|
|
system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)"
|
|
echo ""
|
|
clang -framework Foundation -framework CoreGraphics \
|
|
-o /tmp/create-virtual-display scripts/create-virtual-display.m
|
|
/tmp/create-virtual-display &
|
|
VDISPLAY_PID=$!
|
|
echo "VDISPLAY_PID=$VDISPLAY_PID" >> "$GITHUB_ENV"
|
|
sleep 3
|
|
echo "=== Display after ==="
|
|
system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)"
|
|
|
|
- name: Install ffmpeg
|
|
if: ${{ inputs.record_video }}
|
|
run: |
|
|
brew install --quiet ffmpeg
|
|
FFMPEG_PATH=$(which ffmpeg)
|
|
echo "ffmpeg: $FFMPEG_PATH"
|
|
ffmpeg -version | head -1
|
|
echo "FFMPEG_PATH=$FFMPEG_PATH" >> "$GITHUB_ENV"
|
|
|
|
- name: Grant TCC screen recording permission
|
|
if: ${{ inputs.record_video }}
|
|
continue-on-error: true
|
|
run: |
|
|
FFMPEG_BIN="${FFMPEG_PATH:-/opt/homebrew/bin/ffmpeg}"
|
|
|
|
# System-level TCC database (where kTCCServiceScreenCapture lives)
|
|
SYS_TCC="/Library/Application Support/com.apple.TCC/TCC.db"
|
|
if [ -f "$SYS_TCC" ]; then
|
|
echo "Granting screen capture in system TCC database"
|
|
for client in "$FFMPEG_BIN" /opt/homebrew/bin/ffmpeg /usr/local/bin/ffmpeg /usr/sbin/screencapture; do
|
|
sudo sqlite3 "$SYS_TCC" \
|
|
"INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version, csreq, policy_id, indirect_object_identifier_type, indirect_object_identifier, indirect_object_code_identity, flags, last_modified) VALUES ('kTCCServiceScreenCapture', '$client', 1, 2, 4, 1, NULL, NULL, 0, 'UNUSED', NULL, 0, $(date +%s));" 2>&1 || echo " (failed for $client)"
|
|
done
|
|
fi
|
|
|
|
# User-level TCC database (fallback)
|
|
USER_TCC="$HOME/Library/Application Support/com.apple.TCC/TCC.db"
|
|
if [ -f "$USER_TCC" ]; then
|
|
echo "Granting screen capture in user TCC database"
|
|
for client in "$FFMPEG_BIN" /opt/homebrew/bin/ffmpeg /usr/local/bin/ffmpeg; do
|
|
sqlite3 "$USER_TCC" \
|
|
"INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version, csreq, policy_id, indirect_object_identifier_type, indirect_object_identifier, indirect_object_code_identity, flags, last_modified) VALUES ('kTCCServiceScreenCapture', '$client', 1, 2, 4, 1, NULL, NULL, 0, 'UNUSED', NULL, 0, $(date +%s));" 2>&1 || echo " (failed for $client)"
|
|
done
|
|
fi
|
|
|
|
# Suppress Sequoia's ScreenCaptureApprovals prompt by pre-dating approval
|
|
APPROVALS_PLIST="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist"
|
|
if [ -d "$(dirname "$APPROVALS_PLIST")" ]; then
|
|
echo "Pre-dating ScreenCaptureApprovals"
|
|
# Set approval date far in the future so the monthly prompt never fires
|
|
defaults write "$APPROVALS_PLIST" "$FFMPEG_BIN" -date "3000-01-01T00:00:00Z" 2>&1 || echo " (failed)"
|
|
fi
|
|
|
|
- name: Clean DerivedData
|
|
run: rm -rf ~/Library/Developer/Xcode/DerivedData/GhosttyTabs-*
|
|
|
|
- name: Cache Swift packages
|
|
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
|
|
with:
|
|
path: .ci-source-packages
|
|
key: spm-${{ inputs.runner || 'depot-macos-latest' }}-${{ hashFiles('GhosttyTabs.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
|
|
restore-keys: spm-${{ inputs.runner || 'depot-macos-latest' }}-
|
|
|
|
- name: Resolve Swift packages
|
|
run: |
|
|
set -euo pipefail
|
|
SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
|
|
mkdir -p "$SOURCE_PACKAGES_DIR"
|
|
for attempt in 1 2 3; do
|
|
if xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux-unit -configuration Debug \
|
|
-clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
|
|
-resolvePackageDependencies; then
|
|
exit 0
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "Failed to resolve Swift packages after 3 attempts" >&2
|
|
exit 1
|
|
fi
|
|
echo "Package resolution failed on attempt $attempt, retrying..."
|
|
sleep $((attempt * 5))
|
|
done
|
|
|
|
- name: Run UI tests
|
|
id: tests
|
|
env:
|
|
TEST_FILTER: ${{ inputs.test_filter }}
|
|
TEST_TIMEOUT: ${{ inputs.test_timeout || '120' }}
|
|
RECORD_VIDEO: ${{ inputs.record_video }}
|
|
run: |
|
|
set -euo pipefail
|
|
SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
|
|
ONLY_TESTING="-only-testing:cmuxUITests/$TEST_FILTER"
|
|
DISPLAY_ENV_PREFIX=()
|
|
|
|
if [ "$TEST_FILTER" = "DisplayResolutionRegressionUITests" ]; then
|
|
PREFIX="/tmp/cmux-display-churn-${{ github.run_id }}-${{ github.run_attempt }}"
|
|
READY_PATH="${PREFIX}.ready"
|
|
DISPLAY_ID_PATH="${PREFIX}.id"
|
|
START_PATH="${PREFIX}.start"
|
|
DONE_PATH="${PREFIX}.done"
|
|
LOG_PATH="${PREFIX}.log"
|
|
MANIFEST_PATH="/tmp/cmux-ui-test-display-harness.json"
|
|
|
|
rm -f "$READY_PATH" "$DISPLAY_ID_PATH" "$START_PATH" "$DONE_PATH" "$LOG_PATH" "$MANIFEST_PATH"
|
|
|
|
clang -framework Foundation -framework CoreGraphics \
|
|
-o /tmp/create-virtual-display scripts/create-virtual-display.m
|
|
|
|
/tmp/create-virtual-display \
|
|
--modes 1920x1080,1728x1117,1600x900,1440x810 \
|
|
--ready-path "$READY_PATH" \
|
|
--display-id-path "$DISPLAY_ID_PATH" \
|
|
--start-path "$START_PATH" \
|
|
--done-path "$DONE_PATH" \
|
|
--iterations 40 \
|
|
--interval-ms 40 \
|
|
>"$LOG_PATH" 2>&1 &
|
|
DISPLAY_VDISPLAY_PID=$!
|
|
trap 'kill "${DISPLAY_VDISPLAY_PID:-}" >/dev/null 2>&1 || true; rm -f "$MANIFEST_PATH"' EXIT
|
|
|
|
for _ in {1..120}; do
|
|
[ -f "$READY_PATH" ] && break
|
|
sleep 0.25
|
|
done
|
|
[ -f "$READY_PATH" ] || {
|
|
echo "Display harness failed to start" >&2
|
|
cat "$LOG_PATH" >&2 || true
|
|
exit 1
|
|
}
|
|
|
|
cat >"$MANIFEST_PATH" <<EOF
|
|
{"readyPath":"$READY_PATH","displayIDPath":"$DISPLAY_ID_PATH","startPath":"$START_PATH","donePath":"$DONE_PATH","logPath":"$LOG_PATH"}
|
|
EOF
|
|
echo "Display harness manifest: $MANIFEST_PATH"
|
|
cat "$MANIFEST_PATH"
|
|
|
|
DISPLAY_ENV_PREFIX=(
|
|
CMUX_UI_TEST_DISPLAY_READY_PATH="$READY_PATH"
|
|
CMUX_UI_TEST_DISPLAY_ID_PATH="$DISPLAY_ID_PATH"
|
|
CMUX_UI_TEST_DISPLAY_START_PATH="$START_PATH"
|
|
CMUX_UI_TEST_DISPLAY_DONE_PATH="$DONE_PATH"
|
|
CMUX_UI_TEST_DISPLAY_LOG_PATH="$LOG_PATH"
|
|
CMUX_UI_TEST_DISPLAY_HARNESS_MANIFEST_PATH="$MANIFEST_PATH"
|
|
)
|
|
fi
|
|
|
|
# Start recording right before the test (after build/resolve).
|
|
# The display churn regression creates its own virtual display above,
|
|
# so recording must start after that harness is ready.
|
|
if [ "$RECORD_VIDEO" = "true" ]; then
|
|
DEVLIST=$( ffmpeg -f avfoundation -list_devices true -i "" 2>&1 || true )
|
|
echo "Available devices:"
|
|
echo "$DEVLIST" | grep -E "AVFoundation|Capture screen"
|
|
|
|
SCREEN_INDEX=$( echo "$DEVLIST" | grep "Capture screen" | head -1 \
|
|
| sed 's/.*\[\([0-9]*\)\].*/\1/' )
|
|
SCREEN_INDEX="${SCREEN_INDEX:-0}"
|
|
echo "Using screen device index: $SCREEN_INDEX"
|
|
|
|
ffmpeg -f avfoundation -framerate 10 -capture_cursor 1 \
|
|
-i "${SCREEN_INDEX}:none" \
|
|
-c:v libx264 -preset ultrafast -pix_fmt yuv420p \
|
|
/tmp/test-recording-raw.mp4 </dev/null >/tmp/ffmpeg.log 2>&1 &
|
|
RECORD_PID=$!
|
|
echo "RECORD_PID=$RECORD_PID" >> "$GITHUB_ENV"
|
|
sleep 2
|
|
|
|
if kill -0 "$RECORD_PID" 2>/dev/null; then
|
|
echo "Recording started (PID $RECORD_PID)"
|
|
else
|
|
echo "::warning::ffmpeg screen recording failed to start"
|
|
cat /tmp/ffmpeg.log
|
|
fi
|
|
fi
|
|
|
|
set +e
|
|
OUTPUT=$(env "${DISPLAY_ENV_PREFIX[@]}" xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug \
|
|
-clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
|
|
-disableAutomaticPackageResolution \
|
|
-destination "platform=macOS" \
|
|
-maximum-test-execution-time-allowance "$TEST_TIMEOUT" \
|
|
$ONLY_TESTING test 2>&1)
|
|
EXIT_CODE=$?
|
|
set -e
|
|
|
|
echo "$OUTPUT"
|
|
|
|
# Save summary for the issue
|
|
SUMMARY=$(echo "$OUTPUT" | grep -E "(Test Suite|Executed|FAIL|PASS)" | tail -20)
|
|
{
|
|
echo "test_summary<<EOFSUM"
|
|
echo "$SUMMARY"
|
|
echo "EOFSUM"
|
|
} >> "$GITHUB_OUTPUT"
|
|
|
|
if [ "$EXIT_CODE" -eq 0 ]; then
|
|
echo "test_result=passed" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "test_result=failed" >> "$GITHUB_OUTPUT"
|
|
# Save full output for the issue body
|
|
{
|
|
echo "test_output<<EOFOUT"
|
|
echo "$OUTPUT" | tail -200
|
|
echo "EOFOUT"
|
|
} >> "$GITHUB_OUTPUT"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Stop recording and trim
|
|
if: ${{ always() && inputs.record_video && env.RECORD_PID != '' }}
|
|
run: |
|
|
# Stop ffmpeg cleanly
|
|
kill -INT "$RECORD_PID" 2>/dev/null || true
|
|
for i in $(seq 1 15); do
|
|
if ! kill -0 "$RECORD_PID" 2>/dev/null; then
|
|
echo "Recording stopped after ${i}s"
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
kill -9 "$RECORD_PID" 2>/dev/null || true
|
|
|
|
echo "=== raw recording ==="
|
|
ls -lh /tmp/test-recording-raw.mp4 2>/dev/null || { echo "No recording file"; exit 0; }
|
|
|
|
# Trim: detect first non-black frame and cut from there
|
|
BLACK_END=$(ffmpeg -i /tmp/test-recording-raw.mp4 \
|
|
-vf "blackdetect=d=0.3:pic_th=0.95:pix_th=0.1" \
|
|
-an -f null - 2>&1 \
|
|
| grep "black_end" | tail -1 \
|
|
| sed 's/.*black_end:\([0-9.]*\).*/\1/' || true)
|
|
|
|
if [ -n "$BLACK_END" ] && [ "$BLACK_END" != "0" ]; then
|
|
echo "Trimming ${BLACK_END}s of black frames from start"
|
|
ffmpeg -y -i /tmp/test-recording-raw.mp4 -ss "$BLACK_END" \
|
|
-c:v libx264 -preset ultrafast -pix_fmt yuv420p \
|
|
/tmp/test-recording.mp4 2>/dev/null
|
|
else
|
|
echo "No black frames detected, using raw recording"
|
|
mv /tmp/test-recording-raw.mp4 /tmp/test-recording.mp4
|
|
fi
|
|
|
|
echo "=== final recording ==="
|
|
ls -lh /tmp/test-recording.mp4
|
|
# Print duration
|
|
ffprobe -v error -show_entries format=duration \
|
|
-of default=noprint_wrappers=1:nokey=1 /tmp/test-recording.mp4 2>/dev/null \
|
|
| xargs -I{} echo "Duration: {}s"
|
|
|
|
- name: Upload recording artifact
|
|
if: ${{ always() && inputs.record_video }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: test-recording
|
|
path: /tmp/test-recording.mp4
|
|
if-no-files-found: warn
|
|
|
|
- name: Post results to cmux-dev-artifacts
|
|
if: always()
|
|
env:
|
|
GH_TOKEN: ${{ secrets.DEV_ARTIFACTS_TOKEN }}
|
|
TEST_RESULT: ${{ steps.tests.outputs.test_result || 'failed' }}
|
|
TEST_SUMMARY: ${{ steps.tests.outputs.test_summary }}
|
|
TEST_OUTPUT: ${{ steps.tests.outputs.test_output }}
|
|
TEST_FILTER: ${{ inputs.test_filter }}
|
|
COMMIT_SHA: ${{ steps.sha.outputs.sha }}
|
|
RUN_ID: ${{ github.run_id }}
|
|
RECORD_VIDEO: ${{ inputs.record_video }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
LABEL="$TEST_RESULT"
|
|
if [ "$TEST_RESULT" = "passed" ]; then
|
|
STATUS_EMOJI="PASSED"
|
|
else
|
|
STATUS_EMOJI="FAILED"
|
|
fi
|
|
|
|
REF_DISPLAY="${{ inputs.ref || github.ref_name }}"
|
|
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID"
|
|
ARTIFACT_URL="$RUN_URL#artifacts"
|
|
|
|
BODY="**Status:** $STATUS_EMOJI
|
|
**Ref:** \`$REF_DISPLAY\`
|
|
**SHA:** [\`${COMMIT_SHA:0:12}\`](https://github.com/${{ github.repository }}/commit/$COMMIT_SHA)
|
|
**Test:** \`$TEST_FILTER\`
|
|
**Workflow run:** $RUN_URL"
|
|
|
|
if [ "$RECORD_VIDEO" = "true" ]; then
|
|
BODY="$BODY
|
|
**Recording:** [Download from artifacts]($ARTIFACT_URL)"
|
|
fi
|
|
|
|
if [ -n "$TEST_OUTPUT" ]; then
|
|
BODY="$BODY
|
|
|
|
<details><summary>Test output (last 200 lines)</summary>
|
|
|
|
\`\`\`
|
|
$TEST_OUTPUT
|
|
\`\`\`
|
|
|
|
</details>"
|
|
fi
|
|
|
|
if [ -n "$TEST_SUMMARY" ]; then
|
|
BODY="$BODY
|
|
|
|
\`\`\`
|
|
$TEST_SUMMARY
|
|
\`\`\`"
|
|
fi
|
|
|
|
ISSUE_URL=$(gh issue create \
|
|
--repo manaflow-ai/cmux-dev-artifacts \
|
|
--title "[$STATUS_EMOJI] $TEST_FILTER @ ${COMMIT_SHA:0:7} ($REF_DISPLAY)" \
|
|
--body "$BODY" \
|
|
--label "$LABEL")
|
|
|
|
echo "Issue posted: $ISSUE_URL"
|
|
echo "::notice title=Test Result Issue::$ISSUE_URL"
|