Skip to content

Commit feb609d

Browse files
waleedlatif1waleedlatif1
committed
test(ci): benchmark production runtime on free runners
1 parent 9499b73 commit feb609d

1 file changed

Lines changed: 245 additions & 0 deletions

File tree

‎.github/workflows/ci-runtime-benchmark.yml‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
name: CI runtime runner benchmark
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
source_run:
6+
description: Run containing the same-code next-build artifact
7+
required: true
8+
type: string
9+
permissions:
10+
contents: read
11+
actions: read
12+
jobs:
13+
desktop-live:
14+
name: desktop-live (${{ matrix.tests }})
15+
runs-on: ubuntu-24.04
16+
timeout-minutes: 30
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
include:
21+
- tests: lease-lapse
22+
filter: --grep @lease-lapse
23+
- tests: rest
24+
filter: --grep-invert @lease-lapse
25+
services:
26+
postgres:
27+
image: pgvector/pgvector:pg17
28+
env:
29+
POSTGRES_USER: postgres
30+
POSTGRES_PASSWORD: postgres
31+
POSTGRES_DB: sim_test
32+
ports:
33+
- 5432:5432
34+
options: --health-cmd "pg_isready -U postgres -d sim_test" --health-interval 5s --health-timeout 5s --health-retries 10
35+
redis:
36+
image: redis:8.2-alpine
37+
ports:
38+
- 6379:6379
39+
options: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 5s --health-retries 10
40+
env:
41+
DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test
42+
BETTER_AUTH_SECRET: desktop-live-e2e-ci-secret-at-least-32-characters
43+
ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000"
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
47+
- name: Setup workspace
48+
uses: ./.github/actions/setup
49+
with:
50+
provider: github
51+
- name: Provision the database through migrations
52+
working-directory: packages/db
53+
run: bun run db:migrate
54+
- name: Download the production build
55+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
56+
with:
57+
name: next-build
58+
path: ${{ runner.temp }}
59+
run-id: ${{ inputs.source_run }}
60+
github-token: ${{ github.token }}
61+
- name: Extract the production build
62+
run: |
63+
tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim
64+
jq -e '[.routes[].srcRoute] | any(. == "/blog/[slug]" or . == "/library/[slug]") | not' \
65+
apps/sim/.next/prerender-manifest.json > /dev/null ||
66+
{ echo "::error::The build prerendered posts (a hosted build) or has no prerender manifest."; exit 1; }
67+
- name: Verify desktop tools in the Electron app against a local app
68+
env:
69+
NEXT_PUBLIC_APP_URL: http://127.0.0.1:3020
70+
BETTER_AUTH_URL: http://127.0.0.1:3020
71+
REDIS_URL: redis://127.0.0.1:6379
72+
SIM_AGENT_API_URL: http://127.0.0.1:3022
73+
SIM_DESKTOP_E2E_APP_SERVER: start
74+
NEXT_PUBLIC_SOCKET_URL: http://127.0.0.1:3023
75+
SOCKET_SERVER_URL: http://127.0.0.1:3023
76+
NEXT_PUBLIC_FORCE_HOSTED: "false"
77+
COPILOT_API_KEY: desktop-tools-e2e-ci-local-copilot-key
78+
COPILOT_TOOL_PERMISSIONS_ENABLED: "true"
79+
MOTHERSHIP_SIM_TRANSPORT: direct
80+
INTERNAL_API_SECRET: desktop-tools-e2e-ci-local-secret-at-least-32-characters
81+
DISABLE_TELEMETRY: "true"
82+
NEXT_TELEMETRY_DISABLED: "1"
83+
READY_TIMEOUT_SECONDS: 300
84+
run: |
85+
report_dir="$RUNNER_TEMP/e2e"
86+
next_log="$report_dir/desktop-tools-next.log"
87+
mkdir -p "$report_dir"
88+
# Each app runs in its own session under an E2E_APP tag, and stop-session.sh returns once
89+
# every process it started has exited.
90+
realtime_tag="desktop-realtime-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
91+
server_tag="desktop-tools-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
92+
start_sim() {
93+
(cd apps/sim && E2E_APP="$server_tag" LOG_LEVEL=INFO exec setsid node ../../node_modules/next/dist/bin/next start --hostname 127.0.0.1 \
94+
--port 3021 >> "$next_log" 2>&1) &
95+
server_pid=$!
96+
}
97+
stop_sim() {
98+
bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag"
99+
}
100+
(cd apps/realtime && PORT=3023 SIM_DB_ROLE=realtime ALLOWED_ORIGINS="$NEXT_PUBLIC_APP_URL" \
101+
E2E_APP="$realtime_tag" exec setsid bun src/index.ts > "$report_dir/desktop-tools-realtime.log" 2>&1) &
102+
realtime_pid=$!
103+
start_sim
104+
finish() {
105+
status=$?
106+
stop_sim || status=1
107+
bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$realtime_pid" "$realtime_tag" || status=1
108+
wait "$server_pid" "$realtime_pid" 2>/dev/null || true
109+
exit "$status"
110+
}
111+
trap finish EXIT
112+
# The apps boot while the runner installs Electron's libraries and bundles the shell.
113+
sudo apt-get update -q
114+
sudo apt-get install -yq xvfb libgtk-3-0t64 libnss3 libasound2t64 libgbm1 libxss1 \
115+
libxtst6 libatk-bridge2.0-0t64 libxkbcommon0 > /dev/null
116+
# Bundle only: `bun run build` also fetches the macOS node-pty prebuilds for packaging,
117+
# which a Linux run does not use.
118+
(cd apps/desktop && bun run scripts/build.ts)
119+
started=$SECONDS
120+
until curl --fail --silent --max-time 10 http://127.0.0.1:3021/api/health > /dev/null &&
121+
curl --fail --silent --max-time 10 http://127.0.0.1:3023/health > /dev/null; do
122+
if ! kill -0 "$server_pid" 2>/dev/null; then
123+
tail -n 200 "$next_log"
124+
exit 1
125+
fi
126+
kill -0 "$realtime_pid" 2>/dev/null || { tail -n 200 "$report_dir/desktop-tools-realtime.log"; exit 1; }
127+
[ $((SECONDS - started)) -lt "$READY_TIMEOUT_SECONDS" ] || { echo '::error::Local app did not become ready'; exit 1; }
128+
sleep 2
129+
done
130+
echo "Local apps ready $((SECONDS - started))s after the Electron bundle"
131+
cd apps/desktop
132+
SIM_DESKTOP_E2E_SIM_URL=http://127.0.0.1:3021 \
133+
SIM_DESKTOP_E2E_PROXY_PORT=3020 \
134+
SIM_DESKTOP_E2E_AGENT_PORT=3022 \
135+
PLAYWRIGHT_JSON_OUTPUT_FILE="$report_dir/desktop-tests.json" \
136+
SIM_DESKTOP_E2E_DATABASE_URL="$DATABASE_URL" \
137+
SIM_DESKTOP_E2E_REDIS_URL="$REDIS_URL" \
138+
SIM_DESKTOP_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \
139+
xvfb-run -a -s '-screen 0 1920x1200x24' bunx playwright test e2e/desktop-tools-live-sim.spec.ts \
140+
${{ matrix.filter }} --output "$report_dir/desktop-tools-results" --retries=0 --reporter=line,json
141+
- name: Upload Electron coverage and timings
142+
if: always()
143+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
144+
with:
145+
name: desktop-reports-${{ matrix.tests }}
146+
path: ${{ runner.temp }}/e2e/desktop-tests.json
147+
if-no-files-found: ignore
148+
retention-days: 7
149+
- name: Upload Electron E2E results and server logs
150+
if: failure()
151+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
152+
with:
153+
name: desktop-live-e2e-results-${{ matrix.tests }}
154+
path: ${{ runner.temp }}/e2e/
155+
if-no-files-found: ignore
156+
retention-days: 7
157+
e2e-production:
158+
name: e2e (${{ matrix.name }})
159+
runs-on: ubuntu-24.04
160+
timeout-minutes: 25
161+
strategy:
162+
fail-fast: false
163+
matrix:
164+
include:
165+
- group: project-files
166+
name: project-files
167+
browser: ""
168+
shard: ""
169+
artifact: project-files
170+
task: ci:e2e-project-files
171+
services:
172+
postgres:
173+
image: pgvector/pgvector:pg17
174+
env:
175+
POSTGRES_USER: postgres
176+
POSTGRES_PASSWORD: postgres
177+
POSTGRES_DB: sim_test
178+
ports:
179+
- 5432:5432
180+
options: --health-cmd "pg_isready -U postgres -d sim_test" --health-interval 5s --health-timeout 5s --health-retries 10
181+
redis:
182+
image: ${{ matrix.group == 'project-files' && 'redis:8.2-alpine' || '' }}
183+
ports:
184+
- 6379:6379
185+
options: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 5s --health-retries 10
186+
env:
187+
DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test
188+
BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters
189+
ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000"
190+
steps:
191+
- name: Checkout code
192+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
193+
- name: Setup workspace
194+
uses: ./.github/actions/setup
195+
with:
196+
provider: github
197+
- name: Provision the database through migrations
198+
if: matrix.group != 'project-files'
199+
working-directory: packages/db
200+
run: bun run db:migrate
201+
- name: Download the production build
202+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
203+
with:
204+
name: next-build
205+
path: ${{ runner.temp }}
206+
run-id: ${{ inputs.source_run }}
207+
github-token: ${{ github.token }}
208+
- name: Extract the production build
209+
run: |
210+
tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim
211+
jq -e '[.routes[].srcRoute] | any(. == "/blog/[slug]" or . == "/library/[slug]") | not' \
212+
apps/sim/.next/prerender-manifest.json > /dev/null ||
213+
{ echo "::error::The build prerendered posts (a hosted build) or has no prerender manifest."; exit 1; }
214+
- name: Build native isolate addon for Project file rendering
215+
if: matrix.group == 'project-files'
216+
working-directory: apps/sim
217+
env:
218+
JOBS: 4
219+
run: |
220+
addon_dir="$(node -p 'require("node:path").dirname(require.resolve("isolated-vm/package.json", { paths: ["."] }))')"
221+
cd "$addon_dir"
222+
bun run install
223+
- name: Run end-to-end suites
224+
working-directory: apps/sim
225+
env:
226+
MOBILE_E2E_BROWSER: ${{ matrix.browser }}
227+
MOBILE_E2E_SHARD: ${{ matrix.shard }}
228+
PROJECT_FILES_E2E_APP_SERVER: start
229+
run: bash ../../.github/scripts/http-e2e.sh "${{ matrix.group }}"
230+
- name: Upload end-to-end reports and server logs
231+
if: failure()
232+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
233+
with:
234+
name: http-e2e-${{ matrix.artifact }}
235+
path: ${{ runner.temp }}/e2e/
236+
if-no-files-found: ignore
237+
retention-days: 7
238+
- name: Upload acceptance timing and coverage reports
239+
if: always()
240+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
241+
with:
242+
name: e2e-reports-${{ matrix.artifact }}
243+
path: ${{ runner.temp }}/e2e/**/*.json
244+
if-no-files-found: ignore
245+
retention-days: 7

0 commit comments

Comments
 (0)