Skip to content

Commit c795c47

Browse files
committed
perf(test): optimize CLI suites and shard CI tests
1 parent 6f0cd3d commit c795c47

7 files changed

Lines changed: 597 additions & 48 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 102 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,13 @@ concurrency:
99
group: ci-${{ github.ref }}
1010
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
1111

12+
permissions:
13+
contents: read
14+
actions: read
15+
1216
jobs:
13-
check:
14-
name: lint + typecheck + test
17+
quality:
18+
name: lint + typecheck + build
1519
runs-on: ubuntu-latest
1620
steps:
1721
- uses: actions/checkout@v6
@@ -23,7 +27,32 @@ jobs:
2327
node-version: "22"
2428
cache: pnpm
2529

26-
# Keep native-fetch and compiled Binary tests on the release Bun version.
30+
- run: pnpm install --frozen-lockfile
31+
32+
- run: pnpm run sync:skill-assets
33+
34+
- run: pnpm -r --filter "./packages/*" build
35+
36+
- run: pnpm run check
37+
38+
test:
39+
name: test shard ${{ matrix.shard }}/4
40+
runs-on: ubuntu-latest
41+
strategy:
42+
fail-fast: false
43+
matrix:
44+
shard: [1, 2, 3, 4]
45+
steps:
46+
- uses: actions/checkout@v6
47+
48+
- uses: pnpm/action-setup@v6
49+
50+
- uses: actions/setup-node@v6
51+
with:
52+
node-version: "22"
53+
cache: pnpm
54+
55+
# Every shard keeps the real Node/Bun/compiled-binary coverage.
2756
- uses: oven-sh/setup-bun@v2
2857
with:
2958
bun-version: "1.3.14"
@@ -34,6 +63,74 @@ jobs:
3463

3564
- run: pnpm -r --filter "./packages/*" build
3665

37-
- run: pnpm run check
66+
- name: Run test shard
67+
run: >-
68+
pnpm exec vp test --shard=${{ matrix.shard }}/4
69+
--reporter=default --reporter=blob --reporter=json
70+
--outputFile.blob=.ci-test-results/blob/shard-${{ matrix.shard }}.json
71+
--outputFile.json=.ci-test-results/json/shard-${{ matrix.shard }}.json
3872
39-
- run: pnpm test
73+
- name: Upload shard reports
74+
if: ${{ always() }}
75+
uses: actions/upload-artifact@v7
76+
with:
77+
name: test-shard-${{ matrix.shard }}
78+
path: .ci-test-results/
79+
include-hidden-files: true
80+
if-no-files-found: error
81+
retention-days: 3
82+
83+
# Keep the existing required-check identity. Upstream failure/skip/cancellation
84+
# must produce a failing check, rather than silently skipping the gate.
85+
check:
86+
name: lint + typecheck + test
87+
if: ${{ always() }}
88+
needs: [quality, test]
89+
runs-on: ubuntu-latest
90+
steps:
91+
- uses: actions/checkout@v6
92+
93+
- uses: pnpm/action-setup@v6
94+
95+
- uses: actions/setup-node@v6
96+
with:
97+
node-version: "22"
98+
cache: pnpm
99+
100+
- run: pnpm install --frozen-lockfile
101+
102+
- name: Download shard reports
103+
id: download
104+
uses: actions/download-artifact@v8
105+
with:
106+
pattern: test-shard-*
107+
path: .ci-test-results
108+
merge-multiple: true
109+
110+
# list --filesOnly discovers the full inventory, independently of sharding.
111+
- name: Discover complete test inventory
112+
run: pnpm exec vp test list --filesOnly --json=.ci-test-inventory.json
113+
114+
- name: Verify complete shard coverage
115+
if: ${{ always() }}
116+
env:
117+
QUALITY_RESULT: ${{ needs.quality.result }}
118+
TEST_RESULT: ${{ needs.test.result }}
119+
run: >-
120+
node tools/ci/verify-shard-reports.mjs
121+
--reports .ci-test-results --inventory .ci-test-inventory.json --shards 4
122+
123+
- name: Merge test reports
124+
if: ${{ always() && steps.download.outcome == 'success' }}
125+
run: >-
126+
pnpm exec vp test --mergeReports=.ci-test-results/blob
127+
--reporter=default --reporter=json --outputFile.json=.ci-test-summary.json
128+
129+
- name: Upload merged report
130+
if: ${{ always() }}
131+
uses: actions/upload-artifact@v7
132+
with:
133+
name: test-summary
134+
path: .ci-test-summary.json
135+
include-hidden-files: true
136+
retention-days: 3

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ tools/generated
4646

4747
# Test & scene outputs
4848
/test/
49+
/.ci-test-results/
50+
/.ci-test-inventory.json
51+
/.ci-test-summary.json
4952
packages/cli/scene/**/outputs/
5053

5154
# Environment variables (sensitive data)

‎docs/agents/cli-e2e-tests.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,14 @@ pnpm exec vp test packages/commands/tests/e2e/managed-agent-webhook.e2e.test.ts
9393

9494
需要三轮对比时为每轮分别命名 `before-1/2/3` 和 `after-1/2/3`,从报告读取套件耗时与总墙钟;worker 数可按环境选择,但前后必须一致。
9595

96+
### D. CI 分片与汇总门槛
97+
98+
- [ ] `.github/workflows/ci.yml` 使用 4 个独立 runner 的原生 `--shard=N/4`,保留完整测试范围、默认文件隔离、既有门控及每片相同的 Node/Bun 版本;不能用新增 skip、文件过滤或同一 checkout 多进程共享全局状态替代分片
99+
- [ ] 质量检查与测试矩阵并行;矩阵 `fail-fast: false`,失败时仍上传已有报告。每片输出独立命名的 blob + JSON 报告,保留 3 天
100+
- [ ] `jobs.check` 和名称 `lint + typecheck + test` 保持现有 required check 身份;汇总 job 用 `always()` 执行,并显式要求质量检查和整个测试矩阵均成功,不能让失败、取消或整个 job 跳过变成绿灯
101+
- [ ] `tools/ci/verify-shard-reports.mjs` 在合并前核对 4 片报告完整,动态发现的全部测试文件恰好出现一次,报告及用例没有失败;缺报告、漏跑、重复或元数据不一致必须使门槛失败。不要硬编码文件数或用例数
102+
- [ ] 用 `--mergeReports` 汇总 blob 报告;即使校验失败仍尽可能合并已有报告以便诊断,合并成功不能覆盖前面的失败结果。调整该链路时验证成功路径与失败/取消/跳过、缺报告、漏跑和重复路径
103+
96104
## 离线与真实集成分组
97105

98106
```ts

‎packages/commands/tests/e2e/deploy.e2e.test.ts‎

Lines changed: 131 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { fileURLToPath } from "node:url";
2-
import { describe, expect, test } from "vite-plus/test";
2+
import { httpTransport } from "bailian-cli-core";
3+
import { describe, expect, test, vi } from "vite-plus/test";
34
import {
45
isDashScopeE2EReady,
56
parseStdoutJson,
@@ -22,14 +23,120 @@ const require = createRequire(${JSON.stringify(fileURLToPath(new URL("../../../c
2223
require("undici").fetch = globalThis.fetch = async () => { throw new Error("Network disabled for deploy query E2E"); };`)}`,
2324
};
2425

26+
const IN_PROCESS_ENV: NodeJS.ProcessEnv = {
27+
...READ_ONLY_ENV,
28+
NODE_OPTIONS: "",
29+
NODE_USE_ENV_PROXY: "0",
30+
HTTP_PROXY: "",
31+
HTTPS_PROXY: "",
32+
ALL_PROXY: "",
33+
http_proxy: "",
34+
https_proxy: "",
35+
all_proxy: "",
36+
DASHSCOPE_API_KEY: "",
37+
BAILIAN_API_KEY: "",
38+
DASHSCOPE_BASE_URL: "",
39+
BAILIAN_BASE_URL: "",
40+
BAILIAN_WORKSPACE_ID: "",
41+
DASHSCOPE_OUTPUT: "",
42+
DASHSCOPE_QUIET: "0",
43+
DASHSCOPE_VERBOSE: "0",
44+
};
45+
46+
async function runOfflineDeploy(args: string[]) {
47+
// Keep the no-network contract without NODE_OPTIONS forcing every matrix case
48+
// into a new process. Fail even if command code catches a forbidden request.
49+
const transportFetch = vi
50+
.spyOn(httpTransport, "fetch")
51+
.mockRejectedValue(new Error("Network disabled for deploy query E2E"));
52+
const nativeFetch = vi
53+
.spyOn(globalThis, "fetch")
54+
.mockRejectedValue(new Error("Network disabled for deploy query E2E"));
55+
try {
56+
const result = await runCommandInProcess(DEPLOY_ROUTES, args, IN_PROCESS_ENV);
57+
expect(transportFetch).not.toHaveBeenCalled();
58+
expect(nativeFetch).not.toHaveBeenCalled();
59+
return result;
60+
} finally {
61+
transportFetch.mockRestore();
62+
nativeFetch.mockRestore();
63+
}
64+
}
65+
2566
function runReadOnlyDeploy(args: string[]) {
26-
return runCommandInProcess(
27-
DEPLOY_ROUTES,
28-
["deploy", ...args, "--api-key", "e2e-dummy-key", "--dry-run", "--output", "json", "--quiet"],
29-
READ_ONLY_ENV,
30-
);
67+
return runOfflineDeploy([
68+
"deploy",
69+
...args,
70+
"--api-key",
71+
"e2e-dummy-key",
72+
"--dry-run",
73+
"--output",
74+
"json",
75+
"--quiet",
76+
]);
3177
}
3278

79+
test.each([
80+
{
81+
label: "read-only query preview",
82+
args: [
83+
"deploy",
84+
"capacity",
85+
"list",
86+
"--deployed-model",
87+
"dep-test",
88+
"--dry-run",
89+
"--api-key",
90+
"e2e-dummy-key",
91+
],
92+
exitCode: 0,
93+
},
94+
{
95+
label: "missing required query flag",
96+
args: [
97+
"deploy",
98+
"capacity",
99+
"get",
100+
"--deployed-model",
101+
"dep-test",
102+
"--dry-run",
103+
"--api-key",
104+
"e2e-dummy-key",
105+
],
106+
exitCode: 2,
107+
},
108+
{
109+
label: "risky write confirmation",
110+
args: [
111+
"deploy",
112+
"capacity",
113+
"delete",
114+
"--deployed-model",
115+
"dep-test",
116+
"--instance-id",
117+
"instance-test",
118+
"--api-key",
119+
"e2e-dummy-key",
120+
],
121+
exitCode: 7,
122+
},
123+
{
124+
label: "refund link without credentials",
125+
args: ["deploy", "capacity", "unsubscribe", "--instance-id", "a b&c"],
126+
exitCode: 0,
127+
},
128+
])("$label matches the network-blocked subprocess exactly", async ({ args, exitCode }) => {
129+
const commandArgs = [...args, "--output", "json"];
130+
const inProcess = await runOfflineDeploy(commandArgs);
131+
// Preserve real startup/preload coverage and block networking in that process.
132+
const subprocess = await runCommandE2e(DEPLOY_ROUTES, commandArgs, {
133+
...IN_PROCESS_ENV,
134+
NODE_OPTIONS: READ_ONLY_ENV.NODE_OPTIONS,
135+
});
136+
expect(subprocess.exitCode, subprocess.stderr).toBe(exitCode);
137+
expect(inProcess).toEqual(subprocess);
138+
});
139+
33140
// Deliberately ungated: these checks must also run on machines without credentials.
34141
describe("e2e: deploy read-only queries (offline, no credentials)", () => {
35142
test.each([
@@ -204,11 +311,7 @@ describe("e2e: deploy capacity unsubscribe (offline, no credentials)", () => {
204311
const REFUND_URL = `https://billing-cost.console.aliyun.com/refund/refund?instanceId=${INSTANCE_ID}`;
205312

206313
function runUnsubscribe(args: string[]) {
207-
return runCommandE2e(
208-
DEPLOY_ROUTES,
209-
["deploy", "capacity", "unsubscribe", ...args, "--output", "json"],
210-
READ_ONLY_ENV,
211-
);
314+
return runOfflineDeploy(["deploy", "capacity", "unsubscribe", ...args, "--output", "json"]);
212315
}
213316

214317
test("--help exposes --instance-id, no --yes and no --api-key requirement", async () => {
@@ -239,11 +342,14 @@ describe("e2e: deploy capacity unsubscribe (offline, no credentials)", () => {
239342
});
240343

241344
test("--quiet retains the complete refund-link result", async () => {
242-
const { stdout, stderr, exitCode } = await runCommandE2e(
243-
DEPLOY_ROUTES,
244-
["deploy", "capacity", "unsubscribe", "--instance-id", INSTANCE_ID, "--quiet"],
245-
READ_ONLY_ENV,
246-
);
345+
const { stdout, stderr, exitCode } = await runOfflineDeploy([
346+
"deploy",
347+
"capacity",
348+
"unsubscribe",
349+
"--instance-id",
350+
INSTANCE_ID,
351+
"--quiet",
352+
]);
247353
expect(exitCode, stderr).toBe(0);
248354
expect(JSON.parse(stdout)).toEqual({
249355
action: "deploy.capacity.unsubscribe",
@@ -798,11 +904,15 @@ describe("e2e: deploy capacity writes (offline, no credentials)", () => {
798904
},
799905
{ path: ["overflow"], args: ["--deployed-model", "dep-test", "--strategy", "disable"] },
800906
])("deploy $path without --yes requires confirmation (7)", async ({ path, args }) => {
801-
const { stdout, stderr, exitCode } = await runCommandE2e(
802-
DEPLOY_ROUTES,
803-
["deploy", ...path, ...args, "--api-key", "e2e-dummy-key", "--output", "json"],
804-
READ_ONLY_ENV,
805-
);
907+
const { stdout, stderr, exitCode } = await runOfflineDeploy([
908+
"deploy",
909+
...path,
910+
...args,
911+
"--api-key",
912+
"e2e-dummy-key",
913+
"--output",
914+
"json",
915+
]);
806916
expect(exitCode, stderr).toBe(7);
807917
expect(stdout.trim()).toBe("");
808918
expect(JSON.parse(stderr)).toMatchObject({

0 commit comments

Comments
 (0)