Skip to content

Commit ef8cefb

Browse files
authored
Merge pull request #423 from flashcatcloud/feat/ai-sre
Feat/ai sre
2 parents 35d6d8e + 69afbea commit ef8cefb

11 files changed

Lines changed: 553 additions & 81 deletions

File tree

‎.github/workflows/upload.yml‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
# Upload documentation to Meilisearch for AI Q&A bot indexing.
1+
# Publish the docs for AI Q&A: sync them into Meilisearch, and publish
2+
# flashduty-docs.tar.gz (the same docs as Markdown files, plus the OpenAPI
3+
# specs and glossary) for tools that grep and read the docs directly.
24
#
3-
# Every run syncs the whole doc set: all docs are upserted and index documents
4-
# whose source file is gone are deleted.
5+
# The Meilisearch sync covers the whole doc set: all docs are upserted and
6+
# index documents whose source file is gone are deleted.
57
#
68
# Uses GitHub Environments for dev/prod separation. Create two environments in
79
# Settings > Environments: "development" and "production".
@@ -10,10 +12,13 @@
1012
# MEILI_ENDPOINT - Meilisearch instance URL
1113
# MEILI_API_KEY - API key with documents read/write permission
1214
# MEILI_INDEX - Target index name
15+
# Repository secrets for the tarball: CDN_ACCESS_KEY, CDN_SECRET_KEY,
16+
# CDN_BUCKET, CDN_REGION, CDN_ENDPOINT.
1317
#
14-
# Branch mapping: main -> production, test -> development
18+
# Branch mapping: main -> production (https://docs-cdn.flashcat.cloud/docs/),
19+
# test -> development (https://docs-cdn.flashcat.cloud/test/docs/)
1520

16-
name: Upload docs to Meilisearch
21+
name: Publish docs for AI search
1722

1823
on:
1924
push:
@@ -22,6 +27,13 @@ on:
2227
- 'zh/**'
2328
- 'en/**'
2429
- 'docs.json'
30+
- 'api-reference/**'
31+
- 'glossary.md'
32+
- 'integration-docs/scripts/*docs-bundle*'
33+
- 'integration-docs/scripts/oss-cdn.mjs'
34+
- 'integration-docs/scripts/cdn-url.mjs'
35+
- 'integration-docs/package.json'
36+
- '.github/workflows/upload.yml'
2537
workflow_dispatch:
2638
inputs:
2739
environment:
@@ -60,3 +72,40 @@ jobs:
6072
run: |
6173
chmod +x ./scripts/upload.sh
6274
bash ./scripts/upload.sh
75+
76+
bundle:
77+
runs-on: ubuntu-latest
78+
environment: ${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'development') }}
79+
# Every run uploads the whole tarball, so a newer run supersedes an older
80+
# one; cancelling the older keeps it from finishing last with stale docs.
81+
concurrency:
82+
group: docs-bundle-${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'development') }}
83+
cancel-in-progress: true
84+
steps:
85+
- name: Checkout
86+
uses: actions/checkout@v4
87+
88+
- name: Setup Node.js
89+
uses: actions/setup-node@v4
90+
with:
91+
node-version: 18
92+
93+
- name: Install dependencies
94+
working-directory: integration-docs
95+
run: npm install
96+
97+
- name: Test docs bundle scripts
98+
working-directory: integration-docs
99+
run: npm run test:docs-bundle
100+
101+
- name: Build and upload docs bundle
102+
working-directory: integration-docs
103+
env:
104+
CDN_ACCESS_KEY: ${{ secrets.CDN_ACCESS_KEY }}
105+
CDN_SECRET_KEY: ${{ secrets.CDN_SECRET_KEY }}
106+
CDN_BUCKET: ${{ secrets.CDN_BUCKET }}
107+
CDN_REGION: ${{ secrets.CDN_REGION }}
108+
CDN_ENDPOINT: ${{ secrets.CDN_ENDPOINT }}
109+
CDN_URL: 'https://docs-cdn.flashcat.cloud'
110+
CDN_DIR: ${{ (github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'development')) == 'production' && '/docs' || '/test/docs' }}
111+
run: npm run upload:docs-bundle

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88
integration-docs/dist/
99
integration-docs/node_modules/
1010

11+
# Docs tarball build output (staging dir + flashduty-docs.tar.gz)
12+
.docs-bundle-build/
13+
1114
# Superpowers plugin docs (auto-generated)
1215
docs/
1316

‎integration-docs/package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@
1212
"upload": "npm run build && node scripts/upload.mjs",
1313
"test:upload-url": "node --test scripts/cdn-url.test.mjs",
1414
"test:openapi-upload": "node --test scripts/cdn-url.test.mjs scripts/upload-openapi.test.mjs",
15-
"upload:openapi": "node scripts/upload-openapi.mjs"
15+
"upload:openapi": "node scripts/upload-openapi.mjs",
16+
"test:docs-bundle": "node --test scripts/build-docs-bundle.test.mjs scripts/upload-docs-bundle.test.mjs",
17+
"build:docs-bundle": "node scripts/build-docs-bundle.mjs",
18+
"upload:docs-bundle": "npm run build:docs-bundle && node scripts/upload-docs-bundle.mjs"
1619
},
1720
"exports": {
1821
"./zh": {
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// Build flashduty-docs.tar.gz: an offline copy of the docs (zh/, en/,
2+
// api-reference/, glossary.md) plus a generated INDEX.md per locale, for
3+
// tools that grep/read Markdown directly instead of calling a search API.
4+
//
5+
// Page selection mirrors scripts/upload.sh (the Meilisearch sync): every
6+
// .md/.mdx file under zh/ or en/, except index pages.
7+
import fs from 'node:fs';
8+
import path from 'node:path';
9+
import { execFileSync } from 'node:child_process';
10+
import { fileURLToPath, pathToFileURL } from 'node:url';
11+
import { listOpenapiJsonFiles } from './upload-openapi.mjs';
12+
13+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
14+
const packageRoot = path.resolve(__dirname, '..');
15+
const defaultRepoRoot = path.resolve(packageRoot, '..');
16+
const defaultBuildDir = path.join(defaultRepoRoot, '.docs-bundle-build');
17+
const docsBaseUrl = 'https://docs.flashduty.com';
18+
const locales = ['zh', 'en'];
19+
20+
export function listDocPages(repoRoot, locale) {
21+
const localeDir = path.join(repoRoot, locale);
22+
const pages = [];
23+
24+
const walk = (dir) => {
25+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
26+
const entryPath = path.join(dir, entry.name);
27+
if (entry.isDirectory()) {
28+
walk(entryPath);
29+
continue;
30+
}
31+
if (!entry.isFile() || !/\.mdx?$/.test(entry.name)) continue;
32+
if (entry.name === 'index.md' || entry.name === 'index.mdx') continue;
33+
pages.push(path.relative(repoRoot, entryPath).split(path.sep).join('/'));
34+
}
35+
};
36+
walk(localeDir);
37+
38+
return pages.sort();
39+
}
40+
41+
export function parseFrontmatter(text) {
42+
const match = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);
43+
const fields = {};
44+
if (!match) return fields;
45+
46+
for (const line of match[1].split(/\r?\n/)) {
47+
const fieldMatch = /^([A-Za-z_]+):\s*(.*)$/.exec(line);
48+
if (!fieldMatch) continue;
49+
fields[fieldMatch[1]] = fieldMatch[2].trim().replace(/^["']|["']$/g, '');
50+
}
51+
52+
return fields;
53+
}
54+
55+
export function derivePublicUrl(relPath, frontmatter) {
56+
if (frontmatter.url) return frontmatter.url;
57+
return `${docsBaseUrl}/${relPath.replace(/\.mdx?$/, '')}`;
58+
}
59+
60+
export function buildIndexLine(relPath, frontmatter) {
61+
const title = frontmatter.title || '';
62+
const description = frontmatter.description ? `: ${frontmatter.description}` : '';
63+
return `- \`${relPath}\` — ${title}${description} ${derivePublicUrl(relPath, frontmatter)}`;
64+
}
65+
66+
export function buildLocaleIndex(repoRoot, locale) {
67+
const relPaths = listDocPages(repoRoot, locale);
68+
const lines = relPaths.map((relPath) => {
69+
const text = fs.readFileSync(path.join(repoRoot, relPath), 'utf8');
70+
return buildIndexLine(relPath, parseFrontmatter(text));
71+
});
72+
return { relPaths, lines };
73+
}
74+
75+
export function buildDocsBundle({
76+
repoRoot,
77+
stagingDir,
78+
outDir,
79+
apiReferenceDir = path.join(repoRoot, 'api-reference'),
80+
tarBin = 'tar'
81+
}) {
82+
fs.rmSync(stagingDir, { recursive: true, force: true });
83+
fs.mkdirSync(stagingDir, { recursive: true });
84+
fs.mkdirSync(outDir, { recursive: true });
85+
86+
const pageCounts = {};
87+
for (const locale of locales) {
88+
const { relPaths, lines } = buildLocaleIndex(repoRoot, locale);
89+
for (const relPath of relPaths) {
90+
const dest = path.join(stagingDir, relPath);
91+
fs.mkdirSync(path.dirname(dest), { recursive: true });
92+
fs.copyFileSync(path.join(repoRoot, relPath), dest);
93+
}
94+
const header = `# Flashduty docs index (${locale})\n\nOne line per page: file path — title: description public URL.\n\n`;
95+
fs.writeFileSync(path.join(stagingDir, locale, 'INDEX.md'), `${header}${lines.join('\n')}\n`);
96+
pageCounts[locale] = relPaths.length;
97+
}
98+
99+
const apiReferenceStagingDir = path.join(stagingDir, 'api-reference');
100+
fs.mkdirSync(apiReferenceStagingDir, { recursive: true });
101+
const apiReferenceFiles = listOpenapiJsonFiles(apiReferenceDir);
102+
for (const file of apiReferenceFiles) {
103+
fs.copyFileSync(path.join(apiReferenceDir, file), path.join(apiReferenceStagingDir, file));
104+
}
105+
106+
fs.copyFileSync(path.join(repoRoot, 'glossary.md'), path.join(stagingDir, 'glossary.md'));
107+
108+
const tarPath = path.join(outDir, 'flashduty-docs.tar.gz');
109+
execFileSync(tarBin, ['-czf', tarPath, '-C', stagingDir, 'zh', 'en', 'api-reference', 'glossary.md']);
110+
111+
return {
112+
tarPath,
113+
size: fs.statSync(tarPath).size,
114+
zhPages: pageCounts.zh,
115+
enPages: pageCounts.en,
116+
apiReferenceFiles: apiReferenceFiles.length
117+
};
118+
}
119+
120+
async function main() {
121+
const result = buildDocsBundle({
122+
repoRoot: defaultRepoRoot,
123+
stagingDir: path.join(defaultBuildDir, 'staging'),
124+
outDir: defaultBuildDir
125+
});
126+
console.log(
127+
`Built ${result.tarPath} (${result.size} bytes): `
128+
+ `zh=${result.zhPages} en=${result.enPages} api-reference=${result.apiReferenceFiles}`
129+
);
130+
}
131+
132+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
133+
await main();
134+
}
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
import assert from 'node:assert/strict';
2+
import fs from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
import { execFileSync } from 'node:child_process';
6+
import test from 'node:test';
7+
8+
import {
9+
buildDocsBundle,
10+
buildIndexLine,
11+
derivePublicUrl,
12+
listDocPages,
13+
parseFrontmatter
14+
} from './build-docs-bundle.mjs';
15+
16+
test('parseFrontmatter reads quoted and unquoted fields', () => {
17+
const text = [
18+
'---',
19+
'title: "飞书/Lark"',
20+
'description: "通过集成飞书自建应用"',
21+
'url: https://status.flashcat.cloud',
22+
'---',
23+
'',
24+
'# body'
25+
].join('\n');
26+
27+
assert.deepEqual(parseFrontmatter(text), {
28+
title: '飞书/Lark',
29+
description: '通过集成飞书自建应用',
30+
url: 'https://status.flashcat.cloud'
31+
});
32+
});
33+
34+
test('parseFrontmatter returns no fields without a frontmatter block', () => {
35+
assert.deepEqual(parseFrontmatter('# just a heading\n'), {});
36+
});
37+
38+
test('derivePublicUrl prefers frontmatter url, else derives from the path', () => {
39+
assert.equal(
40+
derivePublicUrl('zh/on-call/integration/instant-messaging/lark.mdx', {}),
41+
'https://docs.flashduty.com/zh/on-call/integration/instant-messaging/lark'
42+
);
43+
assert.equal(
44+
derivePublicUrl('zh/stsatuspage.mdx', { url: 'https://status.flashcat.cloud' }),
45+
'https://status.flashcat.cloud'
46+
);
47+
});
48+
49+
test('buildIndexLine omits the description separator when there is none', () => {
50+
assert.equal(
51+
buildIndexLine('zh/foo.mdx', { title: '标题', description: '描述' }),
52+
'- `zh/foo.mdx` — 标题: 描述 https://docs.flashduty.com/zh/foo'
53+
);
54+
assert.equal(
55+
buildIndexLine('zh/foo.mdx', { title: '标题' }),
56+
'- `zh/foo.mdx` — 标题 https://docs.flashduty.com/zh/foo'
57+
);
58+
});
59+
60+
test('listDocPages selects only .md/.mdx pages, excludes index pages and media', () => {
61+
const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'docs-bundle-pages-'));
62+
fs.mkdirSync(path.join(repoRoot, 'zh', 'on-call'), { recursive: true });
63+
fs.writeFileSync(path.join(repoRoot, 'zh', 'a.mdx'), '# a\n');
64+
fs.writeFileSync(path.join(repoRoot, 'zh', 'on-call', 'b.md'), '# b\n');
65+
fs.writeFileSync(path.join(repoRoot, 'zh', 'index.mdx'), '# index\n');
66+
fs.writeFileSync(path.join(repoRoot, 'zh', 'on-call', 'diagram.png'), 'not a page');
67+
fs.writeFileSync(path.join(repoRoot, 'zh', '.DS_Store'), 'junk');
68+
69+
assert.deepEqual(listDocPages(repoRoot, 'zh'), ['zh/a.mdx', 'zh/on-call/b.md']);
70+
});
71+
72+
function hasTar() {
73+
try {
74+
execFileSync('tar', ['--version']);
75+
return true;
76+
} catch {
77+
return false;
78+
}
79+
}
80+
81+
test('buildDocsBundle produces the fixed archive layout with no media files', { skip: !hasTar() }, () => {
82+
const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'docs-bundle-repo-'));
83+
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'docs-bundle-work-'));
84+
85+
fs.mkdirSync(path.join(repoRoot, 'zh', 'on-call'), { recursive: true });
86+
fs.mkdirSync(path.join(repoRoot, 'en'), { recursive: true });
87+
fs.mkdirSync(path.join(repoRoot, 'api-reference'), { recursive: true });
88+
89+
fs.writeFileSync(
90+
path.join(repoRoot, 'zh', 'on-call', 'lark.mdx'),
91+
'---\ntitle: "飞书/Lark"\ndescription: "接入飞书"\n---\n\nbody\n'
92+
);
93+
fs.writeFileSync(path.join(repoRoot, 'zh', 'on-call', 'diagram.png'), 'not a page');
94+
fs.writeFileSync(
95+
path.join(repoRoot, 'en', 'lark.mdx'),
96+
'---\ntitle: "Lark"\n---\n\nbody\n'
97+
);
98+
fs.writeFileSync(path.join(repoRoot, 'api-reference', 'on-call.openapi.zh.json'), '{}\n');
99+
fs.writeFileSync(path.join(repoRoot, 'api-reference', 'README.md'), 'not json\n');
100+
fs.writeFileSync(path.join(repoRoot, 'glossary.md'), '# glossary\n');
101+
102+
const stagingDir = path.join(work, 'staging');
103+
const outDir = path.join(work, 'out');
104+
const result = buildDocsBundle({ repoRoot, stagingDir, outDir });
105+
106+
assert.equal(result.zhPages, 1);
107+
assert.equal(result.enPages, 1);
108+
assert.equal(result.apiReferenceFiles, 1);
109+
assert.ok(result.size > 0);
110+
111+
const extractDir = path.join(work, 'extract');
112+
fs.mkdirSync(extractDir, { recursive: true });
113+
execFileSync('tar', ['-xzf', result.tarPath, '-C', extractDir]);
114+
115+
assert.deepEqual(fs.readdirSync(extractDir).sort(), ['api-reference', 'en', 'glossary.md', 'zh']);
116+
assert.deepEqual(fs.readdirSync(path.join(extractDir, 'api-reference')), ['on-call.openapi.zh.json']);
117+
assert.ok(fs.existsSync(path.join(extractDir, 'zh', 'INDEX.md')));
118+
assert.ok(fs.existsSync(path.join(extractDir, 'en', 'INDEX.md')));
119+
assert.ok(!fs.existsSync(path.join(extractDir, 'zh', 'on-call', 'diagram.png')));
120+
121+
const zhIndex = fs.readFileSync(path.join(extractDir, 'zh', 'INDEX.md'), 'utf8');
122+
assert.match(zhIndex, /`zh\/on-call\/lark\.mdx` — 飞书\/Lark: 接入飞书 https:\/\/docs\.flashduty\.com\/zh\/on-call\/lark/);
123+
});

0 commit comments

Comments
 (0)