From 2e68d9b575b49d16097bdc0dc516a3267a91c0ff Mon Sep 17 00:00:00 2001 From: Y1hsiaochunnn <66012385+Y1hsiaochunnn@users.noreply.github.com> Date: Mon, 7 Sep 2026 13:52:20 +0800 Subject: [PATCH] Publish new component versions automatically after merging to main --- .github/workflows/ci.yml | 14 ++++- .github/workflows/publish.yml | 88 +++++++++++++++++++++++++--- ci/scripts/check_release.py | 19 ++++-- ci/scripts/github_release.cjs | 34 +++++++++++ ci/scripts/test_github_release.cjs | 39 ++++++++++++ ci/scripts/test_repository_checks.py | 35 +++++++++-- docs/PUBLISHING.md | 41 +++++++++---- docs/PUBLISHING_ZH.md | 34 ++++++++--- 8 files changed, 263 insertions(+), 41 deletions(-) create mode 100644 ci/scripts/github_release.cjs create mode 100644 ci/scripts/test_github_release.cjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 311b66a..77b38ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,11 @@ on: branches: [main] workflow_dispatch: workflow_call: + inputs: + full_validation: + description: Build every board before publishing a new version + type: boolean + default: false permissions: contents: read @@ -37,7 +42,9 @@ jobs: - name: Install validation tools run: python -m pip install PyYAML==6.0.3 idf-component-manager==2.5.0 - name: Test CI helpers - run: python -m unittest discover -s ci/scripts -p 'test_*.py' -v + run: | + python -m unittest discover -s ci/scripts -p 'test_*.py' -v + node --test ci/scripts/test_github_release.cjs - name: Check board definitions and documentation navigation run: | python ci/scripts/board_pack.py check @@ -48,10 +55,13 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + FULL_VALIDATION: ${{ inputs.full_validation || false }} shell: bash run: | set -euo pipefail - if [[ "$EVENT_NAME" == "pull_request" || "$EVENT_NAME" == "push" ]]; then + if [[ "$FULL_VALIDATION" == true ]]; then + python ci/scripts/board_pack.py matrix --all + elif [[ "$EVENT_NAME" == "pull_request" || "$EVENT_NAME" == "push" ]]; then python ci/scripts/board_pack.py matrix --base "$BASE_SHA" else python ci/scripts/board_pack.py matrix --all diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3a3172f..d2c183f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,8 @@ name: Publish board pack on: + push: + branches: [main] workflow_dispatch: inputs: dry_run: @@ -16,14 +18,50 @@ concurrency: group: board-pack-publish cancel-in-progress: false +env: + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }} + jobs: + prepare: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} + publish: ${{ steps.plan.outputs.result }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install PyYAML==6.0.3 + - id: version + name: Check release source and manifest + run: python ci/scripts/check_release.py + - id: plan + name: Skip versions with a completed release + uses: actions/github-script@v7 + env: + RELEASE_TAG: ${{ steps.version.outputs.tag }} + with: + script: | + const release = require('./ci/scripts/github_release.cjs'); + return release.shouldPublish({github, context, core}, process.env.RELEASE_TAG, process.env.DRY_RUN === 'true'); + validate: + needs: prepare + if: needs.prepare.outputs.publish == 'true' uses: ./.github/workflows/ci.yml + with: + full_validation: true upload: - needs: validate + needs: [prepare, validate] runs-on: ubuntu-latest environment: component-registry + permissions: + contents: write steps: - uses: actions/checkout@v4 with: @@ -33,24 +71,56 @@ jobs: python-version: '3.12' - name: Install component tooling run: python -m pip install PyYAML==6.0.3 idf-component-manager==2.5.0 - - name: Check release reference + - name: Check release source and credentials env: - DRY_RUN: ${{ inputs.dry_run }} + IDF_COMPONENT_API_TOKEN: ${{ secrets.IDF_COMPONENT_API_TOKEN }} run: | python ci/scripts/check_release.py + test -n "$IDF_COMPONENT_API_TOKEN" || { echo 'Set IDF_COMPONENT_API_TOKEN in Actions secrets.'; exit 1; } + - name: Create or verify immutable version tag + if: env.DRY_RUN == 'false' + uses: actions/github-script@v7 + env: + RELEASE_TAG: ${{ needs.prepare.outputs.tag }} + with: + script: | + const release = require('./ci/scripts/github_release.cjs'); + await release.ensureTag({github, context}, process.env.RELEASE_TAG); - name: Upload to waveshare namespace env: IDF_COMPONENT_API_TOKEN: ${{ secrets.IDF_COMPONENT_API_TOKEN }} - DRY_RUN: ${{ inputs.dry_run }} shell: bash run: | set -euo pipefail - if [[ -z "$IDF_COMPONENT_API_TOKEN" ]]; then - echo 'Set IDF_COMPONENT_API_TOKEN in the component-registry environment or repository Actions secrets.' - exit 1 - fi options=() if [[ "$DRY_RUN" == true ]]; then options+=(--dry-run) fi - compote component upload --namespace waveshare --name waveshare_boards "${options[@]}" + compote component upload --namespace waveshare --name waveshare_boards --commit-sha "$GITHUB_SHA" "${options[@]}" + + release: + needs: [prepare, upload] + if: ${{ success() && (github.event_name == 'push' || !inputs.dry_run) }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Publish GitHub Release after registry upload + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ needs.prepare.outputs.tag }} + VERSION: ${{ needs.prepare.outputs.version }} + shell: bash + run: | + set -euo pipefail + cat > release-notes.md < tag after merging to main") + if not isinstance(version, str) or not re.fullmatch(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)", version): + raise ValueError("release version must be a stable major.minor.patch string") + if dry_run == "true": + return version + if ref not in {"refs/heads/main", f"refs/tags/v{version}"}: + raise ValueError("publish from main or its v tag") subprocess.run(["git", "merge-base", "--is-ancestor", "HEAD", "origin/main"], cwd=root, check=True) + return version if __name__ == "__main__": try: - check(ROOT, os.environ.get("DRY_RUN"), os.environ.get("GITHUB_REF")) + version = check(ROOT, os.environ.get("DRY_RUN"), os.environ.get("GITHUB_REF")) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"version={version}\ntag=v{version}\n") except Exception as error: print(f"release check failed: {error}", file=sys.stderr) sys.exit(1) diff --git a/ci/scripts/github_release.cjs b/ci/scripts/github_release.cjs new file mode 100644 index 0000000..e3de418 --- /dev/null +++ b/ci/scripts/github_release.cjs @@ -0,0 +1,34 @@ +// GitHub release operations are separate from registry credentials and uploads. +async function shouldPublish({github, context, core}, tag, dryRun) { + if (dryRun) return true; + try { + const {data} = await github.rest.repos.getReleaseByTag({...context.repo, tag}); + if (data.draft) throw new Error(`Finish or remove the existing draft release ${tag} before publishing`); + core.info(`${tag} already has a completed release; skipping publication`); + return false; + } catch (error) { + if (error.status === 404) return true; + throw error; + } +} + +async function ensureTag({github, context}, tag) { + let target; + try { + const {data} = await github.rest.git.getRef({...context.repo, ref: `tags/${tag}`}); + target = data.object; + } catch (error) { + if (error.status !== 404) throw error; + await github.rest.git.createRef({...context.repo, ref: `refs/tags/${tag}`, sha: context.sha}); + return; + } + while (target.type === 'tag') { + const {data} = await github.rest.git.getTag({...context.repo, tag_sha: target.sha}); + target = data.object; + } + if (target.type !== 'commit' || target.sha !== context.sha) { + throw new Error(`${tag} points to another commit; retry the tagged run or use a new version`); + } +} + +module.exports = {shouldPublish, ensureTag}; diff --git a/ci/scripts/test_github_release.cjs b/ci/scripts/test_github_release.cjs new file mode 100644 index 0000000..ce0721b --- /dev/null +++ b/ci/scripts/test_github_release.cjs @@ -0,0 +1,39 @@ +const {test} = require('node:test'); +const assert = require('node:assert/strict'); +const {shouldPublish, ensureTag} = require('./github_release.cjs'); +const context = {repo: {owner: 'vendor', repo: 'boards'}, sha: 'tested-commit'}; +const missing = () => { throw Object.assign(new Error('Missing'), {status: 404}); }; +const unavailable = () => { throw Object.assign(new Error('Unavailable'), {status: 503}); }; +function fixture(repos = {}, git = {}) { + return {context, github: {rest: {repos, git}}, core: {info() {}}}; +} + +test('dry run does not query or mutate releases', async () => { + assert.equal(await shouldPublish(fixture(), 'v0.1.0', true), true); +}); +test('new version proceeds, completed version skips', async () => { + assert.equal(await shouldPublish(fixture({getReleaseByTag: missing}), 'v0.1.0', false), true); + assert.equal(await shouldPublish(fixture({getReleaseByTag: async () => ({data: {draft: false}})}), 'v0.1.0', false), false); +}); +test('draft and unavailable release status do not authorize publishing', async () => { + await assert.rejects(shouldPublish(fixture({getReleaseByTag: async () => ({data: {draft: true}})}), 'v0.1.0', false), /draft/); + await assert.rejects(shouldPublish(fixture({getReleaseByTag: unavailable}), 'v0.1.0', false), /Unavailable/); +}); +test('new tag points only to the tested workflow commit', async () => { + const calls = []; + await ensureTag(fixture({}, {getRef: missing, createRef: async args => calls.push(args)}), 'v0.1.0'); + assert.deepEqual(calls, [{...context.repo, ref: 'refs/tags/v0.1.0', sha: context.sha}]); +}); +test('matching lightweight and annotated tags are reusable without mutation', async () => { + await ensureTag(fixture({}, {getRef: async () => ({data: {object: {type: 'commit', sha: context.sha}}})}), 'v0.1.0'); + await ensureTag(fixture({}, { + getRef: async () => ({data: {object: {type: 'tag', sha: 'annotated'}}}), + getTag: async () => ({data: {object: {type: 'commit', sha: context.sha}}}) + }), 'v0.1.0'); +}); +test('different source cannot move an existing version tag', async () => { + await assert.rejects(ensureTag(fixture({}, {getRef: async () => ({data: {object: {type: 'commit', sha: 'other'}}})}), 'v0.1.0'), /another commit/); +}); +test('tag lookup network failure cannot be treated as missing', async () => { + await assert.rejects(ensureTag(fixture({}, {getRef: unavailable}), 'v0.1.0'), /Unavailable/); +}); diff --git a/ci/scripts/test_repository_checks.py b/ci/scripts/test_repository_checks.py index e3364b2..f7c6b69 100644 --- a/ci/scripts/test_repository_checks.py +++ b/ci/scripts/test_repository_checks.py @@ -56,19 +56,37 @@ def test_catalog_replaces_stale_boards_and_check_never_writes(self): with self.assertRaises(ValueError): catalog.update(self.root) - def test_real_publish_requires_matching_tag_and_main_ancestry(self): + def test_real_publish_requires_main_or_matching_tag_and_main_ancestry(self): (self.root / 'idf_component.yml').write_text('version: "0.1.0"\n') check_release.check(self.root, 'true', 'refs/heads/feature') - for ref in ('refs/heads/main', 'refs/tags/v0.2.0'): + for ref in ('refs/heads/feature', 'refs/tags/v0.2.0'): with self.assertRaises(ValueError): check_release.check(self.root, 'false', ref) with patch('check_release.subprocess.run') as run: + self.assertEqual('0.1.0', check_release.check(self.root, 'false', 'refs/heads/main')) check_release.check(self.root, 'false', 'refs/tags/v0.1.0') self.assertEqual(['git', 'merge-base', '--is-ancestor', 'HEAD', 'origin/main'], run.call_args.args[0]) run.side_effect = subprocess.CalledProcessError(1, 'git') with self.assertRaises(subprocess.CalledProcessError): check_release.check(self.root, 'false', 'refs/tags/v0.1.0') + def test_release_rejects_invalid_versions_and_modes(self): + for version in ('01.0.0', '0.1.0-rc1', 'bad;command'): + (self.root / 'idf_component.yml').write_text(f'version: {version!r}\n') + with self.assertRaises(ValueError): + check_release.check(self.root, 'true', 'refs/heads/main') + with self.assertRaises(ValueError): + check_release.check(self.root, 'yes', 'refs/heads/main') + + def test_release_cli_emits_manifest_version(self): + output = self.root / 'outputs' + result = subprocess.run([sys.executable, 'ci/scripts/check_release.py'], cwd=ROOT, + env={**os.environ, 'DRY_RUN': 'true', 'GITHUB_OUTPUT': str(output)}, + capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stderr) + version = yaml.safe_load((ROOT / 'idf_component.yml').read_text())['version'] + self.assertEqual(f'version={version}\ntag=v{version}\n', output.read_text()) + def test_exact_lightweight_workflow_commands(self): for command in [['ci/scripts/board_pack.py', 'check'], ['scripts/update_supported_boards_table.py', '--check'], @@ -88,10 +106,19 @@ def test_workflow_consumes_classifier_and_separates_credentials(self): self.assertIn('always()', jobs['result']['if']) self.assertNotIn('secrets.', (ROOT / '.github/workflows/ci.yml').read_text()) publish = yaml.load((ROOT / '.github/workflows/publish.yml').read_text(), Loader=yaml.BaseLoader) - self.assertEqual(['workflow_dispatch'], list(publish['on'])) + self.assertEqual(['main'], publish['on']['push']['branches']) self.assertEqual('true', publish['on']['workflow_dispatch']['inputs']['dry_run']['default']) self.assertEqual('component-registry', publish['jobs']['upload']['environment']) - self.assertEqual('validate', publish['jobs']['upload']['needs']) + self.assertEqual(['prepare', 'validate'], publish['jobs']['upload']['needs']) + self.assertEqual('true', publish['jobs']['validate']['with']['full_validation']) + self.assertIn('needs.prepare.outputs.publish', publish['jobs']['validate']['if']) + self.assertEqual(['prepare', 'upload'], publish['jobs']['release']['needs']) + self.assertIn('!inputs.dry_run', publish['jobs']['release']['if']) + self.assertEqual('read', publish['permissions']['contents']) + self.assertEqual('write', publish['jobs']['upload']['permissions']['contents']) + scope = next(step for step in jobs['validate']['steps'] if step.get('id') == 'scope') + self.assertIn('FULL_VALIDATION', scope['env']) + self.assertIn('matrix --all', scope['run']) if __name__ == '__main__': diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 572d063..a01d873 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -10,7 +10,9 @@ The GitHub repository is `waveshareteam/waveshare_boards`. The registry identity Keep the `esp_board_manager`, `board_manager`, and `boards` tags in the manifest so Board Manager can discover the pack. -No registry version is created by a pull request, a merge, or normal CI. +Pull requests run CI without publishing. A push or merge to `main` automatically +starts the release workflow. A completed GitHub Release for the manifest version +makes the publication jobs skip; publishing a new version runs the full build matrix. `compote component pack --name waveshare_boards` validates the package locally without a token. Generated archives remain under ignored `dist/`. @@ -46,13 +48,30 @@ it does not affect ordinary PR CI. ## Publish a version 1. Update `version` in `idf_component.yml`, keep both language pages current, - and merge the PR after all required checks pass. -2. Create a `v` tag on that reviewed commit, for example `v0.1.0`. -3. Run **Publish board pack** from that tag and clear **dry_run**. The workflow - checks that the tag matches the manifest and that the commit belongs to `main`, - reruns all board builds, then uploads `waveshare/waveshare_boards`. -4. Verify the registry version page and install it into a fresh application. - -Published versions are immutable. A duplicate version fails rather than silently -accepting different source. Make a new version for subsequent changes. See the -[Compote upload reference](https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/compote_cli.html#upload). + and merge the PR after all required checks pass. The initial version is `0.1.0`. +2. **Publish board pack** runs automatically on `main`. For a new version, it + compiles every board, creates `v` on that exact commit, uploads + `waveshare/waveshare_boards`, and creates the GitHub Release after the registry + confirms success. Release validation always builds the full matrix, including + when the ordinary CI change classifier selects fewer boards. +3. Verify the registry version page and install it into a fresh application: + + ```bash + idf.py add-dependency "waveshare/waveshare_boards==0.1.0" + ``` + +Merging with an already released version does not publish again. Increase the +manifest version when subsequent changes should reach registry users. Published +versions and existing tags are never overwritten or moved. An existing tag for +another commit stops the upload instead of publishing different source under it. + +For an upload failure, fix credentials or permissions and rerun the failed jobs +on the same workflow commit. If a tag was already created, a manual real upload +can also run from that tag with **dry_run** cleared. Manual real uploads accept +`main` or the matching version tag, and require the source to belong to `main`. +If the upload succeeded but GitHub Release creation failed, rerun only the failed +`release` job; the successful upload job is kept. Do not re-upload an immutable +registry version. Release runs are serialized and are not cancelled by later merges. + +The [Compote upload reference](https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/compote_cli.html#upload) +describes registry validation and duplicate-version behavior. diff --git a/docs/PUBLISHING_ZH.md b/docs/PUBLISHING_ZH.md index fd1331a..9b483d1 100644 --- a/docs/PUBLISHING_ZH.md +++ b/docs/PUBLISHING_ZH.md @@ -9,7 +9,8 @@ GitHub 仓库为 `waveshareteam/waveshare_boards`，在线组件库名称为 **` 使用的命名空间。清单中保留 `esp_board_manager`、`board_manager` 和 `boards` 标签， 便于 Board Manager 发现组件包。 -提交 PR、合并或普通 CI 都不会发布组件库版本。 +PR 只运行 CI，不发布组件。推送或合并到 `main` 后会自动启动发布流程。 +如果清单版本已有完整的 GitHub Release，则跳过发布；新版本先运行完整编译矩阵。 `compote component pack --name waveshare_boards` 可以在无令牌情况下验证本地打包， 生成的归档存放在已忽略的 `dist/` 目录。 @@ -41,11 +42,26 @@ GitHub 自动提供的 `GITHUB_TOKEN` 不能替代组件库令牌。 ## 发布版本 1. 更新 `idf_component.yml` 中的 `version`，维护双语文档，所有必需检查通过后合并 PR。 -2. 在审核通过的提交上创建 `v` 标签，例如 `v0.1.0`。 -3. 从该标签运行 **Publish board pack**，取消勾选 **dry_run**。 - 工作流检查标签与清单版本一致且提交属于 `main`，重新编译全部板卡， - 再上传 `waveshare/waveshare_boards`。 -4. 检查组件库版本页面，并在全新应用中验证安装。 - -已发布版本不可覆盖。重复版本会失败，不会把不同源码静默视为成功。 -后续变更使用新版本号，参阅 [Compote 上传说明](https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/compote_cli.html#upload)。 + 初始版本为 `0.1.0`。 +2. `main` 上自动运行 **Publish board pack**。对于新版本，工作流先编译全部板卡， + 再在该确切提交上创建 `v` 标签，上传 `waveshare/waveshare_boards`， + 组件库确认成功后创建 GitHub Release。发布前始终运行完整矩阵， + 即使普通 CI 的变更分类器只选择了部分板卡。 +3. 检查组件库版本页面，并在全新应用中验证安装： + + ```bash + idf.py add-dependency "waveshare/waveshare_boards==0.1.0" + ``` + +合并时若版本已经发布，不会再次上传。后续修改需要交付给组件库用户时，必须增加清单版本号。 +已发布版本不可覆盖，已有标签不会被移动。如果版本标签指向其他提交，工作流会停止上传， +不会用同一标签发布不同源码。 + +上传失败时，修正凭据或权限，然后在原工作流提交上重新运行失败的任务。 +如果标签已经创建，也可从该标签手动运行并取消勾选 **dry_run**。 +手动真实发布只接受 `main` 或匹配版本的标签，并要求提交属于 `main`。 +如果组件上传成功而 GitHub Release 创建失败，只重跑失败的 `release` 任务， +保留已经成功的上传任务，避免重复上传不可变版本。发布流程串行执行，不会被后续合并取消。 + +[Compote 上传说明](https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/compote_cli.html#upload) +介绍了组件库校验和重复版本的处理方式。