diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 0bf3b8cff..9dfb2002b 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -33,7 +33,12 @@ concurrency:
 
 jobs:
   build_wheels:
-    if: github.repository_owner == 'mlcommons'
+    # A manual release pushes its tag with the App token, which starts this
+    # workflow again; that run would rebuild and re-upload the same release. The
+    # below check ensures that does not happen
+    if: >-
+      github.repository_owner == 'mlcommons'
+      && !(github.event_name == 'push' && github.actor == 'mlc-automations[bot]')
     name: Build wheel
     runs-on: ubuntu-latest
     environment: release
@@ -47,7 +52,7 @@ jobs:
     steps:
       - name: Generate GitHub App token
         id: app-token
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         with:
           app-id: ${{ secrets.MLC_AUTOMATIONS_APP_ID }}
           private-key: ${{ secrets.MLC_AUTOMATIONS_PRIVATE_KEY }}
diff --git a/tests/test_build_wheels_workflow.py b/tests/test_build_wheels_workflow.py
index 7e5383fa4..738357285 100644
--- a/tests/test_build_wheels_workflow.py
+++ b/tests/test_build_wheels_workflow.py
@@ -56,9 +56,8 @@ def test_checkout_uses_app_token_for_protected_branch_pushes(self):
             step for step in self.steps if step.get("uses", "").startswith("actions/checkout@")
         )
 
-        self.assertEqual(
-            token_step["uses"],
-            "actions/create-github-app-token@v1")
+        self.assertTrue(
+            token_step["uses"].startswith("actions/create-github-app-token@"))
         self.assertEqual(
             token_step["with"]["app-id"],
             "${{ secrets.MLC_AUTOMATIONS_APP_ID }}",
@@ -116,6 +115,22 @@ def test_release_step_uses_app_token_for_github_release_mutations(self):
             "${{ steps.app-token.outputs.token }}",
         )
 
+    def test_tag_push_from_app_token_does_not_start_second_release(self):
+        job_condition = self.workflow["jobs"]["build_wheels"]["if"]
+
+        self.assertIn("github.repository_owner == 'mlcommons'", job_condition)
+        self.assertIn(
+            "!(github.event_name == 'push' && github.actor == 'mlc-automations[bot]')",
+            job_condition,
+        )
+
+    def test_actions_are_pinned_to_commit_shas(self):
+        for step in self.steps:
+            uses = step.get("uses")
+            if uses:
+                ref = uses.split("@", 1)[1]
+                self.assertRegex(ref, r"^[0-9a-f]{40}$", uses)
+
     def test_workflow_serializes_release_runs(self):
         concurrency = self.workflow["concurrency"]