Land nine open scanner and GitHub-import fixes (#466, #467, #469, #470, #472, #473, #474, #475, #476)

apps/cursor/src/app/admin/plugins/stuck-scan-list.tsx

@@ -61,7 +61,7 @@ function StuckCard({ plugin }: { plugin: PluginRow }) {
           </div>
           <div className="mt-3 flex flex-wrap items-center gap-3 text-xs text-muted-foreground">
             <span className="text-text-tertiary">
-              created {new Date(plugin.created_at).toLocaleString()}
+              last change {new Date(plugin.updated_at).toLocaleString()}
             </span>
             {plugin.scan_run_id && (
               <span className="font-mono text-text-tertiary">

apps/cursor/src/app/api/cron/recover-stuck-scans/route.ts

@@ -38,22 +38,45 @@ export async function GET(request: NextRequest) {
       id: string;
       slug: string;
       ok: boolean;
+      action: "reset" | "enqueued";
       error?: string;
     }> = [];
 
     for (const plugin of stuck ?? []) {
-      // Reset back to pending so the worker's loadPlugin → markScanning
-      // transition is idempotent (the prevActive snapshot is recomputed).
+      // A row still at `scanning` after the threshold was left by a worker
+      // that died mid-run. Its queue message was never archived (the drain
+      // archives only after runPluginScan has written a terminal status), so
+      // it comes back on its own once the visibility timeout expires, and
+      // the minute drain picks it up. Enqueueing here as well would put a
+      // second message for the same plugin in the queue: two agent runs
+      // racing to write the verdict. So only reset the status, which makes
+      // the worker's loadPlugin → markScanning transition idempotent (the
+      // prevActive snapshot is recomputed). If the message really is gone
+      // (a row from before the queue existed), the next tick sees this row
+      // as stale `pending` and enqueues it then.
       if (plugin.scan_status === "scanning") {
-        await supabase
+        const { error: resetError } = await supabase
           .from("plugins")
           .update({ scan_status: "pending" })
           .eq("id", plugin.id);
+        results.push({
+          id: plugin.id,
+          slug: plugin.slug,
+          ok: !resetError,
+          action: "reset",
+          ...(resetError ? { error: resetError.message } : {}),
+        });
+        continue;
       }
 
       try {
         await enqueuePluginScan(plugin.id);
-        results.push({ id: plugin.id, slug: plugin.slug, ok: true });
+        results.push({
+          id: plugin.id,
+          slug: plugin.slug,
+          ok: true,
+          action: "enqueued",
+        });
       } catch (err) {
         const message = err instanceof Error ? err.message : String(err);
         console.error(
@@ -64,6 +87,7 @@ export async function GET(request: NextRequest) {
           id: plugin.id,
           slug: plugin.slug,
           ok: false,
+          action: "enqueued",
           error: message,
         });
       }
@@ -72,7 +96,8 @@ export async function GET(request: NextRequest) {
     return NextResponse.json({
       ok: true,
       considered: stuck?.length ?? 0,
-      retried: results.filter((r) => r.ok).length,
+      retried: results.filter((r) => r.ok && r.action === "enqueued").length,
+      reset: results.filter((r) => r.ok && r.action === "reset").length,
       failed: results.filter((r) => !r.ok).length,
       results,
     });

apps/cursor/src/data/queries.ts

@@ -538,11 +538,15 @@ export async function getStuckScans() {
   const supabase = await createClient();
   const fifteenMinutesAgo = new Date(Date.now() - 15 * 60 * 1000).toISOString();
 
+  // A scan counts as stuck 15 minutes after the row last changed, the same
+  // test the recover-stuck-scans cron and the MCP rescan tool apply. Every
+  // status write bumps updated_at through the plugins_set_updated_at trigger,
+  // so a fresh re-scan of an old plugin is not stuck until that time passes.
   const { data, error } = await supabase
     .from("plugins")
     .select("*, plugin_components(*)")
     .or(
-      `scan_status.eq.error,and(scan_status.in.(pending,scanning),created_at.lt.${fifteenMinutesAgo})`,
+      `scan_status.eq.error,and(scan_status.in.(pending,scanning),updated_at.lt.${fifteenMinutesAgo})`,
     )
     .order("created_at", { ascending: false })
     .limit(200);

apps/cursor/src/lib/github-plugin/parse.ts

@@ -132,6 +132,24 @@ export function githubAuthHeaders(): Record<string, string> {
 
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
+/**
+ * Normalize a sub-plugin dir to `a/b/` form: no leading `./` or `/`, exactly
+ * one trailing `/`. Marketplace manifests commonly write `./plugins/foo`.
+ */
+function normalizeSubPluginDir(source: string): string {
+  const trimmed = source.replace(/^(?:\.\/)+/, "").replace(/^\/+|\/+$/g, "");
+  return trimmed ? `${trimmed}/` : "";
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Match tree paths under `prefix` (a literal dir) followed by `pattern` (a regex). */
+function pathUnder(prefix: string, pattern: string): RegExp {
+  return new RegExp(`^${escapeRegExp(prefix)}${pattern}`);
+}
+
 /**
  * Fetch with bounded retry on GitHub rate-limit (429 / 403 + retry-after).
  *
@@ -154,19 +172,34 @@ export async function fetchWithRateLimit(
 
     if (res.status !== 429 && res.status !== 403) return res;
 
-    // Distinguish "rate-limited" 403 from "forbidden" 403 by checking the
-    // X-RateLimit-Remaining header. A real auth/permission 403 has remaining > 0.
+    const retryAfterHeader = res.headers.get("retry-after");
+    const retryAfter = Number(retryAfterHeader ?? "");
     const remaining = Number(res.headers.get("x-ratelimit-remaining") ?? "");
-    if (res.status === 403 && Number.isFinite(remaining) && remaining > 0) {
+
+    // Distinguish "rate-limited" 403 from "forbidden" 403. A primary-limit 403
+    // has remaining === 0; a secondary-limit 403 (too many concurrent requests,
+    // too many points per minute) carries a retry-after header while remaining
+    // can still be > 0. Only a 403 with budget left and no retry-after is a real
+    // auth/permission answer.
+    if (
+      res.status === 403 &&
+      retryAfterHeader === null &&
+      Number.isFinite(remaining) &&
+      remaining > 0
+    ) {
       return res;
     }
 
-    const retryAfter = Number(res.headers.get("retry-after") ?? "");
     const reset = Number(res.headers.get("x-ratelimit-reset") ?? "0") * 1000;
+    // GitHub's documented order: honour retry-after; else, if remaining is 0,
+    // wait for the reset; else (secondary limit without a retry-after) wait a
+    // minute before retrying.
     let waitMs =
       Number.isFinite(retryAfter) && retryAfter > 0
         ? retryAfter * 1000
-        : Math.max(reset - Date.now(), 1000);
+        : remaining === 0
+          ? Math.max(reset - Date.now(), 1000)
+          : 60_000;
     waitMs = Math.min(waitMs, maxWaitMs - totalWait);
 
     if (waitMs <= 0 || attempt === maxAttempts) return res;
@@ -187,12 +220,25 @@ export async function fetchWithRateLimit(
  */
 export type FetchOptions = { maxWaitMs?: number };
 
+/**
+ * Raw-content URL for a repo path. Tree paths arrive unencoded, so a `#`
+ * would start a URL fragment, a `?` a query, and a `%` a broken escape; each
+ * segment is encoded while the `/` between them is kept.
+ */
+function rawGitHubUrl(owner: string, repo: string, path: string): string {
+  const encodedPath = path
+    .split("/")
+    .map((segment) => encodeURIComponent(segment))
+    .join("/");
+  return `https://raw.githubusercontent.com/${owner}/${repo}/HEAD/${encodedPath}`;
+}
+
 async function fetchGitHubFile(
   owner: string,
   repo: string,
   path: string,
 ): Promise<string | null> {
-  const url = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD/${path}`;
+  const url = rawGitHubUrl(owner, repo, path);
   try {
     const res = await fetch(url, { cache: "no-store" });
     if (!res.ok) return null;
@@ -344,7 +390,8 @@ export async function parseGitHubPlugin(
       if (entry && typeof entry === "object") {
         const source = (entry as Record<string, unknown>).source;
         if (typeof source === "string" && source && !source.includes("..")) {
-          subPluginDirs.add(`${source.replace(/^\/+|\/+$/g, "")}/`);
+          const dir = normalizeSubPluginDir(source);
+          if (dir) subPluginDirs.add(dir);
         }
       }
     }
@@ -382,14 +429,17 @@ export async function parseGitHubPlugin(
   }
 
   const components: ParsedComponent[] = [];
+  // Which sub-plugin dir each component came from ("" for the repo root), so
+  // a slug that collides across sub-plugins can be told apart below.
+  const prefixOf = new Map<ParsedComponent, string>();
 
   const prefixes = ["", ...subPluginDirs];
 
   for (const prefix of prefixes) {
+    const firstIndex = components.length;
     const ruleFiles = tree.filter(
       (f) =>
-        f.type === "blob" &&
-        new RegExp(`^${prefix}rules/.*\\.mdc$`).test(f.path),
+        f.type === "blob" && pathUnder(prefix, "rules/.*\\.mdc$").test(f.path),
     );
     for (const rf of ruleFiles) {
       const raw = await fetchGitHubFile(owner, repo, rf.path);
@@ -451,8 +501,7 @@ export async function parseGitHubPlugin(
 
     const agentFiles = tree.filter(
       (f) =>
-        f.type === "blob" &&
-        new RegExp(`^${prefix}agents/.*\\.md$`).test(f.path),
+        f.type === "blob" && pathUnder(prefix, "agents/.*\\.md$").test(f.path),
     );
     for (const af of agentFiles) {
       const raw = await fetchGitHubFile(owner, repo, af.path);
@@ -473,7 +522,7 @@ export async function parseGitHubPlugin(
     const skillFiles = tree.filter(
       (f) =>
         f.type === "blob" &&
-        new RegExp(`^${prefix}skills/[^/]+/SKILL\\.md$`).test(f.path),
+        pathUnder(prefix, "skills/[^/]+/SKILL\\.md$").test(f.path),
     );
     for (const sf of skillFiles) {
       const raw = await fetchGitHubFile(owner, repo, sf.path);
@@ -495,7 +544,7 @@ export async function parseGitHubPlugin(
     const commandFiles = tree.filter(
       (f) =>
         f.type === "blob" &&
-        new RegExp(`^${prefix}commands/.*\\.md$`).test(f.path),
+        pathUnder(prefix, "commands/.*\\.md$").test(f.path),
     );
     for (const cf of commandFiles) {
       const raw = await fetchGitHubFile(owner, repo, cf.path);
@@ -552,15 +601,28 @@ export async function parseGitHubPlugin(
         // ignore
       }
     }
+
+    for (const c of components.slice(firstIndex)) prefixOf.set(c, prefix);
   }
 
+  // Slugs come from file or directory names, so two sub-plugins that both
+  // ship, say, `skills/setup/SKILL.md` produce the same `skill:setup` key. The
+  // first occurrence keeps its slug; a later one from a sub-plugin dir gets
+  // the dir as a prefix (the hooks component already does this). Only a key
+  // that still collides after that is dropped.
   const seen = new Set<string>();
-  const deduped = components.filter((c) => {
-    const key = `${c.type}:${c.slug}`;
-    if (seen.has(key)) return false;
-    seen.add(key);
-    return true;
-  });
+  const deduped: ParsedComponent[] = [];
+  for (const c of components) {
+    let slug = c.slug;
+    if (seen.has(`${c.type}:${slug}`)) {
+      const prefix = prefixOf.get(c) ?? "";
+      if (!prefix) continue;
+      slug = slugify(`${prefix} ${c.slug}`);
+      if (seen.has(`${c.type}:${slug}`)) continue;
+    }
+    seen.add(`${c.type}:${slug}`);
+    deduped.push(slug === c.slug ? c : { ...c, slug });
+  }
 
   if (deduped.length === 0) {
     const scannedPrefixes = prefixes
@@ -593,7 +655,7 @@ export async function parseGitHubPlugin(
         logo = manifest.logo;
       }
     } catch {
-      logo = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD/${manifest.logo.replace(/^\.?\//, "")}`;
+      logo = rawGitHubUrl(owner, repo, manifest.logo.replace(/^\.?\//, ""));
     }
   }
 

apps/cursor/src/lib/mcp/tools.ts

@@ -39,9 +39,6 @@ const UNTRUSTED_NOTE =
 const UUID_RE =
   /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 
-// Matches the stuck-scan threshold in getStuckScans(): older pending/scanning
-// rows are treated as stuck and may be re-scanned.
-const SCAN_IN_PROGRESS_MS = 15 * 60 * 1000;
 const MAX_COMPONENT_CONTENT_CHARS = 4000;
 const MAX_SUMMARY_CHARS = 500;
 
@@ -534,20 +531,15 @@ export function registerModerationTools(server: McpServer) {
       annotations: { ...write, idempotentHint: false },
     },
     adminTool(async ({ plugin }, admin) => {
-      const state = await getPluginModerationState(
-        await resolvePluginId(plugin),
-      );
-      const inProgress =
-        (state.scan_status === "pending" || state.scan_status === "scanning") &&
-        Date.now() - new Date(state.updated_at).getTime() < SCAN_IN_PROGRESS_MS;
-      if (inProgress) {
-        throw new ToolError(
-          `A scan for "${state.slug}" is already ${state.scan_status}. Check again in a few minutes.`,
-        );
-      }
+      // rescanPlugin refuses with a ModerationError while a scan is queued or
+      // running (same check the admin UI buttons go through); adminTool turns
+      // that into the error text.
       return changed(
         "Re-scan queued",
-        await rescanPlugin(await beginWrite(admin), state.id),
+        await rescanPlugin(
+          await beginWrite(admin),
+          await resolvePluginId(plugin),
+        ),
       );
     }),
   );

apps/cursor/src/lib/plugins/moderation.ts

@@ -203,7 +203,36 @@ export function blockPlugin(
   );
 }
 
+/**
+ * A pending/scanning row younger than this still has a live queue message or
+ * a running worker behind it. Matches the stuck-scan threshold shared by
+ * getStuckScans() and the recover-stuck-scans cron: an older row is treated
+ * as stuck and may be re-scanned.
+ */
+export const SCAN_IN_PROGRESS_MS = 15 * 60 * 1000;
+
+export function isScanInProgress(
+  state: Pick<PluginModerationState, "scan_status" | "updated_at">,
+  now = Date.now(),
+): boolean {
+  if (state.scan_status !== "pending" && state.scan_status !== "scanning") {
+    return false;
+  }
+  return now - new Date(state.updated_at).getTime() < SCAN_IN_PROGRESS_MS;
+}
+
 export async function rescanPlugin(ctx: ModerationContext, pluginId: string) {
+  // Every re-scan entry point (admin UI buttons, MCP tool) lands here, so the
+  // in-progress check lives here too. Without it a second click while the
+  // first scan is queued or running enqueues a second message for the same
+  // plugin: two billable agent runs whose verdicts race each other.
+  const state = await getPluginModerationState(pluginId);
+  if (isScanInProgress(state)) {
+    throw new ModerationError(
+      `A scan for "${state.slug}" is already ${state.scan_status}. Check again in a few minutes.`,
+    );
+  }
+
   // The status reset must reach cached readers (detail banner, leaderboard)
   // before enqueueing, so a queue failure can't leave cached views showing a
   // stale status while the row is already pending. The drain route only