Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,10 @@ follows the fixer's Git alternates. Pack indexes are rebuilt with
checks every object and link, since the import enables `fetch.fsckObjects=true`. It records `mechanical fix by <recorded agent/model>: <reason>` and the new head in
`## Review`, and enables head-pinned auto-merge again on the fixed head, which keeps
the place of the pick it descends from. Healthy PRs go back to batches unchanged,
without a comment. Forks refusing maintainer edits, refused pushes, invalid results,
without a comment. A PR the daemon moved to its own recorded base merge before
parking it is collected at that merge: a requeue pins it, and a fix, built on the
pin, replaces it with a lease, since building on it would revert its newer base.
Any other moved head hands back. Forks refusing maintainer edits, refused pushes, invalid results,
crashes, or the 60-minute fixer deadline hand back with a marked comment naming the exit
status or signal, with the tail of the job's retained log. A validated fix whose push
GitHub refuses, such as a merged base's `.github/workflows` changes that the App lacks
Expand Down Expand Up @@ -645,7 +648,10 @@ and is rebuilt and checked on the new base.
After an API refusal, the daemon re-reads mergeability and briefly retries
pending mergeability or a base refresh. It also retries while GitHub still
reports the pin as the head of a PR it just prepared, since GitHub can refuse the
merge for that stale head; only another writer's head ejects `head_moved`.
merge for that stale head; only another writer's head ejects `head_moved`. It
likewise retries while GitHub calls the merge it just pushed conflicting, since
that merge holds the tested base; if GitHub still does after about 15 seconds,
the next round rebuilds the queue on the base as it then is.
A confirmed conflict parks only that PR for the fixer and a moved head ejects
only that PR; the rest stays queued. A persistent policy refusal ejects as `merge_refused` without the
fixer. Already-merged PRs are skipped. An unexpected API squash parent means the
Expand Down
5 changes: 5 additions & 0 deletions quest/m0/land-fixer/mergeability.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,8 @@ Since batch triage ([moq-dev/quest#355](https://github.com/moq-dev/quest/pull/35
`merge_conflict` already parks the PR for the fixer (`fail` in
`src/cli/land-serve.ts`) instead of ejecting it, and the rest of the batch
stays queued; this quest adds the local merge before parking.

GitHub also flags the gate's own freshly pushed base merge as conflicting, from
the pin's stale mergeability ([moq-dev/quest#342](https://github.com/moq-dev/quest/pull/342)).
`mergePinned` now retries that case, since the merge holds the tested base. What
remains is a head the gate did not merge in that round, which GitHub flags anyway.
25 changes: 19 additions & 6 deletions src/cli/land-fixer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import { COMMENT_LIMIT, resolution, resolutionSection, type Resolution } from ".
import { readFixResult, parseBaseFix, parseFixResult, type BaseFix, type FixResult } from "./land-fix";
import { readRemote, type RemoteRead } from "./land";
import { importFixer } from "./land-fixer-repository";
import { push, remoteHead } from "./land-head";
import { isPrepared, push, remoteHead } from "./land-head";
import { BASE_JOB, type Halt, haltNotice, readHalt, saveHalt } from "./land-halt";
import { isParked, parked } from "./land-park";
import { excerpt, type Failure } from "./land-serve";
Expand Down Expand Up @@ -725,7 +725,12 @@ async function collectJob(
);
const auto = object(queued.node).autoMergeRequest;
let previous = attempt(state, number)!;
const head = result?.action === "fixed" ? result.head : entry.head;
// The head the PR stands at: its pinned head, or the gate's own base merge on it for this pick, as
// when the gate pushed that merge and then failed to land it (moq-dev/quest#342). The gate's merge is
// known only from its private record, never from its forgeable author or message.
const standing =
current !== entry.head && isPrepared(root, state, { ...entry, head: current }) ? current : entry.head;
const head = result?.action === "fixed" ? result.head : standing;
const actor = { agent: result?.agent, model: result?.model };
const queuedAs = auto == null ? null : string(object(auto).enabledAt);
// This collection already enabled auto-merge again on the fixed head, before a restart.
Expand All @@ -739,7 +744,7 @@ async function collectJob(
if (!samePick(previous, entry))
error = "fixer was superseded by a newer maintainer pick; keeping its work untouched";
// Never displace a newer maintainer pick, closed PR, or writer's commit.
if (pull.state !== "open" || (current !== entry.head && !(job.published.includes(number) && current === head)))
if (pull.state !== "open" || (current !== standing && !(job.published.includes(number) && current === head)))
error = "PR closed or head moved while fixing; inspect and pick it again";
const left = job.left?.includes(number) ?? false;
// A parked PR keeps its auto-merge; one an older job's ejection left without it is requeued below.
Expand Down Expand Up @@ -781,19 +786,27 @@ async function collectJob(
save(path, job);
}
if (result?.action === "fixed" && current !== head) {
// The fix descends from the pin, not from a gate merge the PR stands at: it replaces that
// merge, as the gate replaces its own, since building on it would revert the newer base it
// merged. The lease still refuses any writer's commit.
const pushed = await push(
root,
`https://github.com/${entry.repository}.git`,
entry.head,
head,
`refs/heads/${entry.branch}`,
token,
standing,
);
if (pushed.status !== 0) {
// GitHub refuses an App's push of a workflow change it lacks the permission for, and a
// fork's branch rules may refuse any. The maintainer's own push may still pass.
// fork's branch rules may refuse any. The maintainer's own push may still pass, but
// `quest land apply` rebuilds only on the head the fixer worked from.
try {
const ready = resolution(root, entry.head, job.plan.base.sha, head);
const ready =
standing === entry.head
? resolution(root, entry.head, job.plan.base.sha, head)
: undefined;
if (ready) (job.resolutions ??= {})[number] = ready;
} catch (error) {
deps.warn(`quest land fixer: cannot hand back #${number}'s resolution: ${String(error)}`);
Expand All @@ -805,7 +818,7 @@ async function collectJob(
if (result?.action === "fixed") {
// GitHub shows a push a moment later. Wait for our own head; only a third head is a real move.
let latest = object(await (await github.request(pullPath)).json());
for (let wait = 0; object(latest.head).sha === entry.head && wait < SYNC_PAUSES; wait++) {
for (let wait = 0; object(latest.head).sha === standing && wait < SYNC_PAUSES; wait++) {
await github.pause(1000);
latest = object(await (await github.request(pullPath)).json());
}
Expand Down
7 changes: 5 additions & 2 deletions src/cli/land-head.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ export function forgetHeads(state: string, number: number): void {
if (name.startsWith(`head-${number}-`) && name.endsWith(".json")) unlinkSync(join(state, name));
}

function isPrepared(root: string, state: string, entry: Entry): boolean {
export function isPrepared(root: string, state: string, entry: Entry): boolean {
const path = recordPath(state, entry.number, entry.head);
if (!existsSync(path)) return false;
const record = JSON.parse(readFileSync(path, "utf8")) as Prepared;
Expand All @@ -94,7 +94,10 @@ export async function push(
/** What `ref` must be before the push; empty when it must not exist yet. */
lease = from,
) {
const workflows = git(root, ["diff", "--name-only", from, to, "--", ".github/workflows"]) !== "";
// A push that replaces another head than the one it builds on can change workflows relative to either.
const workflows = [...new Set([from, lease])].some(
(old) => old !== "" && git(root, ["diff", "--name-only", old, to, "--", ".github/workflows"]) !== "",
);
const credential = await token(workflows);
return tryGit(root, ["push", "--quiet", remote, `${to}:${ref}`, `--force-with-lease=${ref}:${lease}`], {
env: gitAuthentication(credential),
Expand Down
21 changes: 17 additions & 4 deletions src/cli/land-serve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,9 +341,16 @@ async function reportStalled(github: GitHub, plan: Plan, state: string, now: num
type MergeResult =
{ sha: string } | { reason: "head_moved" | "merge_conflict" | "merge_refused"; detail: string } | null;

/**
* How many times a merge is retried while GitHub catches up on a push, 15.5 s of backoff in all. A
* merge still unsettled after them throws, and the next round rebuilds the queue as GitHub then shows it.
*/
const SETTLE_RETRIES = 5;

/**
* Retry only GitHub's pending mergeability or base refresh. Any other refusal, such as squash being
* disabled or a rule the bot cannot bypass, would refuse every round after a full check, so it ejects.
* `previous` is the head before the gate pushed its own base merge as `head`, if it did.
*/
async function mergePinned(github: GitHub, path: string, head: string, previous = head): Promise<MergeResult> {
for (let attempt = 0; ; attempt++) {
Expand All @@ -365,14 +372,20 @@ async function mergePinned(github: GitHub, path: string, head: string, previous
// for it: a 409, or a 405 when the pin conflicts with the base that push merged. Neither is a writer.
const current = object(pull.head).sha;
if (previous !== head && (current === previous || (response.status === 409 && current === head))) {
if (attempt >= 3) throw new Error(`${detail}; prepared head has not propagated`);
if (attempt >= SETTLE_RETRIES) throw new Error(`${detail}; prepared head has not propagated`);
await github.pause(500 * 2 ** attempt);
continue;
}
if (current !== head) return { reason: "head_moved", detail };
if (pull.mergeable === false || pull.mergeable_state === "dirty") return { reason: "merge_conflict", detail };
if (pull.mergeable === null || /base branch was modified/i.test(String(result.message))) {
if (attempt >= 3) throw new Error(`${detail}; mergeability did not settle`);
const conflict = pull.mergeable === false || pull.mergeable_state === "dirty";
// Once GitHub shows the gate's own merge, it can still call it conflicting from the pin's stale
// mergeability, in its refusal or in the PR (moq-dev/quest#342). That merge already holds the
// base it was checked on, so it cannot conflict with it: wait. Only a base that moved since can
// conflict, and then the retries run out and the next round rebuilds on that base.
const stale = previous !== head && (conflict || /merge conflict/i.test(String(result.message)));
if (conflict && !stale) return { reason: "merge_conflict", detail };
if (stale || pull.mergeable === null || /base branch was modified/i.test(String(result.message))) {
if (attempt >= SETTLE_RETRIES) throw new Error(`${detail}; mergeability did not settle`);
await github.pause(500 * 2 ** attempt);
continue;
}
Expand Down
Loading
Loading