diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index 108a2de903310c..b265fe35cbfc51 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -20,7 +20,7 @@ jobs: jobname: ClangFormat runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml index ea6f49f742108e..3379f89a814abd 100644 --- a/.github/workflows/check-whitespace.yml +++ b/.github/workflows/check-whitespace.yml @@ -19,7 +19,7 @@ jobs: check-whitespace: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml new file mode 100644 index 00000000000000..778a67abf28b4f --- /dev/null +++ b/.github/workflows/codex-release.yml @@ -0,0 +1,582 @@ +name: Codex Git release + +on: + push: + branches: + - codex + - codex-unstable + +permissions: + contents: read + +concurrency: + group: codex-git-release-${{ github.sha }} + cancel-in-progress: false + +jobs: + publication: + name: Verify controller publication + runs-on: ubuntu-24.04 + if: github.event.deleted == false + outputs: + published: ${{ steps.verify.outputs.published }} + steps: + - name: Check the published controller output + id: verify + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + case "$GITHUB_REF" in + refs/heads/codex) + output_key=codex.output-tip + ;; + refs/heads/codex-unstable) + output_key=codex-unstable.output-tip + ;; + *) + printf 'unexpected release ref: %s\n' "$GITHUB_REF" >&2 + exit 1 + ;; + esac + + meta=$(gh api \ + "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq '.object.sha') + recorded=$(gh api \ + "repos/$GITHUB_REPOSITORY/contents/codex.config?ref=$meta" \ + -H 'Accept: application/vnd.github.raw+json' | + git config --no-includes --file /dev/stdin \ + --get "$output_key") + + if test "$GITHUB_SHA" = "$recorded" + then + printf 'published=true\n' >>"$GITHUB_OUTPUT" + printf 'Releasing controller-published commit %s.\n' "$GITHUB_SHA" + else + printf 'published=false\n' >>"$GITHUB_OUTPUT" + printf 'Skipping non-controller publication %s.\n' "$GITHUB_SHA" + fi + + version: + name: Determine version + needs: publication + if: needs.publication.outputs.published == 'true' + runs-on: ubuntu-24.04 + outputs: + describe: ${{ steps.version.outputs.describe }} + upstream_tag: ${{ steps.version.outputs.upstream_tag }} + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Derive OpenAI version from git describe + id: version + shell: bash + run: | + set -euo pipefail + describe="$(git describe \ + --match 'v[0-9]*' \ + --exclude 'v*-openai.*' \ + --long \ + --always \ + --abbrev=12 \ + "$GITHUB_SHA")" + + if [[ "$describe" =~ ^(.+)-([0-9]+)-g([0-9a-f]+)$ ]] + then + upstream_tag="${BASH_REMATCH[1]}" + version="$upstream_tag-openai.${BASH_REMATCH[2]}.g${BASH_REMATCH[3]}" + else + upstream_tag= + version="openai-$describe" + fi + git check-ref-format "refs/tags/$version" + printf 'describe=%s\n' "$describe" | tee -a "$GITHUB_OUTPUT" + printf 'upstream_tag=%s\n' "$upstream_tag" | tee -a "$GITHUB_OUTPUT" + printf 'version=%s\n' "$version" | tee -a "$GITHUB_OUTPUT" + + build: + name: ${{ matrix.name }} + needs: version + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - name: macOS arm64 + os: macos-15 + target_platform: macOS + asset_platform: macOS + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable arm64 + has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata + max_tar_bytes: 67108864 + - name: macOS x64 + os: macos-15-intel + target_platform: macOS + asset_platform: macOS + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable x86_64 + has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata + max_tar_bytes: 67108864 + # Keep arm64 builds native so release smoke tests can execute them. + - name: Linux arm64 + os: ubuntu-22.04-arm + target_platform: ubuntu + asset_platform: ubuntu + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*ARM aarch64 + has_gcm: false + lto: auto + profile_format: GCC + max_tar_bytes: 67108864 + - name: Linux x64 + os: ubuntu-22.04 + target_platform: ubuntu + asset_platform: ubuntu + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*x86-64 + has_gcm: false + lto: auto + profile_format: GCC + max_tar_bytes: 67108864 + - name: Windows arm64 + os: windows-11-arm + target_platform: win32 + asset_platform: windows + arch: arm64 + binary: /tmp/build/git/clangarm64/bin/git.exe + file_pattern: PE32\+.*ARM64 + has_gcm: true + lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata + max_tar_bytes: 134217728 + sdk_arch: aarch64 + sdk_flavor: full + mingw_dir: clangarm64 + mingit_arch: arm64 + mingit_filename: MinGit-2.55.0.2-arm64.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-arm64.zip + mingit_sha256: 0b2b81fdce284efd174cbb51b886ccea2fd271679c4b5c21f07d9e03bae51413 + - name: Windows x64 + os: windows-2025 + target_platform: win32 + asset_platform: windows + arch: x64 + binary: /tmp/build/git/mingw64/bin/git.exe + file_pattern: PE32\+.*x86-64 + has_gcm: true + lto: auto + profile_format: GCC + max_tar_bytes: 134217728 + sdk_arch: x86_64 + sdk_flavor: full + mingw_dir: mingw64 + mingit_arch: amd64 + mingit_filename: MinGit-2.55.0.2-64-bit.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-64-bit.zip + mingit_sha256: e3ea2944cea4b3fabcd69c7c1669ef69b1b66c05ac7806d81224d0abad2dec31 + + steps: + # Keep the packaging contract, dependency pins, and platform build logic + # aligned with the artifacts already consumed by Codex and GitHub Desktop. + - name: Check out Dugite Native + uses: actions/checkout@v6 + with: + repository: dreynaud-oai/dugite-native + ref: b6f4473557acb85433fdf9deffe0854a34fd9cc5 + path: dugite-native + fetch-depth: 0 + persist-credentials: false + + - name: Check out this Git revision + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + path: dugite-native/git + fetch-depth: 1 + persist-credentials: false + + - name: Give the source an immutable package version + shell: bash + working-directory: dugite-native/git + env: + VERSION: ${{ needs.version.outputs.version }} + run: | + git \ + -c 'user.name=github-actions[bot]' \ + -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ + tag -a "$VERSION" -m "$VERSION" + + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + + # Match Dugite Native's compatibility choice for its macOS x64 build. + - name: Select Xcode 16.4 + if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' + run: | + sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer/ + sudo rm -rf /Library/Developer/CommandLineTools + + - name: Install Linux build dependencies + if: matrix.target_platform == 'ubuntu' + run: | + sudo apt-get update + sudo apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + curl \ + gettext \ + jq \ + lsb-release \ + pkg-config + + - name: Install Linux x64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'x64' + run: | + sudo apt-get install -y \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + - name: Install Linux arm64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' + run: | + sudo apt-get install -y \ + binutils-aarch64-linux-gnu \ + gcc-aarch64-linux-gnu \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + # Dugite Native currently pins MinGit 2.53. Keep its build script and + # dependency schema, but match the runtime to the Git series we compile. + - name: Select the matching MinGit runtime + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native + env: + MINGIT_ARCH: ${{ matrix.mingit_arch }} + MINGIT_FILENAME: ${{ matrix.mingit_filename }} + MINGIT_SHA256: ${{ matrix.mingit_sha256 }} + MINGIT_URL: ${{ matrix.mingit_url }} + MINGIT_VERSION: v2.55.0 + SOURCE_UPSTREAM_TAG: ${{ needs.version.outputs.upstream_tag }} + run: | + set -euo pipefail + test "$SOURCE_UPSTREAM_TAG" = "$MINGIT_VERSION" + updated="$(mktemp)" + jq \ + --arg arch "$MINGIT_ARCH" \ + --arg checksum "$MINGIT_SHA256" \ + --arg filename "$MINGIT_FILENAME" \ + --arg url "$MINGIT_URL" \ + --arg version "$MINGIT_VERSION" \ + '.git.version = $version | + (.git.files[] | + select(.platform == "windows" and .arch == $arch)) |= + (.filename = $filename | + .url = $url | + .checksum = $checksum)' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + # Codex does not configure or invoke GCM on macOS or Linux. The + # self-contained .NET payload accounts for most of those bundles, while + # Windows MinGit configures credential.helper=manager and must retain it. + - name: Omit unused GCM from POSIX bundles + if: matrix.target_platform != 'win32' + shell: bash + working-directory: dugite-native + run: | + set -euo pipefail + updated="$(mktemp)" + jq '."git-credential-manager".files = []' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. + - name: Build the Dugite Native distribution + shell: bash + working-directory: dugite-native + env: + NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + script/build.sh + + - name: Set up Git for Windows SDK + if: matrix.target_platform == 'win32' + uses: git-for-windows/setup-git-for-windows-sdk@v2 + with: + architecture: ${{ matrix.sdk_arch }} + flavor: ${{ matrix.sdk_flavor }} + cache: false + + # Dugite Native compiles its Git submodule on macOS and Linux. On + # Windows it starts from MinGit, so replace MinGit's Git programs with + # the build from this repository while retaining the portable runtime. + # MinGit omits dashed builtin aliases; installing them as copies would + # add hundreds of redundant MiB to the archive. + - name: Install this Git build into the Windows distribution + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native/git + env: + MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} + run: | + set -euo pipefail + + make_args=( + "prefix=/$MINGW_DIR" + NO_PERL=YesPlease + NO_RUST=YesPlease + NO_TCLTK=YesPlease + NO_GETTEXT=YesPlease + NO_INSTALL_HARDLINKS=YesPlease + NO_CROSS_DIRECTORY_HARDLINKS=YesPlease + SKIP_DASHED_BUILT_INS=YesPlease + ) + jobs="${NUMBER_OF_PROCESSORS:-2}" + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install + + - name: Verify distribution layout and provenance + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + GIT_BINARY: ${{ matrix.binary }} + FILE_PATTERN: ${{ matrix.file_pattern }} + HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} + run: | + set -euo pipefail + if test "$TARGET_PLATFORM" = win32 + then + test -f /tmp/build/git/cmd/git.exe + test -f "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-lfs.exe" + test -d "/tmp/build/git/$MINGW_DIR/share/git-core/templates" + test ! -e "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-add.exe" + if test "$HAS_GCM" = true + then + test -f "/tmp/build/git/$MINGW_DIR/bin/git-credential-manager.exe" + fi + else + test -x /tmp/build/git/libexec/git-core/git-lfs + test -d /tmp/build/git/share/git-core/templates + if test "$HAS_GCM" = true + then + test -x /tmp/build/git/libexec/git-core/git-credential-manager + else + test ! -e /tmp/build/git/libexec/git-core/git-credential-manager + fi + fi + test -f /tmp/build/git/etc/gitconfig + + file "$GIT_BINARY" | tee /tmp/git-file-type + grep -E "$FILE_PATTERN" /tmp/git-file-type + strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi + + - name: Smoke-test the native distribution + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + HAS_GCM: ${{ matrix.has_gcm }} + run: | + set -euo pipefail + smoke=/tmp/codex-git-smoke + mkdir -p "$smoke/home" + + if test "$TARGET_PLATFORM" = win32 + then + git_binary=/tmp/build/git/cmd/git.exe + git_env=( + "PATH=/tmp/build/git/cmd:/tmp/build/git/$MINGW_DIR/bin:/tmp/build/git/usr/bin:$PATH" + ) + else + git_binary=/tmp/build/git/bin/git + git_env=( + GIT_CONFIG_SYSTEM=/tmp/build/git/etc/gitconfig + GIT_EXEC_PATH=/tmp/build/git/libexec/git-core + GIT_TEMPLATE_DIR=/tmp/build/git/share/git-core/templates + ) + if test "$TARGET_PLATFORM" = ubuntu + then + git_env+=( + GIT_SSL_CAINFO=/tmp/build/git/ssl/cacert.pem + PREFIX=/tmp/build/git + ) + fi + fi + git_env+=("HOME=$smoke/home" GIT_TERMINAL_PROMPT=0) + + build_options="$(env "${git_env[@]}" "$git_binary" --version --build-options)" + printf '%s\n' "$build_options" + grep -F "built from commit: $GITHUB_SHA" <<<"$build_options" + env "${git_env[@]}" "$git_binary" lfs version + if test "$HAS_GCM" = true + then + env "${git_env[@]}" "$git_binary" credential-manager --version + fi + + env "${git_env[@]}" "$git_binary" init --quiet "$smoke/repo" + echo test >"$smoke/repo/file" + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" add file + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" \ + -c user.name='Codex Git CI' \ + -c user.email='codex-git-ci@openai.com' \ + commit --quiet -m initial + test -z "$(env "${git_env[@]}" "$git_binary" -C "$smoke/repo" status --porcelain)" + + - name: Package with Dugite Native + shell: bash + working-directory: dugite-native + env: + ASSET_PLATFORM: ${{ matrix.asset_platform }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + VERSION: ${{ needs.version.outputs.version }} + MAX_TAR_BYTES: ${{ matrix.max_tar_bytes }} + run: | + set -euo pipefail + script/package.sh + + for extension in tar.gz lzma + do + matches=( + output/dugite-native-"$VERSION"-*-"$ASSET_PLATFORM"-"$TARGET_ARCH.$extension" + ) + test "${#matches[@]}" -eq 1 + test -f "${matches[0]}" + + destination="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.$extension" + mv "${matches[0]}" "$destination" + mv "${matches[0]}.sha256" "$destination.sha256" + done + + for checksum in output/*.sha256 + do + archive="${checksum%.sha256}" + expected="$(tr -d '\r\n' <"$checksum")" + if command -v sha256sum >/dev/null 2>&1 + then + actual="$(sha256sum "$archive" | awk '{print $1}')" + else + actual="$(shasum -a 256 "$archive" | awk '{print $1}')" + fi + test "$actual" = "$expected" + done + + tarball="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.tar.gz" + tar_bytes="$(wc -c <"$tarball")" + printf '%s bytes: %s\n' "$tar_bytes" "$tarball" + test "$tar_bytes" -le "$MAX_TAR_BYTES" + + - name: Upload release assets + uses: actions/upload-artifact@v7 + with: + name: git-${{ matrix.asset_platform }}-${{ matrix.arch }} + path: dugite-native/output/git-* + if-no-files-found: error + retention-days: 7 + + # The arm64 SDK puts its target Git first on PATH, but action cleanup + # runs on the x64 host and therefore needs the runner's native Git. + - name: Restore native Git for action cleanup + if: always() && matrix.target_platform == 'win32' + shell: pwsh + run: | + "C:\Program Files\Git\cmd" | + Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + release: + name: Publish GitHub prerelease + needs: + - version + - build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Download release assets + uses: actions/download-artifact@v8 + with: + pattern: git-* + path: artifacts + merge-multiple: true + + - name: Publish immutable prerelease + env: + GH_TOKEN: ${{ github.token }} + SOURCE_DESCRIPTION: ${{ needs.version.outputs.describe }} + SOURCE_REF: ${{ github.ref }} + VERSION: ${{ needs.version.outputs.version }} + run: | + set -euo pipefail + assets=(artifacts/git-*) + release_notes=$( + printf '%s\n' \ + "source_ref=$SOURCE_REF" \ + "source_sha=$GITHUB_SHA" \ + "" \ + "OpenAI Git release artifacts for $SOURCE_DESCRIPTION, built from $GITHUB_SHA for Codex." + ) + + if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 + then + gh release upload "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + else + gh release create "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$VERSION" \ + --notes "$release_notes" \ + --prerelease + fi diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml new file mode 100644 index 00000000000000..650cfd55baef14 --- /dev/null +++ b/.github/workflows/codex.yml @@ -0,0 +1,243 @@ +name: Refresh codex + +on: + schedule: + - cron: '*/5 * * * *' + workflow_dispatch: + inputs: + operation: + description: Refresh, scan, remove, or reorder a pinned topic + type: choice + options: + - refresh + - scan + - remove + - reorder + default: refresh + lane: + description: codex or codex-unstable for a plan operation + required: false + type: string + topic: + description: Exact topic branch for a plan operation + required: false + type: string + after: + description: Existing topic or root for reorder + required: false + type: string + plan_branch: + description: Optional codex-plan/* branch name + required: false + type: string + pull_request_target: + branches: + - meta + types: + - opened + - reopened + - synchronize + - ready_for_review + +permissions: + actions: read + contents: read + pull-requests: read + +jobs: + refresh: + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'refresh' + uses: openai/git/.github/workflows/codex.yml@meta + topic_plan_scan: + name: Find one approved topic plan + if: >- + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'scan') + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + concurrency: + group: codex-topic-plan-scan + cancel-in-progress: false + outputs: + lane: ${{ steps.reviewed.outputs.lane }} + topic: ${{ steps.reviewed.outputs.topic }} + source_tip: ${{ steps.reviewed.outputs.source_tip }} + review_pr: ${{ steps.reviewed.outputs.review_pr }} + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Pin trusted meta + id: meta + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex + sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq .object.sha) + case "$sha" in + ''|*[!0-9a-f]*) exit 1 ;; + esac + test "${#sha}" = 40 + printf 'sha=%s\n' "$sha" >>"$GITHUB_OUTPUT" + + - name: Check out trusted meta + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ github.repository }} + ref: ${{ steps.meta.outputs.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Find one exact approved topic PR + id: reviewed + env: + META_SHA: ${{ steps.meta.outputs.sha }} + run: | + set -euo pipefail + + die () { + printf '%s\n' "$*" >&2 + exit 1 + } + + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex || + die "topic scan must run from the trusted default branch" + test "$(git rev-parse HEAD)" = "$META_SHA" || + die "trusted checkout does not match pinned meta" + gh auth setup-git + mkdir -p "$RUNNER_TEMP/codex-plan-scan" + for lane in codex codex-unstable + do + case "$lane" in + codex) plan=codex.plan ;; + codex-unstable) plan=codex-unstable.plan ;; + esac + test -f "$plan" || + die "trusted meta has no $plan" + gh pr list --repo "$GITHUB_REPOSITORY" --state open \ + --base "$lane" --limit 1000 \ + --json number,isDraft,headRefName,headRefOid,headRepository,reviewDecision | + jq -r --arg lane "$lane" ' + .[] | + select(.isDraft | not) | + select(.reviewDecision == "APPROVED") | + select(.headRepository.nameWithOwner == "openai/git") | + [$lane, .headRefName, .headRefOid, + (.number | tostring)] | @tsv + ' + done | sort -k4,4n >"$RUNNER_TEMP/codex-plan-scan/candidates" + + while IFS=$'\t' read -r lane topic source_tip review_pr + do + test -n "$review_pr" || continue + case "$review_pr" in + *[!0-9]*) die "approved topic PR has invalid number '$review_pr'" ;; + esac + case "$source_tip" in + *[!0-9a-f]*|'') die "approved topic PR has invalid source SHA" ;; + esac + test "${#source_tip}" = 40 || + die "approved topic PR has invalid source SHA" + git check-ref-format "refs/heads/$topic" >/dev/null 2>&1 || + die "approved topic PR has invalid branch '$topic'" + case "$topic" in + ??/codex/*) ;; + *) continue ;; + esac + suffix=${topic#??/codex/} + case "$suffix" in + ''|*/*|*-wip|*-stale) continue ;; + esac + case "$lane" in + codex) + case "$topic" in + *-unstable) continue ;; + esac + plan=codex.plan + ;; + codex-unstable) + case "$topic" in + *-unstable) ;; + *) continue ;; + esac + plan=codex-unstable.plan + ;; + *) die "approved topic PR has invalid lane '$lane'" ;; + esac + pinned=$(git config --no-includes \ + --file "$plan" \ + --get "branch.$topic.source-tip" || :) + test "$pinned" = "$source_tip" && continue + short=$(printf '%.12s' "$source_tip") + slug=${topic##*/} + plan_branch=codex-plan/$lane-$slug-$short + pending=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --state open --base meta --head "$plan_branch" \ + --json number --jq '.[0].number // empty') || + die "could not inspect pending Codex plan PR" + test -n "$pending" && continue + if ! sh .github/workflows/codex-branch.sh propose-plan \ + --remote origin --lane "$lane" --topic "$topic" \ + --action auto --source-tip "$source_tip" \ + --review-pr "$review_pr" --expected-meta "$META_SHA" \ + --no-push >/dev/null + then + printf 'skipping approved topic PR #%s: preflight failed\n' \ + "$review_pr" >&2 + continue + fi + { + printf 'lane=%s\n' "$lane" + printf 'topic=%s\n' "$topic" + printf 'source_tip=%s\n' "$source_tip" + printf 'review_pr=%s\n' "$review_pr" + } >>"$GITHUB_OUTPUT" + exit 0 + done <"$RUNNER_TEMP/codex-plan-scan/candidates" + topic_plan_propose: + name: Propose reviewed topic plan + needs: topic_plan_scan + if: needs.topic_plan_scan.outputs.review_pr != '' + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ needs.topic_plan_scan.outputs.lane }} + topic: ${{ needs.topic_plan_scan.outputs.topic }} + action: auto + source_tip: ${{ needs.topic_plan_scan.outputs.source_tip }} + review_pr: ${{ needs.topic_plan_scan.outputs.review_pr }} + policy_plan_propose: + name: Propose explicit plan policy + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + (inputs.operation == 'remove' || inputs.operation == 'reorder') + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ inputs.lane }} + topic: ${{ inputs.topic }} + action: ${{ inputs.operation }} + after: ${{ inputs.after }} + plan_branch: ${{ inputs.plan_branch }} + plan_admission: + name: Codex plan admission + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.base.ref == 'meta' + permissions: + contents: read + pull-requests: write + uses: openai/git/.github/workflows/codex-plan-admission.yml@meta diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 205325eb33b064..485e3be6658151 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,7 +63,7 @@ jobs: echo "skip_concurrent=$skip_concurrent" >>$GITHUB_OUTPUT - name: skip if the commit or tree was already tested id: skip-if-redundant - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 if: steps.check-ref.outputs.enabled == 'yes' with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -112,8 +112,8 @@ jobs: group: windows-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: git-for-windows/setup-git-for-windows-sdk@335917db02da4280d3d5e87915d7b86196677f9f # v2 - name: build shell: bash env: @@ -123,7 +123,7 @@ jobs: - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: windows-artifacts path: artifacts @@ -140,14 +140,14 @@ jobs: cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - name: download tracked files and build artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: windows-artifacts path: ${{github.workspace}} - name: extract tracked files and build artifacts shell: bash run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: git-for-windows/setup-git-for-windows-sdk@335917db02da4280d3d5e87915d7b86196677f9f # v2 - name: test shell: bash run: . /etc/profile && ci/run-test-slice.sh $((${{matrix.nr}} + 1)) 10 @@ -157,7 +157,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-windows-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -173,20 +173,20 @@ jobs: group: vs-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: git-for-windows/setup-git-for-windows-sdk@335917db02da4280d3d5e87915d7b86196677f9f # v2 - name: initialize vcpkg - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: 'microsoft/vcpkg' path: 'compat/vcbuild/vcpkg' - name: download vcpkg artifacts - uses: git-for-windows/get-azure-pipelines-artifact@v0 + uses: git-for-windows/get-azure-pipelines-artifact@2e424b98e1251b725d6064c5f1a9f854a33e5b88 # v0 with: repository: git/git definitionId: 9 - name: add msbuild to PATH - uses: microsoft/setup-msbuild@v3 + uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # v3 - name: copy dlls to root shell: cmd run: compat\vcbuild\vcpkg_copy_dlls.bat release @@ -208,7 +208,7 @@ jobs: - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: vs-artifacts path: artifacts @@ -224,9 +224,9 @@ jobs: group: vs-test-${{ matrix.nr }}-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: git-for-windows/setup-git-for-windows-sdk@335917db02da4280d3d5e87915d7b86196677f9f # v2 - name: download tracked files and build artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: vs-artifacts path: ${{github.workspace}} @@ -244,7 +244,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-windows-vs-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -258,8 +258,8 @@ jobs: group: windows-meson-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - name: Set up dependencies shell: pwsh run: pip install meson ninja @@ -270,7 +270,7 @@ jobs: shell: pwsh run: meson compile -C build - name: Upload build artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: windows-meson-artifacts path: build @@ -286,13 +286,13 @@ jobs: group: windows-meson-test-${{ matrix.nr }}-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - name: Set up dependencies shell: pwsh run: pip install meson ninja - name: Download build artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: windows-meson-artifacts path: build @@ -305,7 +305,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-windows-meson-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -341,7 +341,7 @@ jobs: TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - name: print test failures @@ -349,7 +349,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -362,7 +362,7 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-build-and-minimal-fuzzers.sh dockerized: @@ -441,7 +441,7 @@ jobs: else apt-get -q update && apt-get -q -y install git fi - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: useradd builder --create-home - run: chown -R builder . @@ -451,7 +451,7 @@ jobs: run: sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -466,7 +466,7 @@ jobs: group: static-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh - run: ci/check-directional-formatting.bash @@ -482,7 +482,7 @@ jobs: group: rust-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-rust-checks.sh sparse: @@ -496,7 +496,7 @@ jobs: group: sparse-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install other dependencies run: ci/install-dependencies.sh - run: make sparse @@ -512,6 +512,6 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh diff --git a/Documentation/config/fetch.adoc b/Documentation/config/fetch.adoc index 00435e9a16d9f9..be055c35813d57 100644 --- a/Documentation/config/fetch.adoc +++ b/Documentation/config/fetch.adoc @@ -94,6 +94,16 @@ A value of 0 will give some reasonable default. If unset, it defaults to 1. For submodules, this setting can be overridden using the `submodule.fetchJobs` config setting. +`fetch.packfileUriJobs`:: + Specifies the maximum number of packfile URI downloads and indexers + to run at once. The default is 1, which preserves advertised URI + order. ++ +Values greater than 1 are used only when the server advertises and the +client requests the `no-ref-delta` promise. Each URI pack is then checked +with `index-pack --no-ref-delta` before it is accepted. +Responses with one URI retain the serial path. + `fetch.writeCommitGraph`:: Set to true to write a commit-graph after every `git fetch` command that downloads a pack-file from a remote. Using the `--split` option, diff --git a/Documentation/config/uploadpack.adoc b/Documentation/config/uploadpack.adoc index 0e1dda944a5490..c45f3c10645419 100644 --- a/Documentation/config/uploadpack.adoc +++ b/Documentation/config/uploadpack.adoc @@ -86,3 +86,14 @@ uploadpack.allowRefInWant:: is intended for the benefit of load-balanced servers which may not have the same view of what OIDs their refs point to due to replication delay. + +uploadpack.allowNoRefDelta:: + If this option is set, `upload-pack` may advertise the + `no-ref-delta` feature of the protocol version 2 `fetch` + command. When a client requests the feature, `upload-pack` passes + `--no-ref-delta` to `pack-objects` for the inline pack and promises + that every pack named in a `packfile-uris` response section also + contains no `REF_DELTA` entries. `upload-pack` does not inspect + configured URI packs, so the server administrator must create each + pack with `pack-objects --no-ref-delta` or otherwise verify this + property. The default is `false`. diff --git a/Documentation/git-index-pack.adoc b/Documentation/git-index-pack.adoc index 18036953c06b22..0076d6c183c220 100644 --- a/Documentation/git-index-pack.adoc +++ b/Documentation/git-index-pack.adoc @@ -9,8 +9,10 @@ git-index-pack - Build pack index file for an existing packed archive SYNOPSIS -------- [verse] -'git index-pack' [-v] [-o ] [--[no-]rev-index] -'git index-pack' --stdin [--fix-thin] [--keep] [-v] [-o ] +'git index-pack' [-v] [-o ] [--no-ref-delta] + [--[no-]rev-index] +'git index-pack' --stdin [--fix-thin] [--keep] [--no-ref-delta] + [-v] [-o ] [--[no-]rev-index] [] @@ -60,6 +62,11 @@ OPTIONS excluded objects the deltified objects are based on to the pack. This option only makes sense in conjunction with --stdin. +--no-ref-delta:: + Reject a pack containing a `REF_DELTA` entry. `OFS_DELTA` entries + are accepted. This option can be used to verify a protocol promise + that a pack contains no `REF_DELTA` entries. + --keep:: Before moving the index into its final destination create an empty .keep file for the associated pack file. diff --git a/Documentation/git-pack-objects.adoc b/Documentation/git-pack-objects.adoc index 65cd00c152f495..5e42e4429d0b8e 100644 --- a/Documentation/git-pack-objects.adoc +++ b/Documentation/git-pack-objects.adoc @@ -10,7 +10,8 @@ SYNOPSIS -------- [verse] 'git pack-objects' [-q | --progress | --all-progress] [--all-progress-implied] - [--no-reuse-delta] [--delta-base-offset] [--non-empty] + [--no-reuse-delta] [--delta-base-offset] [--no-ref-delta] + [--non-empty] [--local] [--incremental] [--window=] [--depth=] [--revs [--unpacked | --all]] [--keep-pack=] [--cruft] [--cruft-expiration=