feat(build): source kernels from edera-dev/linux branches

.github/workflows/build.yml

@@ -4,9 +4,10 @@ on:
   schedule:
     - cron: "0 0 * * 1"
   # A merge that changes a kernel config snippet changes the built kernel
-  # without bumping any version, so republish the release set now rather than
-  # waiting for the weekly cron. On main the tags carry no branch suffix, so
-  # this overwrites the canonical release tags with the new config content.
+  # without moving any kernel branch, so `new` would see nothing to do. Force a
+  # rebuild now rather than waiting for the weekly cron. On main the tags carry
+  # no branch suffix, so this overwrites the canonical tags with the new config
+  # content.
   push:
     branches:
       - main
@@ -17,7 +18,7 @@ on:
       spec:
         description: 'Build Specification'
         type: string
-        default: "only-latest-lts"
+        default: "new"
         required: true
       publish:
         description: 'Publish Builds'
@@ -35,8 +36,9 @@ jobs:
     uses: ./.github/workflows/matrix.yml
     secrets: inherit
     with:
-      # schedule/push carry no inputs, so this fallback is what the cron and
-      # the configs/** push actually build. Keep it equal to the
-      # workflow_dispatch default above.
-      spec: ${{ inputs.spec || 'only-latest-lts' }}
+      # schedule/push carry no inputs, so these fallbacks are what the cron and
+      # the configs/** push actually build. The cron only wants branches that
+      # have moved (`new`); a configs/** merge changed the build inputs without
+      # moving anything, so it has to force `rebuild`.
+      spec: ${{ inputs.spec || (github.event_name == 'push' && 'rebuild' || 'new') }}
       publish: ${{ inputs.publish == '' || inputs.publish }}

.github/workflows/matrix.yml

@@ -52,8 +52,9 @@ jobs:
       run: |
         # Split matrix.json into two strategy-matrix-shaped outputs: one keyed
         # on `builds` (per-arch build jobs) and one on `merges` (per
-        # (version, flavor) manifest-list assembly jobs). Each iteration of the
-        # build job sees matrix.builds.* and each merge sees matrix.merges.*.
+        # (branch, version, flavor) manifest-list assembly jobs). Each iteration
+        # of the build job sees matrix.builds.* and each merge sees
+        # matrix.merges.*.
         BUILDS=$(jq -c '{builds: .builds}' matrix.json)
         MERGES=$(jq -c '{merges: .merges}' matrix.json)
         echo "builds=${BUILDS}" >> "${GITHUB_OUTPUT}"
@@ -62,7 +63,7 @@ jobs:
       builds: "${{ steps.capture-matrix.outputs.builds }}"
       merges: "${{ steps.capture-matrix.outputs.merges }}"
   build:
-    name: "build ${{ matrix.builds.version }} ${{ matrix.builds.flavor }} ${{ matrix.builds.arch }}"
+    name: "build ${{ matrix.builds.branch }} ${{ matrix.builds.version }} ${{ matrix.builds.flavor }} ${{ matrix.builds.arch }}"
     needs: matrix
     strategy:
       fail-fast: false
@@ -72,6 +73,9 @@ jobs:
       KERNEL_PUBLISH: "${{ inputs.publish }}"
       KERNEL_VERSION: "${{ matrix.builds.version }}"
       KERNEL_SRC_URL: "${{ matrix.builds.source }}"
+      KERNEL_SRC_REPO: "${{ matrix.builds.repo }}"
+      KERNEL_SRC_REF: "${{ matrix.builds.ref }}"
+      KERNEL_SRC_COMMIT: "${{ matrix.builds.commit }}"
       FIRMWARE_URL: "${{ matrix.builds.firmware_url }}"
       FIRMWARE_SIG_URL: "${{ matrix.builds.firmware_sig_url }}"
       KERNEL_FLAVOR: "${{ matrix.builds.flavor }}"
@@ -158,7 +162,7 @@ jobs:
       - name: upload docker script
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: "build-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}.sh"
+          name: "build-${{ matrix.builds.branch }}-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}.sh"
           path: "docker.sh"
           compression-level: 0
       - name: run docker script
@@ -174,15 +178,15 @@ jobs:
         if: ${{ inputs.publish }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: "digests-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}"
+          name: "digests-${{ matrix.builds.branch }}-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}"
           path: "digests.json"
           if-no-files-found: error
           compression-level: 0
           retention-days: 1
   merge:
     # Stitch the per-arch single-platform pushes from `build` into multi-arch
     # manifest lists. Only runs when publishing; no-op when nothing was pushed.
-    name: "merge ${{ matrix.merges.version }} ${{ matrix.merges.flavor }}"
+    name: "merge ${{ matrix.merges.branch }} ${{ matrix.merges.version }} ${{ matrix.merges.flavor }}"
     needs: [matrix, build]
     if: ${{ inputs.publish && needs.matrix.outputs.merges != '' }}
     strategy:
@@ -197,6 +201,9 @@ jobs:
       KERNEL_FLAVOR: "${{ matrix.merges.flavor }}"
       KERNEL_PRODUCES: "${{ join(matrix.merges.produces, ',') }}"
       KERNEL_SRC_URL: "${{ matrix.merges.source }}"
+      KERNEL_SRC_REPO: "${{ matrix.merges.repo }}"
+      KERNEL_SRC_REF: "${{ matrix.merges.ref }}"
+      KERNEL_SRC_COMMIT: "${{ matrix.merges.commit }}"
       FIRMWARE_URL: "${{ matrix.merges.firmware_url }}"
       DIGESTS_DIR: digests
     steps:
@@ -230,14 +237,14 @@ jobs:
           # Each per-arch build uploads its digests under a uniquely-named
           # artifact; pattern + default merge-multiple=false drops each artifact
           # into its own subdirectory under digests/.
-          pattern: "digests-${{ matrix.merges.version }}-${{ matrix.merges.flavor }}-*"
+          pattern: "digests-${{ matrix.merges.branch }}-${{ matrix.merges.version }}-${{ matrix.merges.flavor }}-*"
           path: digests
       - name: generate merge script
         run: uv run ./hack/build/generate-merge-script.py
       - name: upload merge script
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: "merge-${{ matrix.merges.version }}-${{ matrix.merges.flavor }}.sh"
+          name: "merge-${{ matrix.merges.branch }}-${{ matrix.merges.version }}-${{ matrix.merges.flavor }}.sh"
           path: "merge.sh"
           compression-level: 0
       - name: run merge script

.github/workflows/test.yml

@@ -19,5 +19,5 @@ jobs:
     uses: ./.github/workflows/matrix.yml
     secrets: inherit
     with:
-      spec: "only-latest-lts:flavor=host,zone,zone-nvidiagpu"
+      spec: "rebuild:branch=6.18-lts;flavor=host,zone,zone-nvidiagpu"
       publish: false

Dockerfile

@@ -1,6 +1,9 @@
+# KERNEL_SRC_URL is an archive of one commit of the Edera Linux tree (see
+# config.yaml and hack/build/matrix.py). It is addressed by commit, so this
+# ADD's buildkit cache entry can never go stale against a moving branch.
 FROM --platform=$BUILDPLATFORM scratch AS kernelsrc
 ARG KERNEL_SRC_URL=
-ADD ${KERNEL_SRC_URL} /src.tar.xz
+ADD ${KERNEL_SRC_URL} /src.tar.gz
 
 FROM --platform=$BUILDPLATFORM scratch AS firmware
 ARG FIRMWARE_URL=
@@ -24,7 +27,7 @@ WORKDIR /build
 RUN chmod +x hack/build/docker-build-internal.sh
 
 FROM buildenv AS build-staged
-COPY --from=kernelsrc --chown=build:build /src.tar.xz /build/override-kernel-src.tar.xz
+COPY --from=kernelsrc --chown=build:build /src.tar.gz /build/override-kernel-src.tar.gz
 
 FROM build-staged AS build-staged-amdgpu
 COPY --from=firmware --chown=build:build /firmware.tar.xz /build/override-firmware.tar.xz

README.md

@@ -4,21 +4,74 @@ Builds the Linux kernel into OCI images.
 
 ## Build overview
 
+Kernels are built from [github.com/edera-dev/linux](https://github.com/edera-dev/linux), Edera's downstream
+Linux tree. Every Edera change is a commit on a branch there, so this repo carries no patch series of its
+own: whatever the branch says is what gets built.
+
 This repo is a series of helper scripts and Github Actions that
 
-1. Use [config.yaml](/config.yaml) and various scripts in `hack/build` to generate a matrix of kernel upstream versions, variants, and flavors to be built.
-1. Fetches `kernel.org` tarballs, applies custom config flags, and patches, depending on version, flavor and variant.
-1. Builds those kernels for x86_64/arm64.
+1. Resolve each branch listed in [config.yaml](/config.yaml) to the commit it currently points at, and read
+   the kernel version out of that commit's `Makefile`.
+1. Use that, plus the flavors and architectures in `config.yaml`, to generate a build matrix.
+1. Fetch the source archive for that exact commit, apply the Edera kconfig fragments, and build for
+   x86_64/aarch64.
 
-### Variant and flavor configs
+### Branches
 
-See the [configs](/configs/README.md) directory for more info.
+| Branch in `config.yaml` | Ref on `edera-dev/linux` | What it is |
+| --- | --- | --- |
+| `6.18-lts` | `edera/6.18-lts` | The 6.18 LTS series with the Edera stack. Also publishes `latest`. |
+| `mainline` | `edera/mainline` | Current mainline (including `-rc`) with the Edera stack. |
+
+Adding a branch is three lines in `config.yaml`; nothing else needs to know about it.
+
+### Image tags
+
+Each build publishes one immutable tag and a set of moving aliases. For example, `edera/6.18-lts` at
+6.18.52, commit `efb09285bd95`:
+
+| Tag | Moves? |
+| --- | --- |
+| `zone-kernel:6.18.52-gefb09285bd95` | no - one commit, forever |
+| `zone-kernel:6.18.52` | yes |
+| `zone-kernel:6.18` | yes |
+| `zone-kernel:6.18-lts` | yes |
+| `zone-kernel:latest` | yes |
+
+and `edera/mainline` at 7.3-rc3, commit `391f6f12ecf5`, publishes `7.3.0-rc3-g391f6f12ecf5`, `7.3.0-rc3` and
+`mainline`. A prerelease deliberately does not claim the bare series tag (`7.3`), which belongs to the
+eventual 7.3 release.
+
+The immutable tag names the *source tree*, not every build input. A change to a kconfig fragment in
+[configs](/configs) rebuilds and republishes the same tags against the same commit; the kernel inside
+changes, the tag does not. `config.gz` and the image's `metadata` (which records
+`KERNEL_SRC_REPO`/`KERNEL_SRC_REF`/`KERNEL_SRC_COMMIT` alongside a hash of the kconfig) are what
+distinguish two such builds.
+
+### Build specifications
+
+The `Build Kernels` action takes a build spec of `<type>[:<constraints>]`:
 
-### Patches
+- `new` (default) - build only what the registry does not already have. Since each build carries an
+  immutable `<version>-g<commit>` tag, this means "build every branch that has moved". This is what the
+  weekly cron runs.
+- `rebuild` - build everything the config selects, published or not. Use this when something other than the
+  kernel source changed: a kconfig fragment, the buildenv, the packaging. A merge touching `configs/**`
+  triggers this automatically.
 
-See the [patches](/patches) directory for the current set of patches Edera carries against upstream kernels.
+Constraints are semicolon-separated `key=value` pairs over `branch`, `flavor` and `arch`, with
+comma-separated values:
 
-Note that not all the patches will be applied to all kernel versions, this is driven by version constraints in [config.yaml](/config.yaml).
+```
+new
+rebuild:branch=mainline
+rebuild:branch=6.18-lts;flavor=host,zone
+new:flavor=zone;arch=aarch64
+```
+
+### Variant and flavor configs
+
+See the [configs](/configs/README.md) directory for more info.
 
 ## Building your own kernels with custom KConfig (Using Github Actions)
 
@@ -30,14 +83,14 @@ Note that not all the patches will be applied to all kernel versions, this is dr
    1. Edit the line `imageNameFormat: "ghcr.io/edera-dev/[image]:[tag]"` and change it to `imageNameFormat: "ghcr.io/<your GH org>/[image]:[tag]"`
    1. Add or remove any `Kconfig` options you want to the `flavor` and `variant` Kconfig fragments in [configs](/configs), as outlined by the [README](/configs/README.md) in that folder.
    1. Commit those changes to `main` in your fork.
-1. From your forked repository's `Actions` tab, run the `Build Kernels` job with a Build Specification like: `stable:flavor=zone,host`. This will build `zone` and `host` kernel flavors using `stable` kernel.org releases for all current LTS kernels.
+1. From your forked repository's `Actions` tab, run the `Build Kernels` job with a Build Specification like: `rebuild:flavor=zone,host`. This will build the `zone` and `host` flavors of every branch in `config.yaml`.
    1. ![Example](/images/job-example.png)
 
 ## Building your own kernels with custom KConfig (Locally, for debugging)
 
 For most of the kernels in this registry, debugging symbols and features are disabled, to keep the kernel artifacts small.
 
-You may want or need to build your own debugging kernel with custom patches/options locally, and publish it to a transient OCI registry (like [ttl.sh](ttl.sh)) for testing purposes.
+You may want or need to build your own debugging kernel with custom options locally, and publish it to a transient OCI registry (like [ttl.sh](ttl.sh)) for testing purposes.
 
 To do this, you will need `docker` installed and configured correctly to support cross-builds (`docker buildx`) in your local environment.
 
@@ -46,42 +99,56 @@ The simplest way to do that is to
 1. Clone this repo locally: `git clone git@github.com:edera-dev/linux-kernel-oci.git`
 1. Manually edit [config.yaml](/config.yaml) on-disk:
    - to change the `imageNameFormat` key to push to an OCI registry you have access to.
-   - to change the `architectures` YAML key to only include the architectures you care about (x86_64, arm64, or both - `docker buildx` is used so you can build arm64 on x86_64 and vice-versa).
+   - to change the `architectures` YAML key to only include the architectures you care about (x86_64, aarch64, or both - `docker buildx` is used so you can build aarch64 on x86_64 and vice-versa).
    - to change the `flavors` YAML key to only include the flavors you care about (host, or zone, or both).
-   - to change the `versions` YAML key to only include the `kernel.org` versions you care about. For instance, to only build the latest `5.4` kernel.org upstream release, use:
-
-        ```yaml
-        versions:
-            - series: '5.4'
-        ```
-
-   - For example, if I wanted to only build the `zone` kernel flavor, only for `x86_64`, only the latest `6.15` point release, and tag the result for a custom `ttl.sh/hackben` registry, the final result would look something like this:
+   - to change the `branches` YAML key to only include the branches you care about. You can also point
+     `source.repo` at your own fork of `edera-dev/linux` and list a branch on it.
+   - For example, if I wanted to only build the `zone` kernel flavor, only for `x86_64`, only from the LTS
+     branch, and tag the result for a custom `ttl.sh/hackben` registry, the final result would look
+     something like this:
 
         ```yaml
             imageNameFormat: "ttl.sh/hackben/[image]:[tag]"
+            source:
+              repo: https://github.com/edera-dev/linux
+            branches:
+            - name: 6.18-lts
+              ref: edera/6.18-lts
             architectures:
             - x86_64
             flavors:
-            - name: zone-debu
-            constraints:
-                series:
-                - '6.15'
-            versions:
-            - current: true
+            - name: zone
         ```
 
 1. Add or remove any `Kconfig` options you want to the `flavor` and `variant` Kconfig fragments in [configs](/configs), as outlined by the [README](/configs/README.md) in that folder.
 1. Run [hack/build/docker-build.sh](hack/build/docker-build.sh)
-   - It is **important** you follow the previous step, and edit the [config.yaml](config.yaml) locally to reduce the number of kernels the script will try to build, or you may end up building 15+ different kernels in parallel on your local box, which will take a very, very long time.
+   - It is **important** you follow the previous step, and edit the [config.yaml](config.yaml) locally to reduce the number of kernels the script will try to build, or you may end up building many different kernels in parallel on your local box, which will take a very, very long time.
    - When this command runs, it will generate a build matrix and print out what it will build.
+   - Pass a build spec as the first argument to narrow it further, e.g.
+     `./hack/build/docker-build.sh 'rebuild:branch=6.18-lts;flavor=zone'` (quote it - `;` is a shell
+     metacharacter).
 1. When the above command finishes, you can see the local OCI images that were built by running `docker image list`. The images will be tagged with the repo you specified in `imageNameFormat` in the [config.yaml](/config.yaml).
 1. From this point, you may push those images to an OCI registry with standard commands like `docker image push <image tag>`, and consume them how you wish.
 1. If you wish to unpack and inspect the final image (for instance, to make sure certain modules or firmware exist in the correct paths, or that the final `config.gz` has the options you expect), you can do the following to fetch and extract the image artifact you just pushed to your local disk with [`crane`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/README.md):
 
     ```sh
-    crane export  ttl.sh/hackben/zone-kernel:6.15.6 - --platform=linux/amd64 | tar --keep-directory-symlink -xf - -C .
+    crane export  ttl.sh/hackben/zone-kernel:6.18-lts - --platform=linux/amd64 | tar --keep-directory-symlink -xf - -C .
     cd `kernel`
     zcat config.gz
+    cat metadata
     unsquashfs addons.squashfs
     ...
     ```
+
+### Building a branch that isn't in `config.yaml`
+
+`hack/build/build.sh` takes `KERNEL_SRC_URL` directly and understands a `git::<url>[::<ref>]` form, which
+clones that ref instead of fetching an archive. That is the escape hatch for building a work-in-progress
+branch without listing it in `config.yaml` first. It expects the kbuild toolchain to already be present, so
+run it inside the build environment image (`ghcr.io/edera-dev/kernel-buildenv`) rather than on a bare host:
+
+```sh
+KERNEL_VERSION=6.18.52 KERNEL_FLAVOR=zone \
+  KERNEL_SRC_URL='git::https://github.com/edera-dev/linux::azenla/zone-perf' \
+  ./hack/build/build.sh
+```