ollama, llama-cpp, vllm: add per-hardware compose overrides

apps/llama-cpp/.env.example

@@ -1,8 +1,33 @@
+# ==== hardware: pick exactly one COMPOSE_FILE line ===========================
+# See README "Choosing compose files". Add :docker-compose.nfs.yml to the end of
+# any line to keep models on NFS.
+
+# CPU only
+COMPOSE_FILE=docker-compose.yml
+# NVIDIA discrete GPUs, CUDA 12.8 (driver >= 570)
+# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
+# NVIDIA DGX Spark (GB10), CUDA 13
+# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml
+# NVIDIA Jetson Thor (uses the nvidia runtime instead of GPU reservations)
+# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml
+# AMD Strix Halo / Strix Point (Vulkan), x86_64 only
+# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml
+# Intel Arc (SYCL), x86_64 only
+# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml
+
+# Full image tag. Leave unset: each compose file picks its own default
+#   server (CPU), server-cuda, server-cuda13, server-vulkan, server-rocm, server-intel
+# Set it to pin a build, e.g. server-cuda-b6500, or to switch backend within a
+# file (server-cuda13 on nvidia.yml, server-rocm on strix.yml, server-vulkan on arc.yml).
+# IMAGE_VERSION=
+
+# ==== model ===================================================================
 # Path to the GGUF model file inside the container
 LLAMA_ARG_MODEL=/models/model.gguf
 
-# Number of GPU layers to offload (0 = CPU only, -1 = all layers)
-LLAMA_ARG_N_GPU_LAYERS=0
+# Layers to offload to the GPU (0 = CPU only, 999 = all). Leave unset: the base
+# file runs on the CPU and the GPU files default to 999.
+# LLAMA_ARG_N_GPU_LAYERS=
 
 # Context window size in tokens
 LLAMA_ARG_CTX_SIZE=4096
@@ -14,15 +39,25 @@ LLAMA_ARG_THREADS=4
 LLAMA_ARG_HOST=0.0.0.0
 LLAMA_ARG_PORT=8080
 
-# Image tag controls the hardware backend:
-#   server        - CPU only (default)
-#   server-cuda   - NVIDIA GPU (requires nvidia container runtime)
-#   server-rocm   - AMD GPU (requires ROCm)
-#   server-vulkan - Vulkan (Intel/AMD integrated)
-#IMAGE_VERSION=server
+# LLAMA_ARG_FLASH_ATTN=on
+# LLAMA_ARG_N_PARALLEL=1
+
+# ==== NVIDIA (nvidia.yml, spark.yml, jetson.yml) ==============================
+# GPUs reserved for the container: all, or a number (ignored on Jetson)
+# GPU_COUNT=all
+# CUDA_VISIBLE_DEVICES=0
+
+# ==== AMD Strix (strix.yml) ===================================================
+# Pick one Vulkan device if the host has more than one GPU:
+# GGML_VK_VISIBLE_DEVICES=0
+# Only for IMAGE_VERSION=server-rocm, if ROCm doesn't recognise the APU:
+# HSA_OVERRIDE_GFX_VERSION=11.5.1   # 11.5.0 for Strix Point
+
+# ==== Intel Arc (arc.yml) =====================================================
+# Pick one card on a multi-GPU host
+# ONEAPI_DEVICE_SELECTOR=level_zero:0
 
-# --- Optional: GGUF models on NFS ---
-# COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml
+# --- Optional: GGUF models on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) ---
 # LLAMA_CPP_NFS_HOST=
 # LLAMA_CPP_NFS_PATH=
 # NFS_MOUNT_OPTS=nfsvers=4,hard

apps/llama-cpp/README.md

@@ -6,17 +6,26 @@ on CPU or GPU and exposes them as an OpenAI API endpoint on `llama-net`,
 making it usable by [openwebui](../openwebui/) and
 [home-assistant](../home-assistant/) alongside [ollama](../ollama/).
 
-## Hardware backend
+## Choosing compose files
 
-The Docker image tag selects the hardware backend. Set `IMAGE_VERSION` in
-`.env` to choose:
+`docker-compose.yml` is hardware-neutral and runs the CPU image by itself. Pick
+the hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists every line):
 
-| `IMAGE_VERSION` | Backend |
-|-----------------|---------|
-| `server` _(default)_ | CPU only |
-| `server-cuda` | NVIDIA GPU (requires nvidia container runtime) |
-| `server-rocm` | AMD GPU (requires ROCm) |
-| `server-vulkan` | Vulkan — Intel/AMD integrated graphics |
+| Hardware | `COMPOSE_FILE` | Image tag |
+|----------|----------------|-----------|
+| NVIDIA discrete GPUs (driver >= 570) | `docker-compose.yml:docker-compose.nvidia.yml` | `server-cuda` (CUDA 12.8) |
+| NVIDIA DGX Spark (GB10) | `docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml` | `server-cuda13` |
+| NVIDIA Jetson Thor | `docker-compose.yml:docker-compose.jetson.yml` | `server-cuda13`, `nvidia` runtime |
+| AMD Strix Halo / Strix Point | `docker-compose.yml:docker-compose.strix.yml` | `server-vulkan` |
+| Intel Arc | `docker-compose.yml:docker-compose.arc.yml` | `server-intel` (SYCL) |
+| CPU only | `docker-compose.yml` | `server` |
+
+Notes:
+
+- The GPU files default `LLAMA_ARG_N_GPU_LAYERS` to `999` (offload everything); set it in `.env` to override.
+- `IMAGE_VERSION` is the full image tag and overrides the file's default. Leave it unset unless you want to pin a build (`server-cuda-b6500`) or switch backend within a file: `server-cuda13` on `nvidia.yml` (driver >= 580), `server-rocm` on `strix.yml`, `server-vulkan` on `arc.yml`.
+- **Jetson Thor**: the toolkit runs in CSV mode there and rejects GPU reservations, so `jetson.yml` uses `runtime: nvidia` and replaces `nvidia.yml` rather than layering on it.
+- **Strix**: Vulkan (RADV) is the most reliable and usually fastest llama.cpp backend on these APUs. For large models on Strix Halo, raise the GPU's share of system memory on the host (BIOS UMA setting and/or the `amdttm.pages_limit` kernel parameter).
 
 ## Models
 
@@ -46,21 +55,23 @@ cp .env.example .env
 | Variable | Description | Default |
 |----------|-------------|---------|
 | `LLAMA_ARG_MODEL` | Path to the GGUF model file inside the container | `/models/model.gguf` |
-| `LLAMA_ARG_N_GPU_LAYERS` | Layers to offload to GPU (`0` = CPU only, `-1` = all) | `0` |
+| `LLAMA_ARG_N_GPU_LAYERS` | Layers to offload to GPU (`0` = CPU only, `999` = all) | `999` in the GPU files |
 | `LLAMA_ARG_CTX_SIZE` | Context window size in tokens | `4096` |
 | `LLAMA_ARG_THREADS` | CPU threads to use | `4` |
 | `LLAMA_ARG_HOST` | Bind address inside the container | `0.0.0.0` |
 | `LLAMA_ARG_PORT` | Port inside the container | `8080` |
-| `IMAGE_VERSION` | Image tag / hardware backend (see table above) | `server` |
+| `IMAGE_VERSION` | Full image tag, overrides the compose file's default (see above) | per file |
 
 ## Storage on NFS (optional)
 
 Large data defaults to bind mounts under `../../lib/` in `docker-compose.yml`. To use a NAS export instead, set in **`.env`** (see **`.env.example`**):
 
 ```bash
-COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml
+COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml
 ```
 
+(use your hardware line, with `docker-compose.nfs.yml` last)
+
 plus the `*_NFS_HOST`, `*_NFS_PATH`, and `NFS_MOUNT_OPTS` variables documented in `.env.example`. Compose reads `COMPOSE_FILE` from `.env`, so plain `docker compose up -d` picks up the override.
 
 Prefer **`hard`** NFS mounts, not `soft`. Docker creates named NFS volumes on first start; if the server or export path is wrong, fix `.env` and remove the project volume before retrying (`docker volume ls`, then `docker volume rm …`).

apps/llama-cpp/docker-compose.arc.yml

@@ -0,0 +1,12 @@
+# Intel Arc (Battlemage B580 / Arc Pro B-series, Alchemist A-series) on Linux,
+# x86_64 only. Uses the SYCL (oneAPI) build; IMAGE_VERSION=server-vulkan also
+# works with this file. Needs the xe (or i915) kernel driver.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml
+services:
+  llama-cpp:
+    restart: unless-stopped
+    image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-intel}
+    devices:
+      - /dev/dri:/dev/dri
+    environment:
+      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}

apps/llama-cpp/docker-compose.jetson.yml

@@ -0,0 +1,12 @@
+# NVIDIA Jetson Thor (JetPack 7, arm64, CUDA 13). Use instead of docker-compose.nvidia.yml,
+# not on top of it: Jetson's container toolkit runs in CSV mode, which rejects device
+# reservations (the --gpus equivalent) and only works through the nvidia runtime.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml
+services:
+  llama-cpp:
+    restart: unless-stopped
+    image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda13}
+    runtime: nvidia
+    environment:
+      NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
+      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}

apps/llama-cpp/docker-compose.nfs.yml

@@ -1,5 +1,6 @@
 # Optional override: GGUF models on NFS.
-# Enable: COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml
+# Add it last in COMPOSE_FILE, e.g.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml
 
 services:
   llama-cpp:

apps/llama-cpp/docker-compose.nvidia.yml

@@ -0,0 +1,17 @@
+# NVIDIA discrete GPUs on Linux (x86_64 or arm64), CUDA 12.8 (driver >= 570).
+# For driver >= 580 you can set IMAGE_VERSION=server-cuda13 instead.
+# DGX Spark: also add docker-compose.spark.yml. Jetson Thor: use docker-compose.jetson.yml.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
+services:
+  llama-cpp:
+    restart: unless-stopped
+    image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda}
+    environment:
+      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: ${GPU_COUNT:-all}
+              capabilities: [gpu]

apps/llama-cpp/docker-compose.spark.yml

@@ -0,0 +1,6 @@
+# NVIDIA DGX Spark (GB10, arm64, sm_121): the CUDA 13 build. Layer on nvidia.yml:
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml
+services:
+  llama-cpp:
+    restart: unless-stopped
+    image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda13}

apps/llama-cpp/docker-compose.strix.yml

@@ -0,0 +1,16 @@
+# AMD Strix Halo (Ryzen AI Max, gfx1151) and Strix Point (Ryzen AI 300, gfx1150)
+# on Linux, x86_64 only. Uses the Vulkan (RADV) build, which is the fastest and
+# least fussy llama.cpp backend on these APUs; the host only needs the amdgpu
+# kernel driver. IMAGE_VERSION=server-rocm also works with this file.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml
+services:
+  llama-cpp:
+    restart: unless-stopped
+    image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-vulkan}
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri:/dev/dri
+    group_add:
+      - video
+    environment:
+      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}

apps/llama-cpp/docker-compose.yml

@@ -1,3 +1,10 @@
+# Hardware-neutral base. On its own this runs the CPU-only image; layer one of
+# the hardware overrides on top via COMPOSE_FILE in .env (see README):
+#   docker-compose.nvidia.yml  NVIDIA discrete GPUs, CUDA 12.8 (driver >= 570)
+#   docker-compose.spark.yml   DGX Spark (GB10), CUDA 13, on top of nvidia.yml
+#   docker-compose.jetson.yml  NVIDIA Jetson Thor (nvidia runtime, instead of nvidia.yml)
+#   docker-compose.strix.yml   AMD Strix Halo / Strix Point APUs (Vulkan), x86_64 only
+#   docker-compose.arc.yml     Intel Arc (SYCL), x86_64 only
 services:
   llama-cpp:
     image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server}

apps/ollama/.env.example

@@ -1,8 +1,46 @@
-IMAGE_VERSION=latest
+# ==== hardware: pick exactly one COMPOSE_FILE line ===========================
+# See README "Choosing compose files". Add :docker-compose.nfs.yml to the end of
+# any line to keep models on NFS.
+
+# NVIDIA discrete GPUs and DGX Spark (GB10)
+COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
+# NVIDIA Jetson Thor (uses the nvidia runtime instead of GPU reservations)
+# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml
+# AMD Strix Halo / Strix Point (ROCm), x86_64 only
+# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml
+# Intel Arc (Vulkan), x86_64 only
+# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml
+# CPU only
+# COMPOSE_FILE=docker-compose.yml
+
+# Ollama release (e.g. 0.40.11), or leave unset for latest. The Strix file adds
+# the -rocm suffix itself, so don't include it here.
+# IMAGE_VERSION=
+
+# ==== NVIDIA (docker-compose.nvidia.yml, docker-compose.jetson.yml) ===========
+# GPUs reserved for the container: all, or a number (ignored on Jetson)
 GPU_COUNT=4
+# CUDA_VISIBLE_DEVICES=             # GPU UUIDs or indexes, comma separated
+
+# ==== AMD Strix (docker-compose.strix.yml) ====================================
+# Strix Halo (gfx1151) and Strix Point (gfx1150) are supported by the ROCm
+# build. Only set this if Ollama falls back to the CPU on your APU:
+# HSA_OVERRIDE_GFX_VERSION=11.5.1   # 11.5.0 for Strix Point
+# ROCR_VISIBLE_DEVICES=0
+
+# ==== Intel Arc (docker-compose.arc.yml) ======================================
+# Vulkan is on by default in that file; pick one card on a multi-GPU host:
+# GGML_VK_VISIBLE_DEVICES=0
+
+# ==== Ollama (all backends) ===================================================
+# OLLAMA_KEEP_ALIVE=5m
+# OLLAMA_CONTEXT_LENGTH=4096
+# OLLAMA_NUM_PARALLEL=1
+# OLLAMA_MAX_LOADED_MODELS=1
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
 
-# --- Optional: models on NFS ---
-# COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml
+# --- Optional: models on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) ---
 # OLLAMA_NFS_HOST=
 # OLLAMA_NFS_PATH=
 # NFS_MOUNT_OPTS=nfsvers=4,hard

apps/ollama/README.md

@@ -13,14 +13,36 @@ cp .env.example .env
 # Edit .env with your settings
 ```
 
+## Choosing compose files
+
+`docker-compose.yml` is hardware-neutral and runs on the CPU by itself. Pick the
+hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists every line):
+
+| Hardware | `COMPOSE_FILE` | Backend |
+|----------|----------------|---------|
+| NVIDIA discrete GPUs, DGX Spark (GB10) | `docker-compose.yml:docker-compose.nvidia.yml` | CUDA, via GPU reservation |
+| NVIDIA Jetson Thor | `docker-compose.yml:docker-compose.jetson.yml` | CUDA, via the `nvidia` runtime |
+| AMD Strix Halo / Strix Point | `docker-compose.yml:docker-compose.strix.yml` | ROCm (`ollama/ollama:rocm`) |
+| Intel Arc | `docker-compose.yml:docker-compose.arc.yml` | Vulkan |
+| CPU only | `docker-compose.yml` | CPU |
+
+Notes:
+
+- **NVIDIA**: needs the NVIDIA Container Toolkit. `GPU_COUNT` (`all` or a number) sets how many GPUs are reserved.
+- **Jetson Thor**: the toolkit runs in CSV mode there and rejects GPU reservations, so `jetson.yml` uses `runtime: nvidia` and replaces `nvidia.yml` rather than layering on it.
+- **Strix**: Strix Halo (gfx1151) and Strix Point (gfx1150) are in Ollama's ROCm support list. If `IMAGE_VERSION` is set it is the bare version (`0.40.11`); the file adds `-rocm`. For large models on Strix Halo, raise the GPU's share of system memory on the host (BIOS UMA setting and/or the `amdttm.pages_limit` kernel parameter).
+- **Intel Arc**: Ollama has no oneAPI backend; Arc runs through the Vulkan backend bundled in the stock image.
+
 ## Storage on NFS (optional)
 
 Large data defaults to bind mounts under `../../lib/` in `docker-compose.yml`. To use a NAS export instead, set in **`.env`** (see **`.env.example`**):
 
 ```bash
-COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml
+COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml
 ```
 
+(use your hardware line, with `docker-compose.nfs.yml` last)
+
 plus the `*_NFS_HOST`, `*_NFS_PATH`, and `NFS_MOUNT_OPTS` variables documented in `.env.example`. Compose reads `COMPOSE_FILE` from `.env`, so plain `docker compose up -d` picks up the override.
 
 Prefer **`hard`** NFS mounts, not `soft`. Docker creates named NFS volumes on first start; if the server or export path is wrong, fix `.env` and remove the project volume before retrying (`docker volume ls`, then `docker volume rm …`).

apps/ollama/docker-compose.arc.yml

@@ -0,0 +1,11 @@
+# Intel Arc (Battlemage B580 / Arc Pro B-series, Alchemist A-series) on Linux,
+# x86_64 only. Ollama has no oneAPI/SYCL backend; the stock image drives Arc
+# through its bundled Vulkan backend. Needs the xe (or i915) kernel driver.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml
+services:
+  ollama:
+    restart: unless-stopped
+    devices:
+      - /dev/dri:/dev/dri
+    environment:
+      OLLAMA_VULKAN: ${OLLAMA_VULKAN:-1}

apps/ollama/docker-compose.jetson.yml

@@ -0,0 +1,10 @@
+# NVIDIA Jetson Thor (JetPack 7). Use instead of docker-compose.nvidia.yml, not on top of it:
+# Jetson's container toolkit runs in CSV mode, which rejects device reservations
+# (the --gpus equivalent) and only works through the nvidia runtime.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml
+services:
+  ollama:
+    restart: unless-stopped
+    runtime: nvidia
+    environment:
+      NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}

apps/ollama/docker-compose.nfs.yml

@@ -1,5 +1,6 @@
 # Optional override: Ollama models on NFS.
-# Enable: COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml
+# Add it last in COMPOSE_FILE, e.g.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml
 
 services:
   ollama:

apps/ollama/docker-compose.nvidia.yml

@@ -0,0 +1,14 @@
+# NVIDIA GPUs on Linux (x86_64 or arm64: discrete cards, DGX Spark GB10).
+# Not for Jetson Thor; use docker-compose.jetson.yml there.
+# Needs the NVIDIA Container Toolkit on the host and driver >= 550.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
+services:
+  ollama:
+    restart: unless-stopped
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: ${GPU_COUNT:-all}
+              capabilities: [gpu]

apps/ollama/docker-compose.strix.yml

@@ -0,0 +1,14 @@
+# AMD Strix Halo (Ryzen AI Max, gfx1151) and Strix Point (Ryzen AI 300, gfx1150)
+# on Linux, x86_64 only. Uses the ROCm image; the host only needs the amdgpu
+# kernel driver. IMAGE_VERSION, if set, is the bare Ollama version (e.g. 0.40.11);
+# the -rocm suffix is added here.
+#   COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml
+services:
+  ollama:
+    restart: unless-stopped
+    image: ollama/ollama:${IMAGE_VERSION:+${IMAGE_VERSION}-}rocm
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri:/dev/dri
+    group_add:
+      - video

apps/ollama/docker-compose.yml

@@ -1,3 +1,9 @@
+# Hardware-neutral base. On its own this runs on the CPU; layer one of the
+# hardware overrides on top via COMPOSE_FILE in .env (see README):
+#   docker-compose.nvidia.yml  NVIDIA discrete GPUs and DGX Spark (GB10)
+#   docker-compose.jetson.yml  NVIDIA Jetson Thor (nvidia runtime, instead of nvidia.yml)
+#   docker-compose.strix.yml   AMD Strix Halo / Strix Point APUs (ROCm), x86_64 only
+#   docker-compose.arc.yml     Intel Arc (Vulkan), x86_64 only
 services:
   ollama:
     image: ollama/ollama:${IMAGE_VERSION:-latest}
@@ -18,13 +24,6 @@ services:
       timeout: 10s
       retries: 3
       start_period: 30s
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: "${GPU_COUNT}"
-              capabilities: [gpu]
 networks:
   llama:
     name: llama-net