diff --git a/apps/llama-cpp/.env.example b/apps/llama-cpp/.env.example index 272de12..01498b4 100644 --- a/apps/llama-cpp/.env.example +++ b/apps/llama-cpp/.env.example @@ -1,8 +1,33 @@ +# ==== hardware: pick exactly one COMPOSE_FILE line =========================== +# See README "Choosing compose files". Add :docker-compose.nfs.yml to the end of +# any line to keep models on NFS. + +# CPU only +COMPOSE_FILE=docker-compose.yml +# NVIDIA discrete GPUs, CUDA 12.8 (driver >= 570) +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml +# NVIDIA DGX Spark (GB10), CUDA 13 +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml +# NVIDIA Jetson Thor (uses the nvidia runtime instead of GPU reservations) +# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml +# AMD Strix Halo / Strix Point (Vulkan), x86_64 only +# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml +# Intel Arc (SYCL), x86_64 only +# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml + +# Full image tag. Leave unset: each compose file picks its own default +# server (CPU), server-cuda, server-cuda13, server-vulkan, server-rocm, server-intel +# Set it to pin a build, e.g. server-cuda-b6500, or to switch backend within a +# file (server-cuda13 on nvidia.yml, server-rocm on strix.yml, server-vulkan on arc.yml). +# IMAGE_VERSION= + +# ==== model =================================================================== # Path to the GGUF model file inside the container LLAMA_ARG_MODEL=/models/model.gguf -# Number of GPU layers to offload (0 = CPU only, -1 = all layers) -LLAMA_ARG_N_GPU_LAYERS=0 +# Layers to offload to the GPU (0 = CPU only, 999 = all). Leave unset: the base +# file runs on the CPU and the GPU files default to 999. +# LLAMA_ARG_N_GPU_LAYERS= # Context window size in tokens LLAMA_ARG_CTX_SIZE=4096 @@ -14,15 +39,25 @@ LLAMA_ARG_THREADS=4 LLAMA_ARG_HOST=0.0.0.0 LLAMA_ARG_PORT=8080 -# Image tag controls the hardware backend: -# server - CPU only (default) -# server-cuda - NVIDIA GPU (requires nvidia container runtime) -# server-rocm - AMD GPU (requires ROCm) -# server-vulkan - Vulkan (Intel/AMD integrated) -#IMAGE_VERSION=server +# LLAMA_ARG_FLASH_ATTN=on +# LLAMA_ARG_N_PARALLEL=1 + +# ==== NVIDIA (nvidia.yml, spark.yml, jetson.yml) ============================== +# GPUs reserved for the container: all, or a number (ignored on Jetson) +# GPU_COUNT=all +# CUDA_VISIBLE_DEVICES=0 + +# ==== AMD Strix (strix.yml) =================================================== +# Pick one Vulkan device if the host has more than one GPU: +# GGML_VK_VISIBLE_DEVICES=0 +# Only for IMAGE_VERSION=server-rocm, if ROCm doesn't recognise the APU: +# HSA_OVERRIDE_GFX_VERSION=11.5.1 # 11.5.0 for Strix Point + +# ==== Intel Arc (arc.yml) ===================================================== +# Pick one card on a multi-GPU host +# ONEAPI_DEVICE_SELECTOR=level_zero:0 -# --- Optional: GGUF models on NFS --- -# COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +# --- Optional: GGUF models on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) --- # LLAMA_CPP_NFS_HOST= # LLAMA_CPP_NFS_PATH= # NFS_MOUNT_OPTS=nfsvers=4,hard diff --git a/apps/llama-cpp/README.md b/apps/llama-cpp/README.md index 8534d7e..2d5fa53 100644 --- a/apps/llama-cpp/README.md +++ b/apps/llama-cpp/README.md @@ -6,17 +6,26 @@ on CPU or GPU and exposes them as an OpenAI API endpoint on `llama-net`, making it usable by [openwebui](../openwebui/) and [home-assistant](../home-assistant/) alongside [ollama](../ollama/). -## Hardware backend +## Choosing compose files -The Docker image tag selects the hardware backend. Set `IMAGE_VERSION` in -`.env` to choose: +`docker-compose.yml` is hardware-neutral and runs the CPU image by itself. Pick +the hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists every line): -| `IMAGE_VERSION` | Backend | -|-----------------|---------| -| `server` _(default)_ | CPU only | -| `server-cuda` | NVIDIA GPU (requires nvidia container runtime) | -| `server-rocm` | AMD GPU (requires ROCm) | -| `server-vulkan` | Vulkan — Intel/AMD integrated graphics | +| Hardware | `COMPOSE_FILE` | Image tag | +|----------|----------------|-----------| +| NVIDIA discrete GPUs (driver >= 570) | `docker-compose.yml:docker-compose.nvidia.yml` | `server-cuda` (CUDA 12.8) | +| NVIDIA DGX Spark (GB10) | `docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml` | `server-cuda13` | +| NVIDIA Jetson Thor | `docker-compose.yml:docker-compose.jetson.yml` | `server-cuda13`, `nvidia` runtime | +| AMD Strix Halo / Strix Point | `docker-compose.yml:docker-compose.strix.yml` | `server-vulkan` | +| Intel Arc | `docker-compose.yml:docker-compose.arc.yml` | `server-intel` (SYCL) | +| CPU only | `docker-compose.yml` | `server` | + +Notes: + +- The GPU files default `LLAMA_ARG_N_GPU_LAYERS` to `999` (offload everything); set it in `.env` to override. +- `IMAGE_VERSION` is the full image tag and overrides the file's default. Leave it unset unless you want to pin a build (`server-cuda-b6500`) or switch backend within a file: `server-cuda13` on `nvidia.yml` (driver >= 580), `server-rocm` on `strix.yml`, `server-vulkan` on `arc.yml`. +- **Jetson Thor**: the toolkit runs in CSV mode there and rejects GPU reservations, so `jetson.yml` uses `runtime: nvidia` and replaces `nvidia.yml` rather than layering on it. +- **Strix**: Vulkan (RADV) is the most reliable and usually fastest llama.cpp backend on these APUs. For large models on Strix Halo, raise the GPU's share of system memory on the host (BIOS UMA setting and/or the `amdttm.pages_limit` kernel parameter). ## Models @@ -46,21 +55,23 @@ cp .env.example .env | Variable | Description | Default | |----------|-------------|---------| | `LLAMA_ARG_MODEL` | Path to the GGUF model file inside the container | `/models/model.gguf` | -| `LLAMA_ARG_N_GPU_LAYERS` | Layers to offload to GPU (`0` = CPU only, `-1` = all) | `0` | +| `LLAMA_ARG_N_GPU_LAYERS` | Layers to offload to GPU (`0` = CPU only, `999` = all) | `999` in the GPU files | | `LLAMA_ARG_CTX_SIZE` | Context window size in tokens | `4096` | | `LLAMA_ARG_THREADS` | CPU threads to use | `4` | | `LLAMA_ARG_HOST` | Bind address inside the container | `0.0.0.0` | | `LLAMA_ARG_PORT` | Port inside the container | `8080` | -| `IMAGE_VERSION` | Image tag / hardware backend (see table above) | `server` | +| `IMAGE_VERSION` | Full image tag, overrides the compose file's default (see above) | per file | ## Storage on NFS (optional) Large data defaults to bind mounts under `../../lib/` in `docker-compose.yml`. To use a NAS export instead, set in **`.env`** (see **`.env.example`**): ```bash -COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml ``` +(use your hardware line, with `docker-compose.nfs.yml` last) + plus the `*_NFS_HOST`, `*_NFS_PATH`, and `NFS_MOUNT_OPTS` variables documented in `.env.example`. Compose reads `COMPOSE_FILE` from `.env`, so plain `docker compose up -d` picks up the override. Prefer **`hard`** NFS mounts, not `soft`. Docker creates named NFS volumes on first start; if the server or export path is wrong, fix `.env` and remove the project volume before retrying (`docker volume ls`, then `docker volume rm …`). diff --git a/apps/llama-cpp/docker-compose.arc.yml b/apps/llama-cpp/docker-compose.arc.yml new file mode 100644 index 0000000..44db70c --- /dev/null +++ b/apps/llama-cpp/docker-compose.arc.yml @@ -0,0 +1,12 @@ +# Intel Arc (Battlemage B580 / Arc Pro B-series, Alchemist A-series) on Linux, +# x86_64 only. Uses the SYCL (oneAPI) build; IMAGE_VERSION=server-vulkan also +# works with this file. Needs the xe (or i915) kernel driver. +# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml +services: + llama-cpp: + restart: unless-stopped + image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-intel} + devices: + - /dev/dri:/dev/dri + environment: + LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999} diff --git a/apps/llama-cpp/docker-compose.jetson.yml b/apps/llama-cpp/docker-compose.jetson.yml new file mode 100644 index 0000000..167fb7b --- /dev/null +++ b/apps/llama-cpp/docker-compose.jetson.yml @@ -0,0 +1,12 @@ +# NVIDIA Jetson Thor (JetPack 7, arm64, CUDA 13). Use instead of docker-compose.nvidia.yml, +# not on top of it: Jetson's container toolkit runs in CSV mode, which rejects device +# reservations (the --gpus equivalent) and only works through the nvidia runtime. +# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml +services: + llama-cpp: + restart: unless-stopped + image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda13} + runtime: nvidia + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} + LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999} diff --git a/apps/llama-cpp/docker-compose.nfs.yml b/apps/llama-cpp/docker-compose.nfs.yml index d73697a..3cb3c49 100644 --- a/apps/llama-cpp/docker-compose.nfs.yml +++ b/apps/llama-cpp/docker-compose.nfs.yml @@ -1,5 +1,6 @@ # Optional override: GGUF models on NFS. -# Enable: COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +# Add it last in COMPOSE_FILE, e.g. +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml services: llama-cpp: diff --git a/apps/llama-cpp/docker-compose.nvidia.yml b/apps/llama-cpp/docker-compose.nvidia.yml new file mode 100644 index 0000000..b1765ed --- /dev/null +++ b/apps/llama-cpp/docker-compose.nvidia.yml @@ -0,0 +1,17 @@ +# NVIDIA discrete GPUs on Linux (x86_64 or arm64), CUDA 12.8 (driver >= 570). +# For driver >= 580 you can set IMAGE_VERSION=server-cuda13 instead. +# DGX Spark: also add docker-compose.spark.yml. Jetson Thor: use docker-compose.jetson.yml. +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml +services: + llama-cpp: + restart: unless-stopped + image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda} + environment: + LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${GPU_COUNT:-all} + capabilities: [gpu] diff --git a/apps/llama-cpp/docker-compose.spark.yml b/apps/llama-cpp/docker-compose.spark.yml new file mode 100644 index 0000000..48f9ed4 --- /dev/null +++ b/apps/llama-cpp/docker-compose.spark.yml @@ -0,0 +1,6 @@ +# NVIDIA DGX Spark (GB10, arm64, sm_121): the CUDA 13 build. Layer on nvidia.yml: +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml +services: + llama-cpp: + restart: unless-stopped + image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda13} diff --git a/apps/llama-cpp/docker-compose.strix.yml b/apps/llama-cpp/docker-compose.strix.yml new file mode 100644 index 0000000..7d38136 --- /dev/null +++ b/apps/llama-cpp/docker-compose.strix.yml @@ -0,0 +1,16 @@ +# AMD Strix Halo (Ryzen AI Max, gfx1151) and Strix Point (Ryzen AI 300, gfx1150) +# on Linux, x86_64 only. Uses the Vulkan (RADV) build, which is the fastest and +# least fussy llama.cpp backend on these APUs; the host only needs the amdgpu +# kernel driver. IMAGE_VERSION=server-rocm also works with this file. +# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml +services: + llama-cpp: + restart: unless-stopped + image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-vulkan} + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + group_add: + - video + environment: + LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999} diff --git a/apps/llama-cpp/docker-compose.yml b/apps/llama-cpp/docker-compose.yml index 3fd29de..5536bb4 100644 --- a/apps/llama-cpp/docker-compose.yml +++ b/apps/llama-cpp/docker-compose.yml @@ -1,3 +1,10 @@ +# Hardware-neutral base. On its own this runs the CPU-only image; layer one of +# the hardware overrides on top via COMPOSE_FILE in .env (see README): +# docker-compose.nvidia.yml NVIDIA discrete GPUs, CUDA 12.8 (driver >= 570) +# docker-compose.spark.yml DGX Spark (GB10), CUDA 13, on top of nvidia.yml +# docker-compose.jetson.yml NVIDIA Jetson Thor (nvidia runtime, instead of nvidia.yml) +# docker-compose.strix.yml AMD Strix Halo / Strix Point APUs (Vulkan), x86_64 only +# docker-compose.arc.yml Intel Arc (SYCL), x86_64 only services: llama-cpp: image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server} diff --git a/apps/ollama/.env.example b/apps/ollama/.env.example index f341c54..c2e4911 100644 --- a/apps/ollama/.env.example +++ b/apps/ollama/.env.example @@ -1,8 +1,46 @@ -IMAGE_VERSION=latest +# ==== hardware: pick exactly one COMPOSE_FILE line =========================== +# See README "Choosing compose files". Add :docker-compose.nfs.yml to the end of +# any line to keep models on NFS. + +# NVIDIA discrete GPUs and DGX Spark (GB10) +COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml +# NVIDIA Jetson Thor (uses the nvidia runtime instead of GPU reservations) +# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml +# AMD Strix Halo / Strix Point (ROCm), x86_64 only +# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml +# Intel Arc (Vulkan), x86_64 only +# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml +# CPU only +# COMPOSE_FILE=docker-compose.yml + +# Ollama release (e.g. 0.40.11), or leave unset for latest. The Strix file adds +# the -rocm suffix itself, so don't include it here. +# IMAGE_VERSION= + +# ==== NVIDIA (docker-compose.nvidia.yml, docker-compose.jetson.yml) =========== +# GPUs reserved for the container: all, or a number (ignored on Jetson) GPU_COUNT=4 +# CUDA_VISIBLE_DEVICES= # GPU UUIDs or indexes, comma separated + +# ==== AMD Strix (docker-compose.strix.yml) ==================================== +# Strix Halo (gfx1151) and Strix Point (gfx1150) are supported by the ROCm +# build. Only set this if Ollama falls back to the CPU on your APU: +# HSA_OVERRIDE_GFX_VERSION=11.5.1 # 11.5.0 for Strix Point +# ROCR_VISIBLE_DEVICES=0 + +# ==== Intel Arc (docker-compose.arc.yml) ====================================== +# Vulkan is on by default in that file; pick one card on a multi-GPU host: +# GGML_VK_VISIBLE_DEVICES=0 + +# ==== Ollama (all backends) =================================================== +# OLLAMA_KEEP_ALIVE=5m +# OLLAMA_CONTEXT_LENGTH=4096 +# OLLAMA_NUM_PARALLEL=1 +# OLLAMA_MAX_LOADED_MODELS=1 +# OLLAMA_FLASH_ATTENTION=1 +# OLLAMA_KV_CACHE_TYPE=q8_0 -# --- Optional: models on NFS --- -# COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +# --- Optional: models on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) --- # OLLAMA_NFS_HOST= # OLLAMA_NFS_PATH= # NFS_MOUNT_OPTS=nfsvers=4,hard diff --git a/apps/ollama/README.md b/apps/ollama/README.md index 6835507..aef5dfb 100644 --- a/apps/ollama/README.md +++ b/apps/ollama/README.md @@ -13,14 +13,36 @@ cp .env.example .env # Edit .env with your settings ``` +## Choosing compose files + +`docker-compose.yml` is hardware-neutral and runs on the CPU by itself. Pick the +hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists every line): + +| Hardware | `COMPOSE_FILE` | Backend | +|----------|----------------|---------| +| NVIDIA discrete GPUs, DGX Spark (GB10) | `docker-compose.yml:docker-compose.nvidia.yml` | CUDA, via GPU reservation | +| NVIDIA Jetson Thor | `docker-compose.yml:docker-compose.jetson.yml` | CUDA, via the `nvidia` runtime | +| AMD Strix Halo / Strix Point | `docker-compose.yml:docker-compose.strix.yml` | ROCm (`ollama/ollama:rocm`) | +| Intel Arc | `docker-compose.yml:docker-compose.arc.yml` | Vulkan | +| CPU only | `docker-compose.yml` | CPU | + +Notes: + +- **NVIDIA**: needs the NVIDIA Container Toolkit. `GPU_COUNT` (`all` or a number) sets how many GPUs are reserved. +- **Jetson Thor**: the toolkit runs in CSV mode there and rejects GPU reservations, so `jetson.yml` uses `runtime: nvidia` and replaces `nvidia.yml` rather than layering on it. +- **Strix**: Strix Halo (gfx1151) and Strix Point (gfx1150) are in Ollama's ROCm support list. If `IMAGE_VERSION` is set it is the bare version (`0.40.11`); the file adds `-rocm`. For large models on Strix Halo, raise the GPU's share of system memory on the host (BIOS UMA setting and/or the `amdttm.pages_limit` kernel parameter). +- **Intel Arc**: Ollama has no oneAPI backend; Arc runs through the Vulkan backend bundled in the stock image. + ## Storage on NFS (optional) Large data defaults to bind mounts under `../../lib/` in `docker-compose.yml`. To use a NAS export instead, set in **`.env`** (see **`.env.example`**): ```bash -COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml ``` +(use your hardware line, with `docker-compose.nfs.yml` last) + plus the `*_NFS_HOST`, `*_NFS_PATH`, and `NFS_MOUNT_OPTS` variables documented in `.env.example`. Compose reads `COMPOSE_FILE` from `.env`, so plain `docker compose up -d` picks up the override. Prefer **`hard`** NFS mounts, not `soft`. Docker creates named NFS volumes on first start; if the server or export path is wrong, fix `.env` and remove the project volume before retrying (`docker volume ls`, then `docker volume rm …`). diff --git a/apps/ollama/docker-compose.arc.yml b/apps/ollama/docker-compose.arc.yml new file mode 100644 index 0000000..25fee2b --- /dev/null +++ b/apps/ollama/docker-compose.arc.yml @@ -0,0 +1,11 @@ +# Intel Arc (Battlemage B580 / Arc Pro B-series, Alchemist A-series) on Linux, +# x86_64 only. Ollama has no oneAPI/SYCL backend; the stock image drives Arc +# through its bundled Vulkan backend. Needs the xe (or i915) kernel driver. +# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml +services: + ollama: + restart: unless-stopped + devices: + - /dev/dri:/dev/dri + environment: + OLLAMA_VULKAN: ${OLLAMA_VULKAN:-1} diff --git a/apps/ollama/docker-compose.jetson.yml b/apps/ollama/docker-compose.jetson.yml new file mode 100644 index 0000000..367c1ef --- /dev/null +++ b/apps/ollama/docker-compose.jetson.yml @@ -0,0 +1,10 @@ +# NVIDIA Jetson Thor (JetPack 7). Use instead of docker-compose.nvidia.yml, not on top of it: +# Jetson's container toolkit runs in CSV mode, which rejects device reservations +# (the --gpus equivalent) and only works through the nvidia runtime. +# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml +services: + ollama: + restart: unless-stopped + runtime: nvidia + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} diff --git a/apps/ollama/docker-compose.nfs.yml b/apps/ollama/docker-compose.nfs.yml index 77b5699..f6971df 100644 --- a/apps/ollama/docker-compose.nfs.yml +++ b/apps/ollama/docker-compose.nfs.yml @@ -1,5 +1,6 @@ # Optional override: Ollama models on NFS. -# Enable: COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +# Add it last in COMPOSE_FILE, e.g. +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml services: ollama: diff --git a/apps/ollama/docker-compose.nvidia.yml b/apps/ollama/docker-compose.nvidia.yml new file mode 100644 index 0000000..52fa6f6 --- /dev/null +++ b/apps/ollama/docker-compose.nvidia.yml @@ -0,0 +1,14 @@ +# NVIDIA GPUs on Linux (x86_64 or arm64: discrete cards, DGX Spark GB10). +# Not for Jetson Thor; use docker-compose.jetson.yml there. +# Needs the NVIDIA Container Toolkit on the host and driver >= 550. +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml +services: + ollama: + restart: unless-stopped + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${GPU_COUNT:-all} + capabilities: [gpu] diff --git a/apps/ollama/docker-compose.strix.yml b/apps/ollama/docker-compose.strix.yml new file mode 100644 index 0000000..14bb554 --- /dev/null +++ b/apps/ollama/docker-compose.strix.yml @@ -0,0 +1,14 @@ +# AMD Strix Halo (Ryzen AI Max, gfx1151) and Strix Point (Ryzen AI 300, gfx1150) +# on Linux, x86_64 only. Uses the ROCm image; the host only needs the amdgpu +# kernel driver. IMAGE_VERSION, if set, is the bare Ollama version (e.g. 0.40.11); +# the -rocm suffix is added here. +# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml +services: + ollama: + restart: unless-stopped + image: ollama/ollama:${IMAGE_VERSION:+${IMAGE_VERSION}-}rocm + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + group_add: + - video diff --git a/apps/ollama/docker-compose.yml b/apps/ollama/docker-compose.yml index a625c21..6a241b3 100644 --- a/apps/ollama/docker-compose.yml +++ b/apps/ollama/docker-compose.yml @@ -1,3 +1,9 @@ +# Hardware-neutral base. On its own this runs on the CPU; layer one of the +# hardware overrides on top via COMPOSE_FILE in .env (see README): +# docker-compose.nvidia.yml NVIDIA discrete GPUs and DGX Spark (GB10) +# docker-compose.jetson.yml NVIDIA Jetson Thor (nvidia runtime, instead of nvidia.yml) +# docker-compose.strix.yml AMD Strix Halo / Strix Point APUs (ROCm), x86_64 only +# docker-compose.arc.yml Intel Arc (Vulkan), x86_64 only services: ollama: image: ollama/ollama:${IMAGE_VERSION:-latest} @@ -18,13 +24,6 @@ services: timeout: 10s retries: 3 start_period: 30s - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: "${GPU_COUNT}" - capabilities: [gpu] networks: llama: name: llama-net diff --git a/apps/vllm/.env.example b/apps/vllm/.env.example index ad7d386..8906249 100644 --- a/apps/vllm/.env.example +++ b/apps/vllm/.env.example @@ -1,21 +1,59 @@ +# ==== hardware: pick exactly one COMPOSE_FILE line =========================== +# See README "Choosing compose files". Add :docker-compose.nfs.yml to the end of +# any line to keep the Hugging Face cache on NFS (Intel Arc: see arc.yml). + +# NVIDIA discrete GPUs and DGX Spark, CUDA 13 (driver >= 580) +COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml +# NVIDIA DGX Spark with the community GB10 build +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml +# NVIDIA Jetson Thor (uses the nvidia runtime instead of GPU reservations) +# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml +# AMD Strix Halo / Strix Point (ROCm), x86_64 only +# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml +# Intel Arc (XPU), x86_64 only +# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml +# CPU only (x86_64 with AVX-512 recommended, or arm64); slow, mainly for testing +# COMPOSE_FILE=docker-compose.yml + +# ==== model =================================================================== # Required: Hugging Face model id, or a path to weights visible inside the container VLLM_MODEL=meta-llama/Llama-3.2-3B-Instruct # Optional: extra vLLM engine (CLI) flags — space-separated, same as after `vllm serve` in the docs # VLLM_ARGS=--max-model-len 8192 --tensor-parallel-size 1 -# vLLM image tag (CUDA); use a matching vllm version, e.g. a release tag or latest -# For AMD ROCm, use image vllm/vllm-openai-rocm in docker-compose — see README -# IMAGE_VERSION=latest +# vLLM release tag (e.g. v0.32.0), or leave unset for latest. Applies to every +# image: vllm-openai, -cpu, -rocm and -xpu share version tags. +# NVIDIA with driver 575-579: IMAGE_VERSION=v0.32.0-cu129 +# IMAGE_VERSION= + +# Hugging Face token for gated models (optional) +# HF_TOKEN= -# Number of NVIDIA GPUs to expose to the container (0 may still start the process but is unusual for vLLM) +# ==== NVIDIA (nvidia.yml, spark.yml, jetson.yml) ============================== +# GPUs reserved for the container: all, or a number (ignored on Jetson) # GPU_COUNT=1 +# Tag of the community Spark image (spark.yml only) +# SPARK_IMAGE_VERSION=latest +# Spark and Thor share CPU and GPU memory; leave room for the OS and other services: +# VLLM_ARGS=--gpu-memory-utilization 0.7 -# Hugging Face token for gated models (optional; can also be set in .env without prefix) -# HF_TOKEN= +# ==== AMD Strix (strix.yml) =================================================== +# HIP_VISIBLE_DEVICES=0 +# Only if ROCm doesn't recognise the APU: +# HSA_OVERRIDE_GFX_VERSION=11.5.1 # 11.5.0 for Strix Point + +# ==== Intel Arc (arc.yml) ===================================================== +# Pick one card on a multi-GPU host +# ONEAPI_DEVICE_SELECTOR=level_zero:0 + +# ==== CPU (base file alone) =================================================== +# GB of RAM for the KV cache +# VLLM_CPU_KVCACHE_SPACE=8 +# Cores to pin OpenMP threads to, e.g. 0-15, or auto +# VLLM_CPU_OMP_THREADS_BIND=auto -# --- Optional: Hugging Face cache on NFS --- -# COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +# --- Optional: Hugging Face cache on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) --- # VLLM_NFS_HOST= # VLLM_NFS_PATH= # NFS_MOUNT_OPTS=nfsvers=4,hard diff --git a/apps/vllm/README.md b/apps/vllm/README.md index f0a5f37..e25ac08 100644 --- a/apps/vllm/README.md +++ b/apps/vllm/README.md @@ -16,17 +16,33 @@ cp .env.example .env |----------|-------------| | `VLLM_MODEL` | **Required.** Model id (e.g. `meta-llama/Llama-3.2-3B-Instruct`) or a path to weights inside the container. | | `VLLM_ARGS` | **Optional.** Extra arguments for `vllm serve`, space-separated (e.g. `--max-model-len 8192 --tensor-parallel-size 2`). Same flags as the [vLLM engine / CLI](https://docs.vllm.ai/) documentation. | -| `IMAGE_VERSION` | Docker tag for `vllm/vllm-openai` (default in compose: `latest`). Pick a version that matches your CUDA stack. | -| `GPU_COUNT` | Number of NVIDIA GPUs to assign (default `1` in compose). Set in `.env` to match the host. | +| `IMAGE_VERSION` | vLLM release tag (e.g. `v0.32.0`, default `latest`). The CUDA, CPU, ROCm and XPU images share version tags, so it applies to whichever file you use. | +| `GPU_COUNT` | Number of NVIDIA GPUs to reserve with `nvidia.yml` (default `1`). | | `HF_TOKEN` | Optional. For gated or private Hugging Face models, set the token in `.env` (vLLM reads the standard `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` env vars). | -### AMD / ROCm +## Choosing compose files -The compose file uses `vllm/vllm-openai` (NVIDIA). For AMD GPUs, change the `image` in `docker-compose.yml` to `vllm/vllm-openai-rocm` (and a suitable tag) and follow the vLLM Docker notes for ROCm. Remove or adjust the `deploy.resources` GPU block if your runtime does not use NVIDIA device reservation. +`docker-compose.yml` is hardware-neutral and runs the CPU image by itself. Pick +the hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists every line): -### CPU-only +| Hardware | `COMPOSE_FILE` | Image | +|----------|----------------|-------| +| NVIDIA discrete GPUs, DGX Spark (GB10) | `docker-compose.yml:docker-compose.nvidia.yml` | `vllm/vllm-openai` (CUDA 13) | +| DGX Spark, community GB10 build | `docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml` | `ghcr.io/spark-arena/dgx-vllm-eugr-nightly` | +| NVIDIA Jetson Thor | `docker-compose.yml:docker-compose.jetson.yml` | `vllm/vllm-openai`, `nvidia` runtime | +| AMD Strix Halo / Strix Point | `docker-compose.yml:docker-compose.strix.yml` | `vllm/vllm-openai-rocm` | +| Intel Arc | `docker-compose.yml:docker-compose.arc.yml` | `vllm/vllm-openai-xpu` | +| CPU only | `docker-compose.yml` | `vllm/vllm-openai-cpu` | -vLLM is aimed at GPU inference. Running CPU-only is not the typical path; for CPU inference in this stack, consider [llama-cpp](../llama-cpp/) or [ollama](../ollama/). +Notes: + +- **NVIDIA**: needs the NVIDIA Container Toolkit and driver >= 580 for the CUDA 13 images. With a 575-579 driver, pin a CUDA 12.9 build, e.g. `IMAGE_VERSION=v0.32.0-cu129`. +- **DGX Spark**: the stock image runs on GB10. `spark.yml` swaps in a community build with Spark-specific kernels; its tag is `SPARK_IMAGE_VERSION`, not `IMAGE_VERSION`. +- **Spark and Thor** share CPU and GPU memory, so lower `--gpu-memory-utilization` in `VLLM_ARGS` to leave room for the OS and other services. +- **Jetson Thor**: the stock arm64 CUDA 13 image includes sm_110 kernels. The toolkit runs in CSV mode there and rejects GPU reservations, so `jetson.yml` uses `runtime: nvidia` and replaces `nvidia.yml` rather than layering on it. +- **Strix**: vLLM supports Ryzen AI Max (gfx1151) and Ryzen AI 300 (gfx1150) on ROCm >= 7.0.2, which the official ROCm image bundles. Expect fewer optimized kernels than on Instinct or Radeon cards. +- **Intel Arc**: Battlemage (B580, Arc Pro B-series) is what Intel tests; Alchemist may work but is slow. Needs Intel compute runtime >= 26.18 on the host. Upstream's example runs the container privileged; uncomment `privileged: true` in `arc.yml` if no XPU device is found. With NFS, put `docker-compose.nfs.yml` before `arc.yml` so the `/dev/dri/by-path` mount survives. +- **CPU**: works on x86_64 (AVX-512 recommended, AVX2 limited) and arm64, but is slow; it is mainly useful for testing. Size the KV cache with `VLLM_CPU_KVCACHE_SPACE` (GB). For real CPU inference in this stack, use [llama-cpp](../llama-cpp/) or [ollama](../ollama/). ## Starting and stopping @@ -57,9 +73,11 @@ curl -sS http://vllm:8000/v1/models Large data defaults to bind mounts under `../../lib/` in `docker-compose.yml`. To use a NAS export instead, set in **`.env`** (see **`.env.example`**): ```bash -COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml ``` +(use your hardware line, with `docker-compose.nfs.yml` last, except on Intel Arc) + plus the `*_NFS_HOST`, `*_NFS_PATH`, and `NFS_MOUNT_OPTS` variables documented in `.env.example`. Compose reads `COMPOSE_FILE` from `.env`, so plain `docker compose up -d` picks up the override. Prefer **`hard`** NFS mounts, not `soft`. Docker creates named NFS volumes on first start; if the server or export path is wrong, fix `.env` and remove the project volume before retrying (`docker volume ls`, then `docker volume rm …`). diff --git a/apps/vllm/docker-compose.arc.yml b/apps/vllm/docker-compose.arc.yml new file mode 100644 index 0000000..0d707e0 --- /dev/null +++ b/apps/vllm/docker-compose.arc.yml @@ -0,0 +1,16 @@ +# Intel Arc on Linux, x86_64 only, via vLLM's XPU backend. Battlemage (B580, +# Arc Pro B-series) is what Intel tests; Alchemist (A-series) may work but is slow. +# Needs the xe kernel driver and Intel compute runtime >= 26.18 on the host. +# With NFS, put docker-compose.nfs.yml before this file so the by-path mount survives: +# COMPOSE_FILE=docker-compose.yml:docker-compose.arc.yml +# COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml:docker-compose.arc.yml +services: + vllm: + restart: unless-stopped + image: vllm/vllm-openai-xpu:${IMAGE_VERSION:-latest} + devices: + - /dev/dri:/dev/dri + volumes: + - /dev/dri/by-path:/dev/dri/by-path + # Upstream's example runs privileged; uncomment if no XPU device is found. + # privileged: true diff --git a/apps/vllm/docker-compose.jetson.yml b/apps/vllm/docker-compose.jetson.yml new file mode 100644 index 0000000..63de77e --- /dev/null +++ b/apps/vllm/docker-compose.jetson.yml @@ -0,0 +1,12 @@ +# NVIDIA Jetson Thor (JetPack 7, arm64, sm_110). The stock arm64 CUDA 13 image +# includes Thor kernels. Use instead of docker-compose.nvidia.yml, not on top of it: +# Jetson's container toolkit runs in CSV mode, which rejects device reservations +# (the --gpus equivalent) and only works through the nvidia runtime. +# COMPOSE_FILE=docker-compose.yml:docker-compose.jetson.yml +services: + vllm: + restart: unless-stopped + image: vllm/vllm-openai:${IMAGE_VERSION:-latest} + runtime: nvidia + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} diff --git a/apps/vllm/docker-compose.nfs.yml b/apps/vllm/docker-compose.nfs.yml index 1ecf770..62064a9 100644 --- a/apps/vllm/docker-compose.nfs.yml +++ b/apps/vllm/docker-compose.nfs.yml @@ -1,5 +1,6 @@ # Optional override: Hugging Face model cache on NFS; vLLM cache stays local. -# Enable: COMPOSE_FILE=docker-compose.yml:docker-compose.nfs.yml +# Add it last in COMPOSE_FILE, e.g. +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.nfs.yml services: vllm: diff --git a/apps/vllm/docker-compose.nvidia.yml b/apps/vllm/docker-compose.nvidia.yml new file mode 100644 index 0000000..43d275e --- /dev/null +++ b/apps/vllm/docker-compose.nvidia.yml @@ -0,0 +1,16 @@ +# NVIDIA GPUs on Linux (x86_64 or arm64: discrete cards, DGX Spark GB10). +# Not for Jetson Thor; use docker-compose.jetson.yml there. +# Needs the NVIDIA Container Toolkit on the host and driver >= 580 for CUDA 13; +# for an older driver (>= 575) pin a CUDA 12.9 build, e.g. IMAGE_VERSION=v0.32.0-cu129. +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml +services: + vllm: + restart: unless-stopped + image: vllm/vllm-openai:${IMAGE_VERSION:-latest} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${GPU_COUNT:-1} + capabilities: [gpu] diff --git a/apps/vllm/docker-compose.spark.yml b/apps/vllm/docker-compose.spark.yml new file mode 100644 index 0000000..1ae9424 --- /dev/null +++ b/apps/vllm/docker-compose.spark.yml @@ -0,0 +1,8 @@ +# NVIDIA DGX Spark (GB10): optional community build tuned for GB10 (NVFP4, MoE +# kernels). The stock vllm/vllm-openai image from nvidia.yml also runs on Spark; +# use this when you want the Spark-specific kernels. Layer on nvidia.yml: +# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml:docker-compose.spark.yml +services: + vllm: + restart: unless-stopped + image: ghcr.io/spark-arena/dgx-vllm-eugr-nightly:${SPARK_IMAGE_VERSION:-latest} diff --git a/apps/vllm/docker-compose.strix.yml b/apps/vllm/docker-compose.strix.yml new file mode 100644 index 0000000..b9ba9fd --- /dev/null +++ b/apps/vllm/docker-compose.strix.yml @@ -0,0 +1,15 @@ +# AMD Strix Halo (Ryzen AI Max, gfx1151) and Strix Point (Ryzen AI 300, gfx1150) +# on Linux, x86_64 only. The official ROCm image ships gfx1150/gfx1151 kernels +# (ROCm >= 7.0.2); the host only needs the amdgpu kernel driver. +# COMPOSE_FILE=docker-compose.yml:docker-compose.strix.yml +services: + vllm: + restart: unless-stopped + image: vllm/vllm-openai-rocm:${IMAGE_VERSION:-latest} + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + group_add: + - video + security_opt: + - seccomp:unconfined diff --git a/apps/vllm/docker-compose.yml b/apps/vllm/docker-compose.yml index 2563751..f930307 100644 --- a/apps/vllm/docker-compose.yml +++ b/apps/vllm/docker-compose.yml @@ -1,7 +1,13 @@ +# Hardware-neutral base. On its own this runs the CPU-only image; layer one of +# the hardware overrides on top via COMPOSE_FILE in .env (see README): +# docker-compose.nvidia.yml NVIDIA discrete GPUs and DGX Spark, CUDA 13 (driver >= 580) +# docker-compose.spark.yml DGX Spark community build, on top of nvidia.yml (optional) +# docker-compose.jetson.yml NVIDIA Jetson Thor (nvidia runtime, instead of nvidia.yml) +# docker-compose.strix.yml AMD Strix Halo / Strix Point APUs (ROCm), x86_64 only +# docker-compose.arc.yml Intel Arc (XPU), x86_64 only services: vllm: - image: vllm/vllm-openai:${IMAGE_VERSION:-latest} -# image: ghcr.io/spark-arena/dgx-vllm-eugr-nightly:${IMAGE_VERSION:-latest} + image: vllm/vllm-openai-cpu:${IMAGE_VERSION:-latest} container_name: vllm hostname: vllm restart: unless-stopped @@ -13,13 +19,6 @@ services: entrypoint: ["/bin/sh", "/vllm-entrypoint.sh"] env_file: - .env - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: ${GPU_COUNT:-1} - capabilities: [gpu] ports: - 8000:8000 healthcheck: