ollama, llama-cpp, vllm: per-platform tuning in .env.example

apps/llama-cpp/.env.example

@@ -25,37 +25,118 @@ COMPOSE_FILE=docker-compose.yml
 # Path to the GGUF model file inside the container
 LLAMA_ARG_MODEL=/models/model.gguf
 
-# Layers to offload to the GPU (0 = CPU only, 999 = all). Leave unset: the base
-# file runs on the CPU and the GPU files default to 999.
-# LLAMA_ARG_N_GPU_LAYERS=
-
-# Context window size in tokens
-LLAMA_ARG_CTX_SIZE=4096
-
-# Number of CPU threads to use
-LLAMA_ARG_THREADS=4
-
 # Bind host and port inside the container
 LLAMA_ARG_HOST=0.0.0.0
 LLAMA_ARG_PORT=8080
 
-# LLAMA_ARG_FLASH_ATTN=on
-# LLAMA_ARG_N_PARALLEL=1
+# ==== llama.cpp server (all backends) =========================================
+# Defaults shown. The platform sections below suggest tuned values: uncomment
+# the ones for your hardware (only one value per variable takes effect).
+#
+# GPU layers and context size are fitted to free device memory automatically
+# (--fit on). Setting either one pins it and turns fitting off for it.
+# unset = auto; 0 = CPU only; 999 = all
+# LLAMA_ARG_N_GPU_LAYERS=
+LLAMA_ARG_CTX_SIZE=4096             # 0 = the model's trained context
+# LLAMA_ARG_FIT=on
+# LLAMA_ARG_FIT_TARGET=1024         # MiB to leave free per device when fitting
+# default: all cores; set to physical cores
+# LLAMA_ARG_THREADS=
+# parallel slots (context is split between them)
+# LLAMA_ARG_N_PARALLEL=
+# LLAMA_ARG_FLASH_ATTN=auto         # on | off | auto
+# LLAMA_ARG_CACHE_TYPE_K=f16        # f16 | q8_0 | q4_0 (quantized V needs flash attention)
+# LLAMA_ARG_CACHE_TYPE_V=f16
+# LLAMA_ARG_BATCH=2048              # logical batch size
+# LLAMA_ARG_UBATCH=512              # physical batch size; larger speeds prompt processing on big GPUs
+# LLAMA_ARG_LOAD_MODE=auto          # auto | none (no mmap) | mmap | mlock | mmap+mlock | dio
+# LLAMA_ARG_JINJA=1                 # use the model's chat template (tool calling)
 
-# ==== NVIDIA (nvidia.yml, spark.yml, jetson.yml) ==============================
-# GPUs reserved for the container: all, or a number (ignored on Jetson)
+# ==== NVIDIA discrete GPUs (nvidia.yml) =======================================
+# GPUs reserved for the container: all, or a number
 # GPU_COUNT=all
 # CUDA_VISIBLE_DEVICES=0
+# LLAMA_ARG_FLASH_ATTN=on
+# LLAMA_ARG_CACHE_TYPE_K=q8_0
+# LLAMA_ARG_CACHE_TYPE_V=q8_0
+# LLAMA_ARG_CTX_SIZE=32768
+# LLAMA_ARG_N_PARALLEL=2
+# LLAMA_ARG_UBATCH=1024
+# Multi-GPU: layer (default) | row | tensor; tensor split by VRAM ratio, main GPU index
+# LLAMA_ARG_SPLIT_MODE=layer
+# LLAMA_ARG_TENSOR_SPLIT=3,1
+# LLAMA_ARG_MAIN_GPU=0
+# MoE models larger than VRAM: keep the experts of the first N layers on the CPU
+# LLAMA_ARG_N_CPU_MOE=10
+# Spill to system RAM instead of failing when VRAM runs out (slow)
+# GGML_CUDA_ENABLE_UNIFIED_MEMORY=1
+
+# --- DGX Spark (GB10, 20 cores, 128 GB unified memory; nvidia.yml + spark.yml) -
+# LLAMA_ARG_FLASH_ATTN=on
+# LLAMA_ARG_CTX_SIZE=131072
+# LLAMA_ARG_N_PARALLEL=4
+# LLAMA_ARG_BATCH=2048
+# LLAMA_ARG_UBATCH=2048
+# LLAMA_ARG_THREADS=20
+# LLAMA_ARG_LOAD_MODE=none          # much faster loads on unified memory than mmap
+# LLAMA_ARG_FIT_TARGET=24576        # leave 24 GiB for the OS and co-resident services
+
+# ==== Jetson Thor (14 cores, 128 GB unified memory; jetson.yml) ===============
+# On the host: sudo nvpmodel -m 0 && sudo jetson_clocks
+# NVIDIA_VISIBLE_DEVICES=all
+# LLAMA_ARG_FLASH_ATTN=on
+# LLAMA_ARG_CTX_SIZE=65536
+# LLAMA_ARG_N_PARALLEL=2
+# LLAMA_ARG_UBATCH=1024
+# LLAMA_ARG_THREADS=14
+# LLAMA_ARG_LOAD_MODE=none
+# LLAMA_ARG_FIT_TARGET=16384
 
 # ==== AMD Strix (strix.yml) ===================================================
+# Raise the GPU's share of system memory on the host first (see README).
 # Pick one Vulkan device if the host has more than one GPU:
 # GGML_VK_VISIBLE_DEVICES=0
-# Only for IMAGE_VERSION=server-rocm, if ROCm doesn't recognise the APU:
-# HSA_OVERRIDE_GFX_VERSION=11.5.1   # 11.5.0 for Strix Point
+# Strix Halo (Ryzen AI Max, 16 cores, up to 128 GB shared):
+# LLAMA_ARG_FLASH_ATTN=on
+# LLAMA_ARG_CTX_SIZE=65536
+# LLAMA_ARG_N_PARALLEL=2
+# LLAMA_ARG_UBATCH=1024
+# LLAMA_ARG_THREADS=16
+# LLAMA_ARG_LOAD_MODE=none          # mmap loads are often very slow on these APUs
+# Strix Point (Ryzen AI 300, 12 cores, memory shared with the desktop):
+# LLAMA_ARG_CTX_SIZE=16384
+# LLAMA_ARG_THREADS=12
+# Only with IMAGE_VERSION=server-rocm:
+# HSA_OVERRIDE_GFX_VERSION=11.5.1   # 11.5.0 for Strix Point; only if ROCm doesn't recognise the APU
+# GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 # let HIP allocate from shared system memory
+# ROCBLAS_USE_HIPBLASLT=1           # faster prompt processing on RDNA 3.5
 
 # ==== Intel Arc (arc.yml) =====================================================
+# arc.yml already sets ZES_ENABLE_SYSMAN=1 so --fit can see free VRAM.
 # Pick one card on a multi-GPU host
 # ONEAPI_DEVICE_SELECTOR=level_zero:0
+# LLAMA_ARG_SPLIT_MODE=none
+# LLAMA_ARG_MAIN_GPU=0
+# Arc B580 / A770 (12-16 GB):
+# LLAMA_ARG_FLASH_ATTN=on
+# LLAMA_ARG_CACHE_TYPE_K=q8_0
+# LLAMA_ARG_CACHE_TYPE_V=q8_0
+# LLAMA_ARG_CTX_SIZE=16384
+# LLAMA_ARG_N_PARALLEL=1
+# Only with IMAGE_VERSION=server-vulkan, if output is garbled:
+# GGML_VK_DISABLE_COOPMAT=1
+
+# ==== CPU only (docker-compose.yml alone) =====================================
+# LLAMA_ARG_THREADS=8               # physical cores, not hyperthreads
+# LLAMA_ARG_CTX_SIZE=4096
+# LLAMA_ARG_N_PARALLEL=1
+# LLAMA_ARG_BATCH=512
+# LLAMA_ARG_UBATCH=512
+# LLAMA_ARG_CACHE_TYPE_K=q8_0
+# LLAMA_ARG_CACHE_TYPE_V=q8_0
+# LLAMA_ARG_FLASH_ATTN=on
+# Multi-socket servers:
+# LLAMA_ARG_NUMA=distribute
 
 # --- Optional: GGUF models on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) ---
 # LLAMA_CPP_NFS_HOST=

apps/llama-cpp/README.md

@@ -22,7 +22,8 @@ the hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists ev
 
 Notes:
 
-- The GPU files default `LLAMA_ARG_N_GPU_LAYERS` to `999` (offload everything); set it in `.env` to override.
+- GPU layers and context size are fitted to free device memory automatically (`--fit`, on by default). Setting `LLAMA_ARG_N_GPU_LAYERS` or `LLAMA_ARG_CTX_SIZE` pins that value and turns fitting off for it.
+- `.env.example` has tuned, commented-out settings for each platform; uncomment the block for your hardware.
 - `IMAGE_VERSION` is the full image tag and overrides the file's default. Leave it unset unless you want to pin a build (`server-cuda-b6500`) or switch backend within a file: `server-cuda13` on `nvidia.yml` (driver >= 580), `server-rocm` on `strix.yml`, `server-vulkan` on `arc.yml`.
 - **Jetson Thor**: the toolkit runs in CSV mode there and rejects GPU reservations, so `jetson.yml` uses `runtime: nvidia` and replaces `nvidia.yml` rather than layering on it.
 - **Strix**: Vulkan (RADV) is the most reliable and usually fastest llama.cpp backend on these APUs. For large models on Strix Halo, raise the GPU's share of system memory on the host (BIOS UMA setting and/or the `amdttm.pages_limit` kernel parameter).
@@ -55,9 +56,9 @@ cp .env.example .env
 | Variable | Description | Default |
 |----------|-------------|---------|
 | `LLAMA_ARG_MODEL` | Path to the GGUF model file inside the container | `/models/model.gguf` |
-| `LLAMA_ARG_N_GPU_LAYERS` | Layers to offload to GPU (`0` = CPU only, `999` = all) | `999` in the GPU files |
+| `LLAMA_ARG_N_GPU_LAYERS` | Layers to offload to GPU (`0` = CPU only, `999` = all) | auto (fit to free memory) |
 | `LLAMA_ARG_CTX_SIZE` | Context window size in tokens | `4096` |
-| `LLAMA_ARG_THREADS` | CPU threads to use | `4` |
+| `LLAMA_ARG_THREADS` | CPU threads to use | all cores |
 | `LLAMA_ARG_HOST` | Bind address inside the container | `0.0.0.0` |
 | `LLAMA_ARG_PORT` | Port inside the container | `8080` |
 | `IMAGE_VERSION` | Full image tag, overrides the compose file's default (see above) | per file |

apps/llama-cpp/docker-compose.arc.yml

@@ -1,3 +1,3 @@
 # Intel Arc (Battlemage B580 / Arc Pro B-series, Alchemist A-series) on Linux,
 # x86_64 only. Uses the SYCL (oneAPI) build; IMAGE_VERSION=server-vulkan also
 # works with this file. Needs the xe (or i915) kernel driver.
@@ -9,4 +9,5 @@
     devices:
       - /dev/dri:/dev/dri
     environment:
-      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}
+      # Lets SYCL report free device memory, which the automatic --fit needs.
+      ZES_ENABLE_SYSMAN: ${ZES_ENABLE_SYSMAN:-1}

apps/llama-cpp/docker-compose.jetson.yml

@@ -1,3 +1,3 @@
 # NVIDIA Jetson Thor (JetPack 7, arm64, CUDA 13). Use instead of docker-compose.nvidia.yml,
 # not on top of it: Jetson's container toolkit runs in CSV mode, which rejects device
 # reservations (the --gpus equivalent) and only works through the nvidia runtime.
@@ -9,4 +9,3 @@
     runtime: nvidia
     environment:
       NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
-      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}

apps/llama-cpp/docker-compose.nvidia.yml

@@ -1,3 +1,3 @@
 # NVIDIA discrete GPUs on Linux (x86_64 or arm64), CUDA 12.8 (driver >= 570).
 # For driver >= 580 you can set IMAGE_VERSION=server-cuda13 instead.
 # DGX Spark: also add docker-compose.spark.yml. Jetson Thor: use docker-compose.jetson.yml.
@@ -6,8 +6,6 @@
   llama-cpp:
     restart: unless-stopped
     image: ghcr.io/ggml-org/llama.cpp:${IMAGE_VERSION:-server-cuda}
-    environment:
-      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}
     deploy:
       resources:
         reservations:

apps/llama-cpp/docker-compose.strix.yml

@@ -1,3 +1,3 @@
 # AMD Strix Halo (Ryzen AI Max, gfx1151) and Strix Point (Ryzen AI 300, gfx1150)
 # on Linux, x86_64 only. Uses the Vulkan (RADV) build, which is the fastest and
 # least fussy llama.cpp backend on these APUs; the host only needs the amdgpu
@@ -12,5 +12,3 @@
       - /dev/dri:/dev/dri
     group_add:
       - video
-    environment:
-      LLAMA_ARG_N_GPU_LAYERS: ${LLAMA_ARG_N_GPU_LAYERS:-999}

apps/ollama/.env.example

@@ -17,28 +17,94 @@ COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
 # the -rocm suffix itself, so don't include it here.
 # IMAGE_VERSION=
 
-# ==== NVIDIA (docker-compose.nvidia.yml, docker-compose.jetson.yml) ===========
-# GPUs reserved for the container: all, or a number (ignored on Jetson)
+# ==== Ollama (all backends) ===================================================
+# Defaults shown. The platform sections below suggest tuned values: uncomment
+# the ones for your hardware (only one value per variable takes effect).
+# OLLAMA_KEEP_ALIVE=5m              # how long an idle model stays loaded; -1 = forever
+# default: 4k/32k/256k depending on VRAM
+# OLLAMA_CONTEXT_LENGTH=
+# OLLAMA_NUM_PARALLEL=1             # parallel requests per model (each multiplies KV cache)
+# default: 3 per GPU, 3 on CPU
+# OLLAMA_MAX_LOADED_MODELS=
+# OLLAMA_MAX_QUEUE=512
+# OLLAMA_LOAD_TIMEOUT=5m            # raise for very large models on slow storage (NFS)
+# OLLAMA_FLASH_ATTENTION=1          # needed for a quantized KV cache
+# OLLAMA_KV_CACHE_TYPE=q8_0         # f16 | q8_0 (half the memory, ~no quality loss) | q4_0
+# OLLAMA_DEBUG=1
+
+# ==== NVIDIA discrete GPUs (docker-compose.nvidia.yml) ========================
+# GPUs reserved for the container: all, or a number
 GPU_COUNT=4
-# CUDA_VISIBLE_DEVICES=             # GPU UUIDs or indexes, comma separated
+# GPU UUIDs or indexes, comma separated
+# CUDA_VISIBLE_DEVICES=
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
+# OLLAMA_KEEP_ALIVE=30m
+# OLLAMA_NUM_PARALLEL=2
+# Multi-GPU: spread every model over all GPUs instead of packing onto the fewest
+# OLLAMA_SCHED_SPREAD=1
+# Bytes of VRAM to keep free per GPU (e.g. a display, or another service): 1 GiB
+# OLLAMA_GPU_OVERHEAD=1073741824
+
+# --- DGX Spark (GB10, 128 GB unified memory, nvidia.yml) ----------------------
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
+# OLLAMA_CONTEXT_LENGTH=65536
+# OLLAMA_NUM_PARALLEL=4
+# OLLAMA_MAX_LOADED_MODELS=3
+# OLLAMA_KEEP_ALIVE=1h
+# Leave room for the OS and co-resident services (vLLM, ComfyUI): 24 GiB
+# OLLAMA_GPU_OVERHEAD=25769803776
+
+# ==== Jetson Thor (128 GB unified memory, docker-compose.jetson.yml) ==========
+# On the host: sudo nvpmodel -m 0 && sudo jetson_clocks
+# NVIDIA_VISIBLE_DEVICES=all
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
+# OLLAMA_CONTEXT_LENGTH=32768
+# OLLAMA_NUM_PARALLEL=2
+# OLLAMA_MAX_LOADED_MODELS=2
+# OLLAMA_KEEP_ALIVE=1h
+# OLLAMA_GPU_OVERHEAD=17179869184   # 16 GiB for the OS and other services
 
 # ==== AMD Strix (docker-compose.strix.yml) ====================================
 # Strix Halo (gfx1151) and Strix Point (gfx1150) are supported by the ROCm
 # build. Only set this if Ollama falls back to the CPU on your APU:
 # HSA_OVERRIDE_GFX_VERSION=11.5.1   # 11.5.0 for Strix Point
 # ROCR_VISIBLE_DEVICES=0
+# Strix Halo (up to 128 GB shared; raise the GPU's share on the host, see README):
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
+# OLLAMA_CONTEXT_LENGTH=32768
+# OLLAMA_NUM_PARALLEL=2
+# OLLAMA_MAX_LOADED_MODELS=2
+# OLLAMA_KEEP_ALIVE=1h
+# Strix Point (smaller iGPU, memory shared with the desktop):
+# OLLAMA_CONTEXT_LENGTH=8192
+# OLLAMA_MAX_LOADED_MODELS=1
+# The Vulkan backend (the docker-compose.arc.yml line) also runs on Strix; try it
+# if ROCm misbehaves.
 
 # ==== Intel Arc (docker-compose.arc.yml) ======================================
 # Vulkan is on by default in that file; pick one card on a multi-GPU host:
 # GGML_VK_VISIBLE_DEVICES=0
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
+# Arc B580 / A770 (12-16 GB):
+# OLLAMA_CONTEXT_LENGTH=8192
+# OLLAMA_MAX_LOADED_MODELS=1
+# OLLAMA_NUM_PARALLEL=1
+# Garbled output or crashes on some driver versions: turn off cooperative matrix
+# GGML_VK_DISABLE_COOPMAT=1
 
-# ==== Ollama (all backends) ===================================================
-# OLLAMA_KEEP_ALIVE=5m
+# ==== CPU only (docker-compose.yml alone) =====================================
+# Thread count is a per-model option (num_thread in a Modelfile), not an env var.
+# OLLAMA_FLASH_ATTENTION=1
+# OLLAMA_KV_CACHE_TYPE=q8_0
 # OLLAMA_CONTEXT_LENGTH=4096
 # OLLAMA_NUM_PARALLEL=1
 # OLLAMA_MAX_LOADED_MODELS=1
-# OLLAMA_FLASH_ATTENTION=1
-# OLLAMA_KV_CACHE_TYPE=q8_0
+# OLLAMA_KEEP_ALIVE=30m             # reloading from disk is the slow part on CPU
 
 # --- Optional: models on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) ---
 # OLLAMA_NFS_HOST=

apps/ollama/README.md

@@ -26,6 +26,8 @@ hardware override with `COMPOSE_FILE` in `.env` (the `.env.example` lists every
 | Intel Arc | `docker-compose.yml:docker-compose.arc.yml` | Vulkan |
 | CPU only | `docker-compose.yml` | CPU |
 
+`.env.example` has tuned, commented-out settings for each platform; uncomment the block for your hardware.
+
 Notes:
 
 - **NVIDIA**: needs the NVIDIA Container Toolkit. `GPU_COUNT` (`all` or a number) sets how many GPUs are reserved.

apps/vllm/.env.example

@@ -19,9 +19,6 @@ COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
 # Required: Hugging Face model id, or a path to weights visible inside the container
 VLLM_MODEL=meta-llama/Llama-3.2-3B-Instruct
 
-# Optional: extra vLLM engine (CLI) flags — space-separated, same as after `vllm serve` in the docs
-# VLLM_ARGS=--max-model-len 8192 --tensor-parallel-size 1
-
 # vLLM release tag (e.g. v0.32.0), or leave unset for latest. Applies to every
 # image: vllm-openai, -cpu, -rocm and -xpu share version tags.
 # NVIDIA with driver 575-579: IMAGE_VERSION=v0.32.0-cu129
@@ -30,28 +27,84 @@ VLLM_MODEL=meta-llama/Llama-3.2-3B-Instruct
 # Hugging Face token for gated models (optional)
 # HF_TOKEN=
 
-# ==== NVIDIA (nvidia.yml, spark.yml, jetson.yml) ==============================
-# GPUs reserved for the container: all, or a number (ignored on Jetson)
+# ==== vLLM engine flags =======================================================
+# VLLM_ARGS holds extra `vllm serve` flags, space-separated. It is a single
+# variable: uncomment exactly one VLLM_ARGS line, from your platform's section
+# below or this generic one, and edit it to taste.
+# VLLM_ARGS=--max-model-len 8192 --gpu-memory-utilization 0.9
+#
+# Flags worth knowing on every platform:
+#   --max-model-len N             cap context; the biggest lever on KV cache memory
+#   --gpu-memory-utilization F    fraction of device memory vLLM may claim (default 0.9)
+#   --max-num-seqs N              concurrent sequences (default 256; lower saves memory)
+#   --kv-cache-dtype fp8          halve KV cache memory (NVIDIA, and Instinct on ROCm)
+#   --tensor-parallel-size N      split across N GPUs (match GPU_COUNT)
+#   --enforce-eager               skip CUDA/HIP graph capture: less memory, faster start, slower decode
+#   --served-model-name NAME      name clients see in /v1/models
+
+# ==== NVIDIA discrete GPUs (nvidia.yml) =======================================
+# GPUs reserved for the container: all, or a number (match --tensor-parallel-size)
 # GPU_COUNT=1
+# CUDA_VISIBLE_DEVICES=0
+# PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+# Single 24-32 GB card:
+# VLLM_ARGS=--gpu-memory-utilization 0.92 --max-model-len 32768 --kv-cache-dtype fp8 --max-num-seqs 32
+# Two cards:
+# VLLM_ARGS=--tensor-parallel-size 2 --gpu-memory-utilization 0.92 --max-model-len 32768 --kv-cache-dtype fp8
+# 8-16 GB card:
+# VLLM_ARGS=--gpu-memory-utilization 0.9 --max-model-len 8192 --max-num-seqs 8 --enforce-eager
+# Consumer cards without NVLink that hang at startup with tensor parallel:
+# NCCL_P2P_DISABLE=1
+
+# --- DGX Spark (GB10, 128 GB unified memory; nvidia.yml, optionally spark.yml) -
+# --gpu-memory-utilization is a share of the whole 128 GB; leave room for the OS
+# and co-resident services (ollama, ComfyUI).
+# VLLM_ARGS=--gpu-memory-utilization 0.7 --max-model-len 131072 --max-num-seqs 8 --kv-cache-dtype fp8 --moe-backend marlin --linear-backend cutlass
+# Disable MLA attention (DeepSeek-style models) if it fails on GB10
+# VLLM_MLA_DISABLE=1
 # Tag of the community Spark image (spark.yml only)
 # SPARK_IMAGE_VERSION=latest
-# Spark and Thor share CPU and GPU memory; leave room for the OS and other services:
-# VLLM_ARGS=--gpu-memory-utilization 0.7
+# Older vLLM builds (possibly the community image) take these instead of
+# --moe-backend / --linear-backend; current vLLM ignores them:
+# VLLM_USE_FLASHINFER_MOE_FP4=0
+# VLLM_MOE_BACKEND=marlin
+# VLLM_FP4_BACKEND=VLLM_CUTLASS
+
+# ==== Jetson Thor (128 GB unified memory; jetson.yml) =========================
+# On the host: sudo nvpmodel -m 0 && sudo jetson_clocks
+# NVIDIA_VISIBLE_DEVICES=all
+# VLLM_ARGS=--gpu-memory-utilization 0.6 --max-model-len 32768 --max-num-seqs 8 --kv-cache-dtype fp8
 
 # ==== AMD Strix (strix.yml) ===================================================
+# Raise the GPU's share of system memory on the host first (see README).
 # HIP_VISIBLE_DEVICES=0
 # Only if ROCm doesn't recognise the APU:
 # HSA_OVERRIDE_GFX_VERSION=11.5.1   # 11.5.0 for Strix Point
+# AITER kernels are MI300/MI350-only; keep them off (the default)
+# VLLM_ROCM_USE_AITER=0
+# Strix Halo (up to 128 GB shared):
+# VLLM_ARGS=--gpu-memory-utilization 0.7 --max-model-len 32768 --max-num-seqs 8
+# Strix Point (memory shared with the desktop):
+# VLLM_ARGS=--gpu-memory-utilization 0.5 --max-model-len 8192 --max-num-seqs 4 --enforce-eager
 
 # ==== Intel Arc (arc.yml) =====================================================
 # Pick one card on a multi-GPU host
 # ONEAPI_DEVICE_SELECTOR=level_zero:0
+# ZE_AFFINITY_MASK=0
+# Arc B580 (12 GB):
+# VLLM_ARGS=--dtype bfloat16 --gpu-memory-utilization 0.9 --max-model-len 8192 --max-num-seqs 8
+# Two Arc Pro B60 (24 GB each):
+# VLLM_ARGS=--dtype bfloat16 --tensor-parallel-size 2 --distributed-executor-backend mp --gpu-memory-utilization 0.9 --max-model-len 32768
 
-# ==== CPU (base file alone) ===================================================
-# GB of RAM for the KV cache
-# VLLM_CPU_KVCACHE_SPACE=8
-# Cores to pin OpenMP threads to, e.g. 0-15, or auto
+# ==== CPU only (docker-compose.yml alone) =====================================
+# GiB of RAM for the KV cache (0 = vLLM picks)
+# VLLM_CPU_KVCACHE_SPACE=16
+# Cores for the OpenMP threads: auto (per NUMA node), a list like 0-15, or nobind
 # VLLM_CPU_OMP_THREADS_BIND=auto
+# Cores kept free for the server process when binding is auto
+# VLLM_CPU_NUM_OF_RESERVED_CPU=1
+# float16 is unstable on CPU; bfloat16 is the recommended dtype
+# VLLM_ARGS=--dtype bfloat16 --max-model-len 4096 --max-num-seqs 4
 
 # --- Optional: Hugging Face cache on NFS (append :docker-compose.nfs.yml to COMPOSE_FILE) ---
 # VLLM_NFS_HOST=