From 051a699c00cbe6df2f69c1b8b7ca24c1ac129577 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Fri, 30 Oct 2026 13:08:44 +0100 Subject: [PATCH] [Docs] Fix outdated facts, broken links, and examples Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- .../Project/Details/Settings/constants.tsx | 2 +- mkdocs.yml | 18 +- mkdocs/.justfile | 5 +- mkdocs/assets/stylesheets/extra.css | 7 +- mkdocs/blog/posts/intel-gaudi.md | 2 +- mkdocs/docs/concepts/backends.md | 18 +- mkdocs/docs/concepts/dev-environments.md | 4 +- mkdocs/docs/concepts/events.md | 2 + mkdocs/docs/concepts/exports.md | 9 +- mkdocs/docs/concepts/fleets.md | 43 +- mkdocs/docs/concepts/presets.md | 4 +- mkdocs/docs/concepts/services.md | 115 ++-- .../docs/concepts/snippets/manage-fleets.ext | 2 +- mkdocs/docs/concepts/tasks.md | 64 ++- mkdocs/docs/concepts/volumes.md | 2 +- mkdocs/docs/examples.md | 28 +- mkdocs/docs/examples/accelerators/amd.md | 3 +- .../docs/examples/accelerators/intel/index.md | 0 mkdocs/docs/examples/accelerators/tpu.md | 2 +- mkdocs/docs/examples/clusters/aws.md | 24 +- mkdocs/docs/examples/clusters/crusoe.md | 2 +- mkdocs/docs/examples/clusters/gcp.md | 24 +- mkdocs/docs/examples/clusters/lambda.md | 2 +- .../docs/examples/clusters/nccl-rccl-tests.md | 2 +- mkdocs/docs/examples/inference/dynamo.md | 2 + mkdocs/docs/examples/inference/sglang.md | 17 +- mkdocs/docs/examples/inference/vllm.md | 2 + mkdocs/docs/examples/llms/deepseek/index.md | 0 mkdocs/docs/examples/llms/llama/index.md | 0 .../examples/misc/docker-compose/index.md | 0 mkdocs/docs/examples/models/deepseek-v4.md | 154 ------ mkdocs/docs/examples/models/qwen36.md | 168 ------ mkdocs/docs/examples/models/wan22/index.md | 0 mkdocs/docs/examples/training/axolotl.md | 14 +- mkdocs/docs/examples/training/ray-ragen.md | 2 +- mkdocs/docs/examples/training/trl.md | 2 +- mkdocs/docs/guides/migration/slurm.md | 18 +- mkdocs/docs/guides/protips.md | 522 +++++++++--------- mkdocs/docs/guides/server-deployment.md | 2 +- mkdocs/docs/guides/troubleshooting.md | 31 +- mkdocs/docs/guides/upgrade.md | 41 +- mkdocs/docs/index.md | 2 +- mkdocs/docs/installation.md | 2 + mkdocs/docs/quickstart.md | 12 +- mkdocs/docs/reference/cli/dstack/apply.md | 15 +- mkdocs/docs/reference/cli/dstack/attach.md | 8 +- mkdocs/docs/reference/env.md | 4 +- mkdocs/overrides/partials/tx-footer.html | 1 - skills/dstack/SKILL.md | 8 +- website/src/components/SiteFooter.tsx | 1 - 51 files changed, 620 insertions(+), 796 deletions(-) delete mode 100644 mkdocs/docs/examples/accelerators/intel/index.md delete mode 100644 mkdocs/docs/examples/llms/deepseek/index.md delete mode 100644 mkdocs/docs/examples/llms/llama/index.md delete mode 100644 mkdocs/docs/examples/misc/docker-compose/index.md delete mode 100644 mkdocs/docs/examples/models/deepseek-v4.md delete mode 100644 mkdocs/docs/examples/models/qwen36.md delete mode 100644 mkdocs/docs/examples/models/wan22/index.md diff --git a/README.md b/README.md index 3a9f91bae6..cff230835c 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ It streamlines development, training, and inference, and is compatible with any ## Latest news ✨ - [2026/08] [dstack 0.21.0: Pydantic v2, Gateway replicas](https://github.com/dstackai/dstack/releases/tag/0.21.0) -- [2026/07] [dstack 0.20.29: Presets — agent-driven inference optimization (experimental)](https://dstack.ai/docs/concepts/presets/) +- [2026/07] [dstack 0.20.29: Presets — agent-driven inference optimization](https://dstack.ai/docs/concepts/presets/) - [2026/07] [dstack 0.20.27: Slurm backend](https://github.com/dstackai/dstack/releases/tag/0.20.27) - [2026/05] [dstack 0.20.21: Kubernetes multiple clusters](https://github.com/dstackai/dstack/releases/tag/0.20.21) - [2026/05] [dstack 0.20.20: NVIDIA Dynamo integration](https://github.com/dstackai/dstack/releases/tag/0.20.20) @@ -107,7 +107,7 @@ AI agents like Claude, Codex, and Cursor can now create and manage fleets and su * [Dev environments](https://dstack.ai/docs/concepts/dev-environments) — Launch dev environments to be accessed by agents or from your IDE * [Tasks](https://dstack.ai/docs/concepts/tasks) — Run training, batch or other jobs across a single node or clusters * [Services](https://dstack.ai/docs/concepts/services) — Deploy model inference as secure and scalable endpoints -* [Presets](https://dstack.ai/docs/concepts/presets) — Agent-driven inference optimization (experimental) +* [Presets](https://dstack.ai/docs/concepts/presets) — Agent-driven inference optimization * [Volumes](https://dstack.ai/docs/concepts/volumes) — Managing instance and network volumes for persisting data Configuration can be defined as YAML files within your repo. diff --git a/frontend/src/pages/Project/Details/Settings/constants.tsx b/frontend/src/pages/Project/Details/Settings/constants.tsx index 1f3e6f47ae..38050e99d5 100644 --- a/frontend/src/pages/Project/Details/Settings/constants.tsx +++ b/frontend/src/pages/Project/Details/Settings/constants.tsx @@ -14,7 +14,7 @@ export const CLI_INFO = {

To learn how to install the CLI, refer to the{' '} - + installation {' '} guide. diff --git a/mkdocs.yml b/mkdocs.yml index ff014577cd..dd4d7d0f91 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,12 +107,19 @@ plugins: "docs/reference/cli/dstack/endpoint.md": "docs/reference/cli/dstack/preset.md" "docs/examples/llms/llama31.md": "docs/examples/inference/vllm.md" "docs/examples/llms/llama32.md": "docs/examples/inference/vllm.md" - "docs/examples/llms/qwen36.md": "docs/examples/models/qwen36.md" + "docs/examples/llms/qwen36.md": "docs/examples/inference/sglang.md" "examples/llms/llama31/index.md": "docs/examples/inference/vllm.md" "examples/llms/llama32/index.md": "docs/examples/inference/vllm.md" - "examples/llms/qwen36/index.md": "docs/examples/models/qwen36.md" + "examples/llms/qwen36/index.md": "docs/examples/inference/sglang.md" "docs/examples/deployment/nim/index.md": "docs/examples/inference/nim.md" "docs/examples/deployment/vllm/index.md": "docs/examples/inference/vllm.md" + "docs/examples/models/qwen36.md": "docs/examples/inference/sglang.md" + "docs/examples/models/deepseek-v4.md": "docs/examples/inference/sglang.md" + "docs/examples/models/wan22/index.md": "docs/examples.md" + "docs/examples/accelerators/intel/index.md": "docs/examples.md" + "docs/examples/llms/deepseek/index.md": "docs/examples/inference/sglang.md" + "docs/examples/llms/llama/index.md": "docs/examples/inference/vllm.md" + "docs/examples/misc/docker-compose/index.md": "docs/concepts/dev-environments.md#docker-in-docker" "backends.md": "docs/concepts/backends.md" "blog/monitoring-gpu-usage.md": "blog/posts/dstack-metrics.md" "blog/inactive-dev-environments-auto-shutdown.md": "blog/posts/inactivity-duration.md" @@ -193,8 +200,8 @@ plugins: "examples/inference/vllm.md": "docs/examples/inference/vllm.md" "examples/inference/nim.md": "docs/examples/inference/nim.md" "examples/inference/trtllm.md": "docs/examples/inference/trtllm.md" - "examples/models/deepseek-v4.md": "docs/examples/models/deepseek-v4.md" - "examples/models/qwen36.md": "docs/examples/models/qwen36.md" + "examples/models/deepseek-v4.md": "docs/examples/inference/sglang.md" + "examples/models/qwen36.md": "docs/examples/inference/sglang.md" "examples/accelerators/amd.md": "docs/examples/accelerators/amd.md" "examples/accelerators/tpu.md": "docs/examples/accelerators/tpu.md" "examples/accelerators/tenstorrent.md": "docs/examples/accelerators/tenstorrent.md" @@ -335,9 +342,6 @@ nav: - vLLM: docs/examples/inference/vllm.md - NIM: docs/examples/inference/nim.md - TensorRT-LLM: docs/examples/inference/trtllm.md - - Models: - - DeepSeek V4: docs/examples/models/deepseek-v4.md - - Qwen 3.6: docs/examples/models/qwen36.md - Accelerators: - AMD: docs/examples/accelerators/amd.md - Tenstorrent: docs/examples/accelerators/tenstorrent.md diff --git a/mkdocs/.justfile b/mkdocs/.justfile index a5d019887d..f41633cb04 100644 --- a/mkdocs/.justfile +++ b/mkdocs/.justfile @@ -6,9 +6,10 @@ default: @just --list # Preview the docs site with live-reload -mkdocs-serve: +[arg("port", long="port")] +mkdocs-serve port="8000": # --livereload works around live-reload bugs in recent mkdocs versions - uv run mkdocs serve --livereload -s + uv run mkdocs serve --livereload -s -a 127.0.0.1:{{port}} # Build the docs site to ./site mkdocs-build: diff --git a/mkdocs/assets/stylesheets/extra.css b/mkdocs/assets/stylesheets/extra.css index f93266e3b0..61ff10c02f 100644 --- a/mkdocs/assets/stylesheets/extra.css +++ b/mkdocs/assets/stylesheets/extra.css @@ -1980,10 +1980,9 @@ img.border { a[href^="http"]:not(:where( /* skip if marked with external-skip */ .external-skip, - /* exclude http:// dstack links */ - [href^="http://dstack.ai"], - /* exclude https://dstack.ai links */ - [href^="https://dstack.ai"], + /* exclude dstack.ai links, except non-docs product pages */ + [href^="http://dstack.ai"]:not([href*="dstack.ai/products"]), + [href^="https://dstack.ai"]:not([href*="dstack.ai/products"]), /* exclude md-content__button links */ .md-content__button, )):after { diff --git a/mkdocs/blog/posts/intel-gaudi.md b/mkdocs/blog/posts/intel-gaudi.md index 37b8c383b4..2e3a2d980b 100644 --- a/mkdocs/blog/posts/intel-gaudi.md +++ b/mkdocs/blog/posts/intel-gaudi.md @@ -158,7 +158,7 @@ $ dstack apply -f examples/single-node-training/trl/intel/.dstack.yml -R `dstack` will automatically create containers according to the run configuration and execute them across the fleet. -> Explore our [examples](../../docs/examples/accelerators/intel/index.md) to learn how to train and deploy large models on +> Explore our [examples](../../docs/examples.md) to learn how to train and deploy large models on > Intel Gaudi AI Accelerator. !!! info "Intel Tiber AI Cloud" diff --git a/mkdocs/docs/concepts/backends.md b/mkdocs/docs/concepts/backends.md index a7bf1019bd..20894f9def 100644 --- a/mkdocs/docs/concepts/backends.md +++ b/mkdocs/docs/concepts/backends.md @@ -90,7 +90,7 @@ There are two ways to configure AWS: using an access key or using the default cr "ec2:DeletePlacementGroup", "ec2:DeleteVolume", "ec2:DescribeAvailabilityZones", - "ec2:DescribeCapacityReservations" + "ec2:DescribeCapacityReservations", "ec2:DescribeImages", "ec2:DescribeInstances", "ec2:DescribeInstanceAttribute", @@ -314,7 +314,8 @@ There are two ways to configure Azure: using a client secret or using the defaul ```shell SUBSCRIPTION_ID=... - az ad sp create-for-rbac + DSTACK_ROLE=... + az ad sp create-for-rbac \ --name dstack-app \ --role $DSTACK_ROLE \ --scopes /subscriptions/$SUBSCRIPTION_ID \ @@ -401,6 +402,8 @@ There are two ways to configure Azure: using a client secret or using the defaul - name: main backends: - type: azure + subscription_id: 06c82ce3-28ff-4285-a146-c5e981a9d808 + tenant_id: f84a7584-88e4-4fd2-8e97-623f0a715ee1 creds: type: default regions: [westeurope] @@ -415,6 +418,8 @@ There are two ways to configure Azure: using a client secret or using the defaul - name: main backends: - type: azure + subscription_id: 06c82ce3-28ff-4285-a146-c5e981a9d808 + tenant_id: f84a7584-88e4-4fd2-8e97-623f0a715ee1 creds: type: default regions: [westeurope] @@ -433,6 +438,8 @@ There are two ways to configure Azure: using a client secret or using the defaul - name: main backends: - type: azure + subscription_id: 06c82ce3-28ff-4285-a146-c5e981a9d808 + tenant_id: f84a7584-88e4-4fd2-8e97-623f0a715ee1 creds: type: default regions: [westeurope] @@ -739,6 +746,7 @@ gcloud projects list --format="json(projectId)" - name: main backends: - type: gcp + project_id: gcp-project-id creds: type: default @@ -891,7 +899,7 @@ projects: -### Verda (formerly DataCrunch) { #verda } +### Verda { #verda } Log into your [Verda](https://console.verda.com/signin) account, click Keys in the sidebar, find `REST API Credentials` area and then click the `Generate Credentials` button. @@ -1399,7 +1407,7 @@ projects: This applies to offers shown in `dstack apply` (run plans), during provisioning, and in `dstack offer`. Unlike other backends, offers for the `kubernetes` backend always reflect the lower limit of the range. -> To learn more, see the [Lambda](../examples/clusters/lambda/#kubernetes) and [Crusoe](../examples/clusters/crusoe/#kubernetes) examples. +> To learn more, see the [Lambda](../examples/clusters/lambda.md#kubernetes) and [Crusoe](../examples/clusters/crusoe.md#kubernetes) examples. ### Slurm @@ -1648,5 +1656,3 @@ projects: ``` - -Also, the `vastai` backend supports on-demand instances only. Spot instance support coming soon. diff --git a/mkdocs/docs/concepts/dev-environments.md b/mkdocs/docs/concepts/dev-environments.md index 9a8244b20e..98e7fda216 100644 --- a/mkdocs/docs/concepts/dev-environments.md +++ b/mkdocs/docs/concepts/dev-environments.md @@ -267,13 +267,13 @@ init: -Cannot be used with `python` or `image`. Not supported on `runpod`, `vastai`, or `kubernetes`. +Cannot be used with `python` or `image`. Not supported on `daytona`, `lium`, `runpod`, `slurm`, or `vastai`. #### Privileged mode To enable privileged mode, set [`privileged`](../reference/dstack.yml/dev-environment.md#privileged) to `true`. -Not supported with `runpod`, `vastai`, and `kubernetes`. +Not supported with `daytona`, `lium`, `runpod`, `slurm`, and `vastai`. #### Private registry diff --git a/mkdocs/docs/concepts/events.md b/mkdocs/docs/concepts/events.md index d70133fcc1..fbed1a831f 100644 --- a/mkdocs/docs/concepts/events.md +++ b/mkdocs/docs/concepts/events.md @@ -26,6 +26,8 @@ In the UI, the `Events` tab is included on the run, job, fleet, and instance pag ![](https://dstack.ai/static-assets/static-assets/images/dstack-ui-events-run.png){ width=800 } +> The global `Events` page with advanced filtering is only available in [dstack Sky](https://dstack.ai/products/sky/) and [dstack Factory](https://dstack.ai/products/factory/). + ## CLI To query events via the CLI, use the `dstack event` command. diff --git a/mkdocs/docs/concepts/exports.md b/mkdocs/docs/concepts/exports.md index e198628393..d102835cee 100644 --- a/mkdocs/docs/concepts/exports.md +++ b/mkdocs/docs/concepts/exports.md @@ -8,8 +8,7 @@ description: Exporting resources across projects Exports allow making resources from one project available to other projects. When a project exports a resource, the specified importer projects can see and use it as if it were their own. -!!! info "Export support" - Exports are only supported in `dstack` Sky and `dstack` Factory. +> Exports are only supported in [dstack Sky](https://dstack.ai/products/sky/) and [dstack Factory](https://dstack.ai/products/factory/). An export is created in the exporter project and specifies the resources to export and the importer projects that will gain access to them. @@ -20,7 +19,7 @@ for running tasks, dev environments, and services. Imported resources appear wit Currently, [SSH fleets](fleets.md#ssh-fleets) and [gateways](gateways.md) can be exported. -!!! info "Required project role" +??? info "Required role" The user creating or updating an export must have the project admin role on both the exporter project and any importer project they add. Alternatively, a global admin can add any project as an importer. @@ -128,10 +127,10 @@ $ dstack export create global-export --gateway shared-gateway --global ``` -Only promoting an export to global requires the global admin role. Regular project admins can add or remove resources, remove global status, or delete the export. - +Only promoting an export to global requires the global admin role. Regular project admins can add or remove resources, remove global status, or delete the export. + ## Access imported resources From the importer project's perspective, use `dstack import list` (or simply `dstack import`) to list all imports in the project — i.e., all exports from other projects that this project has been granted access to: diff --git a/mkdocs/docs/concepts/fleets.md b/mkdocs/docs/concepts/fleets.md index 630f23b2e3..ba9700e8e5 100644 --- a/mkdocs/docs/concepts/fleets.md +++ b/mkdocs/docs/concepts/fleets.md @@ -7,7 +7,7 @@ description: Managing pools of compute instances Before submitting runs, you must create a fleet. Fleets act as both pools of instances and templates for how those instances are provisioned. -> `dstack` supports two fleet types: [backend fleets](#backend-fleet) (which are provisioned dynamically in the cloud or on Kubernetes), and [SSH fleets](#ssh-fleet) (which use existing on-prem servers). +> `dstack` supports two fleet types: [backend fleets](#backend-fleets) (which are provisioned dynamically in the cloud or on Kubernetes), and [SSH fleets](#ssh-fleets) (which use existing on-prem servers). ## Apply a configuration @@ -103,10 +103,10 @@ To create a fleet, define its configuration in a YAML file. The filename must en `dstack apply` automatically connects to on-prem servers, installs the required dependencies, and adds them to the created fleet. ??? info "Host requirements" - 1. Hosts must be Linux-based and have Docker pre-installed. + 1. Hosts must be Linux-based with systemd, have a 64-bit x86 or ARM CPU, and have Docker pre-installed. === "NVIDIA" - 2. Hosts with NVIDIA GPUs must also be pre-installed with CUDA 12.1 and + 2. Hosts with NVIDIA GPUs must also be pre-installed with NVIDIA drivers that support CUDA 13.0 and [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). === "AMD" @@ -120,7 +120,7 @@ To create a fleet, define its configuration in a YAML file. The filename must en 3. The user specified must have passwordless `sudo` access. - 4. The SSH server must be running and configured with `AllowTcpForwarding yes` in `/etc/ssh/sshd_config`. + 4. The SSH server must be running and configured with `AllowTcpForwarding yes` in `/etc/ssh/sshd_config`. The SSH key must be RSA, ECDSA, or Ed25519. 5. The firewall must allow SSH and should forbid any other connections from external networks. @@ -134,7 +134,7 @@ Backend fleets support [many options](../reference/dstack.yml/fleet.md); see som ### Cluster placement -Both [backend fleets](#backend-fleet) and [SSH fleets](#ssh-fleet) allow the `placement` property to be set to `cluster`. +Both [backend fleets](#backend-fleets) and [SSH fleets](#ssh-fleets) allow the `placement` property to be set to `cluster`. This property ensures that instances are interconnected. This is required for running [multi-node tasks](tasks.md#nodes). @@ -320,7 +320,10 @@ Backend fleets allow you to set `backends` to specify which backends are allowed Instances of a backend fleet that aren't running any jobs are `idle` and can be reused by new runs. `dstack` terminates idle instances above [`nodes.min`](#nodes) after -[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration), which defaults to 3 days. +[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration). If `idle_duration` is not set, +it defaults to 3 days for pre-provisioned instances and to 5 minutes for instances provisioned +when you submit runs. Instances of [container-based](backends.md#container-based) backends are +terminated as soon as they become idle. For `idle_duration` to have an effect, `nodes` must be a range. Set `min` to `0` and `target` to the number of instances to pre-provision: @@ -498,6 +501,34 @@ $ dstack fleet +### Update fleets + +To update a fleet, change its configuration and pass it to `dstack apply`. `dstack` will automatically detect changes and suggest an in-place update. + +

+ +```shell +$ dstack apply -f fleet.dstack.yml + +Found fleet my-fleet. Detected changes that can be updated in-place: +- Configuration properties: + - nodes + - max_price + +Update the fleet in-place? [y/n]: +``` + +
+ +If approved, `dstack` updates the fleet without re-creating it. For example, if `nodes.min` is increased, `dstack` provisions the missing instances, and if `nodes.max` is decreased, `dstack` terminates idle instances above it. + +??? info "Supported properties" + Backend fleets support in-place changes to `nodes`, `resources`, `backends`, `regions`, `availability_zones`, `instance_types`, `spot_policy`, `max_price`, `reservation`, and `tags`. Except for `nodes`, these changes only apply to instances provisioned after the update. + + SSH fleets support adding and removing `hosts`. Added hosts are provisioned, and removed hosts are terminated. + + Changes to other properties require re-creating the fleet. In this case, `dstack apply` offers to delete the fleet and create it again. + ### Delete fleets When a fleet isn't used by a run, you can delete it by passing the fleet configuration to `dstack delete`: diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index 5a0ef995da..b01c63f62b 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -7,8 +7,6 @@ description: Creating and reusing optimized model inference configurations Presets offer a toolkit that streamlines agent-based model inference optimization and a portable format to deploy the optimized inference endpoint to any cloud, Kubernetes cluster, or on-prem fleet. -> The presets feature is experimental and may change. - ??? info "Prerequisites" Before using presets, make sure you’ve [installed](../installation.md) the server and CLI, and created a [fleet](fleets.md). @@ -70,7 +68,7 @@ Create the preset dsv4-flash? [y/n]: y The command executes entirely locally and uses the locally installed `claude` or `codex` CLI along with `dstack`'s bundled skills (see [Agent](#agent)). The agent uses a `dstack` task to find the best serving configuration for the available fleet offers, then submits it as a `dstack` service for a final benchmark. -You can stop watching with `Ctrl`+`C` at any time. The agent keeps running, and `dstack preset logs -f` follows it again. Resume an interrupted creation with `dstack preset resume`: +Pressing `Ctrl`+`C` asks whether to stop the creation. If you don't stop it, the agent keeps running, and `dstack preset logs -f` follows it again. Resume an interrupted creation with `dstack preset resume`:
diff --git a/mkdocs/docs/concepts/services.md b/mkdocs/docs/concepts/services.md index 5080f9416a..b782dd8b02 100644 --- a/mkdocs/docs/concepts/services.md +++ b/mkdocs/docs/concepts/services.md @@ -5,11 +5,13 @@ description: Deploying models and web apps as endpoints # Services -Services allow you to deploy models or web apps as secure and scalable endpoints. +Services allow you to deploy models as secure and scalable inference endpoints, with support for replica groups, routers, and PD disaggregation. ??? info "Prerequisites" Before running a service, make sure you’ve [installed](../installation.md) the server and CLI, and created a [fleet](fleets.md). +> To optimize model inference with an agent, it's recommended to use [presets](presets.md). + ## Apply a configuration First, define a service configuration as a YAML file in your project folder. @@ -120,7 +122,7 @@ Provisioning... Service is published at: http://localhost:3000/proxy/services/main/qwen36/ Model Qwen/Qwen3.6-27B is published at: - http://localhost:3000/proxy/models/main/ + http://localhost:3000/proxy/services/main/qwen36/v1 ```
@@ -296,7 +298,7 @@ Below are examples for running `Qwen/Qwen3.8-Flash-Next` with SMG and `zai-org/G -[`groups`](../reference/dstack.yml/service.md#groups) and top-level [`replicas`](../reference/dstack.yml/service.md#replicas) are mutually exclusive. +[`groups`](../reference/dstack.yml/service.md#groups) and top-level [`replicas`](../reference/dstack.yml/service.md#replicas) are mutually exclusive. If `groups` is set, top-level `resources` is ignored. Set `resources` in each group. > Properties such as `regions`, `port`, `env` and some other cannot be configured per replica group. This support is coming soon. @@ -359,6 +361,7 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: --model-path $MODEL_ID \ --disaggregation-mode prefill \ --disaggregation-transfer-backend nixl \ + --host 0.0.0.0 \ --port 8000 \ --disaggregation-bootstrap-port 8998 resources: @@ -371,18 +374,13 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: --model-path $MODEL_ID \ --disaggregation-mode decode \ --disaggregation-transfer-backend nixl \ + --host 0.0.0.0 \ --port 8000 resources: gpu: H200 port: 8000 model: zai-org/GLM-4.5-Air-FP8 - - # Custom probe is required for PD disaggregation. - probes: - - type: http - url: /health - interval: 15s ``` @@ -568,12 +566,6 @@ groups: port: 30000 model: Qwen/Qwen2.5-72B-Instruct -# Custom probe is required for PD disaggregation. -probes: - - type: http - url: /health - interval: 15s - volumes: - /usr/lib64/libibverbs/libbnxt_re-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so ``` @@ -635,16 +627,18 @@ $ curl https://llama31.example.com/v1/chat/completions \ ```yaml type: service -name: llama-2-7b-service +name: qwen3-service python: 3.12 env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -712,7 +706,7 @@ $ dstack ps --verbose If multiple probes are configured for the service, their statuses are displayed in the order in which the probes appear in the configuration. -Probes are executed for each service replica while the replica is `running`. A probe execution is considered successful if the replica responds with a `2xx` status code. Probe statuses do not affect how `dstack` handles replicas, except during [rolling deployments](#rolling-deployment). +Probes are executed for each service replica while the replica is `running`. A probe execution is considered successful if the replica responds with a `2xx` status code. A replica starts receiving requests only after each of its probes succeeds [`ready_after`](../reference/dstack.yml/service.md#ready_after) times in a row. After that, probe statuses do not affect how `dstack` handles the replica, except during [rolling deployments](#rolling-deployment). ??? info "HTTP request configuration" You can configure the HTTP request method, headers, and other properties. To include secret values in probe requests, use environment variable interpolation, which is enabled for the `url`, `headers[i].value`, and `body` properties. @@ -756,6 +750,8 @@ In this case, `dstack` will use the service's `/v1/chat/completions` service. When `model` is set, `dstack` automatically configures [`probes`](#probes) to verify model health. To customize or disable this, set `probes` explicitly. +> The `model` property is optional. Services can run any kind of web application. + ### Scaling To scale a service automatically, set [`replicas`](#replicas) to a range and configure @@ -961,13 +957,13 @@ port: 9000 -Cannot be used with `python` or `image`. Not supported on `runpod`, `vastai`, or `kubernetes`. +Cannot be used with `python` or `image`. Not supported on `daytona`, `lium`, `runpod`, `slurm`, or `vastai`. #### Privileged mode To enable privileged mode, set [`privileged`](../reference/dstack.yml/dev-environment.md#privileged) to `true`. -Not supported with `runpod`, `vastai`, and `kubernetes`. +Not supported with `daytona`, `lium`, `runpod`, `slurm`, and `vastai`. #### Private registry @@ -1016,7 +1012,7 @@ Sometimes, when you run a service, you may want to mount local files. This is po ```yaml type: service -name: llama-2-7b-service +name: qwen3-service files: - .:examples # Maps the directory with `.dstack.yml` to `/examples` @@ -1026,10 +1022,12 @@ python: 3.12 env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1049,7 +1047,7 @@ The container path is optional. If not specified, it will be automatically calcu ```yaml type: service -name: llama-2-7b-service +name: qwen3-service files: - ../examples # Maps the parent directory of `.dstack.yml` to `/../examples` @@ -1059,10 +1057,12 @@ python: 3.12 env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1088,7 +1088,7 @@ Imagine you have a Git repo (clonned locally) containing an `examples` subdirect ```yaml type: service -name: llama-2-7b-service +name: qwen3-service repos: # Clones the repo from the parent directory (`examples/..`) to `` @@ -1098,10 +1098,12 @@ python: 3.12 env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1125,7 +1127,7 @@ The local path can be either relative to the configuration file or absolute. ```yaml type: service - name: llama-2-7b-service + name: qwen3-service repos: # Clones the repo in the parent directory (`examples/..`) to `/my-repo` @@ -1135,10 +1137,12 @@ The local path can be either relative to the configuration file or absolute. env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1154,7 +1158,7 @@ The local path can be either relative to the configuration file or absolute. ```yaml type: service - name: llama-2-7b-service + name: qwen3-service repos: - local_path: .. @@ -1165,10 +1169,12 @@ The local path can be either relative to the configuration file or absolute. env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1189,7 +1195,7 @@ The local path can be either relative to the configuration file or absolute. ```yaml type: service - name: llama-2-7b-service + name: qwen3-service repos: # Clone the repo to `` @@ -1199,10 +1205,12 @@ The local path can be either relative to the configuration file or absolute. env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1238,8 +1246,8 @@ env: commands: - uv pip install vllm - | - vllm serve $MODEL_ID - --max-model-len $MAX_MODEL_LEN + vllm serve $MODEL_ID \ + --max-model-len $MAX_MODEL_LEN \ --tensor-parallel-size $DSTACK_GPUS_NUM port: 8000 @@ -1342,6 +1350,9 @@ auth: false # Do not strip the path prefix strip_prefix: false +repos: + - . + env: # Configure Dash to work with a path prefix # Replace `main` with your dstack project name @@ -1407,15 +1418,17 @@ Below is an example of a service that auto-terminate if any GPU stays below 10% ```yaml type: service -name: llama-2-7b-service +name: qwen3-service python: 3.12 env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1436,15 +1449,17 @@ Specify `schedule` to start a service periodically at specific UTC times using t ```yaml type: service -name: llama-2-7b-service +name: qwen3-service python: 3.12 env: - HF_TOKEN - - MODEL=NousResearch/Llama-2-7b-chat-hf + - MODEL=Qwen/Qwen3-4B commands: - uv pip install vllm - - python -m vllm.entrypoints.openai.api_server --model $MODEL --port 8000 + - | + vllm serve $MODEL \ + --port 8000 port: 8000 resources: @@ -1518,7 +1533,7 @@ Update the run? [y/n]: If approved, `dstack` gradually updates the service replicas. To update a replica, `dstack` starts a new replica, waits for it to become `running` and for all of its [probes](#probes) to pass, then terminates the old replica. This process is repeated for each replica, one at a time. -You can track the progress of rolling deployment in both `dstack apply` or `dstack ps`. +You can track the progress of rolling deployment in both `dstack apply` and `dstack ps`. Older replicas have lower `deployment` numbers; newer ones have higher. 100% FLEET INSTANCE BACKEND INSTANCE TYPE GPU PRICE STATUS CREATED - efa-fleet 0 aws (us-west-2) p4d.24xlarge H100:8:80GB $98.32 idle 3 mins ago - 1 aws (us-west-2) p4d.24xlarge H100:8:80GB $98.32 idle 3 mins ago + efa-fleet 0 aws (us-west-2) p5.48xlarge H100:8:80GB $98.32 idle 3 mins ago + 1 aws (us-west-2) p5.48xlarge H100:8:80GB $98.32 idle 3 mins ago ``` @@ -85,6 +85,26 @@ Provisioning... instance_types: ["g6.8xlarge"] # If not specified, g6.xlarge is used (won't have EFA) ``` + +??? info "Reservations" + To use an existing [Capacity Reservation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-capacity-reservations.html) + or [Capacity Block](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-capacity-blocks.html), + specify its ID via [`reservation`](../../reference/dstack.yml/fleet.md#reservation): + + ```yaml + type: fleet + name: efa-fleet + + nodes: 2 + placement: cluster + + resources: + gpu: H100:8 + + reservation: cr-0123456789abcdef0 + ``` + + `dstack` then provisions on-demand instances of the reservation's instance type in its availability zone. ## Run NCCL tests diff --git a/mkdocs/docs/examples/clusters/crusoe.md b/mkdocs/docs/examples/clusters/crusoe.md index b393eda926..c51d10696e 100644 --- a/mkdocs/docs/examples/clusters/crusoe.md +++ b/mkdocs/docs/examples/clusters/crusoe.md @@ -276,5 +276,5 @@ $ dstack apply -f crusoe-nccl-tests.dstack.yml ## What's next 1. Learn about [dev environments](../../concepts/dev-environments.md), [tasks](../../concepts/tasks.md), [services](../../concepts/services.md) -2. Check out [backends](../../concepts/backends.md#crusoe-cloud) and [fleets](../../concepts/fleets.md#cloud-fleets) +2. Check out [backends](../../concepts/backends.md#crusoe) and [fleets](../../concepts/fleets.md#backend-fleets) 3. Check the docs on [Crusoe's networking](https://docs.crusoecloud.com/networking/infiniband/) and ["Crusoe Managed" Kubernetes](https://docs.crusoecloud.com/orchestration/cmk/index.html) diff --git a/mkdocs/docs/examples/clusters/gcp.md b/mkdocs/docs/examples/clusters/gcp.md index e88e292a07..19e64f6c8b 100644 --- a/mkdocs/docs/examples/clusters/gcp.md +++ b/mkdocs/docs/examples/clusters/gcp.md @@ -53,7 +53,7 @@ Despite hiding most of the complexity, `dstack` still requires instance-specific Ensure VPCs allow internal traffic between nodes for MPI/NCCL to function. === "A3 Mega" - A3 Edge/High require at least 4 `extra_vpcs` for data NICs. + A3 Mega requires at least 8 `extra_vpcs` for data NICs.
@@ -310,6 +310,28 @@ Once you've configured the `gcp` backend, create the fleet configuration:
+??? info "Reservations" + To use an existing [reservation](https://cloud.google.com/compute/docs/instances/reservations-overview), + specify its name via [`reservation`](../../reference/dstack.yml/fleet.md#reservation): + + ```yaml + type: fleet + name: a3mega-fleet + + placement: cluster + nodes: 2 + + instance_types: + - a3-megagpu-8g + + reservation: my-reservation + ``` + + For a [shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared) owned by another project, + specify `projects//reservations/`. + + `dstack` then provisions on-demand instances of the reservation's machine type in its zone. + Once the fleet is created, you can run multi-node tasks, in addition to dev environments, services, and regular tasks. ## Run tasks diff --git a/mkdocs/docs/examples/clusters/lambda.md b/mkdocs/docs/examples/clusters/lambda.md index f89c8ecaa6..12d3364ca0 100644 --- a/mkdocs/docs/examples/clusters/lambda.md +++ b/mkdocs/docs/examples/clusters/lambda.md @@ -8,7 +8,7 @@ description: Setting up Lambda clusters using Kubernetes or 1-Click Clusters wit `dstack` allows using Lambda clusters with fast interconnect via two ways: * [Kubernetes](#kubernetes) – If you create a Kubernetes cluster on Lambda and configure a `kubernetes` backend and create a backend fleet in `dstack`, `dstack` lets you fully use this cluster through `dstack`. -* [VMs](#vms) – If you create a 1CC cluster on Lambda and create an SSH fleet in `dstack`, `dstack` lets you fully use this cluster through `dstack`. +* [VMs](#1-click-clusters) – If you create a 1CC cluster on Lambda and create an SSH fleet in `dstack`, `dstack` lets you fully use this cluster through `dstack`. ## Kubernetes diff --git a/mkdocs/docs/examples/clusters/nccl-rccl-tests.md b/mkdocs/docs/examples/clusters/nccl-rccl-tests.md index 8d516591b0..4c1b86ff61 100644 --- a/mkdocs/docs/examples/clusters/nccl-rccl-tests.md +++ b/mkdocs/docs/examples/clusters/nccl-rccl-tests.md @@ -8,7 +8,7 @@ description: Running NCCL and RCCL tests to validate cluster network bandwidth This example shows how to run [NCCL](https://github.com/NVIDIA/nccl-tests) or [RCCL](https://github.com/ROCm/rccl-tests) tests on a cluster using [multi-node tasks](../../concepts/tasks.md#nodes). !!! info "Prerequisites" - Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#ssh-placement)). + Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#cluster-placement)). ## Running as a task diff --git a/mkdocs/docs/examples/inference/dynamo.md b/mkdocs/docs/examples/inference/dynamo.md index 998b00b47c..8add566ea5 100644 --- a/mkdocs/docs/examples/inference/dynamo.md +++ b/mkdocs/docs/examples/inference/dynamo.md @@ -156,6 +156,8 @@ curl http://127.0.0.1:3000/proxy/services/main/dynamo-pd/v1/chat/completions \ Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics are coming soon. +> The `scaling` property requires creating a [gateway](../../concepts/gateways.md). + !!! info "Cluster" PD disaggregation requires the service to run in a fleet with `placement` set to `cluster`, because the replicas require an interconnect between instances. diff --git a/mkdocs/docs/examples/inference/sglang.md b/mkdocs/docs/examples/inference/sglang.md index 8ca795a250..e37d3bef69 100644 --- a/mkdocs/docs/examples/inference/sglang.md +++ b/mkdocs/docs/examples/inference/sglang.md @@ -8,9 +8,6 @@ description: Deploying Qwen3.6-27B using SGLang on NVIDIA and AMD GPUs This example shows how to deploy `Qwen/Qwen3.6-27B` using [SGLang](https://github.com/sgl-project/sglang) and `dstack`. -> For a `DeepSeek-V4-Pro` deployment on `B200:8`, see the -[DeepSeek V4](../models/deepseek-v4.md) model page. - ## Apply a configuration Here's an example of a service that deploys @@ -201,12 +198,6 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ port: 8000 model: zai-org/GLM-4.5-Air-FP8 - - # Custom probe is required for PD disaggregation. - probes: - - type: http - url: /health - interval: 15s ``` @@ -378,12 +369,6 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ port: 30000 model: Qwen/Qwen2.5-72B-Instruct - # Custom probe is required for PD disaggregation. - probes: - - type: http - url: /health - interval: 15s - volumes: - /usr/lib64/libibverbs/libbnxt_re-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so ``` @@ -395,6 +380,8 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics are coming soon. +> The `scaling` property requires creating a [gateway](../../concepts/gateways.md). + !!! info "Cluster" PD disaggregation requires the service to run in a fleet with `placement` set to `cluster`, because the replicas require an interconnect between instances. diff --git a/mkdocs/docs/examples/inference/vllm.md b/mkdocs/docs/examples/inference/vllm.md index 08f45f82ab..83c6901038 100644 --- a/mkdocs/docs/examples/inference/vllm.md +++ b/mkdocs/docs/examples/inference/vllm.md @@ -207,6 +207,8 @@ Use the same SMG version for the router and the worker images: the worker image Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics are coming soon. +> The `scaling` property requires creating a [gateway](../../concepts/gateways.md). + !!! info "Cluster" PD disaggregation requires the service to run in a fleet with `placement` set to `cluster`, because the replicas require an interconnect between instances. diff --git a/mkdocs/docs/examples/llms/deepseek/index.md b/mkdocs/docs/examples/llms/deepseek/index.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mkdocs/docs/examples/llms/llama/index.md b/mkdocs/docs/examples/llms/llama/index.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mkdocs/docs/examples/misc/docker-compose/index.md b/mkdocs/docs/examples/misc/docker-compose/index.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mkdocs/docs/examples/models/deepseek-v4.md b/mkdocs/docs/examples/models/deepseek-v4.md deleted file mode 100644 index 1d13b8d6ae..0000000000 --- a/mkdocs/docs/examples/models/deepseek-v4.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -title: DeepSeek V4 -description: Deploying DeepSeek-V4-Pro using SGLang on NVIDIA B200:8 ---- - -# DeepSeek V4 - -This example shows how to deploy `deepseek-ai/DeepSeek-V4-Pro` as a -[service](../../concepts/services.md) using -[SGLang](https://github.com/sgl-project/sglang) and `dstack`. - -## Apply a configuration - -Save the following configuration as `deepseek-v4.dstack.yml`. - -
- -```yaml -type: service -name: deepseek-v4 - -image: lmsysorg/sglang:deepseek-v4-blackwell - -env: - - HF_TOKEN - - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256 - - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 - -commands: - - | - sglang serve \ - --trust-remote-code \ - --model-path deepseek-ai/DeepSeek-V4-Pro \ - --tp 8 \ - --dp 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.82 \ - --cuda-graph-max-bs 64 \ - --max-running-requests 256 \ - --deepep-config '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --host 0.0.0.0 \ - --port 30000 - -port: 30000 -model: deepseek-ai/DeepSeek-V4-Pro - -volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true - -resources: - gpu: B200:8 - shm_size: 32GB - disk: 2TB.. -``` - -
- -This configuration uses the single-node Blackwell `DeepSeek-V4-Pro` recipe -shape for `8 x NVIDIA B200`. - -Export your Hugging Face token and apply the configuration with -[`dstack apply`](../../reference/cli/dstack/apply.md). - -
- -```shell -$ export HF_TOKEN= -$ dstack apply -f deepseek-v4.dstack.yml -``` - -
- -If no gateway is created, the service endpoint will be available at -`/proxy/services///`. - -
- -```shell -curl http://127.0.0.1:3000/proxy/services/main/deepseek-v4/v1/chat/completions \ - -X POST \ - -H 'Authorization: Bearer <user token>' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Pro", - "messages": [ - { - "role": "user", - "content": "What is 15% of 240? Reply with just the number." - } - ], - "temperature": 0, - "max_tokens": 32 - }' -``` - -
- -## Reasoning mode - -To separate the model's reasoning into `reasoning_content`, keep -`--reasoning-parser deepseek-v4` in the server command and send -`chat_template_kwargs` in the request body. - -For raw HTTP requests, `chat_template_kwargs` and `separate_reasoning` must be -top-level JSON fields. - -
- -```shell -curl http://127.0.0.1:3000/proxy/services/main/deepseek-v4/v1/chat/completions \ - -X POST \ - -H 'Authorization: Bearer <user token>' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Pro", - "messages": [ - { - "role": "user", - "content": "Solve step by step: If 3x + 5 = 20, what is x?" - } - ], - "temperature": 0, - "max_tokens": 256, - "chat_template_kwargs": { - "thinking": true - }, - "separate_reasoning": true - }' -``` - -
- -This returns both: - -- `reasoning_content`: a separate reasoning trace -- `content`: the final user-visible answer - -## Deployment notes - -- The first startup can take several minutes while the model loads and SGLang - finishes initialization. -- The optional `/root/.cache` instance volume helps reuse the model cache on - backends that support instance volumes. - -## What's next? - -1. Read the [DeepSeek-V4-Pro model card](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro) -2. Read the [DeepSeek-V4 SGLang cookbook](https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4) -3. Browse the dedicated [SGLang](../inference/sglang.md) and [vLLM](../inference/vllm.md) examples diff --git a/mkdocs/docs/examples/models/qwen36.md b/mkdocs/docs/examples/models/qwen36.md deleted file mode 100644 index e2b2fa84db..0000000000 --- a/mkdocs/docs/examples/models/qwen36.md +++ /dev/null @@ -1,168 +0,0 @@ ---- -title: Qwen 3.6 -description: Deploying Qwen3.6-27B using SGLang on NVIDIA and AMD GPUs ---- - -# Qwen 3.6 - -This example shows how to deploy `Qwen/Qwen3.6-27B` as a -[service](../../concepts/services.md) using -[SGLang](https://github.com/sgl-project/sglang) and `dstack`. - -## Apply a configuration - -Save one of the following configurations as `qwen36.dstack.yml`. - -=== "NVIDIA" - -
- - ```yaml - type: service - name: qwen36 - - image: lmsysorg/sglang:v0.5.10.post1 - - commands: - - | - sglang serve \ - --model-path Qwen/Qwen3.6-27B \ - --host 0.0.0.0 \ - --port 30000 \ - --tp $DSTACK_GPUS_NUM \ - --reasoning-parser qwen3 \ - --mem-fraction-static 0.8 \ - --context-length 262144 - - port: 30000 - model: Qwen/Qwen3.6-27B - - volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true - - resources: - shm_size: 16GB - gpu: H100:4 - ``` - -
- -=== "AMD" - -
- - ```yaml - type: service - name: qwen36 - - image: lmsysorg/sglang:v0.5.10-rocm720-mi30x - - commands: - - | - sglang serve \ - --model-path Qwen/Qwen3.6-27B \ - --host 0.0.0.0 \ - --port 30000 \ - --tp $DSTACK_GPUS_NUM \ - --reasoning-parser qwen3 \ - --mem-fraction-static 0.8 \ - --context-length 262144 - - port: 30000 - model: Qwen/Qwen3.6-27B - - volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true - - resources: - cpu: 52.. - memory: 896GB.. - shm_size: 16GB - disk: 450GB.. - gpu: MI300X:4 - ``` - -
- -The NVIDIA and AMD configurations above use pinned SGLang images and the same -straightforward 4-GPU layout used across the Qwen 3.6 docs and examples. - -Apply the configuration with -[`dstack apply`](../../reference/cli/dstack/apply.md). - -
- -```shell -$ dstack apply -f qwen36.dstack.yml -``` - -
- -If no gateway is created, the service endpoint will be available at -`/proxy/services///`. - -
- -```shell -curl http://127.0.0.1:3000/proxy/services/main/qwen36/v1/chat/completions \ - -X POST \ - -H 'Authorization: Bearer <user token>' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "Qwen/Qwen3.6-27B", - "messages": [ - { - "role": "user", - "content": "A bat and a ball cost $1.10 total. The bat costs $1.00 more than the ball. How much does the ball cost? Answer with just the dollar amount." - } - ], - "max_tokens": 1024 - }' -``` - -
- -## Thinking mode - -Qwen3.6 uses thinking mode by default. With SGLang, the reasoning stream is -returned separately as `reasoning_content`. - -To disable thinking, pass `chat_template_kwargs` in the request body. - -
- -```shell -curl http://127.0.0.1:3000/proxy/services/main/qwen36/v1/chat/completions \ - -X POST \ - -H 'Authorization: Bearer <user token>' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "Qwen/Qwen3.6-27B", - "messages": [ - { - "role": "user", - "content": "Summarize the benefits of container images in one sentence." - } - ], - "max_tokens": 256, - "chat_template_kwargs": { - "enable_thinking": false - } - }' -``` - -
- -## What's next? - -1. Read the [Qwen/Qwen3.6-27B model card](https://huggingface.co/Qwen/Qwen3.6-27B) -2. Read the [Qwen 3.6 SGLang cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.6) -3. Read the [Qwen 3.5 & 3.6 vLLM recipe](https://docs.vllm.ai/projects/recipes/en/latest/Qwen/Qwen3.5.html) -4. Browse the dedicated [SGLang](../inference/sglang.md) - and [vLLM](../inference/vllm.md) examples -5. Check the [AMD](../accelerators/amd.md) example for - more AMD deployment and training configurations diff --git a/mkdocs/docs/examples/models/wan22/index.md b/mkdocs/docs/examples/models/wan22/index.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mkdocs/docs/examples/training/axolotl.md b/mkdocs/docs/examples/training/axolotl.md index 5dd2cae0fb..e68804cf3e 100644 --- a/mkdocs/docs/examples/training/axolotl.md +++ b/mkdocs/docs/examples/training/axolotl.md @@ -84,7 +84,7 @@ Provisioning... ## Distributed training !!! info "Prerequisites" - Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#ssh-placement)). + Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#cluster-placement)). This section walks through running distributed fine-tuning of `Llama-3.1-70B` with QLoRA and FSDP across multiple nodes. @@ -125,16 +125,16 @@ commands: - | accelerate launch \ --config_file=fsdp1.yaml \ - -m axolotl.cli.train qlora-fsdp-70b.yaml \ - --hub-model-id $HUB_MODEL_ID \ - --output-dir /checkpoints/qlora-llama3-70b \ - --wandb-project $WANDB_PROJECT \ - --wandb-name $DSTACK_RUN_NAME \ --main_process_ip=$DSTACK_MASTER_NODE_IP \ --main_process_port=8008 \ --machine_rank=$DSTACK_NODE_RANK \ --num_processes=$DSTACK_GPUS_NUM \ - --num_machines=$DSTACK_NODES_NUM + --num_machines=$DSTACK_NODES_NUM \ + -m axolotl.cli.train qlora-fsdp-70b.yaml \ + --hub-model-id $HUB_MODEL_ID \ + --output-dir /checkpoints/qlora-llama3-70b \ + --wandb-project $WANDB_PROJECT \ + --wandb-name $DSTACK_RUN_NAME resources: gpu: 80GB:8 diff --git a/mkdocs/docs/examples/training/ray-ragen.md b/mkdocs/docs/examples/training/ray-ragen.md index 9df779e1c9..a2b304fdc2 100644 --- a/mkdocs/docs/examples/training/ray-ragen.md +++ b/mkdocs/docs/examples/training/ray-ragen.md @@ -11,7 +11,7 @@ to fine-tune an agent on multiple nodes. Under the hood `RAGEN` uses [verl](https://github.com/volcengine/verl) for Reinforcement Learning and [Ray](https://docs.ray.io/en/latest/) for distributed training. !!! info "Prerequisites" - Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#ssh-placement)). + Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#cluster-placement)). ## Run a Ray cluster diff --git a/mkdocs/docs/examples/training/trl.md b/mkdocs/docs/examples/training/trl.md index 11a819a1ad..16ab4376fb 100644 --- a/mkdocs/docs/examples/training/trl.md +++ b/mkdocs/docs/examples/training/trl.md @@ -110,7 +110,7 @@ Provisioning... ## Distributed training !!! info "Prerequisites" - Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#ssh-placement)). + Before running a multi-node task, make sure to create a fleet with `placement` set to `cluster` (can be a [managed fleet](../../concepts/fleets.md#cluster-placement) or an [SSH fleet](../../concepts/fleets.md#cluster-placement)). ### Define a configuration diff --git a/mkdocs/docs/guides/migration/slurm.md b/mkdocs/docs/guides/migration/slurm.md index 9a11770456..6c9ce2aac1 100644 --- a/mkdocs/docs/guides/migration/slurm.md +++ b/mkdocs/docs/guides/migration/slurm.md @@ -348,9 +348,9 @@ Slurm explicitly controls both `nodes` and processes/tasks. --epochs=10 resources: - gpu: A100:80GB:8 - memory: 200GB.. - shm_size: 24GB + gpu: A100:80GB:8 + memory: 200GB.. + shm_size: 24GB max_duration: 24h ``` @@ -385,11 +385,11 @@ Slurm explicitly controls both `nodes` and processes/tasks. fi resources: - gpu: nvidia:1..8 - shm_size: 16GB + gpu: nvidia:1..8 + shm_size: 16GB ``` - If `startup_order` and `stop_criteria` are not configured (as in the PyTorch DDP example above), the master worker starts first and waits until all workers terminate. For MPI workloads, we need to change this. + If `startup_order` and `stop_criteria` are not configured (as in the PyTorch DDP example above), the master and worker nodes start in any order, and the run finishes when all nodes are done. For MPI workloads, we need to change this. #### Nodes and processes comparison @@ -601,7 +601,7 @@ schedule: #### Monitoring commands === "Queue status" - The `dstack ps` command displays runs and jobs sorted by priority, reflecting the order in which they will be scheduled. + The `dstack ps` command displays runs and their jobs, with the most recently submitted runs first.
@@ -1520,7 +1520,7 @@ Both systems support fault tolerance for long-running training jobs that may be | | Slurm | dstack | |---|-------|--------| | **Retry** | `--requeue` flag requeues jobs on node failure (hardware crash) or preemption, not application failures (software crashes); all nodes requeued together (all-or-nothing) | `retry` property with `on_events` (`error`, `interruption`) and `duration`; all jobs stopped and run resubmitted if any job fails (all-or-nothing) | -| **Graceful stop** | Grace period with `SIGTERM` before `SIGKILL`; `--signal` sends signal before time limit (e.g., `--signal=B:USR1@300`) | Not supported | +| **Graceful stop** | Grace period with `SIGTERM` before `SIGKILL`; `--signal` sends signal before time limit (e.g., `--signal=B:USR1@300`) | `dstack stop` sends `SIGINT`, then `SIGHUP` after 5 seconds and `SIGKILL` after 10 seconds | | **Checkpointing** | Application-based; save to shared filesystem | Application-based; save to persistent volumes | | **Instance health** | `HealthCheckProgram` in `slurm.conf` runs custom scripts (DCGM/RVS); non-zero exit drains node (excludes from new scheduling, running jobs continue) | Automatic GPU health monitoring via DCGM; unhealthy instances excluded from scheduling | @@ -1845,7 +1845,7 @@ fi ### dstack -`dstack` does not support heterogeneous jobs natively. Use separate runs with [workflow orchestration tools (Prefect, Airflow)](#dstack-workflow-orchestration) or submit multiple runs programmatically to coordinate components with different resource requirements. +`dstack` supports heterogeneous jobs via task [node groups](../../concepts/tasks.md#node-groups). Each group has its own `nodes` count, `resources`, `commands`, and `ports`. ## What's next? diff --git a/mkdocs/docs/guides/protips.md b/mkdocs/docs/guides/protips.md index 4a80ff6d20..fafa9c539c 100644 --- a/mkdocs/docs/guides/protips.md +++ b/mkdocs/docs/guides/protips.md @@ -7,35 +7,244 @@ description: Tips and tricks for using dstack efficiently Below are tips and tricks to use `dstack` more efficiently. -## Dev environments +## Agents -Before running a task or service, it's recommended that you first start with a dev environment. Dev environments -allow you to run commands interactively. +It's highly recommended to use `dstack` with AI agents like Claude, Codex, and Cursor. +To help them use the CLI and edit configuration files, install [`dstack` skills](../installation.md#skills): -Once the commands work, go ahead and run them as a task or a service. +
-??? info "Notebooks" - **VS Code** - - When you access a dev environment using your desktop VS Code, it allows you to work with Jupyter notebooks via its - pre-configured and easy-to-use extension. +```shell +$ npx skills add dstackai/dstack +``` - **JupyterLab** +
- If you prefer to use JupyterLab, you can run it as a task: +## Fleets + +By default, `dstack` picks a suitable fleet for the run. To use a specific fleet, pass `--fleet` to `dstack apply`, +or set [`fleets`](../reference/dstack.yml/dev-environment.md#_fleets) in the run configuration. + +
+ +```shell +$ dstack apply -f .dstack.yml --fleet my-fleet +``` + +
+ +> To see the offers available through a fleet, use `dstack offer --fleet my-fleet`. + +!!! info "Clusters" + To run multi-node tasks, use a fleet with [`placement`](../concepts/fleets.md#cluster-placement) set to `cluster`. + To check the interconnect, run [NCCL/RCCL tests](../examples/clusters/nccl-rccl-tests.md). For cluster setup on specific clouds, + see the [AWS](../examples/clusters/aws.md), [GCP](../examples/clusters/gcp.md), [Lambda](../examples/clusters/lambda.md), + [Crusoe](../examples/clusters/crusoe.md), and [Nebius](../examples/clusters/nebius.md) guides. + +??? info "Instances" + Use `instances` to restrict a run to particular existing fleet instances. You can specify + instance names, hostnames or IP addresses, or a fleet name with an instance number. + For the `fleet` form, use `/` when the fleet belongs to another project: + +
```yaml - type: task - - commands: - - pip install jupyterlab - - jupyter lab --allow-root - - ports: - - 8888 - + type: dev-environment + name: vscode + ide: vscode + + instances: + - name: my-fleet-3 + - hostname: 203.0.113.10 + - fleet: my-fleet + instance: 3 + ``` + +
+ + ??? info "Short syntax" + + To match by instance name, you can use the string shorthand: + + ```yaml + instances: + - my-fleet-3 + ``` + + When `instances` is set, the run is only placed on a matching existing instance and + `dstack` never provisions new instances. If no matching instance is available, the run + fails with a no-capacity error (use [`retry`](../reference/dstack.yml/dev-environment.md#retry) + to wait for a targeted instance to free up). + +!!! info "Idle duration" + If the run is submitted to a fleet with `nodes` set to a range and a new instance is provisioned, the shorter of the fleet's and run's `idle_duration` is used. + If the run reuses an existing fleet instance, only the fleet's + [`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) applies. + + If an instance remains `idle`, it is automatically terminated after `idle_duration`, + unless the fleet is already at `nodes.min`. + + > Not applied for [container-based](../concepts/backends.md#container-based) backends. + +## Working directory + +By default, runs use the working directory set in the Docker image. With the [default image](../concepts/dev-environments.md#default-image), +it's `/dstack/run`. With a custom image that doesn't set one, it's `/`. + +This is also where `dstack` clones [`repos`](../concepts/dev-environments.md#repos) by default and mounts +[`files`](../concepts/dev-environments.md#files) with relative paths. In commands, you can reference it via `DSTACK_WORKING_DIR`. + +If you'd like to use a different directory, set [`working_dir`](../concepts/dev-environments.md#working-directory) +to an absolute path (`~` is supported). + +## Files and repos + +To mount local files or directories into the run's container, use [`files`](../concepts/dev-environments.md#files). + +To clone a Git repo into the run's [working directory](../concepts/dev-environments.md#working-directory), use +[`repos`](../concepts/dev-environments.md#repos). It accepts the path to a local repo, in which case `dstack` also +applies your local changes, or the URL of a remote repo. + +> Each `files` entry and the local changes of a repo are limited to 2MB. To exclude files, use `.gitignore` or `.dstackignore`. +> To increase the limit, set [`DSTACK_SERVER_CODE_UPLOAD_LIMIT`](../reference/env.md#DSTACK_SERVER_CODE_UPLOAD_LIMIT). + +## Volumes + +To persist data across runs, it is recommended to use volumes. +`dstack` supports two types of volumes: [network](../concepts/volumes.md#network-volumes) +(for persisting data even if the instance is interrupted) +and [instance](../concepts/volumes.md#instance-volumes) (useful for persisting cached data across runs while the instance remains active). + +> If you use [SSH fleets](../concepts/fleets.md#ssh-fleets), you can mount network storage (e.g., NFS or SMB) to the hosts and access it in runs via instance volumes. + +## Environment variables + +If a configuration requires an environment variable that you don't want to hardcode in the YAML, you can define it +without assigning a value: + +
+ +```yaml +type: dev-environment +name: vscode + +python: 3.12 + +env: + - HF_TOKEN +ide: vscode +``` + +
+ +Then, you can pass the environment variable either via the shell: + +
+ +```shell +$ HF_TOKEN=... +$ dstack apply -f .dstack.yml +``` + +
+ +Or via the `-e` option of the `dstack apply` command: + +
+ +```shell +$ dstack apply -e HF_TOKEN=... -f .dstack.yml +``` + +
+ +??? info ".envrc" + A better way to configure environment variables not hardcoded in YAML is by specifying them in a `.envrc` file: + +
+ + ```shell + export HF_TOKEN=... ``` +
+ + If you install [`direnv`](https://direnv.net/), + it will automatically apply the environment variables from the `.envrc` file to the `dstack apply` command. + + Remember to add `.envrc` to `.gitignore` to avoid committing it to the repo. + +[//]: # (## Profiles) +[//]: # () +[//]: # (If you don't want to specify the same parameters for each configuration, you can define them once via [profiles](../reference/profiles.yml.md)) +[//]: # (and reuse them across configurations.) +[//]: # () +[//]: # (This can be handy, for example, for configuring parameters such as `max_duration`, `max_price`, `termination_idle_time`,) +[//]: # (`regions`, etc.) +[//]: # () +[//]: # (Set `default` to `true` in your profile, and it will be applied automatically to any run.) + +## Retry policy + +By default, if `dstack` can't find available capacity, the run will fail. + +If you'd like `dstack` to automatically retry, configure the +[retry](../reference/dstack.yml/task.md#retry) property accordingly: + + + +
+ +```yaml +type: task +name: trl-sft + +python: 3.12 + +env: + - MODEL=Qwen/Qwen2.5-0.5B + - DATASET=stanfordnlp/imdb + +commands: + - uv pip install trl + - | + trl sft \ + --model_name_or_path $MODEL --dataset_name $DATASET \ + --num_processes $DSTACK_GPUS_PER_NODE + +retry: + on_events: [no-capacity] + # Retry for up to 1 hour + duration: 1h +``` + +
+ +## Utilization policy + +If you want your run to automatically terminate if any of GPUs are underutilized, you can specify `utilization_policy`. + +Below is an example of a dev environment that auto-terminate if any GPU stays below 10% utilization for 1 hour. + +
+ +```yaml +type: dev-environment +name: my-dev + +python: 3.12 +ide: cursor + +resources: + gpu: H100:8 + +utilization_policy: + min_gpu_utilization: 10 + time_window: 1h +``` + +
+ ## Tasks Tasks can be used not only for batch jobs but also for web applications. @@ -91,30 +300,34 @@ This allows you to access the remote `8501` port on `localhost:8501` while the C [Services](../concepts/services.md) provide external access, `https`, replicas with autoscaling, OpenAI-compatible endpoint and other service features. If you don't need them, you can use [tasks](../concepts/tasks.md) for running apps. -## Utilization policy - -If you want your run to automatically terminate if any of GPUs are underutilized, you can specify `utilization_policy`. - -Below is an example of a dev environment that auto-terminate if any GPU stays below 10% utilization for 1 hour. +## Dev environments -
+Before running a task or service, it's recommended that you first start with a dev environment. Dev environments +allow you to run commands interactively. -```yaml -type: dev-environment -name: my-dev +Once the commands work, go ahead and run them as a task or a service. -python: 3.12 -ide: cursor +??? info "Notebooks" + **VS Code** + + When you access a dev environment using your desktop VS Code, it allows you to work with Jupyter notebooks via its + pre-configured and easy-to-use extension. -resources: - gpu: H100:8 + **JupyterLab** -utilization_policy: - min_gpu_utilization: 10 - time_window: 1h -``` + If you prefer to use JupyterLab, you can run it as a task: -
+ ```yaml + type: task + + commands: + - pip install jupyterlab + - jupyter lab --allow-root + + ports: + - 8888 + + ``` ## Docker in Docker @@ -183,7 +396,7 @@ Set `docker` to `true` to enable the `docker` CLI in your dev environment, e.g., ide: vscode volumes: - - name: /docker-volume + - instance_path: /docker-volume path: /var/lib/docker optional: true ``` @@ -208,178 +421,6 @@ commands: > Besides inspecting runs, you can submit new runs with `dstack apply` and attach to them with `dstack attach`. -## Fleets - -### Creation policy - -By default, when you run `dstack apply` with a dev environment, task, or service, -if no `idle` instances from the available fleets meet the requirements, `dstack` provisions a new instance using configured backends. - -To ensure `dstack apply` doesn't provision a new instance but reuses an existing one, -pass `-R` (or `--reuse`) to `dstack apply`. - -
- -```shell -$ dstack apply -R -f examples/.dstack.yml -``` - -
- -Or, set [`creation_policy`](../reference/dstack.yml/dev-environment.md#creation_policy) to `reuse` in the run configuration. - -### Instances - -Use `instances` to restrict a run to particular existing fleet instances. You can specify -instance names, hostnames or IP addresses, or a fleet name with an instance number. -For the `fleet` form, use `/` when the fleet belongs to another project: - -
- -```yaml -type: dev-environment -name: vscode -ide: vscode - -instances: - - name: my-fleet-3 - - hostname: 203.0.113.10 - - fleet: my-fleet - instance: 3 -``` - -
- -??? info "Short syntax" - - To match by instance name, you can use the string shorthand: - - ```yaml - instances: - - my-fleet-3 - ``` - -When `instances` is set, the run is only placed on a matching existing instance and -`dstack` never provisions new instances. If no matching instance is available, the run -fails with a no-capacity error (use [`retry`](../reference/dstack.yml/dev-environment.md#retry) -to wait for a targeted instance to free up). - -### Idle duration - -If the run is submitted to a fleet with `nodes` set to a range and a new instance is provisioned, the shorter of the fleet's and run's `idle_duration` is used. -If the run reuses an existing fleet instance, only the fleet's -[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) applies. - -If an instance remains `idle`, it is automatically terminated after `idle_duration`, -unless the fleet is already at `nodes.min`. - -> Not applied for container-based backends (Kubernetes, Vast.ai, Runpod). - -## Volumes - -To persist data across runs, it is recommended to use volumes. -`dstack` supports two types of volumes: [network](../concepts/volumes.md#network-volumes) -(for persisting data even if the instance is interrupted) -and [instance](../concepts/volumes.md#instance-volumes) (useful for persisting cached data across runs while the instance remains active). - -> If you use [SSH fleets](../concepts/fleets.md#ssh-fleets), you can mount network storage (e.g., NFS or SMB) to the hosts and access it in runs via instance volumes. - -## Environment variables - -If a configuration requires an environment variable that you don't want to hardcode in the YAML, you can define it -without assigning a value: - -
- -```yaml -type: dev-environment -name: vscode - -python: 3.12 - -env: - - HF_TOKEN -ide: vscode -``` - -
- -Then, you can pass the environment variable either via the shell: - -
- -```shell -$ HF_TOKEN=... -$ dstack apply -f .dstack.yml -``` - -
- -Or via the `-e` option of the `dstack apply` command: - -
- -```shell -$ dstack apply -e HF_TOKEN=... -f .dstack.yml -``` - -
- -??? info ".envrc" - A better way to configure environment variables not hardcoded in YAML is by specifying them in a `.envrc` file: - -
- - ```shell - export HF_TOKEN=... - ``` - -
- - If you install [`direnv`](https://direnv.net/), - it will automatically apply the environment variables from the `.envrc` file to the `dstack apply` command. - - Remember to add `.envrc` to `.gitignore` to avoid committing it to the repo. - -[//]: # (## Profiles) -[//]: # () -[//]: # (If you don't want to specify the same parameters for each configuration, you can define them once via [profiles](../reference/profiles.yml.md)) -[//]: # (and reuse them across configurations.) -[//]: # () -[//]: # (This can be handy, for example, for configuring parameters such as `max_duration`, `max_price`, `termination_idle_time`,) -[//]: # (`regions`, etc.) -[//]: # () -[//]: # (Set `default` to `true` in your profile, and it will be applied automatically to any run.) - -## Retry policy - -By default, if `dstack` can't find available capacity, the run will fail. - -If you'd like `dstack` to automatically retry, configure the -[retry](../reference/dstack.yml/task.md#retry) property accordingly: - - - -
- -```yaml -type: task -name: train - -python: 3.12 - -commands: - - uv pip install -r fine-tuning/qlora/requirements.txt - - python fine-tuning/qlora/train.py - -retry: - on_events: [no-capacity] - # Retry for up to 1 hour - duration: 1h -``` - -
- ## Profiles Sometimes, you may want to reuse parameters across runs or set defaults so you don’t have to repeat them in every configuration. You can do this by defining a profile. @@ -551,36 +592,11 @@ To inspect offers available through a specific fleet, pass `--fleet NAME`. The `offer` command allows you to filter and group offers with various [advanced options](../reference/cli/dstack/offer.md#usage). - ## Metrics `dstack` tracks essential metrics accessible via the CLI and UI. To access advanced metrics like DCGM, configure the server to export metrics to Prometheus. See [Metrics](../concepts/metrics.md) for details. -## Pull progress - -When using a VM-based backend or an SSH instance, you can see the Docker image pull progress in the CLI while the job is in the `pulling` state. - -
- -```shell -$ dstack apply -y - -Launching orange-yak-1... - NAME BACKEND STATUS SUBMITTED - orange-yak-1 aws (eu-west-3) pulling 1.19/3.44/4.76GB 3 mins ago -``` - -
- -The three slash-separated indicators represent: - -- Image data already extracted. -- Image data already downloaded. -- Total target image size. - -The exact total size is not always known initially, in which case a lower estimate is displayed with the `≥` sign. All values exclude any image layers already cached on the host prior to the pull. - -## Service quotas +## Cloud quotas If you're using your own AWS, GCP, Azure, or OCI accounts, before you can use GPUs or spot instances, you have to request the corresponding service quotas for each type of instance in each region. @@ -589,29 +605,28 @@ corresponding service quotas for each type of instance in each region. Check this [guide ](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-resource-limits.html) on EC2 service quotas. The relevant service quotas include: - - `Running On-Demand P instances` (on-demand V100, A100 80GB x8) - - `All P4, P3 and P2 Spot Instance Requests` (spot V100, A100 80GB x8) + - `Running On-Demand P instances` (on-demand V100, A100 80GB x8, H100) + - `All P Spot Instance Requests` (spot V100, A100 80GB x8, H100) - `Running On-Demand G and VT instances` (on-demand T4, A10G, L4) - `All G and VT Spot Instance Requests` (spot T4, A10G, L4) - - `Running Dedicated p5 Hosts` (on-demand H100) - - `All P5 Spot Instance Requests` (spot H100) ??? info "GCP" Check this [guide ](https://cloud.google.com/compute/resource-usage) on Compute Engine service quotas. The relevant service quotas include: - `NVIDIA V100 GPUs` (on-demand V100) - - `Preemtible V100 GPUs` (spot V100) + - `Preemptible V100 GPUs` (spot V100) - `NVIDIA T4 GPUs` (on-demand T4) - - `Preemtible T4 GPUs` (spot T4) + - `Preemptible T4 GPUs` (spot T4) - `NVIDIA L4 GPUs` (on-demand L4) - - `Preemtible L4 GPUs` (spot L4) + - `Preemptible L4 GPUs` (spot L4) - `NVIDIA A100 GPUs` (on-demand A100) - - `Preemtible A100 GPUs` (spot A100) + - `Preemptible A100 GPUs` (spot A100) - `NVIDIA A100 80GB GPUs` (on-demand A100 80GB) - - `Preemtible A100 80GB GPUs` (spot A100 80GB) + - `Preemptible A100 80GB GPUs` (spot A100 80GB) - `NVIDIA H100 GPUs` (on-demand H100) - - `Preemtible H100 GPUs` (spot H100) + - `Preemptible H100 GPUs` (spot H100) + - `GPUs (all regions)` (global quota for all GPU types) ??? info "Azure" Check this [guide ](https://learn.microsoft.com/en-us/azure/quotas/quickstart-increase-quota-portal) on Azure service quotas. @@ -634,6 +649,17 @@ corresponding service quotas for each type of instance in each region. - `GPUs for GPU2 based VM and BM instances` (on-demand P100) - `GPUs for GPU3 based VM and BM instances` (on-demand V100) -Note, for AWS, GCP, and Azure, service quota values are measured with the number of CPUs rather than GPUs. +Note, for AWS and Azure, service quota values are measured with the number of CPUs rather than GPUs. [//]: # (TODO: Mention spot policy) + +## Reservations + +To use existing reservations on AWS or GCP, set +[`reservation`](../reference/dstack.yml/fleet.md#reservation) in the fleet configuration. +See the [AWS](../examples/clusters/aws.md) and [GCP](../examples/clusters/gcp.md) cluster guides. + +On Nebius and Crusoe, reservations are applied automatically, with no configuration needed. + +If you already have a reserved cluster provisioned in a cloud or data center, you can use it with `dstack` via +[SSH fleets](../concepts/fleets.md#ssh-fleets). diff --git a/mkdocs/docs/guides/server-deployment.md b/mkdocs/docs/guides/server-deployment.md index caf85f9da7..bd26538acb 100644 --- a/mkdocs/docs/guides/server-deployment.md +++ b/mkdocs/docs/guides/server-deployment.md @@ -581,7 +581,7 @@ When upgrading the `dstack` server, follow these guidelines to ensure a smooth t ### Before upgrading -1. **Check the changelog**: Review the [release notes](https://github.com/dstackai/dstack/releases) for breaking changes, new features, and migration notes. +1. **Check the changelog**: Review the [upgrade guide](upgrade.md) and the [release notes](https://github.com/dstackai/dstack/releases) for breaking changes, new features, and migration notes. 2. **Review backward compatibility**: Understand the [backward compatibility](#backward-compatibility) policy. 3. **Back up your data**: Ensure you always create a backup before upgrading. diff --git a/mkdocs/docs/guides/troubleshooting.md b/mkdocs/docs/guides/troubleshooting.md index bcfeeeea0c..6f5dc7cac2 100644 --- a/mkdocs/docs/guides/troubleshooting.md +++ b/mkdocs/docs/guides/troubleshooting.md @@ -108,14 +108,14 @@ properties in your configuration. #### Cause 5: GPU requirements -By default, `dstack` only selects instances with no GPUs or a single NVIDIA GPU. -If you want to use non-NVIDIA GPUs or multi-GPU instances, set the `gpu` property -in your configuration. +By default, `dstack` only selects instances with no GPUs or with NVIDIA GPUs. +If you want to use other accelerators, such as AMD or Tenstorrent, set the `gpu` and `image` +properties in your configuration. -Examples: `gpu: amd` (one AMD GPU), `gpu: A10:4..8` (4 to 8 A10 GPUs), +Examples: `gpu: amd` (one or more AMD GPUs), `gpu: A10:4..8` (4 to 8 A10 GPUs), `gpu: tt:32` (32 Tenstorrent devices). -> If you don't specify the number of GPUs, `dstack` will only select single-GPU instances. +> If you don't specify the number of GPUs, it defaults to `1..`. #### Cause 6: Network volumes @@ -128,14 +128,15 @@ For AWS, the availability zone of the volume and the instance should also match. Some `dstack` features are not supported by all backends. If your configuration uses one of these features, `dstack` will only select offers from the backends that support it. -- [Backend fleets](../concepts/fleets.md#backend-fleets) configurations, - [Instance volumes](../concepts/volumes.md#instance-volumes), +- [Pre-provisioning](../concepts/fleets.md#pre-provisioning) in backend fleets + is only supported by [VM-based](../concepts/backends.md#vm-based) backends. +- [Instance volumes](../concepts/volumes.md#instance-volumes) and [Privileged containers](../reference/dstack.yml/dev-environment.md#privileged) - are supported by all backends except `runpod`, `vastai`, and `kubernetes`. + are supported by all backends except `daytona`, `lium`, `runpod`, `slurm`, and `vastai`. - [Clusters](../concepts/fleets.md#cluster-placement) and [multi-node tasks](../concepts/tasks.md#nodes) - are only supported by the `aws`, `azure`, `gcp`, `nebius`, `oci`, and `vultr` backends, - as well as SSH fleets. + are only supported by the `aws`, `azure`, `crusoe`, `gcp`, `jarvislabs`, `kubernetes`, `nebius`, + `oci`, `runpod`, `seeweb`, `slurm`, and `vultr` backends, as well as SSH fleets. - [Reservations](../reference/dstack.yml/fleet.md#reservation) are only supported by the `aws` and `gcp` backends. @@ -166,10 +167,10 @@ fleet, how many of its offers were tried, and what each attempt returned. Repeat reported once; every attempt is logged by the server, so check the [server logs](#server-logs) for the full list. -#### Cause 1: Insufficient service quotas +#### Cause 1: Insufficient cloud quotas -If some runs fail to provision, it may be due to an insufficient service quota. For cloud providers like AWS, GCP, -Azure, and OCI, you often need to request an increased [service quota](protips.md#service-quotas) before you can use +If some runs fail to provision, it may be due to an insufficient cloud quota. For cloud providers like AWS, GCP, +Azure, and OCI, you often need to request an increased [quota](protips.md#cloud-quotas) before you can use specific instances. ### Run starts but fails @@ -199,14 +200,14 @@ was using spot instances and was interrupted. To address this, you can either se #### Cause 1: Gateway misconfiguration If all services fail to start with a specific gateway, make sure a -[correct DNS record](../concepts/gateways.md#update-dns-records) +[correct DNS record](../concepts/gateways.md#domain) pointing to the gateway's hostname is configured. ### Service endpoint doesn't work #### Cause 1: Bad Authorization -If the service endpoint returns a 403 error, it is likely because the [`Authorization`](../concepts/services.md#access-the-endpoint) +If the service endpoint returns a 403 error, it is likely because the [`Authorization`](../concepts/services.md#authorization) header with the correct `dstack` token was not provided. [//]: # (#### Other) diff --git a/mkdocs/docs/guides/upgrade.md b/mkdocs/docs/guides/upgrade.md index 7ccd623434..5c69bef9fe 100644 --- a/mkdocs/docs/guides/upgrade.md +++ b/mkdocs/docs/guides/upgrade.md @@ -5,16 +5,36 @@ description: Upgrading to newer versions of dstack # Upgrade guide - + -## 0.21.* { #0_21 } +!!! info "Backward compatibility" + `dstack` follows the `{major}.{minor}.{patch}` versioning scheme. The sections below cover each minor version and list the changes that may require action when upgrading. + + - The server is backward-compatible with CLIs of the same minor version or older. + - CLIs of a newer minor version are rejected by older servers (e.g., a `0.22.*` CLI with a `0.21.*` server). + +!!! info "Rolling upgrades" + - The server applies database migrations automatically on startup. + - Rolling upgrades are supported from older versions, including when skipping releases (e.g., `0.21.0` to `0.22.6`). + +Exceptions are noted in the sections below. + +## 0.22.* { #0_22 } + +!!! info "Rolling upgrades" + Rolling upgrades to `0.22.*` are supported from `0.21.*`. If upgrading from `0.20.*`, scale server replicas down to `1` before upgrading. + +### Exports and Events -### CLI compatibility +Support for [exports](../concepts/exports.md) and the global `Events` page has moved from the open-source version to [dstack Sky](https://dstack.ai/products/sky/) and [dstack Factory](https://dstack.ai/products/factory/). The `dstack event` CLI command and the `Events` tabs on resource pages remain available in the open-source version. -- CLI versions `0.20.*` and later remain backward compatible with the `0.21.*` `dstack` server. -- CLI versions `0.21.*` are not compatible with server versions prior to `0.21.*`. +### Deprecations + +The `DSTACK_ROUTER_INTERNAL_IP` environment variable is deprecated in favor of [`${{ groups[i].replicas[j].IP_ADDRESS }}`](../concepts/services.md#replica-groups). + +## 0.21.* { #0_21 } -> Upgrade the server before upgrading the CLI, or upgrade both at the same time. CLI versions prior to `0.20.0` must be upgraded along with the server. +> CLI versions prior to `0.20.0` must be upgraded along with the server. ### Pydantic v2 @@ -50,7 +70,7 @@ Use the corresponding replacements: ### Deprecations -The following API response fields are no longer populated by the server and will be removed in **0.22**: +The following API response fields are no longer populated by the server and will be removed in a future release: - `Resources.description` - `Gateway.backend` @@ -62,13 +82,6 @@ The following API response fields are no longer populated by the server and will ## 0.20.* { #0_20 } -### CLI compatibility - -- CLI versions `0.19.*` and earlier remain backward compatible with the `0.20.*` `dstack` server. -- CLI versions `0.20.` are not compatible with server versions prior to `0.20.*`. - -> Do not upgrade the CLI to `0.20.*` until the server has been upgraded. - ### Fleets * Prior to `0.20`, `dstack` automatically provisioned a fleet if one did not exist at run time. diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 34c3f664db..d161cec85f 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -34,7 +34,7 @@ It streamlines development, training, and inference, and is compatible with any * [Dev environments](concepts/dev-environments.md) — Launch dev environments to be accessed by agents or from your IDE * [Tasks](concepts/tasks.md) — Run training, batch or other jobs across a single node or clusters * [Services](concepts/services.md) — Deploy model inference as secure and scalable endpoints -* [Presets](concepts/presets.md) — Agent-driven inference optimization (experimental) +* [Presets](concepts/presets.md) — Agent-driven inference optimization * [Volumes](concepts/volumes.md) — Managing instance and network volumes for persisting data Configuration can be defined as YAML files within your repo. diff --git a/mkdocs/docs/installation.md b/mkdocs/docs/installation.md index 9203f1879b..a3082692ee 100644 --- a/mkdocs/docs/installation.md +++ b/mkdocs/docs/installation.md @@ -74,6 +74,8 @@ For more details on server deployment options, see the [Server deployment](guide To orchestrate compute across GPU clouds or Kubernetes clusters, you need to configure [backends](concepts/backends.md). + + ## Install the CLI { #cli } Once the server is up, you can access it via the `dstack` CLI. diff --git a/mkdocs/docs/quickstart.md b/mkdocs/docs/quickstart.md index 9a9b4a611c..a90e930b71 100644 --- a/mkdocs/docs/quickstart.md +++ b/mkdocs/docs/quickstart.md @@ -10,10 +10,10 @@ description: Quick guide to creating fleets and submitting runs ## Create a fleet -> Before submitting runs, you must create a [fleet](concepts/fleets.md). +Before you submit runs, you have to create [fleets](concepts/fleets.md). === "Backend fleet" - If you're using cloud providers or Kubernetes clusters and have configured the corresponding [backends](concepts/backends.md), create a fleet as follows: + If you're using cloud providers or Kubernetes clusters, create a fleet as follows:
@@ -34,6 +34,8 @@ description: Quick guide to creating fleets and submitting runs
+ > To use backend fleets, ensure you have configured [backends](concepts/backends.md). + Pass the fleet configuration to `dstack apply`:
@@ -96,7 +98,7 @@ description: Quick guide to creating fleets and submitting runs > Hosts must have Docker and GPU drivers installed and meet the other [requirements](concepts/fleets.md#ssh-fleets). - If the fleet needs to be a cluster, the [placement](concepts/fleets.md#ssh-placement) property must be set to `cluster`. + If the fleet needs to be a cluster, the [placement](concepts/fleets.md#cluster-placement) property must be set to `cluster`. ## Submit your first run @@ -265,7 +267,7 @@ description: Quick guide to creating fleets and submitting runs Service is published at: http://localhost:3000/proxy/services/main/qwen36-service/ Model Qwen/Qwen3.6-27B is published at: - http://localhost:3000/proxy/models/main/ + http://localhost:3000/proxy/services/main/qwen36-service/v1 ```
@@ -279,6 +281,6 @@ description: Quick guide to creating fleets and submitting runs Something not working? See the [troubleshooting](guides/troubleshooting.md) guide. !!! info "What's next?" - 1. Read about [backends](concepts/backends.md), [dev environments](concepts/dev-environments.md), [tasks](concepts/tasks.md), [services](concepts/services.md), and [fleets](concepts/services.md) + 1. Read about [backends](concepts/backends.md), [dev environments](concepts/dev-environments.md), [tasks](concepts/tasks.md), [services](concepts/services.md), and [fleets](concepts/fleets.md) 2. Browse [examples](examples.md) 3. Join [Discord](https://discord.gg/u8SmfwPpMd) diff --git a/mkdocs/docs/reference/cli/dstack/apply.md b/mkdocs/docs/reference/cli/dstack/apply.md index 4cc6215a0e..6f26fea63e 100644 --- a/mkdocs/docs/reference/cli/dstack/apply.md +++ b/mkdocs/docs/reference/cli/dstack/apply.md @@ -3,8 +3,12 @@ This command applies a given configuration. If a resource does not exist, `dstack apply` creates the resource. If a resource exists, `dstack apply` updates the resource in-place or re-creates the resource if the update is not possible. -To mount a Git repo to the run's container, `dstack apply` requires that you run `dstack init` first, -or specify a repo to work with via `-P` (or `--repo`), or specify `--no-repo` if you don't need any repo for the run. +To mount a Git repo to the run's container, specify it via [`repos`](../../../concepts/dev-environments.md#repos) in the configuration +or via `-P` (or `--repo`). + +If a Git repo is private, `dstack` will automatically try to use your default Git credentials (from +`~/.ssh/config` or `~/.config/gh/hosts.yml`). +If you want to use custom credentials, you can provide them with `--token` or `--git-identity`, or with [`dstack init`](init.md). ## Usage @@ -19,7 +23,10 @@ $ dstack apply --help ## User SSH key -By default, `dstack` uses its own SSH key to attach to runs (`~/.dstack/ssh/id_rsa`). -It is possible to override this key via the `--ssh-identity` argument. +By default, `dstack` attaches to runs using the user's SSH key automatically created and managed by `dstack`. The CLI downloads it from the server and stores it under `~/.dstack/ssh/`. +To use a different key for the run, pass the path to its private key via `--ssh-identity`. + +!!! info "SSH proxy" + If the [SSH proxy](../../../guides/server-deployment.md#ssh-proxy) is configured, you can also use any key uploaded under **SSH keys** in the user settings. [//]: # (TODO: Provide examples) diff --git a/mkdocs/docs/reference/cli/dstack/attach.md b/mkdocs/docs/reference/cli/dstack/attach.md index eae8c7f176..5d22dc64c0 100644 --- a/mkdocs/docs/reference/cli/dstack/attach.md +++ b/mkdocs/docs/reference/cli/dstack/attach.md @@ -15,7 +15,11 @@ $ dstack attach --help ## User SSH key -By default, `dstack` uses its own SSH key to attach to runs (`~/.dstack/ssh/id_rsa`). -It is possible to override this key via the `--ssh-identity` argument. +By default, `dstack` attaches to runs using the user's SSH key automatically created and managed by `dstack`. The CLI downloads it from the server and stores it under `~/.dstack/ssh/`. +To attach with a different key, pass the path to its private key via `--ssh-identity`. +If the run was submitted with `dstack apply --ssh-identity`, pass the same key. + +!!! info "SSH proxy" + If the [SSH proxy](../../../guides/server-deployment.md#ssh-proxy) is configured, you can also use any key uploaded under **SSH keys** in the user settings. [//]: # (TODO: Provide examples) diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 4cc395a5fa..810645974f 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -47,7 +47,7 @@ tasks, and services: - `DSTACK_NODES_NUM`{ #DSTACK_NODES_NUM } – The number of nodes in the run - `DSTACK_GPUS_PER_NODE`{ #DSTACK_GPUS_PER_NODE } – The number of GPUs per node - `DSTACK_NODE_RANK`{ #DSTACK_NODE_RANK } – The rank of the node -- `DSTACK_MASTER_NODE_IP`{ #DSTACK_NODE_RANK } – The internal IP address of the master node. +- `DSTACK_MASTER_NODE_IP`{ #DSTACK_MASTER_NODE_IP } – The internal IP address of the master node. Below is an example of using `DSTACK_NODES_NUM`, `DSTACK_GPUS_PER_NODE`, `DSTACK_NODE_RANK`, and `DSTACK_MASTER_NODE_IP` for distributed training: @@ -121,6 +121,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_SERVER_ELASTICSEARCH_INDEX`{ #DSTACK_SERVER_ELASTICSEARCH_INDEX } – The Elasticsearch/OpenSearch index pattern. Defaults to `dstack-logs`. - `DSTACK_SERVER_ELASTICSEARCH_API_KEY`{ #DSTACK_SERVER_ELASTICSEARCH_API_KEY } – The Elasticsearch/OpenSearch API key for authentication. - `DSTACK_ENABLE_PROMETHEUS_METRICS`{ #DSTACK_ENABLE_PROMETHEUS_METRICS } — Enables Prometheus metrics collection and export. +- `DSTACK_PROMETHEUS_AUTH_TOKEN`{ #DSTACK_PROMETHEUS_AUTH_TOKEN } – If set, requests to `/metrics` must pass this token in the `Authorization: Bearer ` header. If not set, `/metrics` requires no authentication. - `DSTACK_SENTRY_DSN`{ #DSTACK_SENTRY_DSN } – The Sentry DSN. If set, enables error reporting and tracing via the Sentry SDK. See [observability](../guides/server-deployment.md#observability). - `DSTACK_SENTRY_TRACES_SAMPLE_RATE`{ #DSTACK_SENTRY_TRACES_SAMPLE_RATE } – The Sentry sample rate for API request traces. Defaults to `0.1`. - `DSTACK_SENTRY_TRACES_BACKGROUND_SAMPLE_RATE`{ #DSTACK_SENTRY_TRACES_BACKGROUND_SAMPLE_RATE } – The Sentry sample rate for background task traces. Defaults to `0.01`. @@ -136,6 +137,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY`{ #DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY } – Forbids registering new services without a gateway if set to any value. - `DSTACK_FORBID_DSTACK_IN_RUNS`{ #DSTACK_FORBID_DSTACK_IN_RUNS } – Forbids submitting runs with `dstack: true` (dstack server access inside runs) if set to any value. - `DSTACK_SERVER_CODE_UPLOAD_LIMIT`{ #DSTACK_SERVER_CODE_UPLOAD_LIMIT } - The repo size limit when uploading diffs or local repos, in bytes. Set to `0` to disable size limits. Defaults to `2MiB`. +- `DSTACK_SERVER_LOG_QUOTA_PER_JOB_HOUR`{ #DSTACK_SERVER_LOG_QUOTA_PER_JOB_HOUR } - The maximum size of logs a job can write per hour, in bytes. If exceeded, the job fails with the `log_quota_exceeded` termination reason. Set to `0` to disable the limit. Defaults to `50MiB`. - `DSTACK_SERVER_S3_BUCKET`{ #DSTACK_SERVER_S3_BUCKET } - The bucket that repo diffs will be uploaded to if set. If unset, diffs are uploaded to the database. - `DSTACK_SERVER_S3_BUCKET_REGION`{ #DSTACK_SERVER_S3_BUCKET_REGION } - The region of the S3 Bucket. - `DSTACK_SERVER_GCS_BUCKET`{ #DSTACK_SERVER_GCS_BUCKET } - The bucket that repo diffs will be uploaded to if set. If unset, diffs are uploaded to the database. diff --git a/mkdocs/overrides/partials/tx-footer.html b/mkdocs/overrides/partials/tx-footer.html index c80b1317b4..5ddf8cea03 100644 --- a/mkdocs/overrides/partials/tx-footer.html +++ b/mkdocs/overrides/partials/tx-footer.html @@ -25,7 +25,6 @@ Training Clusters Inference - Models Accelerators
diff --git a/skills/dstack/SKILL.md b/skills/dstack/SKILL.md index eb722e8941..8b910e0cdb 100644 --- a/skills/dstack/SKILL.md +++ b/skills/dstack/SKILL.md @@ -565,7 +565,7 @@ Common issues: ## Additional resources **Core documentation:** -- [Overview](https://dstack.ai/docs/overview.md) +- [Overview](https://dstack.ai/docs/index.md) - [Installation](https://dstack.ai/docs/installation.md) - [Quickstart](https://dstack.ai/docs/quickstart.md) @@ -580,8 +580,8 @@ Common issues: - [Server deployment](https://dstack.ai/docs/guides/server-deployment.md) **Accelerator-specific examples:** -- [AMD](https://dstack.ai/examples/accelerators/amd/index.md) -- [Google TPU](https://dstack.ai/examples/accelerators/tpu/index.md) -- [Tenstorrent](https://dstack.ai/examples/accelerators/tenstorrent/index.md) +- [AMD](https://dstack.ai/docs/examples/accelerators/amd.md) +- [Google TPU](https://dstack.ai/docs/examples/accelerators/tpu.md) +- [Tenstorrent](https://dstack.ai/docs/examples/accelerators/tenstorrent.md) **Full documentation:** https://dstack.ai/llms-full.txt diff --git a/website/src/components/SiteFooter.tsx b/website/src/components/SiteFooter.tsx index ea0792b2de..dc1f6fb582 100644 --- a/website/src/components/SiteFooter.tsx +++ b/website/src/components/SiteFooter.tsx @@ -51,7 +51,6 @@ const footerColumns: FooterColumn[] = [ { label: 'Training', href: docsUrl('examples/training/trl/') }, { label: 'Clusters', href: docsUrl('examples/clusters/aws/') }, { label: 'Inference', href: docsUrl('examples/inference/sglang/') }, - { label: 'Models', href: docsUrl('examples/models/deepseek-v4/') }, { label: 'Accelerators', href: docsUrl('examples/accelerators/amd/') }, ], },