diff --git a/.github/workflows/diffusers-examples.yml b/.github/workflows/diffusers-examples.yml new file mode 100644 index 000000000..a139eda74 --- /dev/null +++ b/.github/workflows/diffusers-examples.yml @@ -0,0 +1,68 @@ +# diffusers examples guard - project thin trigger. +# +# Calls the common engine .github/workflows/examples-template.yml; +# this file only declares what varies per project: schedule, +# concurrency, upstream repo, dispatch input forwarding, and matrix +# parallelism. The guard loop (release + workflow-files monitor / +# manifest-check / run-example matrix / validate-results / +# save-monitor-state) lives in the engine. See examples-guards/README.md +# for the design. + +name: diffusers-examples + +concurrency: + # Three groups: schedule runs serialize (a run cancelled mid-way + # loses its outcome writeback, and the outcome drives the next + # cycle's retry); each PR serializes per number so a new push + # cancels the stale run instead of queueing another matrix on the + # NPU runners; each manual run stands alone. Group names are + # repository-wide, so the PR group carries the workflow name: a bare + # pr- would let two projects' runs on one PR cancel each + # other. + group: ${{ github.event_name == 'schedule' && 'diffusers-examples-schedule' || (github.event_name == 'pull_request' && format('diffusers-examples-pr-{0}', github.event.pull_request.number)) || format('manual-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + schedule: + - cron: '5 22 * * *' + workflow_dispatch: + inputs: + target_ref: + description: Branch, tag, or SHA to test (empty = latest upstream release; falls back to main on API failure) + required: false + default: '' + type: string + # PR runs need no merge: GitHub executes the workflow files from the + # PR merge ref, so a newly added trigger and engine run on the PR + # that adds them. Whether a fork PR waits for a maintainer's approval + # first is the repository's fork-PR policy. `pull_request`, not + # `pull_request_target`: the declared read permissions are all the + # engine needs. paths lists only this project's files, as the + # quick-start triggers do: a PR that touches only the shared engine, + # scripts or schema starts no project's matrix. + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - 'examples-guards/diffusers/**' + - '.github/workflows/diffusers-examples.yml' + +# Engine jobs query the GitHub Job API (validate-results); called +# workflows run with the caller's GITHUB_TOKEN permissions. +permissions: + actions: read + contents: read + +jobs: + diffusers-examples: + uses: ./.github/workflows/examples-template.yml + with: + # Namespaces cache keys (examples-monitor-state-diffusers_*), + # artifact names and the scheduling paths under + # examples-guards/diffusers. + project: diffusers + upstream_repo: huggingface/diffusers + target_ref: ${{ inputs.target_ref }} + # 52 supported entries: cap parallel legs at 4 so cold-cache model + # downloads don't contend for runner bandwidth. + max_parallel: 4 diff --git a/examples-guards/diffusers/constraints-npu.txt b/examples-guards/diffusers/constraints-npu.txt new file mode 100644 index 000000000..5bfae45e7 --- /dev/null +++ b/examples-guards/diffusers/constraints-npu.txt @@ -0,0 +1,41 @@ +torch==2.9.0 +torchvision==0.24.0 +torchaudio<0 +cuda-toolkit<0 +cuda-python<0 +cuda-bindings<0 +cuda-core<0 +cuda-pathfinder<0 +flashinfer-python<0 +nvidia-cublas<0 +nvidia-cuda-runtime<0 +nvidia-cuda-nvrtc<0 +nvidia-cuda-cupti<0 +nvidia-cudnn<0 +nvidia-cudnn-frontend<0 +nvidia-cufft<0 +nvidia-curand<0 +nvidia-cusolver<0 +nvidia-cusparse<0 +nvidia-cutlass-dsl<0 +nvidia-cutlass-dsl-libs-base<0 +nvidia-cutlass-dsl-libs-core<0 +nvidia-cutlass-dsl-libs-cu12<0 +nvidia-ml-py<0 +nvidia-nccl<0 +nvidia-nvjitlink<0 +nvidia-nvtx<0 +nvidia-cublas-cu12<0 +nvidia-cuda-nvdisasm<0 +nvidia-cuda-runtime-cu12<0 +nvidia-cuda-nvrtc-cu12<0 +nvidia-cuda-cupti-cu12<0 +nvidia-cudnn-cu12<0 +nvidia-cufft-cu12<0 +nvidia-curand-cu12<0 +nvidia-cusolver-cu12<0 +nvidia-cusparse-cu12<0 +nvidia-cusparselt-cu12<0 +nvidia-nccl-cu12<0 +nvidia-nvjitlink-cu12<0 +nvidia-nvtx-cu12<0 diff --git a/examples-guards/diffusers/examples_manifest.yaml b/examples-guards/diffusers/examples_manifest.yaml new file mode 100644 index 000000000..bdb888f05 --- /dev/null +++ b/examples-guards/diffusers/examples_manifest.yaml @@ -0,0 +1,1278 @@ +version: 1 +scan: + root: examples + # .yaml is not scanned: the files under examples/ are deepspeed/accelerate + # run configs, not launchable examples. .md/.txt/.json are non-examples. + include_extensions: ['.py', '.sh'] + # files-only 模型（examples-guards/README.md）。2026-10-08 起 + # 对齐收口：原 scan.exclude 的 33 条规则全部迁入 + # unsupported 段（上游自测 / helper / pipeline 模块 / community 整目录等， + # 带分组注释与理由），scan 只保留 root + include_extensions 两个键。 + # unsupported 覆盖全部 308 个扫描文件：supported(52) + unsupported(256)。 + +# 新版 diffusers 需求 huggingface-hub>=1.23，huggingface-hub>=1.23 需求 +# transformers>=5.x，任何 transformers<5.x 的用例都无法跑通 + +supported: + # ${...} 由 setup_example.sh 预下后写入 GITHUB_ENV： + # ${SD15_MODEL_PATH} / ${THREE_D_ICON_PATH} <- profile diffusers-sd15 + # SDXL 模型直接走 stabilityai/stable-diffusion-xl-base-1.0 在线加载。 + + + # FLUX.2 Klein 4B LoRA（t2i）。NPU fork（自带 torch_npu 适配 + + # set_attention_backend("_native_npu")）；模型非 gated，setup 预下 ~15GB。 + # 冻结 transformer ~7.2GB，LoRA + bf16 单卡放得下。 + - path: examples/dreambooth/train_dreambooth_lora_flux2_klein.py + profile: diffusers-flux2-klein + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${FLUX2_KLEIN_MODEL_PATH} + - --instance_data_dir ${FIXTURE_DIR}/images + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --train_batch_size 1 + - --gradient_checkpointing + - --mixed_precision bf16 + - --max_train_steps 1 + timeout_minutes: 600 + + # CogVideoX-5b-I2V LoRA。必须用 5b-I2V（不是 1.5，脚本不处理 1.5 的 ofs 输入）。 + - path: examples/cogvideo/train_cogvideox_image_to_video_lora.py + profile: diffusers-cogvideo-i2v + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path THUDM/CogVideoX-5b-I2V + - --instance_data_root ${COGVIDEOX_DATASET_DIR} + - --caption_column prompt.txt + - --video_column videos.txt + - --max_num_frames 9 + - --gradient_checkpointing + - --enable_slicing --enable_tiling + - --train_batch_size 1 + - --max_train_steps 1 + - --mixed_precision fp16 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 360 + # Kandinsky prior 全量（naruto）。bf16 + ZeRO-2，静态 ~8GB/卡。收尾会在 CPU + # 组装完整 pipeline 再拉 decoder 仓 9.8GB → 长 timeout。 + - path: examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py + profile: diffusers-kandinsky-prior + launcher: accelerate-deepspeed + runner: linux-aarch64-a2-2 + npu_devices: '0,1' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --dataset_name lambdalabs/naruto-blip-captions + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 512 + - --train_batch_size 1 + - --mixed_precision bf16 + - --max_train_steps 1 + timeout_minutes: 360 + # ControlNet webdataset（SDXL）。webdataset 钉 0.2.86（vendored 旧 tar 迭代器 + # 在新版 KeyError）；shard 由 setup 合成 1024 样本（shuffle 缓冲写死 1000， + # 样本不足第一个 batch 永远不来）。单卡 fp16 在 optimizer.step 常驻 ~26.6GB + # 必炸（与分辨率无关），改双卡 ZeRO-2 + bf16（fp16 在 DS 反向有 aclnn 前科）。 + # 训练管线用 ResampledShards（不按 rank 切分），单 shard 双卡各自可读。 + - path: examples/research_projects/controlnet/train_controlnet_webdataset.py + profile: diffusers-controlnet-wds + launcher: accelerate-deepspeed + runner: linux-aarch64-a2-2 + npu_devices: '0,1' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --output_dir ${CI_OUTPUT_DIR} + - --train_shards_path_or_url ${CONTROLNET_WDS_SHARDS} + - --eval_shards_path_or_url ${CONTROLNET_WDS_SHARDS} + - --max_train_samples 1024 + - --max_train_steps 1 + - --resolution 256 + - --train_batch_size 1 + - --gradient_checkpointing + - --mixed_precision bf16 + timeout_minutes: 300 + # Diffusion-ORPO SDXL LoRA（webdataset）。上游偏好数据集转私有，shard 由 setup + # 合成（五件套键名与 get_dataset() rename 对应；1024 填满写死的 690 shuffle + # 缓冲）。--dataloader_num_workers 默认 0 会除零，必须传 1。 + - path: examples/research_projects/diffusion_orpo/train_diffusion_orpo_sdxl_lora_wds.py + profile: diffusers-orpo-wds + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_path ${ORPO_WDS_SHARDS} + - --output_dir ${CI_OUTPUT_DIR} + - --num_train_examples 1024 + - --max_train_steps 1 + - --resolution 512 + - --per_gpu_batch_size 1 + - --global_batch_size 1 + - --dataloader_num_workers 1 + - --gradient_checkpointing + - --mixed_precision fp16 + timeout_minutes: 300 + + # FLUX.2 Klein 4B LoRA（img2img）。同上。必须用带条件图列的数据集 + # （:1541 无条件取 cond_pixel_values）；且 Dataset 会把整个数据集急切物化进 + # 宿主内存（fill50k 1000 对直接吃穿 RAM → SIGKILL），故用 setup 合成的 + # 8 对数据集。 + - path: examples/dreambooth/train_dreambooth_lora_flux2_klein_img2img.py + profile: diffusers-flux2-klein + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${FLUX2_KLEIN_MODEL_PATH} + - --dataset_name ${FLUX2_KLEIN_DATASET} + - --cond_image_column conditioning_image + - --caption_column text + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --train_batch_size 1 + - --gradient_checkpointing + - --mixed_precision bf16 + - --max_train_steps 1 + timeout_minutes: 300 + + # Sana 1600M LoRA（单卡 a2-1） + - path: examples/dreambooth/train_dreambooth_lora_sana.py + profile: diffusers-sana + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path Efficient-Large-Model/Sana_1600M_1024px_BF16_diffusers + - --dataset_name lambdalabs/naruto-blip-captions + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 + - --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + + # PixArt-α ControlNet。T5-XXL 文本编码器，模型由 setup 预下（跳过用不上的 + # fp16 分片省一半）。--train_batch_size 默认 16 必须压 1；bf16 + 梯度检查点 + # 压显存。 + - path: examples/research_projects/pixart/train_pixart_controlnet_hf.py + profile: diffusers-pixart + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${PIXART_MODEL_PATH} + - --dataset_name fusing/fill50k + - --output_dir ${CI_OUTPUT_DIR} + - --train_batch_size 1 + - --mixed_precision bf16 + - --max_train_steps 1 + timeout_minutes: 240 + + - path: examples/t2i_adapter/train_t2i_adapter_sdxl.py + profile: diffusers-t2i-adapter + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name hf-internal-testing/fill10 + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 64 + - --train_batch_size 1 + - --gradient_accumulation_steps 1 + - --max_train_steps 1 + # # 验证阶段--validation_*会走cuda + timeout_minutes: 240 + + # SDXL LoRA + - path: examples/text_to_image/train_text_to_image_lora_sdxl.py + profile: diffusers-text-to-image + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name hf-internal-testing/dummy_image_text_data + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 64 + - --train_batch_size 1 --gradient_accumulation_steps 1 --max_train_steps 2 + timeout_minutes: 240 + + # SDXL + - path: examples/textual_inversion/textual_inversion_sdxl.py + profile: diffusers-textual-inversion + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --train_data_dir ${FIXTURE_DIR}/images + - --learnable_property object + - --placeholder_token + - --initializer_token a + - --output_dir ${CI_OUTPUT_DIR} + - --save_steps 1 --num_vectors 2 + - --resolution 64 + - --train_batch_size 1 --gradient_accumulation_steps 1 --max_train_steps 2 + timeout_minutes: 240 + + # Diffusion-DPO SDXL + - path: examples/research_projects/diffusion_dpo/train_diffusion_dpo_sdxl.py + profile: diffusers-sdxl-online + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name kashif/pickascore + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision fp16 --resolution 512 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + + # Diffusion-ORPO SDXL LoRA + - path: examples/research_projects/diffusion_orpo/train_diffusion_orpo_sdxl_lora.py + profile: diffusers-sdxl-online + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name kashif/pickascore + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision fp16 --resolution 512 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + + # scheduled_huber / dreambooth LoRA SDXL + - path: examples/research_projects/scheduled_huber_loss_training/dreambooth/train_dreambooth_lora_sdxl.py + profile: diffusers-sdxl-online + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --instance_data_dir ${FIXTURE_DIR}/images + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + + # scheduled_huber / text_to_image LoRA SDXL + - path: examples/research_projects/scheduled_huber_loss_training/text_to_image/train_text_to_image_lora_sdxl.py + profile: diffusers-sdxl-online + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name lambdalabs/naruto-blip-captions + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + + # SDXL LoRA（单卡 a2-1） + - path: examples/dreambooth/train_dreambooth_lora_sdxl.py + profile: diffusers-sdxl + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name hf-internal-testing/dummy_image_text_data + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 + - --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + # SDXL ControlNet（fill50k）。fp32：预计算嵌入经 datasets 序列化是 fp32， + # 混精度在 SDPA 处 dtype 冲突；双卡 ZeRO-3 每卡 ~18GB。--max_train_samples 100： + # 全量 1000 行在 map 预计算时一个 batch 全进 CLIP（激活 ~1.5GB），切片发生在 + # map 之前。 + - path: examples/controlnet/train_controlnet_sdxl.py + profile: diffusers-controlnet-sdxl + launcher: accelerate-deepspeed + runner: linux-aarch64-a2-2 + npu_devices: '0,1' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name fusing/fill50k + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 + - --gradient_checkpointing + - --max_train_steps 1 + - --max_train_samples 100 + timeout_minutes: 240 + # SDXL 全量微调。bf16：fp32 在 DS ZeRO-2 初始化平坦缓冲处爆（~19GB/卡且每卡 + # 全量、加卡不减）；脚本 :1120 把嵌入 cast 成 weight_dtype，无 SDPA 冲突。 + - path: examples/text_to_image/train_text_to_image_sdxl.py + profile: diffusers-sdxl-full + launcher: accelerate-deepspeed + runner: linux-aarch64-a2-2 + npu_devices: '0,1' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name hf-internal-testing/dummy_image_text_data + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 + - --gradient_checkpointing + - --mixed_precision bf16 + - --max_train_steps 1 + timeout_minutes: 240 + # InstructPix2Pix SDXL 全量。--seed 必传（:518 无条件 manual_seed）；bf16 + # 必传：脚本不 del 文本编码器/vae，fp32 叠加 ZeRO-2 平坦缓冲 >29.5GB OOM。 + # validation 双重守卫都不传，不跑 pipeline 推理。 + - path: examples/instruct_pix2pix/train_instruct_pix2pix_sdxl.py + profile: diffusers-sdxl-full + launcher: accelerate-deepspeed + runner: linux-aarch64-a2-2 + npu_devices: '0,1' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name fusing/instructpix2pix-1000-samples + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 + - --gradient_checkpointing + - --mixed_precision bf16 + - --max_train_steps 1 + - --seed 42 + timeout_minutes: 240 + + # SDXL LPL（Latent Prompt Learning）。NPU fork：自带 torch_npu 适配 + + # --enable_npu_flash_attention（diffusers 的 NPU 专属接口，唯一覆盖）。 + # 显存账同 t2i_sdxl（a2-2 + ZeRO-2 + bf16）。 + - path: examples/research_projects/lpl/train_sdxl_lpl.py + profile: diffusers-sdxl-full + launcher: accelerate-deepspeed + runner: linux-aarch64-a2-2 + npu_devices: '0,1' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name lambdalabs/naruto-blip-captions + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --mixed_precision bf16 + - --train_batch_size 1 + - --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 240 + + # Kandinsky 2.2 decoder（全量）。fp16：decoder 9.84GB fp32 + Adam 会 OOM。 + # 不传 --validation_prompts（否则加载 26.57GB 的 prior）。 + - path: examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py + profile: diffusers-kandinsky + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --dataset_name lambdalabs/naruto-blip-captions + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --mixed_precision fp16 # decoder 9.84GB fp32 + Adam 状态会 OOM，转 fp16 + - --train_batch_size 1 + - --gradient_accumulation_steps 1 + - --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 180 + + # SD1.5 多主体 inpainting DreamBooth。模型必须是 9 通道 inpainting 版（基础版 + # unet 4 通道，conv_in 报通道数不匹配）。Dataset 用 load_dataset 读本地目录， + # 固定取 train/test split、列 image/prompt/mask，由 setup 生成。 + - path: examples/research_projects/multi_subject_dreambooth_inpainting/train_multi_subject_dreambooth_inpainting.py + profile: diffusers-multi-subject-inpaint + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_INPAINT_PATH} + - --instance_data_dir ${MULTI_SUBJECT_INPAINT_DATASET} + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 512 + - --train_batch_size 1 + - --max_train_steps 1 + timeout_minutes: 180 + + # InstructPix2Pix LoRA（需显式列名 + seed） + - path: examples/research_projects/instructpix2pix_lora/train_instruct_pix2pix_lora.py + profile: diffusers-research-plain + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path timbrooks/instruct-pix2pix + - --dataset_name instruction-tuning-sd/cartoonization + - --original_image_column original_image + - --edited_image_column cartoonized_image + - --edit_prompt_column edit_prompt + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + - --seed 42 + timeout_minutes: 180 + + # GLIGEN text-box（SD1.5）。数据集是 torch.load 的 pth + 图片目录，setup + # 合成极简 fixture（灰底图 + 1 框 + 零向量 embedding，后者是常量条件不进梯度）。 + - path: examples/research_projects/gligen/train_gligen_text.py + profile: diffusers-gligen + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --data_path ${GLIGEN_DATA_PTH} + - --image_path ${GLIGEN_IMAGE_DIR} + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 512 + - --train_batch_size 1 + - --gradient_accumulation_steps 1 + - --max_train_steps 1 + - --mixed_precision no + timeout_minutes: 180 + + # CogVideoX-2b LoRA。模型 ModelScope 预下；Disney 数据集走 HuggingFace 直连。 + # 默认 480x720/49 帧在 29.5GB 卡上 OOM，压到 256x256/9 帧（(n-1)%4==0 约束）。 + - path: examples/cogvideo/train_cogvideox_lora.py + profile: diffusers-cogvideo + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${COGVIDEOX_MODEL_PATH} # ModelScope: AI-ModelScope/CogVideoX-2b + - --instance_data_root ${COGVIDEOX_DATASET_DIR} # HF: Wild-Heart/Disney-VideoGeneration-Dataset + - --caption_column prompt.txt + - --video_column videos.txt + - --height 256 --width 256 # 默认 480x720 在 29.5GB 卡上 OOM + - --max_num_frames 9 # 默认 49；(n-1)%4==0 约束，9 合法 + - --gradient_checkpointing + - --enable_slicing --enable_tiling + - --train_batch_size 1 + - --max_train_steps 1 + - --mixed_precision fp16 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 180 + + # LCM 一致性蒸馏（webdataset 流式）。分片走 ModelScope 镜像的 + # laion/conceptual-captions-12m-webdataset（大文件流式下载，镜像比 HF 直连稳）。 + - path: examples/consistency_distillation/train_lcm_distill_sd_wds.py + profile: diffusers-lcm + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_teacher_model ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision fp16 + - --resolution 256 + - --max_train_steps 1 + - --max_train_samples 1 + - --dataloader_num_workers 1 + - --train_shards_path_or_url "pipe:curl -L -s https://www.modelscope.cn/datasets/laion/conceptual-captions-12m-webdataset/resolve/master/data/{00000..00000}.tar" + - --train_batch_size 1 + - --gradient_accumulation_steps 1 + timeout_minutes: 180 + + # LCM LoRA 蒸馏（SD1.5，webdataset）。同上。 + - path: examples/consistency_distillation/train_lcm_distill_lora_sd_wds.py + profile: diffusers-lcm + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_teacher_model ${SD15_MODEL_PATH} + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision fp16 + - --resolution 512 + - --lora_rank 4 + - --max_train_steps 1 + - --max_train_samples 1 + - --dataloader_num_workers 1 + - --train_shards_path_or_url "pipe:curl -L -s https://www.modelscope.cn/datasets/laion/conceptual-captions-12m-webdataset/resolve/master/data/{00000..00000}.tar" + - --train_batch_size 1 + timeout_minutes: 180 + + # ControlNet（SD1.5）。fill50k 运行时经 HuggingFace 直连取（trust_remote_code 由 + # profile 提供）。 + - path: examples/controlnet/train_controlnet.py + profile: diffusers-controlnet + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --output_dir ${CI_OUTPUT_DIR} + - --dataset_name fusing/fill50k + - --resolution 512 + - --train_batch_size 1 + - --max_train_steps 1 + timeout_minutes: 180 + + # InstructPix2Pix（SD1.5）。fill50k 同款经 HuggingFace 直连取。--seed 必传： + # :458 无条件 manual_seed(args.seed)，默认 None 直接崩。 + - path: examples/instruct_pix2pix/train_instruct_pix2pix.py + profile: diffusers-instruct-pix2pix + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --dataset_name fusing/instructpix2pix-1000-samples + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + - --mixed_precision fp16 + - --seed 42 + timeout_minutes: 180 + + # Diffusion-DPO（SD1.5 + kashif/pickascore） + - path: examples/research_projects/diffusion_dpo/train_diffusion_dpo.py + profile: diffusers-research + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --dataset_name kashif/pickascore + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 180 + + # SD1.5 多主体 DreamBooth。单 instance 目录即普通 dreambooth，DOG fixture； + # 多主体需传 --concepts_list JSON，先按单主体跑。 + - path: examples/research_projects/multi_subject_dreambooth/train_multi_subject_dreambooth.py + profile: diffusers-dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} + - --instance_data_dir ${FIXTURE_DIR}/images + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 512 + - --train_batch_size 1 + - --max_train_steps 1 + - --gradient_checkpointing + timeout_minutes: 180 + + # ReFiLL（SD1.5-inpainting LoRA）。RealFillDataset 认死 + # /{ref/*.png, target/target.png, target/mask.png}，setup 合成。 + # 走 fp32：bf16 下 log_validation 的 vae 是 bf16 而 prompt_embeds 是 fp32， + # prepare_mask_latents 会把输入 cast 成 fp32 喂 bf16 vae → dtype 冲突。 + - path: examples/research_projects/realfill/train_realfill.py + profile: diffusers-realfill + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_INPAINT_PATH} + - --train_data_dir ${REALFILL_DATASET} + - --output_dir ${CI_OUTPUT_DIR} + - --train_batch_size 1 + - --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 180 + + # SD1.5 高级 DreamBooth LoRA（同上 SD1.5 变体）。不传 --validation_prompt： + # validation 会加载 safety_checker/feature_extractor，setup 为省 2.4GB 没下。 + - path: examples/advanced_diffusion_training/train_dreambooth_lora_sd15_advanced.py + profile: diffusers-sd15 + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --dataset_name ${THREE_D_ICON_PATH} # HF: linoyts/3d_icon（本地 imagefolder 目录） + - --instance_prompt "3d icon in the style of TOK" + - --output_dir ${CI_OUTPUT_DIR} + - --caption_column prompt + - --mixed_precision bf16 + - --resolution 128 + - --train_batch_size 1 + - --gradient_checkpointing + - --optimizer prodigy + - --max_train_steps 1 + timeout_minutes: 150 + + # DreamBooth inpainting（SD1.5-inpainting）。数据集用 fixtures/DOG.jpg。 + - path: examples/research_projects/dreambooth_inpaint/train_dreambooth_inpaint.py + profile: diffusers-research-plain + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stable-diffusion-v1-5/stable-diffusion-inpainting + - --instance_data_dir ${FIXTURE_DIR}/images + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 150 + + # SD1.5 dreambooth（全量）。数据 DOG fixture。 + - path: examples/dreambooth/train_dreambooth.py + profile: diffusers-dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # 本地路径：结尾 from_pretrained(repo id) 会调 model_info，在线拉取易超时 + - --instance_data_dir ${FIXTURE_DIR}/images + - --output_dir ${CI_OUTPUT_DIR} + - --instance_prompt "a photo of sks dog" + - --resolution 512 + - --train_batch_size 1 + - --max_train_steps 1 + - --gradient_checkpointing + timeout_minutes: 150 + + # SD1.5 dreambooth LoRA。同上。 + - path: examples/dreambooth/train_dreambooth_lora.py + profile: diffusers-dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # 本地路径，理由同上 + - --instance_data_dir ${FIXTURE_DIR}/images + - --output_dir ${CI_OUTPUT_DIR} + - --instance_prompt "a photo of sks dog" + - --resolution 512 + - --train_batch_size 1 + - --max_train_steps 1 + - --gradient_checkpointing + timeout_minutes: 150 + + # scheduled_huber：SD1.5 naruto t2i / DOG dreambooth 四条。 + - path: examples/research_projects/scheduled_huber_loss_training/text_to_image/train_text_to_image.py + profile: diffusers-research + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --dataset_name lambdalabs/naruto-blip-captions + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 150 + + - path: examples/research_projects/scheduled_huber_loss_training/text_to_image/train_text_to_image_lora.py + profile: diffusers-research + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --dataset_name lambdalabs/naruto-blip-captions + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 150 + + - path: examples/research_projects/scheduled_huber_loss_training/dreambooth/train_dreambooth.py + profile: diffusers-research + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --instance_data_dir ${FIXTURE_DIR}/images + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 150 + + - path: examples/research_projects/scheduled_huber_loss_training/dreambooth/train_dreambooth_lora.py + profile: diffusers-research + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} # ModelScope: AI-ModelScope/stable-diffusion-v1-5 + - --instance_data_dir ${FIXTURE_DIR}/images + - --instance_prompt "a photo of sks dog" + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 256 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 150 + + # SD1.5 全量 + - path: examples/text_to_image/train_text_to_image.py + profile: diffusers-text-to-image + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} + - --dataset_name hf-internal-testing/dummy_image_text_data + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 64 + - --train_batch_size 1 --gradient_accumulation_steps 1 --max_train_steps 2 + timeout_minutes: 150 + + # SD1.5 LoRA + - path: examples/text_to_image/train_text_to_image_lora.py + profile: diffusers-text-to-image + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} + - --dataset_name hf-internal-testing/dummy_image_text_data + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 64 + - --train_batch_size 1 --gradient_accumulation_steps 1 --max_train_steps 2 + timeout_minutes: 150 + + + # SD1.5 + - path: examples/textual_inversion/textual_inversion.py + profile: diffusers-textual-inversion + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path ${SD15_MODEL_PATH} + - --train_data_dir ${FIXTURE_DIR}/images + - --learnable_property object + - --placeholder_token + - --initializer_token a + - --output_dir ${CI_OUTPUT_DIR} + - --save_steps 1 --num_vectors 2 + - --resolution 64 + - --train_batch_size 1 --gradient_accumulation_steps 1 --max_train_steps 2 + timeout_minutes: 150 + # 一致性训练（cifar10，从零）。小步数必须配 --constant_discretization_steps。 + - path: examples/research_projects/consistency_training/train_cm_ct_unconditional.py + profile: diffusers-research-plain + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --dataset_name uoft-cs/cifar10 + - --dataset_image_column_name img + - --constant_discretization_steps + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 32 + - --train_batch_size 1 --gradient_accumulation_steps 1 --gradient_checkpointing + - --max_train_steps 1 + timeout_minutes: 120 + + # VQGAN（判别器 depth=6 下采样 64x，InstanceNorm 需 >1x1，resolution 至少 128） + - path: examples/vqgan/train_vqgan.py + profile: diffusers-vqgan + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --dataset_name hf-internal-testing/dummy_image_text_data + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 128 + - --train_batch_size 1 + - --gradient_accumulation_steps 1 + - --max_train_steps 2 + timeout_minutes: 120 + + # AutoencoderRAE：冻结 encoder 只训 decoder。数据是 ImageFolder 风格 + # （//*.jpg），用 fixtures/imagefolder。--train_batch_size 默认 8 + # 且 drop_last=True，图少会静默 0 步，压到 1。 + - path: examples/research_projects/autoencoder_rae/train_autoencoder_rae.py + profile: diffusers-research-plain + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --train_data_dir ${FIXTURE_DIR}/imagefolder + - --output_dir ${CI_OUTPUT_DIR} + - --train_batch_size 1 + - --max_train_steps 1 + timeout_minutes: 120 + + # LCM LoRA 蒸馏（SDXL teacher + fp16-fix VAE，webdataset）。bf16：fp16 反向 + # aclnn 算子失败。全量 SDXL 版三张 unet 驻留 OOM，故用 LoRA 版。 + - path: examples/consistency_distillation/train_lcm_distill_lora_sdxl_wds.py + profile: diffusers-lcm-sdxl + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_teacher_model stabilityai/stable-diffusion-xl-base-1.0 # 在线 HF 直连（无 --variant，按 fp32 取 ~14GB） + - --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision bf16 # fp16 反向 aclnn 算子失败，改 bf16 + - --resolution 256 + - --lora_rank 4 + - --max_train_steps 1 + - --max_train_samples 1 + - --dataloader_num_workers 1 + - --train_shards_path_or_url "pipe:curl -L -s https://www.modelscope.cn/datasets/laion/conceptual-captions-12m-webdataset/resolve/master/data/{00000..00000}.tar" + - --train_batch_size 1 + timeout_minutes: 120 + # SDXL 高级 DreamBooth LoRA（3D icon example） + - path: examples/advanced_diffusion_training/train_dreambooth_lora_sdxl_advanced.py + profile: diffusers-sdxl + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 + - --variant fp16 + - --dataset_name hf-internal-testing/dummy_image_text_data + - --instance_prompt "a 3d icon in the style of TOK" + - --output_dir ${CI_OUTPUT_DIR} + - --caption_column text + - --mixed_precision bf16 + - --resolution 128 + - --train_batch_size 1 + - --gradient_checkpointing + - --optimizer prodigy + - --max_train_steps 1 + timeout_minutes: 90 + + # unconditional（ddpm_dummy + dummy_image_class_data） + - path: examples/unconditional_image_generation/train_unconditional.py + profile: diffusers-unconditional + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --dataset_name hf-internal-testing/dummy_image_class_data + - --model_config_name_or_path diffusers/ddpm_dummy + - --output_dir ${CI_OUTPUT_DIR} + - --resolution 64 + - --train_batch_size 2 + - --num_epochs 1 + - --gradient_accumulation_steps 1 + - --ddpm_num_inference_steps 2 + timeout_minutes: 90 + + # VAE roundtrip（纯推理；输入图用 fixtures/DOG.jpg） + - path: examples/research_projects/vae/vae_roundtrip.py + profile: diffusers-research-plain + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path stabilityai/sd-vae-ft-mse + - --input_image ${FIXTURE_DIR}/images/DOG.jpg + timeout_minutes: 60 + + + # LLaDA2 block-refinement。Qwen2.5-0.5B + --use_dummy_data，零下载。 + - path: examples/discrete_diffusion/train_llada2.py + profile: diffusers-llada2 + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --model_name_or_path Qwen/Qwen2.5-0.5B + - --use_dummy_data + - --output_dir ${CI_OUTPUT_DIR} + - --max_train_steps 1 + - --gradient_accumulation_steps 1 + timeout_minutes: 60 + + # aMUSED-256 微调。数据用本地 DOG fixture（上游 nouns 数据集在 CI 拉取阶段 + # 反复崩）。--report_to 只能 wandb（tensorboard 收不了它的 list 型参数）， + # 全局 WANDB_MODE=disabled 已让它离线。 + - path: examples/amused/train_amused.py + profile: diffusers-amused + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --pretrained_model_name_or_path amused/amused-256 # HF 直连（ModelScope 无） + - --instance_data_dir ${FIXTURE_DIR}/images # 本地 fixture（fixtures/DOG.jpg），prompt 取文件名 + - --resolution 256 + - --mixed_precision fp16 + - --train_batch_size 1 + - --max_train_steps 1 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 60 + +unsupported: + # ===== Flax/JAX 系列 ======================================================= + # 以下都是训练脚本的 Flax/JAX 版本，跑不通有三层原因： + # 1) diffusers 0.40 起删除了全部 Flax 类（FlaxUNet2DConditionModel 等）， + # transformers 5.x 起删除了 FlaxCLIPTextModel，脚本开头的 import 就失败； + # 2) 版本没有出路：新版 diffusers 强制配 transformers 5.x（两边 hub 版本 + # 要求互斥），降 diffusers 到 0.39 又等于停止看护最新 release； + # 3) JAX/Flax 没有昇腾 NPU 后端，前两条即使解决也上不了卡。 + - examples/controlnet/train_controlnet_flax.py + - examples/dreambooth/train_dreambooth_flax.py + - examples/text_to_image/train_text_to_image_flax.py + - examples/textual_inversion/textual_inversion_flax.py + - examples/research_projects/multi_token_textual_inversion/textual_inversion_flax.py + - examples/research_projects/sdxl_flax/sdxl_single.py + - examples/research_projects/sdxl_flax/sdxl_single_aot.py + # ===== 旧式 LoRA 构造 API 系列：机制已被 diffusers 删除，等上游改写 ========== + # 这 4 个例子还在用"自己构造 LoRAAttnProcessor 再训练"的老写法，而新版 + # diffusers 已把这套机制删掉：这三个类还在，但 __init__ 只剩 pass（空壳）， + # 传 hidden_size / cross_attention_dim / rank 直接 TypeError。就算把调用 + # 改掉也没用——空壳里没有 LoRA 层，AttnProcsLayers 收集不到可训练参数。 + # 官方其他 LoRA 例子早已改用 PEFT 写法，这 4 个是没跟上的遗留。 + # 恢复条件：上游把它们改写成 PEFT LoRA 后一起升格。 + - examples/kandinsky2_2/text_to_image/train_text_to_image_lora_decoder.py + - examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py + - examples/research_projects/lora/train_text_to_image_lora.py + - examples/research_projects/dreambooth_inpaint/train_dreambooth_inpaint_lora.py + # ===== diffusion_policy：教学推理 demo 不适合看护 ========================== + # 1) 它不是训练脚本：201 行里没有优化器/loss/训练循环，只是下载预训练 + # 权重（dorsar/diffusion_policy/push_tblock.pt）跑一次 100 步去噪， + # 打印一条 16 步轨迹； + # 2) device 写死 cpu（__init__ 默认值 + __main__ 裸调用），全文零 CLI + # 参数——overlay 无处下手，不经 NPU，看护无意义； + # 3) checkpoint 是旧结构 state_dict：time_proj.weight [8] vs 当前 API 构造 + # 出的 [128]，load_state_dict 直接崩； + # 4) 无解：repo id 写死换不了 checkpoint，diffusers 是被看护对象不能降版， + # 唯一能跑的办法（ignore_mismatched_sizes / 手术 state_dict）等于把时间 + # 嵌入随机初始化，"预训练"失去意义，且都属 patch。 + - examples/reinforcement_learning/diffusion_policy.py + # ===== IP-Adapter 不支持（第三方包跟新版 diffusers 不兼容）================= + # 1) 这四个用例要用第三方的 ip_adapter 包，这是 diffusers 官方 examples + # 的 requirements.txt 里指定要装的。 + # 2) 这个包从 2023 年 12 月起就没发过新版。它代码里写死了一句导入： + # from diffusers.pipelines.controlnet import MultiControlNetModel + # 而 diffusers 0.41 把这个位置删了（类本身还在，挪到了 + # diffusers.models.controlnets.multicontrolnet），所以脚本一启动就报 + # ImportError。 + # 3) 换版本也救不了：PyPI 上只有 2023 年那一个版本；官方仓库的 main + # 分支也是同样的老代码，2024 年 6 月之后就没提交了。反过来把 + # diffusers 降回旧版倒是能跑，但 diffusers 本身就是我们看护的对象， + # 不能为了四个教程把它降级。 + # 4) 之前能跑绿，是因为 setup 在装完包后会 patch 把那行导入改掉。按 + # no-patch 这个做法已删除。 + # 恢复条件：等 ip_adapter 出适配 diffusers 0.41+ 的新版本（或上游把它 + # 修好）后，把这四条移回 supported。 + # faceid 另需自造人脸数据：JSON 清单 + insightface 的 normed_embedding + - examples/research_projects/ip_adapter/tutorial_train_ip-adapter.py + - examples/research_projects/ip_adapter/tutorial_train_plus.py + - examples/research_projects/ip_adapter/tutorial_train_sdxl.py + - examples/research_projects/ip_adapter/tutorial_train_faceid.py + # ===== wuerstchen prior / lora_prior：模块挪进 deprecated，用例没跟上 ===== + # diffusers 把整条 wuerstchen 管线挪进了 + # pipelines/deprecated/wuerstchen/——WuerstchenPrior、DEFAULT_STAGE_C_TIMESTEPS + # 都还在，顶层的 DDPMWuerstchenScheduler / WuerstchenPriorPipeline 也照常 + # 导出。但这两条例子 import 的还是旧路径 diffusers.pipelines.wuerstchen， + # 直接 ModuleNotFoundError。修复只需把例子里一行 import 改成 + # pipelines.deprecated.wuerstchen，但那属于改源码（patch）；diffusers 是被 + # 看护对象也不能钉旧版。与 transformers 无关，纯 diffusers 内部挪动。 + # 恢复条件：上游更新这两条例子的 import 路径 + - examples/research_projects/wuerstchen/text_to_image/train_text_to_image_prior.py + - examples/research_projects/wuerstchen/text_to_image/train_text_to_image_lora_prior.py + # ==== GATED_MODEL ======================================================== + # 需接受作者设定的许可条款并使用有效的 Hugging Face Token（HF_TOKEN） + # 进行身份验证才能下载模型 + # FLUX.1-dev FLUX.1-Kontext-dev FLUX.2-dev + # FLUX.2-klein-9B SD3-medium SD3.5-medium + # SD3.5-large ideogram-4-nf4 Cosmos-Predict2.5-2B + # Krea-2-Raw Krea-2-Turbo + - examples/controlnet/train_controlnet_flux.py + - examples/controlnet/train_controlnet_sd3.py + - examples/flux-control/train_control_flux.py + - examples/flux-control/train_control_lora_flux.py + - examples/advanced_diffusion_training/train_dreambooth_lora_flux_advanced.py + - examples/dreambooth/train_dreambooth_flux.py + - examples/dreambooth/train_dreambooth_lora_sd3.py + - examples/dreambooth/train_dreambooth_sd3.py + - examples/research_projects/sd3_lora_colab/train_dreambooth_lora_sd3_miniature.py + - examples/dreambooth/train_dreambooth_lora_flux.py + - examples/dreambooth/train_dreambooth_lora_flux_kontext.py + - examples/dreambooth/train_dreambooth_lora_flux2.py + - examples/dreambooth/train_dreambooth_lora_flux2_img2img.py + - examples/dreambooth/train_dreambooth_lora_ideogram4.py + - examples/cosmos/eval_cosmos_predict25_lora.py + - examples/cosmos/eval_lora.sh + - examples/cosmos/train_cosmos_predict25_lora.py + - examples/cosmos/train_lora.sh + # ==== CUDA硬编码类 ======================================================= + - examples/cogview4-control/train_control_cogview4.py + - examples/community/tiled_upscaling.py + - examples/community/dps_pipeline.py + - examples/community/run_onnx_controlnet.py + - examples/community/run_tensorrt_controlnet.py + - examples/research_projects/control_lora/control_lora.py + - examples/cosmos3/inference_cosmos3.py + - examples/profiling/profiling_pipelines.py + - examples/profiling/run_profiling.sh + - examples/research_projects/flux_lora_quantization/compute_embeddings.py # GATED_MODEL + - examples/research_projects/flux_lora_quantization/train_dreambooth_lora_flux_miniature.py # GATED_MODEL + - examples/research_projects/realfill/infer.py + - examples/research_projects/sd3_lora_colab/compute_embeddings.py # GATED_MODEL + - examples/server/server.py # GATED_MODEL + - examples/server-async/serverasync.py # GATED_MODEL + # device 只认 cuda/cpu + - examples/community/cogvideox_ddim_inversion.py + - examples/research_projects/pixart/run_pixart_alpha_controlnet_pipeline.py + # device_map="auto" + trust_remote_code + - examples/discrete_diffusion/sample_llada2.py + # colossalai（依赖 CUDA/NCCL） + - examples/research_projects/colossalai/inference.py + - examples/research_projects/colossalai/train_dreambooth_colossalai.py + # ===== transformers 版本不兼容 =========================================== + # 上游 transformers 5.x 的 API 变动导致这些脚本无法兼容： + # 1) custom_diffusion：新 transformers 把 modifier token 切成多个 token， + # 与 initializer_token 的 [32] 维度不一致 → RuntimeError: expand + # 2) Lumina2/Z-Image/QwenImage/Krea：transformers 5.x 下文本编码器 attention + # 的 q_proj.view() 形状错 → RuntimeError + # 3) multi_token_textual_inversion：transformers 5.x 起 CLIPTextModel 不再有 + # .text_model 属性 → AttributeError + # diffusers 0.4X 必须 transformers 5.x，无法降级 + - examples/custom_diffusion/train_custom_diffusion.py + - examples/dreambooth/train_dreambooth_lora_lumina2.py + - examples/dreambooth/train_dreambooth_lora_z_image.py + - examples/dreambooth/train_dreambooth_lora_qwenimage21.py + - examples/dreambooth/train_dreambooth_lora_qwenimage21_img2img.py + - examples/dreambooth/train_dreambooth_lora_krea2.py # GATED_MODEL + - examples/research_projects/multi_token_textual_inversion/textual_inversion.py + # ===== OOM（显存溢出） =================================================== + # 这些用例在 CI 的 a2-1/a2-4 等规格上因显存不足而 OOM 崩溃： + # 1) LCM 全量蒸馏：teacher + 学生 + EMA target 三张 SDXL unet 同时驻留显存， + # bf16 后仍 ~27.7GB，加梯度 >30GB/卡 > 29.5GB；ZeRO-3 又被 transformers + # 的 zero.Init 冻结模块 bug 打坏 + # 2) scheduled_huber 全量版：fork 缺 .to(dtype=weight_dtype)，DS 的 bf16/fp16 + # 不做 autocast → input fp32 与 bf16 权重在 conv_in 直接 dtype 冲突； + # 纯 fp32 下 DS 初始化平坦缓冲 ~19GB 叠加 base 后 38GB > 29.5GB OOM + # 3) LoRA SDXL LCM 蒸馏：forward 即 OOM（已分配 24.4GiB），DDP 不省单卡显存； + # ZeRO-3 又撞上游 bug——训练循环中 solver.ddim_step(...).to(unet.dtype)， + # 模型被 DeepSpeedEngine 包裹后无 .dtype + # 4) Qwen-Image（20B）+ ZeRO-3：host OOM（Signal 9 SIGKILL，内核 OOM killer）； + # 去掉 offload_param 后仍 OOM（DeepSpeed 初始化仍要把整个模型+文本编码器放到 CPU） + # 5) HiDream-I1-Dev（17B）+ ZeRO-3：:1140 在 prepare/分片之前无条件 + # transformer.to(device)，整只 17B bf16 ≈ 32GB 往单卡搬（a2-4 实测：27.4GB + # 已分配后 34MiB 都要不出）；每个 rank 执行同一整模型搬运，加卡无效。绕法 + # 只有让 zero.Init 在加载时分片，但那会打坏冻结、engine 不接管的 4 个文本 + # 编码器（SDXL CLIP 前科）。另：模型仓 ~44GB + Llama 镜像 ~15GB，无 + # ModelScope 镜像，下载负担也重。回归条件：脚本 DS-aware（zero3 下跳过 + # 整模型 .to）或换更大显存（>36GB）机型 + - examples/consistency_distillation/train_lcm_distill_sdxl_wds.py + - examples/research_projects/scheduled_huber_loss_training/text_to_image/train_text_to_image_sdxl.py + - examples/consistency_distillation/train_lcm_distill_lora_sdxl.py + - examples/dreambooth/train_dreambooth_lora_qwen_image.py + - examples/dreambooth/train_dreambooth_lora_hidream.py + # ===== NPU 算子/功能缺口 ================================================= + # 这些用例需要 NPU 尚未实现的 PyTorch 功能： + # SANA-Sprint 蒸馏：SCM 蒸馏损失硬依赖 torch.func.jvp（前向自动微分）， + # 而 NPU 的 matmul(linear) 未实现前向 AD → NotImplementedError + # 回归条件：torch_npu 补 matmul 的前向 AD，或上游提供非 jvp 的蒸馏实现 + - examples/research_projects/sana/train_sana_sprint_diffusers.py + # ===== 代码BUG ========================================================== + # 这些用例存在明确的代码缺陷，修复后可恢复： + # autoencoderkl：final validation 用 AutoencoderKL.from_pretrained(output_dir) + # 加载到 CPU，未 .to(accelerator.device)，NPU 上 conv 报 "weight is on cpu, + # different from ... npu:0"；该路径无条件执行，去掉 --validation_image 还会 + # enumerate(None) 报错 + - examples/research_projects/autoencoderkl/train_autoencoderkl.py + # ===== 外部依赖/环境不可用 ================================================ + # run_diffuser_locomotion：import d4rl + gym，gym.make("hopper-medium-v2")； + # README 要求 gym==0.24.1 + free-mujoco-py + d4rl（MuJoCo 二进制， + # aarch64/py3.12 装不上）。 + - examples/reinforcement_learning/run_diffuser_locomotion.py + # ===== OTHER_BACKEND（其他训练后端） ===================================== + # 这些用例使用非 PyTorch 原生训练后端，昇腾 NPU 不支持： + # 1) intel_extension_for_pytorch / neural_compressor：Intel 针对 Xeon/Core Ultra/ + # Intel GPU/Gaudi 的扩展，不是昇腾 NPU 后端 + # 2) onnxruntime training（ORT）：ORT-Trainer 原生只深度适配 NVIDIA CUDA 和 + # Intel 硬件，昇腾 NPU 无 ORT 训练后端 + # 3) torch_xla（TPU）：Google TPU 专用后端，与昇腾 NPU 无关 + - examples/research_projects/intel_opts/inference_bf16.py + - examples/research_projects/intel_opts/textual_inversion/textual_inversion_bf16.py + - examples/research_projects/intel_opts/textual_inversion_dfq/text2images.py + - examples/research_projects/intel_opts/textual_inversion_dfq/textual_inversion.py + - examples/research_projects/onnxruntime/text_to_image/train_text_to_image.py + - examples/research_projects/onnxruntime/textual_inversion/textual_inversion.py + - examples/research_projects/onnxruntime/unconditional_image_generation/train_unconditional.py + - examples/research_projects/pytorch_xla/inference/flux/flux_inference.py + - examples/research_projects/pytorch_xla/inference/flux/flux_inference_spmd.py + - examples/research_projects/pytorch_xla/training/text_to_image/train_text_to_image_xla.py + + # ===================================================================== + # ===== 工具类套件 + # ===================================================================== + # 上游自测与 pytest 基础设施（test_*.py / conftest.py），非可守卫 example。 + - examples/advanced_diffusion_training/test_dreambooth_lora_flux_advanced.py + - examples/conftest.py + - examples/consistency_distillation/test_lcm_lora.py + - examples/controlnet/test_controlnet.py + - examples/custom_diffusion/test_custom_diffusion.py + - examples/dreambooth/test_dreambooth.py + - examples/dreambooth/test_dreambooth_flux.py + - examples/dreambooth/test_dreambooth_lora.py + - examples/dreambooth/test_dreambooth_lora_edm.py + - examples/dreambooth/test_dreambooth_lora_flux.py + - examples/dreambooth/test_dreambooth_lora_flux_kontext.py + - examples/dreambooth/test_dreambooth_lora_flux2.py + - examples/dreambooth/test_dreambooth_lora_flux2_klein.py + - examples/dreambooth/test_dreambooth_lora_hidream.py + - examples/dreambooth/test_dreambooth_lora_lumina2.py + - examples/dreambooth/test_dreambooth_lora_qwenimage.py + - examples/dreambooth/test_dreambooth_lora_qwenimage21.py + - examples/dreambooth/test_dreambooth_lora_qwenimage21_img2img.py + - examples/dreambooth/test_dreambooth_lora_sana.py + - examples/dreambooth/test_dreambooth_lora_sd3.py + - examples/dreambooth/test_dreambooth_sd3.py + - examples/instruct_pix2pix/test_instruct_pix2pix.py + - examples/t2i_adapter/test_t2i_adapter.py + - examples/test_examples_utils.py + - examples/text_to_image/test_text_to_image.py + - examples/text_to_image/test_text_to_image_lora.py + - examples/textual_inversion/test_textual_inversion.py + - examples/textual_inversion/test_textual_inversion_sdxl.py + - examples/unconditional_image_generation/test_unconditional.py + - examples/vqgan/test_vqgan.py + # 包标识文件（__init__.py）。 + - examples/research_projects/wuerstchen/text_to_image/__init__.py + - examples/server-async/utils/__init__.py + # helper / 实现模块：被同目录 example import 或调用，非入口（utils、 + # profiling_utils、train_vqgan 的判别器、anytext OCR 实现、 + # promptdiffusion/pixart/rdm 的 pipeline 模块、gligen 的数据处理、 + # lpl 的 loss、multi_token 的 clip 封装、wuerstchen 的 encoder、 + # server-async 的 Pipelines/utils、cosmos3 的并行启动器）。 + - examples/cosmos3/cosmos_parallel.py + - examples/profiling/profiling_utils.py + - examples/research_projects/anytext/anytext.py + - examples/research_projects/anytext/anytext_controlnet.py + - examples/research_projects/anytext/ocr_recog/common.py + - examples/research_projects/anytext/ocr_recog/RecCTCHead.py + - examples/research_projects/anytext/ocr_recog/RecModel.py + - examples/research_projects/anytext/ocr_recog/RecMv1_enhance.py + - examples/research_projects/anytext/ocr_recog/RecSVTR.py + - examples/research_projects/anytext/ocr_recog/RNN.py + - examples/research_projects/gligen/dataset.py + - examples/research_projects/gligen/make_datasets.py + - examples/research_projects/lpl/lpl_loss.py + - examples/research_projects/multi_token_textual_inversion/multi_token_clip.py + - examples/research_projects/pixart/controlnet_pixart_alpha.py + - examples/research_projects/pixart/pipeline_pixart_alpha_controlnet.py + - examples/research_projects/promptdiffusion/pipeline_prompt_diffusion.py + - examples/research_projects/promptdiffusion/promptdiffusioncontrolnet.py + - examples/research_projects/rdm/pipeline_rdm.py + - examples/research_projects/rdm/retriever.py + - examples/research_projects/wuerstchen/text_to_image/modeling_efficient_net_encoder.py + - examples/server-async/Pipelines.py + - examples/server-async/utils/requestscopedpipeline.py + - examples/server-async/utils/scheduler.py + - examples/server-async/utils/utils.py + - examples/server-async/utils/wrappers.py + - examples/vqgan/discriminator.py + # 转换 / 数据预处理 / README 式启动脚本：纯配套工具，非训练入口 + # （sana_sprint 的 .sh 是 README 式下载+启动指南，其 .py 已 supported）。 + - examples/cosmos/create_prompts_for_gr1_dataset.py + - examples/cosmos/download_and_preprocess_datasets.sh + - examples/dreambooth/convert_to_imagefolder.py + - examples/research_projects/promptdiffusion/convert_original_promptdiffusion_to_diffusers.py + - examples/research_projects/sana/train_sana_sprint_diffusers.sh + # 外部服务 / 废弃 shim / 库模块 / HTTP 客户端：不可独立执行 + # （retrieve 需 knn.laion.ai；inference 两条是无 __main__ 的废弃 shim； + # pipeline_easy 是查 civitai API 的库模块；server-async/test.py 需常驻服务）。 + - examples/custom_diffusion/retrieve.py + - examples/inference/image_to_image.py + - examples/inference/inpainting.py + - examples/model_search/pipeline_easy.py + - examples/server-async/test.py + # community/ 其余 105 个无 __main__ 的 custom_pipeline 模块：只能经 + # DiffusionPipeline.from_pretrained(custom_pipeline=...) 加载； + # 有入口的 5 条见上方 CUDA_DEP / OTHER_BACKEND 组。 + - examples/community/adaptive_mask_inpainting.py + - examples/community/bit_diffusion.py + - examples/community/checkpoint_merger.py + - examples/community/clip_guided_images_mixing_stable_diffusion.py + - examples/community/clip_guided_stable_diffusion.py + - examples/community/clip_guided_stable_diffusion_img2img.py + - examples/community/composable_stable_diffusion.py + - examples/community/ddim_noise_comparative_analysis.py + - examples/community/edict_pipeline.py + - examples/community/fresco_v2v.py + - examples/community/gluegen.py + - examples/community/hd_painter.py + - examples/community/iadb.py + - examples/community/imagic_stable_diffusion.py + - examples/community/img2img_inpainting.py + - examples/community/instaflow_one_step.py + - examples/community/interpolate_stable_diffusion.py + - examples/community/ip_adapter_face_id.py + - examples/community/kohya_hires_fix.py + - examples/community/latent_consistency_img2img.py + - examples/community/latent_consistency_interpolate.py + - examples/community/latent_consistency_txt2img.py + - examples/community/llm_grounded_diffusion.py + - examples/community/lpw_stable_diffusion.py + - examples/community/lpw_stable_diffusion_onnx.py + - examples/community/lpw_stable_diffusion_xl.py + - examples/community/magic_mix.py + - examples/community/marigold_depth_estimation.py + - examples/community/masked_stable_diffusion_img2img.py + - examples/community/masked_stable_diffusion_xl_img2img.py + - examples/community/matryoshka.py + - examples/community/mixture_canvas.py + - examples/community/mixture_tiling.py + - examples/community/mixture_tiling_sdxl.py + - examples/community/mod_controlnet_tile_sr_sdxl.py + - examples/community/multilingual_stable_diffusion.py + - examples/community/one_step_unet.py + - examples/community/pipeline_animatediff_controlnet.py + - examples/community/pipeline_animatediff_img2video.py + - examples/community/pipeline_animatediff_ipex.py + - examples/community/pipeline_controlnet_xl_kolors.py + - examples/community/pipeline_controlnet_xl_kolors_img2img.py + - examples/community/pipeline_controlnet_xl_kolors_inpaint.py + - examples/community/pipeline_demofusion_sdxl.py + - examples/community/pipeline_fabric.py + - examples/community/pipeline_faithdiff_stable_diffusion_xl.py + - examples/community/pipeline_flux_differential_img2img.py + - examples/community/pipeline_flux_kontext_multiple_images.py + - examples/community/pipeline_flux_rf_inversion.py + - examples/community/pipeline_flux_semantic_guidance.py + - examples/community/pipeline_flux_with_cfg.py + - examples/community/pipeline_hunyuandit_differential_img2img.py + - examples/community/pipeline_kolors_differential_img2img.py + - examples/community/pipeline_kolors_inpainting.py + - examples/community/pipeline_null_text_inversion.py + - examples/community/pipeline_prompt2prompt.py + - examples/community/pipeline_sdxl_style_aligned.py + - examples/community/pipeline_stable_diffusion_3_differential_img2img.py + - examples/community/pipeline_stable_diffusion_3_instruct_pix2pix.py + - examples/community/pipeline_stable_diffusion_boxdiff.py + - examples/community/pipeline_stable_diffusion_pag.py + - examples/community/pipeline_stable_diffusion_upscale_ldm3d.py + - examples/community/pipeline_stable_diffusion_xl_attentive_eraser.py + - examples/community/pipeline_stable_diffusion_xl_controlnet_adapter.py + - examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py + - examples/community/pipeline_stable_diffusion_xl_differential_img2img.py + - examples/community/pipeline_stable_diffusion_xl_instandid_img2img.py + - examples/community/pipeline_stable_diffusion_xl_instantid.py + - examples/community/pipeline_stable_diffusion_xl_ipex.py + - examples/community/pipeline_stable_diffusion_xl_t5.py + - examples/community/pipeline_stg_cogvideox.py + - examples/community/pipeline_stg_hunyuan_video.py + - examples/community/pipeline_stg_ltx.py + - examples/community/pipeline_stg_ltx_image2video.py + - examples/community/pipeline_stg_mochi.py + - examples/community/pipeline_stg_wan.py + - examples/community/pipeline_z_image_differential_img2img.py + - examples/community/pipeline_zero1to3.py + - examples/community/pipline_flux_fill_controlnet_Inpaint.py + - examples/community/regional_prompting_stable_diffusion.py + - examples/community/rerender_a_video.py + - examples/community/scheduling_ufogen.py + - examples/community/sd_text2img_k_diffusion.py + - examples/community/sde_drag.py + - examples/community/seed_resize_stable_diffusion.py + - examples/community/speech_to_image_diffusion.py + - examples/community/stable_diffusion_comparison.py + - examples/community/stable_diffusion_controlnet_img2img.py + - examples/community/stable_diffusion_controlnet_inpaint.py + - examples/community/stable_diffusion_controlnet_inpaint_img2img.py + - examples/community/stable_diffusion_controlnet_reference.py + - examples/community/stable_diffusion_ipex.py + - examples/community/stable_diffusion_mega.py + - examples/community/stable_diffusion_reference.py + - examples/community/stable_diffusion_repaint.py + - examples/community/stable_diffusion_tensorrt_img2img.py + - examples/community/stable_diffusion_tensorrt_inpaint.py + - examples/community/stable_diffusion_tensorrt_txt2img.py + - examples/community/stable_diffusion_xl_controlnet_reference.py + - examples/community/stable_diffusion_xl_reference.py + - examples/community/stable_unclip.py + - examples/community/text_inpainting.py + - examples/community/unclip_image_interpolation.py + - examples/community/unclip_text_interpolation.py + - examples/community/wildcard_stable_diffusion.py + - examples/research_projects/pixart/train_controlnet_hf_diffusers.sh \ No newline at end of file diff --git a/examples-guards/diffusers/fixtures/configs/ds_zero2.json b/examples-guards/diffusers/fixtures/configs/ds_zero2.json new file mode 100644 index 000000000..7fcd63e8d --- /dev/null +++ b/examples-guards/diffusers/fixtures/configs/ds_zero2.json @@ -0,0 +1,20 @@ +{ + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": 1, + "gradient_accumulation_steps": "auto", + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true + }, + "overlap_comm": true, + "contiguous_gradients": true + }, + "bf16": { + "enabled": "auto" + }, + "fp16": { + "enabled": "auto" + } +} diff --git a/examples-guards/diffusers/fixtures/imagefolder/dog/DOG.jpg b/examples-guards/diffusers/fixtures/imagefolder/dog/DOG.jpg new file mode 100644 index 000000000..4fbe524a9 Binary files /dev/null and b/examples-guards/diffusers/fixtures/imagefolder/dog/DOG.jpg differ diff --git a/examples-guards/diffusers/fixtures/images/DOG.jpg b/examples-guards/diffusers/fixtures/images/DOG.jpg new file mode 100644 index 000000000..4fbe524a9 Binary files /dev/null and b/examples-guards/diffusers/fixtures/images/DOG.jpg differ diff --git a/examples-guards/diffusers/scripts/run_example.sh b/examples-guards/diffusers/scripts/run_example.sh new file mode 100644 index 000000000..6c1d34957 --- /dev/null +++ b/examples-guards/diffusers/scripts/run_example.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Run one diffusers example from a CI working copy of the target tree. +# $1 is the manifest entry path. EXEC, when set, names the launchable +# file relative to the target root; otherwise path itself must be a +# launchable file. Overlay CLI args come from OVERLAY_ARGS (JSON array, +# possibly []). +# +# Python examples are launched with `accelerate launch` (the upstream +# entrypoint for the training scripts). Never git add/commit/push. +# +# Contract: examples-guards/README.md "项目数据" (scripts/run_example.sh). +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +EXAMPLE_REL="$1" +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +CI_OUTPUT_DIR="${CI_OUTPUT_DIR:?CI_OUTPUT_DIR is required}" + +EXAMPLE_PATH="$TARGET_ROOT/$EXAMPLE_REL" +[[ -e "$EXAMPLE_PATH" ]] || { echo "example not found: $EXAMPLE_PATH" >&2; exit 1; } + +# Resolve the launchable file: EXEC (relative to the target root) when +# set, otherwise path itself. +if [[ -n "${EXEC:-}" ]]; then + LAUNCH_PATH="$TARGET_ROOT/$EXEC" +else + LAUNCH_PATH="$EXAMPLE_PATH" +fi +if [[ ! -f "$LAUNCH_PATH" ]]; then + echo "launchable file not found: $LAUNCH_PATH (directory examples need an exec field)" >&2 + exit 1 +fi + +mkdir -p "$CI_OUTPUT_DIR" + +if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +else + PYTHON=python +fi + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +python -c "import torch, torch_npu; print('NPU available:', torch.npu.is_available(), 'devices:', torch.npu.device_count())" + +expand_overlay() { + # The workflow serializes manifest.overlay_args as JSON. Expand each + # item with shell quoting intact, then allow CI paths such as + # ${CI_OUTPUT_DIR} / ${SD15_MODEL_PATH} to resolve only in this job's + # environment. + "$PYTHON" - <<'PY' +import json +import os +import shlex + +raw = os.environ.get('OVERLAY_ARGS', '').strip() +if not raw or raw in ('null', '""'): + raise SystemExit(0) +try: + items = json.loads(raw) +except json.JSONDecodeError as exc: + raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc +if items in (None, ''): + raise SystemExit(0) +if not isinstance(items, list): + raise SystemExit( + f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}') +tokens = [] +for item in items: + if not isinstance(item, str) or not item.strip(): + raise SystemExit('OVERLAY_ARGS items must be non-empty strings') + tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) +print(' '.join(shlex.quote(token) for token in tokens)) +PY +} + +eval "EXTRA_ARGS=( $(expand_overlay) )" + +echo "running $LAUNCH_PATH with ${#EXTRA_ARGS[@]} overlay args" +if ((${#EXTRA_ARGS[@]})); then + printf 'overlay arg: %q\n' "${EXTRA_ARGS[@]}" +fi + +# Visible NPU count for the multi-card launcher. +count_npus() { + "$PYTHON" -c "import torch, torch_npu; print(torch.npu.device_count())" +} + +# Materialize an accelerate+DeepSpeed launch config (num_processes = visible +# NPU count) from the project fixture fixtures/configs/ds_zero2.json. +prepare_deepspeed_configs() { + local ds_name="$1" + local ds_json="${PROJECT_ROOT:?PROJECT_ROOT is required}/fixtures/configs/$ds_name" + local cfg_dir="$GITHUB_WORKSPACE/ci_patch" + local cfg_file="$cfg_dir/${ds_name%.json}.yml" + mkdir -p "$cfg_dir" + local npus + npus=$(count_npus) + cat > "$cfg_file" <&2 + exit 2 + ;; + esac + ;; +esac diff --git a/examples-guards/diffusers/scripts/setup_example.sh b/examples-guards/diffusers/scripts/setup_example.sh new file mode 100644 index 000000000..ec55692e0 --- /dev/null +++ b/examples-guards/diffusers/scripts/setup_example.sh @@ -0,0 +1,859 @@ +#!/usr/bin/env bash +# Prepare the CI environment for one supported diffusers example. +# $1 is the manifest profile. Unknown profiles fail before any install. +# +# diffusers itself is installed from TARGET_ROOT (the release checkout +# under test), so the guarded tag is exactly the code that runs. +# +# Contract: examples-guards/README.md "项目数据" (scripts/setup_example.sh). +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +PROFILE="$1" + +# Validate the profile before installing anything (contract: unknown +# profile must exit non-zero before any install). This array is the single +# profile registry; a new profile also needs a PROFILE_* table entry or a +# bespoke setup_() function (see the Profiles section). +SUPPORTED_PROFILES=( + diffusers-sdxl diffusers-sd15 diffusers-dreambooth diffusers-instruct-pix2pix + diffusers-kandinsky diffusers-research diffusers-research-plain diffusers-t2i-adapter + diffusers-text-to-image diffusers-textual-inversion diffusers-unconditional + diffusers-vqgan diffusers-sdxl-online diffusers-sana diffusers-amused diffusers-cogvideo + diffusers-cogvideo-i2v diffusers-lcm diffusers-lcm-sdxl diffusers-controlnet + diffusers-controlnet-sdxl diffusers-llada2 diffusers-gligen diffusers-controlnet-wds + diffusers-orpo-wds diffusers-sdxl-full + diffusers-kandinsky-prior diffusers-multi-subject-inpaint diffusers-pixart + diffusers-realfill diffusers-flux2-klein +) +profile_known=0 +for _p in "${SUPPORTED_PROFILES[@]}"; do + if [[ "$_p" == "$PROFILE" ]]; then + profile_known=1 + fi +done +if [[ "$profile_known" != 1 ]]; then + echo "unknown profile: ${PROFILE} (supported: ${SUPPORTED_PROFILES[*]})" >&2 + exit 1 +fi + +CLUSTER_PIP_HOST=cache-service.nginx-pypi-cache.svc.cluster.local +export CLUSTER_PIP_INDEX="http://${CLUSTER_PIP_HOST}/pypi/simple" +ASCEND_PIP_INDEX=https://repo.huaweicloud.com/ascend/repos/pypi +ALIYUN_PIP_INDEX=https://mirrors.aliyun.com/pypi/simple/ + +pip_ascend() { + python -m pip install --extra-index-url "$ASCEND_PIP_INDEX" "$@" +} + +select_pip_index() { + # Runners live in mainland China: prefer the cluster pip cache, fall + # back to the Aliyun mirror. The ascend index stays available via + # PIP_EXTRA_INDEX_URL (set by the engine) for torch_npu wheels. + if python -c " +import os +import urllib.error +import urllib.request +try: + urllib.request.urlopen(os.environ['CLUSTER_PIP_INDEX'], timeout=3) +except urllib.error.HTTPError: + pass +" 2>/dev/null; then + export PIP_INDEX_URL="$CLUSTER_PIP_INDEX" + export PIP_TRUSTED_HOST="$CLUSTER_PIP_HOST" + else + export PIP_INDEX_URL="$ALIYUN_PIP_INDEX" + unset PIP_TRUSTED_HOST + fi + echo "pip index: $PIP_INDEX_URL" +} + +ensure_torch_stack() { + # Same torch line as diffusers quick-start (CANN 9.1.0 pairing): + # reuse the image stack when it already matches, otherwise install. + if python -c " +import torch, torch_npu +raise SystemExit( + 0 if torch.__version__.startswith('2.9.0') + and torch_npu.__version__.startswith('2.9.0') else 1) +"; then + echo "reusing image torch stack ($(python -c 'import torch; print(torch.__version__)'))" + return + fi + echo "installing torch==2.9.0 torch_npu==2.9.0.post2" + pip_ascend torch==2.9.0 torch_npu==2.9.0.post2 +} + +prepare_fixtures() { + # Optional: copy project fixtures into the target root so examples can + # read them under $TARGET_ROOT/fixtures/. No-op when FIXTURE_DIR is + # absent or empty (diffusers examples currently pull their datasets + # from ModelScope / HuggingFace during setup instead). + local src="${FIXTURE_DIR:-}" + local dst="$TARGET_ROOT/fixtures" + if [[ -z "$src" || ! -d "$src" ]]; then + echo "no fixtures to prepare ($src)" + return + fi + shopt -s nullglob + local files=("$src"/*) + shopt -u nullglob + if ((${#files[@]} == 0)); then + echo "no fixtures to prepare (empty $src)" + return + fi + echo "preparing fixtures from $src to $dst" + mkdir -p "$dst" + cp -r "${files[@]}" "$dst/" +} + +# --------------------------------------------------------------- +# Shared steps +# --------------------------------------------------------------- + +install_example_stack() { + echo "installing diffusers from $TARGET_ROOT" + python -m pip install -e "$TARGET_ROOT" + # Example stack. transformers 5.x is required by diffusers 0.40+ + # (huggingface-hub>=1.23 vs transformers 4.x's hub<1.0 conflict); + # modelscope pinned to 1.37.0 (hub split started at 1.38 and 1.40.1's + # "modelscope-hub>=0.4.2" floor dies on DEFAULT_CREDENTIALS_PATH); + # torchvision pinned to the torch 2.9.0 pairing; prodigyopt is imported + # by the advanced scripts' --optimizer=prodigy and is not in + # examples/advanced_diffusion_training/requirements.txt. + # datasets<4.0.0: diffusers examples (e.g. controlnet's load_dataset path) + # break on datasets 4.x; the whole project stays below 4. + # wandb: every training script does `if is_wandb_available(): import wandb` + # and several default --report_to to wandb; installed once here, kept + # offline by the global WANDB_MODE=disabled. + python -m pip install \ + "transformers>=5.0,<6.0" "accelerate>=1.0,<2.0" "peft>=0.6" \ + "datasets<4.0.0" ftfy tensorboard Jinja2 sentencepiece "torchvision==0.24.0" \ + prodigyopt "modelscope==1.37.0" wandb + python -c "import diffusers, transformers, accelerate, peft; print('diffusers', diffusers.__version__, '/ transformers',\ + transformers.__version__, '/ accelerate', accelerate.__version__, '/ peft', peft.__version__)" +} + +# download_assets : comma-separated tokens from +# {sd15, sd15-inpaint, 3d-icon, cogvideo, cogvideo-dataset, sana-sprint}. Each +# token exports a path to GITHUB_ENV under the name overlay_args reference: +# sd15 -> SD15_MODEL_PATH (AI-ModelScope/stable-diffusion-v1-5) +# sd15-inpaint -> SD15_INPAINT_PATH (AI-ModelScope/stable-diffusion-inpainting) +# 3d-icon -> THREE_D_ICON_PATH (HF linoyts/3d_icon) +# cogvideo -> COGVIDEOX_MODEL_PATH + COGVIDEOX_DATASET_DIR +# cogvideo-dataset -> COGVIDEOX_DATASET_DIR +# sana-sprint -> SANA_SPRINT_TEACHER_PATH (HF) +download_assets() { + DIFFUSERS_DOWNLOAD="$1" python - <<'PY' +import os +import sys +from pathlib import Path + +os.environ.setdefault("TQDM_MININTERVAL", "15") + +from modelscope import snapshot_download + +WORKSPACE = Path(os.environ["GITHUB_WORKSPACE"]) +ENV_FILE = os.environ["GITHUB_ENV"] +MODEL_CACHE = Path(os.environ.get("MODELSCOPE_CACHE", os.path.expanduser("~/.cache/modelscope"))) +MODEL_CACHE.mkdir(parents=True, exist_ok=True) +WANT = {item.strip() for item in os.environ.get("DIFFUSERS_DOWNLOAD", "").split(",") if item.strip()} + +# ModelScope's snapshot_download gives up on a single file once its own +# retries are exhausted (e.g. "1 file(s) failed to download out of 16"), +# which fails the whole setup. Re-run it a few times: each pass resumes / +# re-fetches only what is still missing. +DOWNLOAD_ATTEMPTS = 3 + +exports: dict[str, str] = {} +failures: list[str] = [] + + +def export(name: str, path: str) -> None: + exports[name] = path + print(f"{name}={path}", flush=True) + + +def corrupt_weights(root: Path) -> list[Path]: + # snapshot_download trusts files that already exist, so a truncated one is + # never re-fetched on its own: detect and delete the bad files here so the + # retry loop re-downloads them. + # * .safetensors -> safe_open fails on a half-written header + # ("incomplete metadata, file not fully covered"); + # * .bin -> diffusers' .bin checkpoints are torch zip archives, so a + # truncated file is no longer a readable zip (stable-diffusion-inpainting + # ships .bin only, and a partial download broke `from_pretrained`). + import zipfile + + from safetensors import safe_open + + bad: list[Path] = [] + for path in sorted(root.rglob("*.safetensors")): + try: + with safe_open(path, framework="pt") as handle: + handle.keys() + except Exception as exc: # noqa: BLE001 + print(f"corrupt {path}: {exc}", flush=True) + bad.append(path) + for path in sorted(root.rglob("*.bin")): + try: + with zipfile.ZipFile(path) as archive: + if archive.testzip() is None: + continue + except (zipfile.BadZipFile, OSError, EOFError): + pass + print(f"corrupt {path}: zip archive is unreadable or has bad entries", flush=True) + bad.append(path) + return bad + + +def snapshot(name: str, ms_id: str, **kwargs) -> None: + last: Exception | None = None + for attempt in range(1, DOWNLOAD_ATTEMPTS + 1): + try: + local = Path(snapshot_download(ms_id, cache_dir=str(MODEL_CACHE), **kwargs)) + bad = corrupt_weights(local) + if bad: + for path in bad: + path.unlink() + raise RuntimeError(f"{len(bad)} corrupt weight file(s) removed; re-downloading") + export(name, str(local)) + return + except Exception as exc: # noqa: BLE001 - retry, then report + last = exc + print(f"retry {attempt}/{DOWNLOAD_ATTEMPTS} {ms_id}: {type(exc).__name__}: {exc}", flush=True) + failures.append(f"{ms_id}: {type(last).__name__}: {last}") + print(f"FAIL {ms_id}: {last}", flush=True) + + +def hf_snapshot(repo_id: str, **kwargs) -> str: + from huggingface_hub import snapshot_download as hf_snapshot_download + + last: Exception | None = None + for attempt in range(1, DOWNLOAD_ATTEMPTS + 1): + try: + return hf_snapshot_download(repo_id, **kwargs) + except Exception as exc: # noqa: BLE001 - retry, then report + last = exc + print(f"retry {attempt}/{DOWNLOAD_ATTEMPTS} {repo_id}: {type(exc).__name__}: {exc}", flush=True) + raise last + + +# SD 1.5 diffusers components only; skip the single-file checkpoints +# (v1-5-pruned*.safetensors, ~4 GB each). Also the LCM SD teacher. +if "sd15" in WANT: + snapshot( + "SD15_MODEL_PATH", + "AI-ModelScope/stable-diffusion-v1-5", + allow_file_pattern=[ + "*.json", "*.txt", "*.model", + "unet/*", "vae/*", + "text_encoder/*", "tokenizer/*", "scheduler/*", + # The training scripts end by building StableDiffusionPipeline from + # this path, which loads the safety_checker / feature_extractor too, + # so both components must be on disk. + "safety_checker/*", "feature_extractor/*", + ], + # Drop the fp16 duplicates and the large unet/vae/text_encoder .bin + # duplicates (the repo ships safetensors for those). safety_checker's + # .bin is kept: some mirrors ship that component as .bin only. + ignore_file_pattern=["*.fp16.*", "unet/*.bin", "vae/*.bin", "text_encoder/*.bin"], + ) + +# SD 1.5 inpainting (9-channel unet: 4 latent + 4 masked-latent + 1 mask). +# Only entries that train an inpainting unet need it. Same component-only +# filter as SD1.5, plus safety_checker: realfill's final +# StableDiffusionInpaintPipeline.from_pretrained only overrides unet/text_encoder +# (no safety_checker=None), so that component must be on disk too. +if "sd15-inpaint" in WANT: + snapshot( + "SD15_INPAINT_PATH", + "AI-ModelScope/stable-diffusion-inpainting", + allow_file_pattern=[ + "*.json", "*.txt", "*.model", + "unet/*", "vae/*", + "text_encoder/*", "tokenizer/*", + "scheduler/*", "feature_extractor/*", "safety_checker/*", + ], + # This repo ships only .bin + *.fp16.safetensors (no full-precision + # safetensors), so keep .bin and drop only the fp16 duplicates. + ignore_file_pattern=["*.fp16.*"], + ) + +# CogVideoX-2b (transformer + T5 text_encoder + VAE). The ModelScope repo +# layout is already clean (component dirs only, no single-file/fp16 dupes). +# Dataset: Wild-Heart/Disney-VideoGeneration-Dataset (69 videos, ~25 MB, +# Steamboat Willie clips) already in the README's first format: +# prompt.txt + videos.txt + videos/. No ModelScope mirror, so pull via +# HuggingFace directly (docs runners reach huggingface.co; the engine +# leaves HF_ENDPOINT / Xet at their defaults, so Xet-backed repos +# download natively). +def download_disney_dataset() -> None: + try: + dataset_dir = WORKSPACE / "datasets" / "disney" + dataset_dir.mkdir(parents=True, exist_ok=True) + hf_snapshot( + "Wild-Heart/Disney-VideoGeneration-Dataset", + repo_type="dataset", + local_dir=str(dataset_dir), + ) + export("COGVIDEOX_DATASET_DIR", str(dataset_dir)) + except Exception as exc: # noqa: BLE001 + failures.append(f"Wild-Heart/Disney-VideoGeneration-Dataset: {type(exc).__name__}: {exc}") + print(f"FAIL Wild-Heart/Disney-VideoGeneration-Dataset: {exc}", flush=True) + + +# cogvideo: CogVideoX-2b (ModelScope) + the Disney dataset. +if "cogvideo" in WANT: + snapshot( + "COGVIDEOX_MODEL_PATH", + "AI-ModelScope/CogVideoX-2b", + allow_file_pattern=[ + "*.json", "*.model", + "scheduler/*", "text_encoder/*", "tokenizer/*", + "transformer/*", "vae/*", + ], + ) + download_disney_dataset() + +# cogvideo-dataset: the Disney dataset only (the I2V entry pulls its own +# CogVideoX-5b-I2V from HuggingFace at run time, so no ModelScope pre-download). +if "cogvideo-dataset" in WANT: + download_disney_dataset() + +# 3d_icon dataset (only the advanced dreambooth entries use it): no +# ModelScope mirror (205 MB imagefolder + metadata.jsonl), so pull it from +# HuggingFace into the workspace and point --dataset_name at the local directory. +if "3d-icon" in WANT: + try: + dataset_dir = WORKSPACE / "datasets" / "3d_icon" + dataset_dir.mkdir(parents=True, exist_ok=True) + hf_snapshot( + "linoyts/3d_icon", + repo_type="dataset", + local_dir=str(dataset_dir), + ignore_patterns=[".gitattributes"], + ) + export("THREE_D_ICON_PATH", str(dataset_dir)) + except Exception as exc: # noqa: BLE001 + failures.append(f"linoyts/3d_icon: {type(exc).__name__}: {exc}") + print(f"FAIL linoyts/3d_icon: {exc}", flush=True) + +# sana-sprint: the SANA-Sprint teacher model for research_projects/sana. The +# script reads /transformer/diffusion_pytorch_model.safetensors directly +# (load_file, not from_pretrained), so --pretrained_model_name_or_path must be +# a LOCAL directory; pre-download it here. +if "sana-sprint" in WANT: + try: + dest = WORKSPACE / "sana_sprint_teacher" + dest.mkdir(parents=True, exist_ok=True) + hf_snapshot( + "Efficient-Large-Model/SANA_Sprint_1.6B_1024px_teacher_diffusers", + local_dir=str(dest), + ) + export("SANA_SPRINT_TEACHER_PATH", str(dest)) + except Exception as exc: # noqa: BLE001 + failures.append(f"SANA_Sprint_1.6B: {type(exc).__name__}: {exc}") + print(f"FAIL SANA_Sprint_1.6B: {exc}", flush=True) + +if exports: + with open(ENV_FILE, "a", encoding="utf-8") as handle: + for key, value in exports.items(): + handle.write(f"{key}={value}\n") + +if failures: + print(f"setup incomplete: {failures}", file=sys.stderr, flush=True) + raise SystemExit(1) +PY +} + +# --------------------------------------------------------------- +# Profiles +# --------------------------------------------------------------- +# +# install_example_stack() is the common stack; a profile declares only its +# delta on top of it in the tables below, and profiles absent from all +# three tables get the base stack alone. When adding a new example, diff +# its requirements.txt against the base, add the profile to the +# SUPPORTED_PROFILES registry at the top, and extend the tables (or add a +# setup_() function for genuinely bespoke steps); never +# `pip install -r` it verbatim (it pins to git main / loose versions and +# pulls CUDA-only deps). + +# deepspeed serves the accelerate-deepspeed (ZeRO-2) launcher; decord2 is the +# Ascend/aarch64 build of decord (plain decord has no NPU wheel). +declare -A PROFILE_PIP=( + [diffusers-text-to-image]="deepspeed>=0.18.2" + [diffusers-vqgan]="timm" + [diffusers-lcm]="webdataset braceexpand" + [diffusers-lcm-sdxl]="webdataset braceexpand" + [diffusers-cogvideo]="decord2 imageio imageio-ffmpeg" + [diffusers-cogvideo-i2v]="decord2 imageio imageio-ffmpeg" + [diffusers-sdxl-full]="deepspeed>=0.18.2" + [diffusers-controlnet-sdxl]="deepspeed>=0.18.2" + [diffusers-kandinsky-prior]="deepspeed>=0.18.2" + # controlnet-wds runs with launcher=accelerate-deepspeed (ZeRO-2), hence + # deepspeed; the rest mirrors its vendored-webdataset requirements. + [diffusers-controlnet-wds]="webdataset==0.2.86 braceexpand opencv-python-headless deepspeed>=0.18.2" + [diffusers-orpo-wds]="webdataset braceexpand" + # hf_transfer: parallel hub downloads for the pre-downloaded models. + [diffusers-pixart]="hf_transfer" + [diffusers-flux2-klein]="hf_transfer" +) + +# download_assets tokens (comma-separated) for the profile's models/datasets. +declare -A PROFILE_ASSETS=( + [diffusers-sd15]="sd15,3d-icon" + [diffusers-dreambooth]="sd15" + [diffusers-instruct-pix2pix]="sd15" + [diffusers-research]="sd15" + [diffusers-text-to-image]="sd15" + [diffusers-textual-inversion]="sd15" + [diffusers-controlnet]="sd15" + [diffusers-lcm]="sd15" + [diffusers-cogvideo]="cogvideo" + [diffusers-cogvideo-i2v]="cogvideo-dataset" + [diffusers-sana]="sana-sprint" + [diffusers-multi-subject-inpaint]="sd15-inpaint" + [diffusers-realfill]="sd15-inpaint" +) + +# fusing/fill50k is a script-based dataset, so the controlnet examples need it. +declare -A PROFILE_ENV=( + [diffusers-controlnet]="HF_DATASETS_TRUST_REMOTE_CODE=1" + [diffusers-controlnet-sdxl]="HF_DATASETS_TRUST_REMOTE_CODE=1" +) + +# --------------------------------------------------------------- +# Shared helper for bespoke profiles: pre-download a hub repo into +# / and export the local path as for the +# run step (local paths also avoid from_pretrained's model_info round-trip). +# allow/ignore are comma-separated glob lists; "-" means "not used". +# Goes straight to huggingface.co (the engine leaves HF_ENDPOINT +# unset); speeds up with hf_transfer when the +# profile pulled it in via PROFILE_PIP (run_profile sets the switch). +# --------------------------------------------------------------- +hub_snapshot() { + local repo="$1" dest="$2" env_var="$3" allow="$4" ignore="$5" + HUB_SNAPSHOT_REPO="$repo" HUB_SNAPSHOT_DEST="$dest" HUB_SNAPSHOT_VAR="$env_var" \ + HUB_SNAPSHOT_ALLOW="$allow" HUB_SNAPSHOT_IGNORE="$ignore" python3 - <<'PY' +import os +from pathlib import Path + +from huggingface_hub import snapshot_download + +repo = os.environ["HUB_SNAPSHOT_REPO"] +dest = Path(os.environ["GITHUB_WORKSPACE"]) / os.environ["HUB_SNAPSHOT_DEST"] +dest.mkdir(parents=True, exist_ok=True) +kwargs = {"local_dir": str(dest)} +allow = os.environ.get("HUB_SNAPSHOT_ALLOW", "-") +ignore = os.environ.get("HUB_SNAPSHOT_IGNORE", "-") +if allow != "-": + kwargs["allow_patterns"] = [p for p in allow.split(",") if p] +if ignore != "-": + kwargs["ignore_patterns"] = [p for p in ignore.split(",") if p] +snapshot_download(repo, **kwargs) +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"{os.environ['HUB_SNAPSHOT_VAR']}={dest}\n") +print(f"pre-downloaded {repo} -> {dest}", flush=True) +PY +} + +# diffusers-gligen: research_projects/gligen text-box training. Base stack only: +# dataset.py needs just torch/PIL/torchvision, and make_datasets.py's +# GroundingDINO + BLIP2 + CLIP pipeline is not used. The model +# (masterful/gligen-1-4-generation-text-box) is fetched online from +# HuggingFace by +# the example itself, so only the dataset is synthesized here: a COCO-style +# torch.load .pth (list of {file_path, captions, annos}) plus the image dir it +# references. Each anno is {bbox: xyxy pixels, text_embeddings_before_projection: +# 768-d}; that embedding is a constant conditioning input (never optimized), so +# zeros are fine. A 512x512 image makes --resolution 512 a no-op. +setup_diffusers_gligen() { + python3 - <<'PY' +import os + +import torch +from PIL import Image + +root = os.path.join(os.environ["GITHUB_WORKSPACE"], "areal_data", "gligen") +image_dir = os.path.join(root, "images") +os.makedirs(image_dir, exist_ok=True) + +samples = [] +for i in range(4): + file_name = f"sample{i}.png" + Image.new("RGB", (512, 512), (110 + i * 20, 130, 150)).save(os.path.join(image_dir, file_name)) + samples.append( + { + "file_path": file_name, + "captions": ["a gray square on a plain background"], + "annos": [ + { + "bbox": [100.0, 100.0, 400.0, 400.0], + "text_embeddings_before_projection": torch.zeros(768), + } + ], + } + ) + +data_path = os.path.join(root, "gligen_train.pth") +torch.save(samples, data_path) + +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"GLIGEN_DATA_PTH={data_path}\n") + fh.write(f"GLIGEN_IMAGE_DIR={image_dir}\n") +print(f"prepared gligen dataset at {data_path} (images in {image_dir})", flush=True) +PY +} + +# controlnet-webdataset: webdataset pinned to 0.2.86 (the example's vendored +# tar iterators predate the 0.2.96 shard-end separator and KeyError on it); +# its dataloader hardcodes a 1000-sample shuffle buffer, so the shard must +# carry >1000 filter-passing samples - synthesized here because the upstream +# laion URL is unreachable. Each sample: jpg+png (image and control_image +# rename from image extensions), txt, and a json with original_width/height +# >= 512 + pwatermark for WebdatasetFilter. NB pwatermark must be truthy AND +# <= 0.5: the filter computes `(x or 1.0)`, so 0.0 evaluates to 1.0 and the +# sample is rejected - a rejected sample never enters the shuffle buffer and +# the run hangs silently (pwatermark=0.0 here cost a 55-min CI hang; 0.1 +# passes). opencv-headless for the canny control image. +setup_diffusers_controlnet_wds() { + python3 - <<'PY' +import io +import json +import os +import tarfile + +from PIL import Image, ImageDraw + +num_samples = 1024 +size = 512 +root = os.path.join(os.environ["GITHUB_WORKSPACE"], "controlnet_wds") +os.makedirs(root, exist_ok=True) +tar_path = os.path.join(root, "train-00000.tar") +meta = json.dumps({"original_width": size, "original_height": size, "pwatermark": 0.1}).encode() + + +def make_image(i): + image = Image.new("RGB", (size, size), (60 + i % 180, 120, 160)) + draw = ImageDraw.Draw(image) + draw.rectangle([64, 64, size - 64, size - 64], outline=(240, 240, 240), width=6) + draw.line([0, i % size, size, (i * 7) % size], fill=(250, 200, 100), width=4) + return image + + +def add(tf, name, data): + info = tarfile.TarInfo(name) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + + +with tarfile.open(tar_path, "w") as tf: + for i in range(num_samples): + key = f"{i:06d}" + image = make_image(i) + for ext, fmt in (("jpg", "JPEG"), ("png", "PNG")): + buf = io.BytesIO() + image.save(buf, format=fmt) + add(tf, f"{key}.{ext}", buf.getvalue()) + add(tf, f"{key}.txt", f"a synthetic line drawing scene {i}".encode()) + add(tf, f"{key}.json", meta) + +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"CONTROLNET_WDS_SHARDS={tar_path}\n") +print(f"prepared {num_samples}-sample webdataset shard at {tar_path}", flush=True) +PY +} + +# orpo-webdataset: the upstream preference dataset (diffusion-preference-opt on +# S3) went private, so the shard is synthesized here. Sample layout mirrors +# get_dataset()'s rename: {key}.original_prompt.txt / jpg_0.jpg / jpg_1.jpg / +# label_0.txt / label_1.txt (base_plus_ext groups by the first dot). The +# dataloader's shuffle buffer is 690, so the shard carries 1024 samples. +# Unlike controlnet-webdataset this uses the stock wds pipeline - no version +# pin needed. label txt values decode to str, so the == 0.5/== 0 comparisons +# never fire and the pair order is deterministically [jpg_0, jpg_1]. +setup_diffusers_orpo_wds() { + python3 - <<'PY' +import io +import os +import tarfile + +from PIL import Image, ImageDraw + +num_samples = 1024 +size = 512 +root = os.path.join(os.environ["GITHUB_WORKSPACE"], "orpo_wds") +os.makedirs(root, exist_ok=True) +tar_path = os.path.join(root, "train-00000.tar") + + +def make_image(i): + image = Image.new("RGB", (size, size), (50 + i % 200, 110, 170)) + draw = ImageDraw.Draw(image) + draw.ellipse([64 + i % 64, 64, size - 64, size - 64 - i % 64], outline=(240, 240, 240), width=6) + return image + + +def add(tf, name, data): + info = tarfile.TarInfo(name) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + + +with tarfile.open(tar_path, "w") as tf: + for i in range(num_samples): + key = f"{i:06d}" + for name, image in (("jpg_0", make_image(i)), ("jpg_1", make_image(i + 256))): + buf = io.BytesIO() + image.save(buf, format="JPEG") + add(tf, f"{key}.{name}.jpg", buf.getvalue()) + add(tf, f"{key}.original_prompt.txt", f"a synthetic ellipse scene {i}".encode()) + add(tf, f"{key}.label_0.txt", b"1.0") + add(tf, f"{key}.label_1.txt", b"0.0") + +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"ORPO_WDS_SHARDS={tar_path}\n") +print(f"prepared {num_samples}-sample orpo webdataset shard at {tar_path}", flush=True) +PY +} + +# sana (sprint distill): teacher from ModelScope/HuggingFace, plus a local parquet +# fixture. The script's __getitem__ does Image.open(io.BytesIO(item["image"])), +# and the upstream dataset's image column is auto-decoded to PIL by datasets +# (TypeError). A hand-built parquet keeps "image" as plain binary so datasets +# returns bytes. Columns must be exactly llava (str) + image (bytes). +setup_diffusers_sana() { + python3 - <<'PY' +import io +import os + +import pyarrow as pa +import pyarrow.parquet as pq +from PIL import Image, ImageDraw + +num_samples = 64 +size = 512 +root = os.path.join(os.environ["GITHUB_WORKSPACE"], "sana_fixture") +os.makedirs(root, exist_ok=True) +path = os.path.join(root, "sana_train.parquet") + +prompts = [] +images = [] +for i in range(num_samples): + image = Image.new("RGB", (size, size), (60 + i % 180, 120, 160)) + draw = ImageDraw.Draw(image) + draw.rectangle([64, 64, size - 64, size - 64], outline=(240, 240, 240), width=6) + buf = io.BytesIO() + image.save(buf, format="JPEG") + images.append(buf.getvalue()) + prompts.append(f"a synthetic line drawing scene {i}") + +table = pa.table( + { + "llava": pa.array(prompts, type=pa.string()), + "image": pa.array(images, type=pa.binary()), + } +) +pq.write_table(table, path) +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"SANA_FIXTURE_PARQUET={path}\n") +print(f"prepared {num_samples}-sample sana fixture parquet at {path}", flush=True) +PY +} + +setup_diffusers_multi_subject_inpaint() { + # multi_subject_dreambooth_inpainting 的 Dataset 用 load_dataset(path) 读本地 + # 目录（不是 load_from_disk），固定取 train/test 两个 split，且列里要有一个 + # 名字含 "mask" 的列。所以仿 Hub 数据集的落盘布局：平铺的 + # train-*.parquet / test-*.parquet，load_dataset 按扩展名推断 parquet builder、 + # 按文件名推断 split，features（Image）存在 schema metadata 里，读回即 PIL。 + # 模型必须是 9 通道的 inpainting 版（不是基础版 SD1.5），由 PROFILE_ASSETS + # 表预下。 + python3 - <<'PY' +import os +from pathlib import Path + +from datasets import Dataset, Features, Image, Value +from PIL import Image as PILImage +from PIL import ImageDraw + +size = 512 +num_samples = 4 +prompts = [f"a photo of sks dog in scene {i}" for i in range(num_samples)] +images = [] +masks = [] +for i in range(num_samples): + image = PILImage.new("RGB", (size, size), (60 + i % 180, 120, 160)) + ImageDraw.Draw(image).rectangle([64, 64, size - 64, size - 64], outline=(240, 240, 240), width=6) + images.append(image) + mask = PILImage.new("L", (size, size), 0) + ImageDraw.Draw(mask).rectangle([64, 64, size - 64, size - 64], fill=255) + masks.append(mask) + +features = Features({"image": Image(), "prompt": Value("string"), "mask": Image()}) +split = Dataset.from_dict({"image": images, "prompt": prompts, "mask": masks}, features=features) + +root = Path(os.environ["GITHUB_WORKSPACE"]) / "multi_subject_inpaint_fixture" +root.mkdir(parents=True, exist_ok=True) +split.to_parquet(str(root / "train-00000-of-00001.parquet")) +split.to_parquet(str(root / "test-00000-of-00001.parquet")) +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"MULTI_SUBJECT_INPAINT_DATASET={root}\n") +print(f"prepared multi-subject inpainting fixture (train+test parquet) at {root}", flush=True) +PY +} + +setup_diffusers_realfill() { + # train_realfill.py 的 RealFillDataset 认死一个目录结构： + # /ref/*.png + /target/target.png + /target/mask.png + # 缺任一就直接 ValueError（:445）。这里现场合成一个最小 fixture（纯合成图，零下载）。 + # 模型是 SD1.5-inpainting（同样 9 通道），由 PROFILE_ASSETS 表预下。 + python3 - <<'PY' +import os +from pathlib import Path + +from PIL import Image, ImageDraw + +size = 512 +root = Path(os.environ["GITHUB_WORKSPACE"]) / "realfill_fixture" +(ref := root / "ref").mkdir(parents=True, exist_ok=True) +(target := root / "target").mkdir(parents=True, exist_ok=True) + + +def scene(seed): + img = Image.new("RGB", (size, size), (40 + seed % 160, 110, 150)) + draw = ImageDraw.Draw(img) + draw.rectangle([80, 80, size - 80, size - 80], outline=(240, 240, 240), width=8) + draw.ellipse([180, 180, 330, 330], fill=(210, 90, 60)) + return img + + +for i in range(3): + scene(i).save(ref / f"ref{i}.png") +scene(9).save(target / "target.png") +mask = Image.new("L", (size, size), 0) +ImageDraw.Draw(mask).rectangle([80, 80, size - 80, size - 80], fill=255) +mask.save(target / "mask.png") + +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"REALFILL_DATASET={root}\n") +print(f"prepared realfill fixture at {root}", flush=True) +PY +} + +setup_diffusers_pixart() { + # PixArt-alpha/PixArt-XL-2-512x512 在 ModelScope 无镜像，只能走 HuggingFace 直连。 + # 它的 text_encoder 是 T5-XXL：脚本按默认（不带 --variant）加载，只会用 fp32 + # 权重（4 分片 ~19GB），另有一套 fp16 分片（~9GB）用不上——预下时跳过 *.fp16.* + # 省掉一半，且只下组件目录（asset/ 里的 README 图/样例文本下载慢、易超时）。 + hub_snapshot "PixArt-alpha/PixArt-XL-2-512x512" pixart_model PIXART_MODEL_PATH \ + "model_index.json,scheduler/*,tokenizer/*,text_encoder/*,transformer/*,vae/*" \ + "*.fp16.*,*.bin,*.ckpt" +} + +setup_diffusers_flux2_klein() { + # FLUX.2-klein-4B：BFL 的开放权重档（实测非 gated、匿名可取）。预下只取组件 + # 目录（跳过顶层 ~7.2GB 单文件 checkpoint 重复）。img2img 条目的 + # DreamBoothDataset 会把整个数据集急切物化进宿主内存（fill50k 的 1000 对 + # 图+条件图直接把 RAM 吃穿 → 内核 SIGKILL），脚本又没有 max_train_samples + # 切片参数，所以现场合成 8 对小数据集（列 image/conditioning_image/text）。 + hub_snapshot "black-forest-labs/FLUX.2-klein-4B" flux2_klein_model FLUX2_KLEIN_MODEL_PATH \ + "model_index.json,scheduler/*,tokenizer/*,text_encoder/*,transformer/*,vae/*" - + python3 - <<'PY' +import os +from pathlib import Path + +from datasets import Dataset, Features, Image as ImageFeature, Value +from PIL import Image, ImageDraw + +size = 512 +images, conds, captions = [], [], [] +for i in range(8): + img = Image.new("RGB", (size, size), (60 + i % 180, 120, 160)) + ImageDraw.Draw(img).rectangle([64, 64, size - 64, size - 64], outline=(240, 240, 240), width=6) + cond = Image.new("RGB", (size, size), (30, 30, 30)) + ImageDraw.Draw(cond).line([0, (i * 61) % size, size, (i * 97) % size], fill=(255, 255, 255), width=8) + images.append(img) + conds.append(cond) + captions.append(f"a synthetic line drawing scene {i}") + +features = Features( + { + "image": ImageFeature(), + "conditioning_image": ImageFeature(), + "text": Value("string"), + } +) +ds = Dataset.from_dict( + {"image": images, "conditioning_image": conds, "text": captions}, features=features +) +data_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "flux2_klein_img2img_dataset" +data_dir.mkdir(parents=True, exist_ok=True) +ds.to_parquet(str(data_dir / "train-00000-of-00001.parquet")) + +with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"FLUX2_KLEIN_DATASET={data_dir}\n") +print(f"prepared img2img dataset at {data_dir}", flush=True) +PY +} + +# Base stack + the three tables ALWAYS apply; a bespoke setup_() +# function, when present, runs ON TOP of them for the genuinely bespoke steps +# (hub model pre-downloads, fixture synthesis) instead of replacing everything. +# This keeps pip/env/assets registration in one place: a profile that needs +# deepspeed just gets a PROFILE_PIP row - no bespoke function can forget it. +run_profile() { + install_example_stack + if [[ -n "${PROFILE_PIP[$PROFILE]:-}" ]]; then + python -m pip install ${PROFILE_PIP[$PROFILE]} + fi + # hf_transfer (pulled in via PROFILE_PIP above): parallel hub downloads for + # both the setup-time hub_snapshot calls and the run step. + if [[ "${PROFILE_PIP[$PROFILE]:-}" == *hf_transfer* ]] && python -c "import hf_transfer" >/dev/null 2>&1; then + export HF_XET_HIGH_PERFORMANCE=1 + echo "HF_XET_HIGH_PERFORMANCE=1" >> "$GITHUB_ENV" + fi + if [[ -n "${PROFILE_ASSETS[$PROFILE]:-}" ]]; then + download_assets "${PROFILE_ASSETS[$PROFILE]}" + fi + local var + for var in ${PROFILE_ENV[$PROFILE]:-}; do + echo "$var" >> "$GITHUB_ENV" + done + local fn="setup_${PROFILE//-/_}" + if declare -F "$fn" >/dev/null 2>&1; then + "$fn" + fi +} + +if [[ -z "${TARGET_ROOT:-}" || -z "${GITHUB_WORKSPACE:-}" || -z "${GITHUB_ENV:-}" ]]; then + echo "TARGET_ROOT / GITHUB_WORKSPACE / GITHUB_ENV must be set" >&2 + exit 2 +fi + +HERE=$(cd "$(dirname "$0")" && pwd) +export PIP_CONSTRAINT="$(cd "$HERE/.." && pwd)/constraints-npu.txt" + +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +select_pip_index +python -m pip install -U pip setuptools wheel +ensure_torch_stack +prepare_fixtures + +# All examples: keep wandb offline. Some entrypoints default --report_to to +# wandb; WANDB_MODE=disabled prevents any network reporting and is harmless +# for the ones that use tensorboard. Persisted to the run step via GITHUB_ENV. +echo "WANDB_MODE=disabled" >> "$GITHUB_ENV" + +# All examples: let HCCL pick a free socket port range (required on some +# Ascend setups to avoid "address already in use" on multi-process init). +echo "HCCL_NPU_SOCKET_PORT_RANGE=auto" >> "$GITHUB_ENV" + +run_profile