Load with ModularPipeline: add modular_model_index.json

README.md

@@ -38,25 +38,33 @@ full Omni checkpoint, and CLIP without its vision tower, since neither is used.
 
 ```python
 import torch
-from diffusers import KandinskyS2VPipeline
+from diffusers import ModularPipeline
 
-pipe = KandinskyS2VPipeline.from_pretrained(
-    "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers", torch_dtype=torch.bfloat16
-).to("cuda")
+pipe = ModularPipeline.from_pretrained("kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers")
+pipe.load_components(dtype=torch.bfloat16)
+pipe.to("cuda")
 
-frames = pipe(image="face.png", audio="speech.wav", num_inference_steps=50).frames[0]
+frames = pipe(image="face.png", audio="speech.wav", output="videos")[0]
 ```
 
-Pro needs `pipe.enable_model_cpu_offload()` in place of `.to("cuda")` on an 80 GB
-card: kept resident beside the 7B speech encoder it leaves the VAE decode of a
-chunk nowhere to go, and that decode asks for 7 GiB in one allocation. Offloading
-brings the peak from 73 GiB to 46 GiB.
+Pro does not fit an 80 GB card with every component resident: beside the 7B speech encoder the VAE
+decode of a chunk has nowhere to go. Keep one model on the GPU at a time instead of `pipe.to("cuda")`,
+which brings the peak to about 46 GB:
+
+```python
+from diffusers import ComponentsManager
+
+manager = ComponentsManager()
+pipe = ModularPipeline.from_pretrained("kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers", components_manager=manager)
+pipe.load_components(dtype=torch.bfloat16)
+manager.enable_auto_cpu_offload(device="cuda", offload_strategy=lambda hooks, **_: hooks)
+```
+
+The pipeline and its blocks come from `modular_model_index.json`. Classifier-free guidance is the `guider` component, not a call argument.
 
 The speech track is not muxed into the result — add it to the frames yourself.
 
-**The Kandinsky S2V 1.0 classes are not part of a released Diffusers version yet.**
-Until they are, `from_pretrained` cannot resolve the class names in
-`model_index.json`; load the components individually against the port instead.
+**Kandinsky S2V 1.0 is not part of a released Diffusers version yet**; it lands as a modular pipeline.
 
 ## Notes
 

model_index.json

@@ -1,32 +0,0 @@
-{
-  "_class_name": "KandinskyS2VPipeline",
-  "_diffusers_version": "0.40.0",
-  "transformer": [
-    "diffusers",
-    "KandinskyS2VTransformer3DModel"
-  ],
-  "vae": [
-    "diffusers",
-    "AutoencoderKLHunyuanVideo"
-  ],
-  "text_encoder": [
-    "transformers",
-    "Qwen2_5OmniThinkerForConditionalGeneration"
-  ],
-  "tokenizer": [
-    "transformers",
-    "Qwen2_5OmniProcessor"
-  ],
-  "text_encoder_2": [
-    "transformers",
-    "CLIPTextModel"
-  ],
-  "tokenizer_2": [
-    "transformers",
-    "CLIPTokenizer"
-  ],
-  "scheduler": [
-    "diffusers",
-    "FlowMatchEulerDiscreteScheduler"
-  ]
-}

modular_model_index.json

@@ -0,0 +1,103 @@
+{
+  "_class_name": "KandinskyS2VModularPipeline",
+  "_diffusers_version": "0.40.0",
+  "_blocks_class_name": "KandinskyS2VAutoBlocks",
+  "transformer": [
+    "diffusers",
+    "KandinskyS2VTransformer3DModel",
+    {
+      "type_hint": [
+        "diffusers",
+        "KandinskyS2VTransformer3DModel"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "transformer",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKLHunyuanVideo",
+    {
+      "type_hint": [
+        "diffusers",
+        "AutoencoderKLHunyuanVideo"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "vae",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "text_encoder": [
+    "transformers",
+    "Qwen2_5OmniThinkerForConditionalGeneration",
+    {
+      "type_hint": [
+        "transformers",
+        "Qwen2_5OmniThinkerForConditionalGeneration"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "text_encoder",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "tokenizer": [
+    "transformers",
+    "Qwen2_5OmniProcessor",
+    {
+      "type_hint": [
+        "transformers",
+        "Qwen2_5OmniProcessor"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "tokenizer",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "text_encoder_2": [
+    "transformers",
+    "CLIPTextModel",
+    {
+      "type_hint": [
+        "transformers",
+        "CLIPTextModel"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "text_encoder_2",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "tokenizer_2": [
+    "transformers",
+    "CLIPTokenizer",
+    {
+      "type_hint": [
+        "transformers",
+        "CLIPTokenizer"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "tokenizer_2",
+      "variant": null,
+      "revision": null
+    }
+  ],
+  "scheduler": [
+    "diffusers",
+    "FlowMatchEulerDiscreteScheduler",
+    {
+      "type_hint": [
+        "diffusers",
+        "FlowMatchEulerDiscreteScheduler"
+      ],
+      "pretrained_model_name_or_path": "kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers",
+      "subfolder": "scheduler",
+      "variant": null,
+      "revision": null
+    }
+  ]
+}