Views
No views yet
1python -m inference.cli_demo \
2 --video_path "resources/bubble.mp4" \
3 --prompt "Close-up shot with soft lighting, focusing sharply on the lower half of a young woman's face. Her lips are slightly parted as she blows an enormous bubblegum bubble. The bubble is semi-transparent, shimmering gently under the light, and surprisingly contains a miniature aquarium inside, where two orange-and-white goldfish slowly swim, their fins delicately fluttering as if in an aquatic universe. The background is a pure light blue color." \
4 --controlnet_type "tile" \
5 --base_model_path Wan-AI/Wan2.2-TI2V-5B-Diffusers \
6 --controlnet_model_path TheDenk/wan2.2-ti2v-5b-controlnet-tile-v11import os
2os.environ['CUDA_VISIBLE_DEVICES'] = "0"
3os.environ["TOKENIZERS_PARALLELISM"] = "false"
4
5import cv2
6from PIL import Image
7import torch
8from diffusers.utils import load_video, export_to_video
9from diffusers import AutoencoderKLWan, UniPCMultistepScheduler
10
11from wan_controlnet import WanControlnet
12from wan_transformer import CustomWanTransformer3DModel
13from wan_t2v_controlnet_pipeline import WanTextToVideoControlnetPipeline
14
15base_model_path = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
16controlnet_model_path = "TheDenk/wan2.2-ti2v-5b-controlnet-tile-v1"
17vae = AutoencoderKLWan.from_pretrained(base_model_path, subfolder="vae", torch_dtype=torch.float32)
18transformer = CustomWanTransformer3DModel.from_pretrained(base_model_path, subfolder="transformer", torch_dtype=torch.bfloat16)
19controlnet = WanControlnet.from_pretrained(controlnet_model_path, torch_dtype=torch.bfloat16)
20pipe = WanTextToVideoControlnetPipeline.from_pretrained(
21 pretrained_model_name_or_path=base_model_path,
22 controlnet=controlnet,
23 transformer=transformer,
24 vae=vae,
25 torch_dtype=torch.bfloat16
26)
27pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=5.0)
28pipe.enable_model_cpu_offload()
29
30img_h = 704 # 704 480
31img_w = 1280 # 1280 832
32num_frames = 121 # 121 81 49
33
34def apply_gaussian_blur(image, ksize=5, sigmaX=1.0):
35 image_np = np.array(image)
36 if ksize % 2 == 0:
37 ksize += 1
38 blurred_image = cv2.GaussianBlur(image_np, (ksize, ksize), sigmaX=sigmaX)
39 return Image.fromarray(blurred_image)
40
41video_path = 'bubble.mp4'
42video_frames = load_video(video_path)[:num_frames]
43ksize = 5
44downscale_coef =4
45controlnet_frames = [x.resize((img_w // downscale_coef, img_h // downscale_coef)) for x in video_frames]
46controlnet_frames = [apply_gaussian_blur(x, ksize=ksize, sigmaX=ksize // 2) for x in controlnet_frames]
47controlnet_frames = [x.resize((img_w, img_h)) for x in controlnet_frames]
48
49prompt = "Close-up shot with soft lighting, focusing sharply on the lower half of a young woman's face. Her lips are slightly parted as she blows an enormous bubblegum bubble. The bubble is semi-transparent, shimmering gently under the light, and surprisingly contains a miniature aquarium inside, where two orange-and-white goldfish slowly swim, their fins delicately fluttering as if in an aquatic universe. The background is a pure light blue color."
50negative_prompt = "bad quality, worst quality"
51
52output = pipe(
53 prompt=prompt,
54 negative_prompt=negative_prompt,
55 height=img_h,
56 width=img_w,
57 num_frames=num_frames,
58 guidance_scale=5,
59 generator=torch.Generator(device="cuda").manual_seed(42),
60 output_type="pil",
61
62 controlnet_frames=controlnet_frames,
63 controlnet_guidance_start=0.0,
64 controlnet_guidance_end=0.8,
65 controlnet_weight=0.8,
66
67 teacache_treshold=0.6,
68).frames[0]
69
70export_to_video(output, "output.mp4", fps=16)@misc{TheDenk,
title={Wan2.2 Controlnet},
author={Karachev Denis},
url={https://github.com/TheDenk/wan2.2-controlnet},
publisher={Github},
year={2025}
}