Views
No views yet
| Model Name | VCtrl-Canny | VCtrl-Mask | VCtrl-Pose |
|---|---|---|---|
| Video Resolution | 720 * 480 | 720 * 480 | 720 * 480 & 480 * 720 |
| Inference Precision | FP16(Recommended) | ||
| Single GPU VRAM Usage | V100: 32GB minimum* | ||
| Inference Speed (Step = 25, FP16) | Single A100: ~300s(49 frames) Single V100: ~400s(49 frames) | ||
| Prompt Language | English* | ||
| Prompt Length Limit | 224 Tokens | ||
| Video Length | T2V model supports only 49 frames, I2V model can extend to any frame count | ||
| Frame Rate | 30 FPS |
1# Clone the PaddleMIX repository
2git clone https://github.com/PaddlePaddle/PaddleMIX.git
3# Install paddlemix
4cd PaddleMIX
5pip install -e .
6# Install ppdiffusers
7pip install -e ppdiffusers
8# Install paddlenlp
9pip install paddlenlp==v3.0.0-beta2
10# Navigate to the vctrl directory
11cd ppdiffusers/examples/ppvctrl
12# Install other required dependencies
13pip install -r requirements.txt
14# Install paddlex
15pip install paddlex==3.0.0b21import os
2import paddle
3import numpy as np
4from decord import VideoReader
5from moviepy.editor import ImageSequenceClip
6from PIL import Image
7from ppdiffusers import (
8 CogVideoXDDIMScheduler,
9 CogVideoXTransformer3DVCtrlModel,
10 CogVideoXVCtrlPipeline,
11 VCtrlModel,
12)
13def write_mp4(video_path, samples, fps=8):
14 clip = ImageSequenceClip(samples, fps=fps)
15 clip.write_videofile(video_path, audio_codec="aac")
16
17
18def save_vid_side_by_side(batch_output, validation_control_images, output_folder, fps):
19 flattened_batch_output = [img for sublist in batch_output for img in sublist]
20 ori_video_path = output_folder + "/origin_predict.mp4"
21 video_path = output_folder + "/test_1.mp4"
22 ori_final_images = []
23 final_images = []
24 outputs = []
25
26 def get_concat_h(im1, im2):
27 dst = Image.new("RGB", (im1.width + im2.width, max(im1.height, im2.height)))
28 dst.paste(im1, (0, 0))
29 dst.paste(im2, (im1.width, 0))
30 return dst
31
32 for image_list in zip(validation_control_images, flattened_batch_output):
33 predict_img = image_list[1].resize(image_list[0].size)
34 result = get_concat_h(image_list[0], predict_img)
35 ori_final_images.append(np.array(image_list[1]))
36 final_images.append(np.array(result))
37 outputs.append(np.array(predict_img))
38 write_mp4(ori_video_path, ori_final_images, fps=fps)
39 write_mp4(video_path, final_images, fps=fps)
40 output_path = output_folder + "/output.mp4"
41 write_mp4(output_path, outputs, fps=fps)
42
43
44def load_images_from_folder_to_pil(folder):
45 images = []
46 valid_extensions = {".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff"}
47
48 def frame_number(filename):
49 new_pattern_match = re.search("frame_(\\d+)_7fps", filename)
50 if new_pattern_match:
51 return int(new_pattern_match.group(1))
52 matches = re.findall("\\d+", filename)
53 if matches:
54 if matches[-1] == "0000" and len(matches) > 1:
55 return int(matches[-2])
56 return int(matches[-1])
57 return float("inf")
58
59 sorted_files = sorted(os.listdir(folder), key=frame_number)
60 for filename in sorted_files:
61 ext = os.path.splitext(filename)[1].lower()
62 if ext in valid_extensions:
63 img = Image.open(os.path.join(folder, filename)).convert("RGB")
64 images.append(img)
65 return images
66
67
68def load_images_from_video_to_pil(video_path):
69 images = []
70 vr = VideoReader(video_path)
71 length = len(vr)
72 for idx in range(length):
73 frame = vr[idx].asnumpy()
74 images.append(Image.fromarray(frame))
75 return images
76
77
78validation_control_images = load_images_from_video_to_pil('your_path')
79prompt = 'Group of fishes swimming in aquarium.'
80vctrl = VCtrlModel.from_pretrained(
81 paddlemix/vctrl-5b-t2v-canny,
82 low_cpu_mem_usage=True,
83 paddle_dtype=paddle.float16
84 )
85pipeline = CogVideoXVCtrlPipeline.from_pretrained(
86 paddlemix/cogvideox-5b-vctrl,
87 vctrl=vctrl,
88 paddle_dtype=paddle.float16,
89 low_cpu_mem_usage=True,
90 map_location="cpu",
91 )
92pipeline.scheduler = CogVideoXDDIMScheduler.from_config(pipeline.scheduler.config, timestep_spacing="trailing")
93pipeline.vae.enable_tiling()
94pipeline.vae.enable_slicing()
95task='canny'
96final_result=[]
97video = pipeline(
98 prompt=prompt,
99 num_inference_steps=25,
100 num_frames=49,
101 guidance_scale=35,
102 generator=paddle.Generator().manual_seed(42),
103 conditioning_frames=validation_control_images[:num_frames],
104 conditioning_frame_indices=list(range(num_frames)),
105 conditioning_scale=1.0,
106 width=720,
107 height=480,
108 task='canny',
109 conditioning_masks=validation_mask_images[:num_frames] if task == "mask" else None,
110 vctrl_layout_type='spacing',
111 ).frames[0]
112 final_result.append(video)
113 save_vid_side_by_side(final_result, validation_control_images[:num_frames], 'save.mp4', fps=30)@article{yang2024cogvideox,
title={VCtrl: Enabling Versatile Controls for Video Diffusion Models},
year={2025}
}