Views
No views yet
1sudo apt-get update && sudo apt-get install ffmpeg git-lfs
2pip install torch torchvision diffusers transformers moviepy==1.0.3 peft safetensors
3git clone https://huggingface.co/svjack/Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora1import torch
2from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
3from diffusers.utils import export_to_video
4from safetensors.torch import load_file
5import os
6
7def infer_video(
8 pretrained_model,
9 prompt,
10 height,
11 width,
12 num_frames,
13 num_inference_steps,
14 seed,
15 output_dir,
16 use_lora=False,
17 lora_path=None,
18 alpha=None,
19):
20 """
21 合并使用和不使用 LoRA 的视频生成函数。
22
23 参数:
24 pretrained_model (str): 预训练模型的路径。
25 prompt (str): 生成视频的提示词。
26 height (int): 生成视频的高度。
27 width (int): 生成视频的宽度。
28 num_frames (int): 生成视频的帧数。
29 num_inference_steps (int): 推断步数。
30 seed (int): 随机种子。
31 output_dir (str): 输出视频的目录。
32 use_lora (bool): 是否使用 LoRA，默认为 False。
33 lora_path (str): LoRA 文件的路径，仅在 use_lora=True 时有效。
34 alpha (int): LoRA 的 alpha 参数，仅在 use_lora=True 时有效。
35 """
36 # 加载模型
37 transformer = HunyuanVideoTransformer3DModel.from_pretrained(
38 pretrained_model,
39 subfolder="transformer",
40 torch_dtype=torch.bfloat16,
41 )
42 # 如果使用 LoRA
43 if use_lora:
44 if lora_path is None:
45 raise ValueError("lora_path must be provided when use_lora is True")
46
47 # 加载 LoRA 权重
48 lora_sd = load_file(lora_path)
49 rank = 0
50 for key in lora_sd.keys():
51 if ".lora_A.weight" in key:
52 rank = lora_sd[key].shape[0]
53
54 alpha = 1 if alpha is None else alpha
55 lora_weight = alpha / rank
56
57 print(f"lora rank = {rank}")
58 print(f"alpha = {alpha}")
59 print(f"lora weight = {lora_weight}")
60
61 # 应用 LoRA
62 transformer.load_lora_adapter(lora_sd, adapter_name="default_lora")
63 transformer.set_adapters(adapter_names="default_lora", weights=lora_weight)
64
65 pipe = HunyuanVideoPipeline.from_pretrained(pretrained_model, transformer=transformer, torch_dtype=torch.float16)
66 pipe.transformer = transformer
67
68 pipe.vae.enable_tiling(
69 tile_sample_min_height=256,
70 tile_sample_min_width=256,
71 tile_sample_min_num_frames=64,
72 tile_sample_stride_height=192,
73 tile_sample_stride_width=192,
74 tile_sample_stride_num_frames=16,
75 )
76 pipe.enable_sequential_cpu_offload()
77
78 # 进行推断
79 output = pipe(
80 prompt=prompt,
81 height=height,
82 width=width,
83 num_frames=num_frames,
84 num_inference_steps=num_inference_steps,
85 generator=torch.Generator(device="cpu").manual_seed(seed),
86 ).frames[0]
87
88 # 导出视频
89 output_filename = "output_lora.mp4" if use_lora else "output_base.mp4"
90 export_to_video(
91 output,
92 os.path.join(output_dir, output_filename),
93 fps=15,
94 )
95
96infer_video(
97 pretrained_model="hunyuanvideo-community/HunyuanVideo",
98 prompt="In the style of Yumemizuki Mizuki, the character stands under moonlight in a serene courtyard, softly illuminated. Translucent curtains sway as she blinks slowly, her eyes open reflecting the moon. Ancient lanterns, moss, and distant cherry blossoms surround her. Her pure, haunting voice blends with the tranquil night, crafting dreamlike serenity. Moonlight weaves through the scene, harmonizing with her melody, creating timeless enchantment.",
99 height=512,
100 width=512,
101 num_frames=33,
102 num_inference_steps=20,
103 seed=42,
104 output_dir="./",
105 use_lora=True,
106 lora_path="Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora/checkpoints/hyv-lora-00000600.safetensors",
107 alpha=16,
108)
109
110infer_video(
111 pretrained_model="hunyuanvideo-community/HunyuanVideo",
112 prompt="In the style of Yumemizuki Mizuki, the character sits in a hot spring under daylight, wearing a swimsuit, softly illuminated. Steam rises gently as she blinks slowly, her eyes open reflecting the sunlight. Ancient rocks, moss, and distant trees surround her. Her pure, soothing voice blends with the warm air, crafting dreamlike tranquility. Sunlight weaves through the scene, harmonizing with her melody, creating timeless relaxation.",
113 height=512,
114 width=512,
115 num_frames=33,
116 num_inference_steps=20,
117 seed=42,
118 output_dir="./",
119 use_lora=True,
120 lora_path="Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora/checkpoints/hyv-lora-00000600.safetensors",
121 alpha=16,
122)1### !cd Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora
2import os
3import torch
4from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
5from pipeline_stg_hunyuan_video import HunyuanVideoSTGPipeline
6from diffusers.utils import export_to_video
7from safetensors.torch import load_file
8
9def infer_video_with_stg(
10 pretrained_model,
11 prompt,
12 height,
13 width,
14 num_frames,
15 num_inference_steps,
16 seed,
17 output_dir,
18 use_lora=False,
19 lora_path=None,
20 alpha=None,
21 stg_mode="STG",
22 stg_applied_layers_idx=[2],
23 stg_scale=0.7,
24 do_rescaling=False,
25):
26 """
27 合并使用和不使用 LoRA 的视频生成函数，并支持 STG 模式。
28
29 参数:
30 pretrained_model (str): 预训练模型的路径。
31 prompt (str): 生成视频的提示词。
32 height (int): 生成视频的高度。
33 width (int): 生成视频的宽度。
34 num_frames (int): 生成视频的帧数。
35 num_inference_steps (int): 推断步数。
36 seed (int): 随机种子。
37 output_dir (str): 输出视频的目录。
38 use_lora (bool): 是否使用 LoRA，默认为 False。
39 lora_path (str): LoRA 文件的路径，仅在 use_lora=True 时有效。
40 alpha (int): LoRA 的 alpha 参数，仅在 use_lora=True 时有效。
41 stg_mode (str): STG 模式，默认为 "STG"。
42 stg_applied_layers_idx (list): STG 应用的层索引，默认为 [2]。
43 stg_scale (float): STG 的缩放比例，默认为 0.7。
44 do_rescaling (bool): 是否进行重新缩放，默认为 False。
45 """
46 # 加载模型
47 transformer = HunyuanVideoTransformer3DModel.from_pretrained(
48 pretrained_model,
49 subfolder="transformer",
50 torch_dtype=torch.bfloat16,
51 )
52
53 # 如果使用 LoRA
54 if use_lora:
55 if lora_path is None:
56 raise ValueError("lora_path must be provided when use_lora is True")
57
58 # 加载 LoRA 权重
59 lora_sd = load_file(lora_path)
60 rank = 0
61 for key in lora_sd.keys():
62 if ".lora_A.weight" in key:
63 rank = lora_sd[key].shape[0]
64
65 alpha = 1 if alpha is None else alpha
66 lora_weight = alpha / rank
67
68 print(f"lora rank = {rank}")
69 print(f"alpha = {alpha}")
70 print(f"lora weight = {lora_weight}")
71
72 # 应用 LoRA
73 transformer.load_lora_adapter(lora_sd, adapter_name="default_lora")
74 transformer.set_adapters(adapter_names="default_lora", weights=lora_weight)
75
76 pipe = HunyuanVideoSTGPipeline.from_pretrained(pretrained_model, transformer=transformer, torch_dtype=torch.float16)
77 pipe.transformer = transformer
78
79 pipe.vae.enable_tiling(
80 tile_sample_min_height=256,
81 tile_sample_min_width=256,
82 tile_sample_min_num_frames=64,
83 tile_sample_stride_height=192,
84 tile_sample_stride_width=192,
85 tile_sample_stride_num_frames=16,
86 )
87 pipe.enable_sequential_cpu_offload()
88
89 # 进行推断
90 output = pipe(
91 prompt=prompt,
92 height=height,
93 width=width,
94 num_frames=num_frames,
95 num_inference_steps=num_inference_steps,
96 stg_applied_layers_idx=stg_applied_layers_idx,
97 stg_scale=stg_scale,
98 do_rescaling=do_rescaling,
99 generator=torch.Generator(device="cpu").manual_seed(seed),
100 ).frames[0]
101
102 # 导出视频
103 if stg_scale == 0:
104 video_name = f"CFG_rescale_{do_rescaling}.mp4"
105 else:
106 layers_str = "_".join(map(str, stg_applied_layers_idx))
107 video_name = f"{stg_mode}_scale_{stg_scale}_layers_{layers_str}_rescale_{do_rescaling}.mp4"
108
109 os.makedirs(output_dir, exist_ok=True)
110 video_path = os.path.join(output_dir, video_name)
111 export_to_video(output, video_path, fps=15)
112
113 print(f"Video saved to {video_path}")
114
115### Yumemizuki Mizuki Moon
116
117prompt = '''
118In the style of Yumemizuki Mizuki, the character stands under moonlight in a serene courtyard, softly illuminated.
119Translucent curtains sway as she blinks slowly, her eyes open reflecting the moon.
120Ancient lanterns, moss, and distant cherry blossoms surround her. Her pure, haunting voice blends with the tranquil night,
121crafting dreamlike serenity. Moonlight weaves through the scene, harmonizing with her melody, creating timeless enchantment.
122'''
123
124infer_video_with_stg(
125 pretrained_model="hunyuanvideo-community/HunyuanVideo",
126 prompt=prompt,
127 height=512,
128 width=512,
129 num_frames=33,
130 num_inference_steps=20,
131 seed=42,
132 output_dir=".",
133 use_lora=True,
134 lora_path="video_checkpoints/hyv-lora-00002750.safetensors",
135 alpha=16,
136 stg_mode="STG",
137 stg_applied_layers_idx=[2],
138 stg_scale=0.7,
139 do_rescaling=False,
140)
141
142### Yumemizuki Mizuki Hot Spring
143
144prompt = '''
145In the style of Yumemizuki Mizuki, the character sits in a hot spring under daylight,
146wearing a swimsuit, softly illuminated. Steam rises gently as she blinks slowly,
147her eyes open reflecting the sunlight. Ancient rocks, moss, and distant trees surround her.
148Her pure, soothing voice blends with the warm air, crafting dreamlike tranquility.
149Sunlight weaves through the scene, harmonizing with her melody, creating timeless relaxation.
150'''
151
152infer_video_with_stg(
153 pretrained_model="hunyuanvideo-community/HunyuanVideo",
154 prompt=prompt,
155 height=512,
156 width=512,
157 num_frames=33,
158 num_inference_steps=20,
159 seed=42,
160 output_dir=".",
161 use_lora=True,
162 lora_path="video_checkpoints/hyv-lora-00002750.safetensors",
163 alpha=16,
164 stg_mode="STG",
165 stg_applied_layers_idx=[2],
166 stg_scale=0.7,
167 do_rescaling=False,
168)
169