Views
No views yet
overfit_step_500.pt| Metric | Value |
|---|---|
| Parameters | 58.6M |
| Training Steps | 500 |
| Final Loss | 0.027 |
| Correlation | 0.989 |
| d_model | 512 |
| n_layers | 12 |
| n_slots | 4 |
1{
2 "d_model": 512,
3 "n_layers": 12,
4 "n_slots": 4,
5 "n_heads": 64,
6 "text_embed_dim": 1024,
7 "num_cross_attn_heads": 8,
8 "use_overlapping_patches": False
9}50M_h100_step_200.pt| Metric | Value |
|---|---|
| Parameters | 50M+ |
| Training Steps | 200 |
| Final Loss | 0.040 |
| d_model | 768 |
| n_layers | 16 |
| n_slots | 64 |
1{
2 "d_model": 768,
3 "n_layers": 16,
4 "n_slots": 64,
5 "n_heads": 64,
6 "headdim": 32,
7 "text_embed_dim": 1024,
8 "num_cross_attn_heads": 8,
9 "use_overlapping_patches": True
10}1import torch
2from eigenvideo.models.student.eigen_switch_model import EigenSwitchModel
3
4# Load checkpoint
5ckpt = torch.load("overfit_step_500.pt", map_location="cpu")
6config = ckpt["config"]
7
8# Create model
9model = EigenSwitchModel(**config)
10model.load_state_dict(ckpt["model_state_dict"])
11model.eval()
12
13# Generate video frames
14# z_0: initial latent frame [B, 16, H, W]
15# text_embed: T5-large embeddings [B, seq_len, 1024]
16output = model(z_0, seq_len=16, text_embed=text_embed)
17z_pred = output.predicted_latents # [B, T, 16, H, W]1from diffusers import AutoencoderKLHunyuanVideo
2
3vae = AutoencoderKLHunyuanVideo.from_pretrained(
4 "hunyuanvideo-community/HunyuanVideo",
5 subfolder="vae",
6 torch_dtype=torch.float16,
7)
8vae = vae.cuda().eval()
9
10# z_pred shape: [B, C, T, H, W] where C=16
11z_pred = z_pred / vae.config.scaling_factor
12frames = vae.decode(z_pred).sample # [B, 3, T*4, H*8, W*8]@misc{eigenvideo2024,
title={EigenVideo: Autoregressive Video Generation with EigenTitan Memory},
year={2024}
}