Views
No views yet
1from videoflextok.wrappers import VideoFlexTokFromHub
2model = VideoFlexTokFromHub.from_pretrained('EPFL-VILAB/videoflextok_d18_d28').eval()model.safetensors checkpoint in this repository manually and loading it using our helper functions:1from hydra.utils import instantiate
2from videoflextok.utils.checkpoint import load_safetensors
3
4ckpt, config = load_safetensors('/path/to/model.safetensors')
5model = instantiate(config).eval()
6model.load_state_dict(ckpt)1from videoflextok.utils.demo import read_mp4
2# Load example video into a float tensor of shape (3, T, 256, 256), normalized to [-1,1]
3# it will sample frames at approx. 8 FPS, ensuring T = 1 + K * 16 for some integer K >= 1,
4# which is required for the chunking mechanism in VideoFlexTok
5video_tensor = read_mp4("./data/video_examples/red_ball.mp4", fps=8) # (C, T, H, W)
6
7# Encode into a list of discrete token sequences, where each sequence is of shape [1, t, 256]
8# this will automatically apply the encoder in the sliding window fashion, and concatenate the resulting tokens along the sequence dimension
9# t = 1 + K * 4 since each chunk of 16 frames is tokenized into 4 tokens, and the first token corresponds to the first frame
10tokens_list = model.tokenize(video_tensor[None])1k_keep = 64 # For example, only keep the first 64 out of 256 tokens for each timestep
2tokens_list = [t[..., :k_keep] for t in tokens_list]1# tokens_list is a list of [1, t, l] discrete token sequences, with l <= 256
2# reconst is a list of RGB videos of shape [1, 3, T, 256, 256] tensor, normalized to [-1,1]
3reconst = model.detokenize(
4 tokens_list,
5 timesteps=30, # Number of denoising steps
6 guidance_scale=20., # Classifier-free guidance scale (15-30 typically works well)
7 perform_norm_guidance=True, # See https://arxiv.org/abs/2410.02416
8)@article{videoflextok,
title={{VideoFlexTok}: Flexible-Length Coarse-to-Fine Video Tokenization},
author={Andrei Atanov and Jesse Allardice and Roman Bachmann and O{\u{g}}uzhan Fatih Kar and Peter Fu and David Griffiths and Devon Hjelm and Afshin Dehghan and Amir Zamir},
journal={arXiv 2026},
year={2026},
}