Views
No views yet
1from transformers import AutoModel, AutoProcessor
2
3model = AutoModel.from_pretrained("sposiboh/videoprism-base-f16r288-pt", trust_remote_code=True)
4processor = AutoProcessor.from_pretrained("sposiboh/videoprism-base-f16r288-pt", trust_remote_code=True)
5
6# Process a video file (or a list of frames / numpy / torch tensor):
7inputs = processor(videos="path/to/video.mp4", return_tensors="pt")
8outputs = model(**inputs)
9embedding = outputs.last_hidden_state # shape: (B, T*N, model_dim) — token sequenceimage_size × image_size. Pixels are scaled to [0, 1].1@inproceedings{zhao2024videoprism,
2 title = {VideoPrism: A Foundational Visual Encoder for Video Understanding},
3 author = {Zhao, Long and Gundavarapu, Nitesh B. and Yuan, Liangzhe and Zhou, Hao and Yan, Shen and Sun, Jennifer J. and Friedman, Luke and Qian, Rui and Weyand, Tobias and Zhao, Yue and Hornung, Rachel and Schroff, Florian and Yang, Ming-Hsuan and Ross, David A. and Wang, Huisheng and Adam, Hartwig and Sirotenko, Mikhail and Liu, Ting and Gong, Boqing},
4 booktitle = {ICML},
5 year = {2024},
6}