Views
No views yet
pip install imageio[ffmpeg], then:1import torch
2# Download the video
3url = 'https://github.com/facebookresearch/co-tracker/raw/refs/heads/main/assets/apple.mp4'
4
5import imageio.v3 as iio
6frames = iio.imread(url, plugin="FFMPEG") # plugin="pyav"
7
8device = 'cuda'
9grid_size = 10
10video = torch.tensor(frames).permute(0, 3, 1, 2)[None].float().to(device) # B T C H W
11
12# Run Offline CoTracker:
13cotracker = torch.hub.load("facebookresearch/co-tracker", "cotracker3_offline").to(device)
14pred_tracks, pred_visibility = cotracker(video, grid_size=grid_size) # B T N 2, B T N 11cotracker = torch.hub.load("facebookresearch/co-tracker", "cotracker3_online").to(device)
2
3# Run Online CoTracker, the same model with a different API:
4# Initialize online processing
5cotracker(video_chunk=video, is_first_step=True, grid_size=grid_size)
6
7# Process the video
8for ind in range(0, video.shape[1] - cotracker.step, cotracker.step):
9 pred_tracks, pred_visibility = cotracker(
10 video_chunk=video[:, ind : ind + cotracker.step * 2]
11 ) # B T N 2, B T N 11@inproceedings{karaev24cotracker3,
2 title = {CoTracker3: Simpler and Better Point Tracking by Pseudo-Labelling Real Videos},
3 author = {Nikita Karaev and Iurii Makarov and Jianyuan Wang and Natalia Neverova and Andrea Vedaldi and Christian Rupprecht},
4 booktitle = {Proc. {arXiv:2410.11831}},
5 year = {2024}
6}