Views
No views yet
pip install imageio[ffmpeg], then:1import torch
2# Download the video
3url = 'https://github.com/facebookresearch/co-tracker/blob/main/assets/apple.mp4'
4
5import imageio.v3 as iio
6frames = iio.imread(url, plugin="FFMPEG") # plugin="pyav"
7
8device = 'cuda'
9grid_size = 10
10video = torch.tensor(frames).permute(0, 3, 1, 2)[None].float().to(device) # B T C H W
11
12# Run Offline CoTracker:
13cotracker = torch.hub.load("facebookresearch/co-tracker", "cotracker2").to(device)
14pred_tracks, pred_visibility = cotracker(video, grid_size=grid_size) # B T N 2, B T N 11cotracker = torch.hub.load("facebookresearch/co-tracker", "cotracker2_online").to(device)
2
3# Run Online CoTracker, the same model with a different API:
4# Initialize online processing
5cotracker(video_chunk=video, is_first_step=True, grid_size=grid_size)
6
7# Process the video
8for ind in range(0, video.shape[1] - cotracker.step, cotracker.step):
9 pred_tracks, pred_visibility = cotracker(
10 video_chunk=video[:, ind : ind + cotracker.step * 2]
11 ) # B T N 2, B T N 11@article{karaev2023cotracker,
2 title={CoTracker: It is Better to Track Together},
3 author={Nikita Karaev and Ignacio Rocco and Benjamin Graham and Natalia Neverova and Andrea Vedaldi and Christian Rupprecht},
4 journal={arXiv:2307.07635},
5 year={2023}
6}