Views
No views yet
EgoGPT-0.5b-Demo is an omni-modal model trained on egocentric datasets, achieving state-of-the-art performance on egocentric video understanding. Built on the foundation of llava-onevision-qwen2-0.5b-ov, it has been finetuned on EgoIT-EgoLife-138k egocentric datasets, which contains EgoIT-99k and depersonalized version of EgoLife-QA (39k).1git clone https://github.com/egolife-ntu/EgoLife
2cd EgoLife/EgoGPT1conda create -n egogpt python=3.10
2conda activate egogpt
3pip install --upgrade pip
4pip install -e .
5
63. Install the dependencies for training and inference.
7
8```shell
9pip install -e ".[train]"
10pip install flash-attn --no-build-isolation1import argparse
2import copy
3import os
4import re
5import sys
6import warnings
7
8import numpy as np
9import requests
10import soundfile as sf
11import torch
12import torch.distributed as dist
13import whisper
14from decord import VideoReader, cpu
15from egogpt.constants import (
16 DEFAULT_IMAGE_TOKEN,
17 DEFAULT_SPEECH_TOKEN,
18 IGNORE_INDEX,
19 IMAGE_TOKEN_INDEX,
20 SPEECH_TOKEN_INDEX,
21)
22from egogpt.conversation import SeparatorStyle, conv_templates
23from egogpt.mm_utils import get_model_name_from_path, process_images
24from egogpt.model.builder import load_pretrained_model
25from PIL import Image
26from scipy.signal import resample
27
28
29def setup(rank, world_size):
30 os.environ["MASTER_ADDR"] = "localhost"
31 os.environ["MASTER_PORT"] = "12355"
32 dist.init_process_group("gloo", rank=rank, world_size=world_size)
33
34
35def load_video(video_path=None, audio_path=None, max_frames_num=16, fps=1):
36 if audio_path is not None:
37 speech, sample_rate = sf.read(audio_path)
38 if sample_rate != 16000:
39 target_length = int(len(speech) * 16000 / sample_rate)
40 speech = resample(speech, target_length)
41 if speech.ndim > 1:
42 speech = np.mean(speech, axis=1)
43 speech = whisper.pad_or_trim(speech.astype(np.float32))
44 speech = whisper.log_mel_spectrogram(speech, n_mels=128).permute(1, 0)
45 speech_lengths = torch.LongTensor([speech.shape[0]])
46 else:
47 speech = torch.zeros(3000, 128)
48 speech_lengths = torch.LongTensor([3000])
49
50 vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
51 total_frame_num = len(vr)
52 avg_fps = round(vr.get_avg_fps() / fps)
53 frame_idx = [i for i in range(0, total_frame_num, avg_fps)]
54 if max_frames_num > 0 and len(frame_idx) > max_frames_num:
55 uniform_sampled_frames = np.linspace(
56 0, total_frame_num - 1, max_frames_num, dtype=int
57 )
58 frame_idx = uniform_sampled_frames.tolist()
59 video = vr.get_batch(frame_idx).asnumpy()
60 return video, speech, speech_lengths
61
62
63def split_text(text, keywords):
64 pattern = "(" + "|".join(map(re.escape, keywords)) + ")"
65 parts = re.split(pattern, text)
66 parts = [part for part in parts if part]
67 return parts
68
69
70def main(
71 pretrained_path="checkpoints/EgoGPT-0.5b-Demo",
72 video_path=None,
73 audio_path=None,
74 query="Please describe the video in detail.",
75):
76 warnings.filterwarnings("ignore")
77 setup(0, 1)
78 device = "cuda"
79 device_map = "cuda"
80
81 tokenizer, model, max_length = load_pretrained_model(
82 pretrained_path, device_map=device_map
83 )
84 model.eval()
85
86 conv_template = "qwen_1_5"
87 question = f"<image>\n<speech>\n\n{query}"
88 conv = copy.deepcopy(conv_templates[conv_template])
89 conv.append_message(conv.roles[0], question)
90 conv.append_message(conv.roles[1], None)
91 prompt_question = conv.get_prompt()
92
93 video, speech, speech_lengths = load_video(
94 video_path=video_path, audio_path=audio_path
95 )
96 speech = torch.stack([speech]).to(device).half()
97 processor = model.get_vision_tower().image_processor
98 processed_video = processor.preprocess(video, return_tensors="pt")["pixel_values"]
99 image = [(processed_video, video[0].size, "video")]
100
101 parts = split_text(prompt_question, ["<image>", "<speech>"])
102 input_ids = []
103 for part in parts:
104 if part == "<image>":
105 input_ids.append(IMAGE_TOKEN_INDEX)
106 elif part == "<speech>":
107 input_ids.append(SPEECH_TOKEN_INDEX)
108 else:
109 input_ids.extend(tokenizer(part).input_ids)
110
111 input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0).to(device)
112 image_tensor = [image[0][0].half()]
113 image_sizes = [image[0][1]]
114 generate_kwargs = {"eos_token_id": tokenizer.eos_token_id}
115
116 cont = model.generate(
117 input_ids,
118 images=image_tensor,
119 image_sizes=image_sizes,
120 speech=speech,
121 speech_lengths=speech_lengths,
122 do_sample=False,
123 temperature=0.5,
124 max_new_tokens=4096,
125 modalities=["video"],
126 **generate_kwargs,
127 )
128 text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
129 print(text_outputs)
130
131
132if __name__ == "__main__":
133 parser = argparse.ArgumentParser()
134 parser.add_argument(
135 "--pretrained_path", type=str, default="lmms-lab/EgoGPT-0.5b-Demo"
136 )
137 parser.add_argument("--video_path", type=str, default=None)
138 parser.add_argument("--audio_path", type=str, default=None)
139 parser.add_argument(
140 "--query", type=str, default="Please describe the video in detail."
141 )
142 args = parser.parse_args()
143 main(args.pretrained_path, args.video_path, args.audio_path, args.query)1@inproceedings{yang2025egolife,
2 title={EgoLife: Towards Egocentric Life Assistant},
3 author={Yang, Jingkang and Liu, Shuai and Guo, Hongming and Dong, Yuhao and Zhang, Xiamengwei and Zhang, Sicheng and Wang, Pengyun and Zhou, Zitang and Xie, Binzhu and Wang, Ziyue and Ouyang, Bei and Lin, Zhengyu and Cominelli, Marco and Cai, Zhongang and Zhang, Yuanhan and Zhang, Peiyuan and Hong, Fangzhou and Widmer, Joerg and Gringoli, Francesco and Yang, Lei and Li, Bo and Liu, Ziwei},
4 booktitle={The IEEE/CVF Conference on Computer Vision and Pattern Recognition},
5 year={2025},
6}