Views
No views yet
1import torch
2from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
3from qwen_vl_utils import process_vision_info
4
5model_path = "ReWatch-R1-SFT"
6
7model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
8 model_path,
9 torch_dtype=torch.bfloat16,
10 trust_remote_code=True,
11 attn_implementation="flash_attention_2",
12)
13
14processor = AutoProcessor.from_pretrained(
15 model_path,
16 trust_remote_code=True,
17 use_fast=True,
18 padding_side="left",
19 truncation_side="right",
20)
21
22video_path = "videos/example.mp4"
23video_duration = 600
24question = "What happened from [05:00] to [05:10]?"
25
26total_pixels = 12288*28*28
27min_pixels = 128*28*28
28max_pixels = 128*28*28
29fps = 2.0
30max_frames = 192
31
32video_config = {
33 "type": "video",
34 "video": video_path,
35 "total_pixels": total_pixels,
36 "min_pixels": min_pixels,
37 "max_pixels": max_pixels,
38 "fps": fps,
39 "max_frames": max_frames
40}
41
42react_prompt = """You are a video understanding expert. You are given a video and a question. You need to answer the question based on the video content. Please answer the question step by step. When you need more video details, you will re-watch the relevant clips and use <action> and </action> to mark the actions, and use <observation> and </observation> to mark the visual details you observe. When you have enough information to determine the final answer, you will wrap the final answer in <answer> and </answer>.
43
44**Video Information and Question:**
45- **Video Duration**: {video_duration}
46- **Question**: {question}"""
47
48def seconds_to_timestamp(seconds):
49 """将秒数转换为时间戳字符串 (MM:SS)"""
50 minutes = seconds // 60
51 seconds = seconds % 60
52 return f"{minutes:02d}:{seconds:02d}"
53
54duration_str = f"00:00-{seconds_to_timestamp(video_duration)}"
55instruction = react_prompt.format(video_duration=duration_str, question=question)
56
57messages = [
58 {"role": "system", "content": "You are a helpful assistant."},
59 {"role": "user", "content": [
60 video_config,
61 {"type": "text", "text": instruction},
62 ]},
63]
64
65text = processor.apply_chat_template(
66 messages,
67 tokenize=False,
68 add_generation_prompt=True,
69)
70
71image_inputs, video_inputs, video_kwargs = process_vision_info(messages, return_video_kwargs=True)
72
73inputs = processor(
74 text=[text],
75 images=image_inputs,
76 videos=video_inputs,
77 padding=True,
78 return_tensors="pt",
79 max_length=16384,
80 truncation=True,
81 do_sample_frames=False,
82 **video_kwargs,
83)
84inputs = inputs.to("cuda")
85
86generated_ids = model.generate(**inputs, do_sample=False, max_new_tokens=4096, use_cache=True)
87generated_ids_trimmed = [
88 out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
89]
90output_text = processor.batch_decode(
91 generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
92)
93print(output_text)