Views
No views yet
pip install transformers1from transformers import Qwen2_5_VLForConditionalGeneration, AutoTokenizer, AutoProcessor
2from qwen_vl_utils import process_vision_info
3
4model_path = "OpenGVLab/VideoChat-R1_5"
5# default: Load the model on the available device(s)
6model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
7 model_path, torch_dtype="auto", device_map="auto",
8 attn_implementation="flash_attention_2"
9)
10
11# default processer
12processor = AutoProcessor.from_pretrained(model_path)
13
14video_path = "your_video.mp4"
15question = "your_qa.mp4"
16num_percptions = 3
17
18QA_THINK_GLUE = """Answer the question: "[QUESTION]" according to the content of the video.
19
20Output your think process within the <think> </think> tags.
21
22Then, provide your answer within the <answer> </answer> tags, output the corresponding letter of the option. At the same time, in the <glue> </glue> tags, present the precise time period in seconds of the video clips on which you base your answer to this question in the format of [(s1, e1), (s2, e2), ...]. For example: <think>...</think><answer>A</answer><glue>[(5.2, 10.4)]</glue>.
23"""
24
25QA_THINK = """Answer the question: "[QUESTION]" according to the content of the video.
26
27Output your think process within the <think> </think> tags.
28
29Then, provide your answer within the <answer> </answer> tags, output the corresponding letter of the option. For example: <think>...</think><answer>A</answer><glue>[(5.2, 10.4)]</glue>.
30"""
31
32
33def inference(video_path, prompt, model, processor, max_new_tokens=2048, device="cuda:0", client = None, pred_glue=None):
34 messages = [
35 {"role": "user", "content": [
36 {"type": "video",
37 "video": video_path,
38 'key_time':pred_glue,
39 "total_pixels": 128*12 * 28 * 28,
40 "min_pixels": 128 * 28 * 28,
41 },
42 {"type": "text", "text": prompt},
43 ]
44 },
45 ]
46 text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
47
48 image_inputs, video_inputs, video_kwargs = process_vision_info(messages, return_video_kwargs=True, client = client)
49 fps_inputs = video_kwargs['fps']
50
51 inputs = processor(text=[text], images=image_inputs, videos=video_inputs, fps=fps_inputs, padding=True, return_tensors="pt")
52 inputs = inputs.to(device)
53
54 with torch.no_grad():
55 output_ids = model.generate(**inputs, max_new_tokens=max_new_tokens, use_cache=True)
56
57 generated_ids = [output_ids[i][len(inputs.input_ids[i]):] for i in range(len(output_ids))]
58 output_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
59 return output_text[0]
60
61
62for percption in range(num_percptions):
63
64 if percption == num_percptions - 1:
65 example_prompt = QA_THINK.replace("[QUESTION]", item["problem"]["question"])
66 else:
67 example_prompt = QA_THINK_GLUE.replace("[QUESTION]", item["problem"]["question"])
68
69
70 ans = inference(video_path, example_prompt, model, processor, device=device, client=client, pred_glue=pred_glue)
71
72 pattern_glue = r'<glue>(.*?)</glue>'
73 match_glue = re.search(pattern_glue, ans, re.DOTALL)
74 # print(f'ann:{ans}')
75 answers.append(ans)
76 pred_glue = None
77 try:
78 if match_glue:
79 glue = match_glue.group(1)
80 pred_glue = ast.literal_eval(glue)
81
82
83 except Exception as e:
84 pred_glue = None
85print(ans)1@article{li2025videochatr1,
2 title={VideoChat-R1: Enhancing Spatio-Temporal
3Perception via Reinforcement Fine-Tuning},
4 author={Li, Xinhao and Yan, Ziang and Meng, Desen and Dong, Lu and Zeng, Xiangyu and He, Yinan and Wang, Yali and Qiao, Yu and Wang, Yi and Wang, Limin},
5 journal={arXiv preprint arXiv:2504.06958},
6 year={2025}
7}
8
9@article{yan2025videochatr15,
10 title={VideoChat-R1.5: Visual Test-Time Scaling to Reinforce Multimodal Reasoning by Iterative Perception},
11 author={Yan, Ziang and Li, Xinhao and He, Yinan and Zhengrong Yue and Zeng, Xiangyu and Wang, Yali and Qiao, Yu and Wang, Limin and Wang, Yi},
12 journal={arXiv preprint arXiv:2509.21100},
13 year={2025}
14}