Views
No views yet
1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor
3
4# Load model
5model_name = "Intel/LongCat-Next-int4-AutoRound"
6model = AutoModelForCausalLM.from_pretrained(
7 model_name,
8 torch_dtype=torch.bfloat16,
9 device_map="auto",
10 trust_remote_code=True,
11)
12model.eval()
13
14tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, fix_mistral_regex=True)
15model.text_tokenizer = tokenizer # Dynamic binding
16processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
17
18# Set messages
19messages = [
20 {"role": "system", "content": "You are a helpful assistant."},
21 {"role": "user", "content": "What book is this?<longcat_img_start>./assets/book.png<longcat_img_end>"}
22]
23
24# Apply chat-template
25text_input = tokenizer.apply_chat_template(
26 messages,
27 tokenize=False,
28 add_generation_prompt=True,
29)
30print(f"{text_input=}")
31
32# Preprocessing
33text_inputs, visual_inputs, audio_inputs = processor(text=text_input, return_tensors="pt")
34text_inputs = text_inputs.to(model.device)
35if visual_inputs is not None:
36 visual_inputs = visual_inputs.to(model.device)
37if audio_inputs is not None:
38 audio_inputs = audio_inputs.to(model.device)
39
40# AR
41with torch.no_grad():
42 outputs = model.generate(
43 input_ids=text_inputs["input_ids"],
44 visual_inputs=visual_inputs,
45 audio_inputs=audio_inputs,
46 return_dict_in_generate=True,
47 )
48
49# Text decoding
50output_input_ids = outputs.sequences
51text_output = tokenizer.decode(output_input_ids[0][len(text_inputs["input_ids"][0]):], skip_special_tokens=True)
52print(f"{text_output=}")
53
54# Images decoding
55output_visual_ids = outputs.visual_ids
56if output_visual_ids.size(0) > 0:
57 image_path_list = model.model.decode_visual_ids_and_save(
58 output_visual_ids,
59 save_prefix="./output_image",
60 **model.generation_config.visual_generation_config["custom_params"],
61 )
62 print(f"{image_path_list=}")
63
64# Audio decoding
65output_audio_text_ids = outputs.audio_text_ids
66output_audio_ids = outputs.audio_ids
67if output_audio_text_ids.size(-1) > 0:
68 audio_text = tokenizer.decode(output_audio_text_ids[0], skip_special_tokens=True)
69 print(f"{audio_text=}")
70if output_audio_ids.size(0) > 0:
71 audio_path_list = model.model.decode_audio_ids_and_save(
72 output_audio_ids,
73 save_prefix="./output_audio",
74 **model.generation_config.audio_generation_config["custom_params"],
75 )
76 print(f"{audio_path_list=}")1# Need https://github.com/intel/auto-round/pull/1637
2AR_CALIB_FORCE_CUDA=1 auto-round --bits 4 --iters 200 --model_name meituan-longcat/LongCat-Next@article{cheng2023optimize, title={Optimize weight rounding via signed gradient descent for the quantization of llms}, author={Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao and Liu, Yi}, journal={arXiv preprint arXiv:2309.05516}, year={2023} }