Views
No views yet
CogAgent-9B-20241220 has been released! Welcome to visit CogAgent GitHub and Technical Report to explore and use our latest model.THUDM/cogagent-chat-hf, and query prompts, please refer to This GitHubcogagent-chat version of CogAgent checkpoint.cogagent-chat: This model has strong capabilities in GUI Agent, visual multi-turn dialogue, visual grounding, etc.cogagent-vqa: This model has stronger capabilities in single-turn visual dialogue.
cli_demo.py:1import torch
2from PIL import Image
3from transformers import AutoModelForCausalLM, LlamaTokenizer
4import argparse
5
6parser = argparse.ArgumentParser()
7parser.add_argument("--quant", choices=[4], type=int, default=None, help='quantization bits')
8parser.add_argument("--from_pretrained", type=str, default="THUDM/cogagent-chat-hf", help='pretrained ckpt')
9parser.add_argument("--local_tokenizer", type=str, default="lmsys/vicuna-7b-v1.5", help='tokenizer path')
10parser.add_argument("--fp16", action="store_true")
11parser.add_argument("--bf16", action="store_true")
12
13args = parser.parse_args()
14MODEL_PATH = args.from_pretrained
15TOKENIZER_PATH = args.local_tokenizer
16DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
17
18tokenizer = LlamaTokenizer.from_pretrained(TOKENIZER_PATH)
19if args.bf16:
20 torch_type = torch.bfloat16
21else:
22 torch_type = torch.float16
23
24print("========Use torch type as:{} with device:{}========\n\n".format(torch_type, DEVICE))
25
26if args.quant:
27 model = AutoModelForCausalLM.from_pretrained(
28 MODEL_PATH,
29 torch_dtype=torch_type,
30 low_cpu_mem_usage=True,
31 load_in_4bit=True,
32 trust_remote_code=True
33 ).eval()
34else:
35 model = AutoModelForCausalLM.from_pretrained(
36 MODEL_PATH,
37 torch_dtype=torch_type,
38 low_cpu_mem_usage=True,
39 load_in_4bit=args.quant is not None,
40 trust_remote_code=True
41 ).to(DEVICE).eval()
42
43while True:
44 image_path = input("image path >>>>> ")
45 if image_path == "stop":
46 break
47
48 image = Image.open(image_path).convert('RGB')
49 history = []
50 while True:
51 query = input("Human:")
52 if query == "clear":
53 break
54 input_by_model = model.build_conversation_input_ids(tokenizer, query=query, history=history, images=[image])
55 inputs = {
56 'input_ids': input_by_model['input_ids'].unsqueeze(0).to(DEVICE),
57 'token_type_ids': input_by_model['token_type_ids'].unsqueeze(0).to(DEVICE),
58 'attention_mask': input_by_model['attention_mask'].unsqueeze(0).to(DEVICE),
59 'images': [[input_by_model['images'][0].to(DEVICE).to(torch_type)]],
60 }
61 if 'cross_images' in input_by_model and input_by_model['cross_images']:
62 inputs['cross_images'] = [[input_by_model['cross_images'][0].to(DEVICE).to(torch_type)]]
63
64 # add any transformers params here.
65 gen_kwargs = {"max_length": 2048,
66 "temperature": 0.9,
67 "do_sample": False}
68 with torch.no_grad():
69 outputs = model.generate(**inputs, **gen_kwargs)
70 outputs = outputs[:, inputs['input_ids'].shape[1]:]
71 response = tokenizer.decode(outputs[0])
72 response = response.split("</s>")[0]
73 print("\nCog:", response)
74 history.append((query, response))python cli_demo_hf.py --bf16@misc{hong2023cogagent,
title={CogAgent: A Visual Language Model for GUI Agents},
author={Wenyi Hong and Weihan Wang and Qingsong Lv and Jiazheng Xu and Wenmeng Yu and Junhui Ji and Yan Wang and Zihan Wang and Yuxiao Dong and Ming Ding and Jie Tang},
year={2023},
eprint={2312.08914},
archivePrefix={arXiv},
primaryClass={cs.CV}
}
@misc{wang2023cogvlm,
title={CogVLM: Visual Expert for Pretrained Language Models},
author={Weihan Wang and Qingsong Lv and Wenmeng Yu and Wenyi Hong and Ji Qi and Yan Wang and Junhui Ji and Zhuoyi Yang and Lei Zhao and Xixuan Song and Jiazheng Xu and Bin Xu and Juanzi Li and Yuxiao Dong and Ming Ding and Jie Tang},
year={2023},
eprint={2311.03079},
archivePrefix={arXiv},
primaryClass={cs.CV}
}