Views
No views yet
1# use phi2
2deepspeed --include localhost:0 moellava/serve/gradio_web_server.py --model-path "LanguageBind/MoE-LLaVA-Phi2-2.7B-4e"
3# use qwen
4deepspeed --include localhost:0 moellava/serve/gradio_web_server.py --model-path "LanguageBind/MoE-LLaVA-Qwen-1.8B-4e"
5# use stablelm
6deepspeed --include localhost:0 moellava/serve/gradio_web_server.py --model-path "LanguageBind/MoE-LLaVA-StableLM-1.6B-4e" 1# use phi2
2deepspeed --include localhost:0 moellava/serve/cli.py --model-path "LanguageBind/MoE-LLaVA-Phi2-2.7B-4e" --image-file "image.jpg"
3# use qwen
4deepspeed --include localhost:0 moellava/serve/cli.py --model-path "LanguageBind/MoE-LLaVA-Qwen-1.8B-4e" --image-file "image.jpg"
5# use stablelm
6deepspeed --include localhost:0 moellava/serve/cli.py --model-path "LanguageBind/MoE-LLaVA-StableLM-1.6B-4e" --image-file "image.jpg"| Model | LLM | Checkpoint | Avg | VQAv2 | GQA | VizWiz | SQA | T-VQA | POPE | MM-Bench | LLaVA-Bench-Wild | MM-Vet |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| MoE-LLaVA-1.6B×4-Top2 | 1.6B | LanguageBind/MoE-LLaVA-StableLM-1.6B-4e | 60.0 | 76.0 | 60.4 | 37.2 | 62.6 | 47.8 | 84.3 | 59.4 | 85.9 | 26.1 |
| MoE-LLaVA-1.8B×4-Top2 | 1.8B | LanguageBind/MoE-LLaVA-Qwen-1.8B-4e | 60.2 | 76.2 | 61.5 | 32.6 | 63.1 | 48.0 | 87.0 | 59.6 | 88.7 | 25.3 |
| MoE-LLaVA-2.7B×4-Top2 | 2.7B | LanguageBind/MoE-LLaVA-Phi2-2.7B-4e | 63.9 | 77.1 | 61.1 | 43.4 | 68.7 | 50.2 | 85.0 | 65.5 | 93.2 | 31.1 |
1git clone https://github.com/PKU-YuanGroup/MoE-LLaVA
2cd MoE-LLaVA
3conda create -n moellava python=3.10 -y
4conda activate moellava
5pip install --upgrade pip # enable PEP 660 support
6pip install -e .
7pip install -e ".[train]"
8pip install flash-attn --no-build-isolation
9
10# Below are optional. For Qwen model.
11git clone https://github.com/Dao-AILab/flash-attention
12cd flash-attention && pip install .
13# Below are optional. Installing them might be slow.
14# pip install csrc/layer_norm
15# If the version of flash-attn is higher than 2.1.1, the following is not needed.
16# pip install csrc/rotaryLanguageBind/MoE-LLaVA) on local, you can use the following code snippets.deepspeed predict.py1import torch
2from moellava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
3from moellava.conversation import conv_templates, SeparatorStyle
4from moellava.model.builder import load_pretrained_model
5from moellava.utils import disable_torch_init
6from moellava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
7
8def main():
9 disable_torch_init()
10 image = 'moellava/serve/examples/extreme_ironing.jpg'
11 inp = 'What is unusual about this image?'
12 model_path = 'LanguageBind/MoE-LLaVA-Phi2-2.7B-4e' # LanguageBind/MoE-LLaVA-Qwen-1.8B-4e or LanguageBind/MoE-LLaVA-StableLM-1.6B-4e
13 device = 'cuda'
14 load_4bit, load_8bit = False, False # FIXME: Deepspeed support 4bit or 8bit?
15 model_name = get_model_name_from_path(model_path)
16 tokenizer, model, processor, context_len = load_pretrained_model(model_path, None, model_name, load_8bit, load_4bit, device=device)
17 image_processor = processor['image']
18 conv_mode = "phi" # qwen or stablelm
19 conv = conv_templates[conv_mode].copy()
20 roles = conv.roles
21 image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].to(model.device, dtype=torch.float16)
22
23 print(f"{roles[1]}: {inp}")
24 inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
25 conv.append_message(conv.roles[0], inp)
26 conv.append_message(conv.roles[1], None)
27 prompt = conv.get_prompt()
28 input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
29 stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
30 keywords = [stop_str]
31 stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
32
33 with torch.inference_mode():
34 output_ids = model.generate(
35 input_ids,
36 images=image_tensor,
37 do_sample=True,
38 temperature=0.2,
39 max_new_tokens=1024,
40 use_cache=True,
41 stopping_criteria=[stopping_criteria])
42
43 outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:], skip_special_tokens=True).strip()
44 print(outputs)
45
46if __name__ == '__main__':
47 main()1@misc{lin2024moellava,
2 title={MoE-LLaVA: Mixture of Experts for Large Vision-Language Models},
3 author={Bin Lin and Zhenyu Tang and Yang Ye and Jiaxi Cui and Bin Zhu and Peng Jin and Junwu Zhang and Munan Ning and Li Yuan},
4 year={2024},
5 eprint={2401.15947},
6 archivePrefix={arXiv},
7 primaryClass={cs.CV}
8}1@article{lin2023video,
2 title={Video-LLaVA: Learning United Visual Representation by Alignment Before Projection},
3 author={Lin, Bin and Zhu, Bin and Ye, Yang and Ning, Munan and Jin, Peng and Yuan, Li},
4 journal={arXiv preprint arXiv:2311.10122},
5 year={2023}
6}