Views
No views yet
pip install torch==1.12.1 tokenizers==0.13.3 git+https://github.com/huggingface/transformers| 需求 Demand | 任务 Task | 系列 Series | 模型 Model | 参数 Parameter | 额外 Extra |
|---|---|---|---|---|---|
| 多模态 Multi-Modal | 通用 General | 姜子牙-多模态 Ziya-Visual | InstructBLIP LLaMA | 14B | English&Chinese |
1import gradio as gr
2from PIL import Image
3import torch
4import random
5from fengshen.models.instruct_ditto.modeling_instruct_ditto import InstructDittoLMForConditionalGeneration, DittoQFromerForPretrain, DittoLMForConditionalGeneration
6from torchvision.transforms import Compose, ToTensor, Resize, Normalize
7from transformers import LlamaTokenizer, BertTokenizer, GenerationConfig
8from torchvision.transforms import Normalize, Compose, RandomResizedCrop, InterpolationMode, ToTensor, RandomHorizontalFlip
9
10OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
11OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
12
13device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
14_MODEL_PATH = "your model path"
15
16transforms = Compose([
17 RandomResizedCrop(
18 224,
19 scale=(0.5, 1.0),
20 interpolation=InterpolationMode.BICUBIC,
21 ),
22 RandomHorizontalFlip(),
23 ToTensor(),
24 Normalize(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
25])
26
27model = InstructDittoLMForConditionalGeneration.from_pretrained(_MODEL_PATH).to(device).eval()
28instruct_tokenizer = BertTokenizer.from_pretrained(os.path.join(_MODEL_PATH, "qformer_tokenizer"))
29tokenizer = LlamaTokenizer.from_pretrained(_MODEL_PATH, use_fast = False)
30
31qformer_prompt = "{prompt}"
32qformer_prompt_list = []
33prompt_prefix = ''
34llm_prompt = "<human>: {prompt}\n<bot>:"
35llm_prompt_list = []
36
37prompt = ["your prompt"]
38
39for i in prompt:
40 qformer_prompt_list.append(qformer_prompt.format_map({"prompt":i}))
41 llm_prompt_list.append(llm_prompt.format_map({"prompt":i}))
42
43image_url = ["your image"]
44
45imgs = []
46for img_url in image_url:
47 imgs.append(transforms(Image.open(img_url).convert('RGB')))
48
49config = GenerationConfig(
50 # do_sample=True, #False
51 # num_beams=3, # 3
52 # min_length=4,
53 max_new_tokens=128,
54 repetition_penalty=1.18,
55 # length_penalty=1,
56 temperature=0.7,
57 top_p=0.1,
58 bos_token_id=1,
59 eos_token_id=2,
60 pad_token_id=39410,
61)
62
63imgs = torch.stack(imgs)
64
65instruct_tokenizer.padding_side = 'right'
66tokenizer.padding_side = 'left'
67
68for i in range(imgs.shape[0]):
69 prompt_prefix_ids = tokenizer(prompt_prefix, return_tensors="pt").input_ids
70 qformer_instruct_ids = instruct_tokenizer(qformer_prompt_list[i], return_tensors="pt").input_ids
71 llm_instruct_ids = tokenizer(llm_prompt_list[i], return_tensors="pt", add_special_tokens=False).input_ids
72 qformer_instruct_atts = instruct_tokenizer(qformer_prompt_list[i], return_tensors="pt").attention_mask
73 llm_instruct_atts = tokenizer(llm_prompt_list[i], return_tensors="pt", add_special_tokens=False).attention_mask
74 captions = model.generate(
75 imgs[i].unsqueeze(0).to('cuda'),
76 qformer_instruct_ids=qformer_instruct_ids.to('cuda'),
77 prompt_prefix_ids = prompt_prefix_ids.to('cuda'),
78 llm_instruct_ids=llm_instruct_ids.to('cuda'),
79 generation_config=config
80 )
81 caption = tokenizer.decode(captions[0])
82 print("问: " + prompt[i] + "\n" + "答: " + caption)
83
841@article{fengshenbang,
2 author = {Jiaxing Zhang and Ruyi Gan and Junjie Wang and Yuxiang Zhang and Lin Zhang and Ping Yang and Xinyu Gao and Ziwei Wu and Xiaoqun Dong and Junqing He and Jianheng Zhuo and Qi Yang and Yongfeng Huang and Xiayu Li and Yanghan Wu and Junyu Lu and Xinyu Zhu and Weifeng Chen and Ting Han and Kunhao Pan and Rui Wang and Hao Wang and Xiaojun Wu and Zhongshen Zeng and Chongpei Chen},
3 title = {Fengshenbang 1.0: Being the Foundation of Chinese Cognitive Intelligence},
4 journal = {CoRR},
5 volume = {abs/2209.02970},
6 year = {2022}
7}1@article{lu2023ziya,
2 title={Ziya-VL: Bilingual Large Vision-Language Model via Multi-Task Instruction Tuning},
3 author={Lu, Junyu and Zhang, Dixiang and Wu, Xiaojun and Gao, Xinyu and Gan, Ruyi and Zhang, Jiaxing and Song, Yan and Zhang, Pingjian},
4 journal={arXiv preprint arXiv:2310.08166},
5 year={2023}
6}1@misc{Fengshenbang-LM,
2 title={Fengshenbang-LM},
3 author={IDEA-CCNL},
4 year={2021},
5 howpublished={\url{https://github.com/IDEA-CCNL/Fengshenbang-LM}},
6}