Views
No views yet
git clone https://github.com/X-PLUG/mPLUG-Owl.git1from mplug_owl.modeling_mplug_owl import MplugOwlForConditionalGeneration
2from mplug_owl.tokenization_mplug_owl import MplugOwlTokenizer
3from mplug_owl.processing_mplug_owl import MplugOwlImageProcessor, MplugOwlProcessor
4
5pretrained_ckpt = 'MAGAer13/mplug-owl-llama-7b'
6model = MplugOwlForConditionalGeneration.from_pretrained(
7 pretrained_ckpt,
8 torch_dtype=torch.bfloat16,
9)
10image_processor = MplugOwlImageProcessor.from_pretrained(pretrained_ckpt)
11tokenizer = MplugOwlTokenizer.from_pretrained(pretrained_ckpt)
12processor = MplugOwlProcessor(image_processor, tokenizer)1# We use a human/AI template to organize the context as a multi-turn conversation.
2# <image> denotes an image placehold.
3prompts = [
4'''The following is a conversation between a curious human and AI assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
5Human: <image>
6Human: Explain why this meme is funny.
7AI: ''']
8
9# The image paths should be placed in the image_list and kept in the same order as in the prompts.
10# We support urls, local file paths and base64 string. You can custom the pre-process of images by modifying the mplug_owl.modeling_mplug_owl.ImageProcessor
11image_list = ['https://xxx.com/image.jpg']1# generate kwargs (the same in transformers) can be passed in the do_generate()
2generate_kwargs = {
3 'do_sample': True,
4 'top_k': 5,
5 'max_length': 512
6}
7from PIL import Image
8images = [Image.open(_) for _ in image_list]
9inputs = processor(text=prompts, images=images, return_tensors='pt')
10inputs = {k: v.bfloat16() if v.dtype == torch.float else v for k, v in inputs.items()}
11inputs = {k: v.to(model.device) for k, v in inputs.items()}
12with torch.no_grad():
13 res = model.generate(**inputs, **generate_kwargs)
14sentence = tokenizer.decode(res.tolist()[0], skip_special_tokens=True)
15print(sentence)