Views
No views yet
1from PIL import Image
2import requests
3import torch
4from transformers import AutoModelForCausalLM, AutoTokenizer
5
6
7tokenizer = AutoTokenizer.from_pretrained("BAAI/Emu2-Chat")
8
9model = AutoModelForCausalLM.from_pretrained(
10 "BAAI/Emu2-Chat",
11 torch_dtype=torch.bfloat16,
12 low_cpu_mem_usage=True,
13 trust_remote_code=True).to('cuda').eval()
14
15
16# `[<IMG_PLH>]` is the image placeholder which will be replaced by image embeddings.
17# the number of `[<IMG_PLH>]` should be equal to the number of input images
18
19query = '[<IMG_PLH>]Describe the image in details:'
20image = Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/blue_black_1_top_left.jpg?raw=true',stream=True).raw).convert('RGB')
21
22
23inputs = model.build_input_ids(
24 text=[query],
25 tokenizer=tokenizer,
26 image=[image]
27)
28
29with torch.no_grad():
30 outputs = model.generate(
31 input_ids=inputs["input_ids"],
32 attention_mask=inputs["attention_mask"],
33 image=inputs["image"].to(torch.bfloat16),
34 max_new_tokens=64,
35 length_penalty=-1)
36
37output_text = tokenizer.batch_decode(outputs, skip_special_tokens=True)1from PIL import Image
2import requests
3import torch
4from transformers import AutoModelForCausalLM, AutoTokenizer
5
6
7tokenizer = AutoTokenizer.from_pretrained("BAAI/Emu2-Chat")
8
9model = AutoModelForCausalLM.from_pretrained(
10 "BAAI/Emu2-Chat",
11 torch_dtype=torch.bfloat16,
12 low_cpu_mem_usage=True,
13 trust_remote_code=True).to('cuda').eval()
14
15# `[<IMG_PLH>]` is the image placeholder which will be replaced by image embeddings.
16# the number of `[<IMG_PLH>]` should be equal to the number of input images
17
18query = "[<IMG_PLH>][red, white, 3, bottom left].[<IMG_PLH>][yellow, white, 2, top left].[<IMG_PLH>][green, black, 4, bottom right][<IMG_PLH>]"
19
20images = [
21 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/red_white_3_bottom_left.jpg?raw=true',stream=True).raw).convert('RGB'),
22 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/yellow_white_2_top_right.jpg?raw=true',stream=True).raw).convert('RGB'),
23 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/green_black_4_bottom_right.jpg?raw=true',stream=True).raw).convert('RGB'),
24 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/blue_black_1_top_left.jpg?raw=true',stream=True).raw).convert('RGB'),
25]
26
27inputs = model.build_input_ids(
28 text=[query],
29 tokenizer=tokenizer,
30 image=images
31
32)
33
34with torch.no_grad():
35 outputs = model.generate(
36 input_ids=inputs["input_ids"],
37 attention_mask=inputs["attention_mask"],
38 image=inputs["image"].to(torch.bfloat16),
39 max_new_tokens=64,
40 length_penalty=-1)
41
42output_text = tokenizer.batch_decode(outputs, skip_special_tokens=True)1from PIL import Image
2import requests
3import torch
4from transformers import AutoModelForCausalLM, AutoTokenizer
5from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch
6
7tokenizer = AutoTokenizer.from_pretrained("BAAI/Emu2-Chat")
8
9with init_empty_weights():
10 model = AutoModelForCausalLM.from_pretrained(
11 "BAAI/Emu2-Chat",
12 torch_dtype=torch.bfloat16,
13 low_cpu_mem_usage=True,
14 trust_remote_code=True)
15
16device_map = infer_auto_device_map(model, max_memory={0:'38GiB',1:'38GiB',}, no_split_module_classes=['Block','LlamaDecoderLayer'])
17# input and output logits should be on same device
18device_map["model.decoder.lm.lm_head"] = 0
19
20model = load_checkpoint_and_dispatch(
21 model,
22 'local/path/to/hf/version/Emu2-Chat/model',
23 device_map=device_map).eval()
24
25# `[<IMG_PLH>]` is the image placeholder which will be replaced by image embeddings.
26# the number of `[<IMG_PLH>]` should be equal to the number of input images
27
28query = '[<IMG_PLH>]Describe the image in details:'
29image = Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/blue_black_1_top_left.jpg?raw=true',stream=True).raw).convert('RGB')
30
31inputs = model.build_input_ids(
32 text=[query],
33 tokenizer=tokenizer,
34 image=[image]
35
36)
37
38with torch.no_grad():
39 outputs = model.generate(
40 input_ids=inputs["input_ids"],
41 attention_mask=inputs["attention_mask"],
42 image=inputs["image"].to(torch.bfloat16),
43 max_new_tokens=64,
44 length_penalty=-1)
45
46output_text = tokenizer.batch_decode(outputs, skip_special_tokens=True)1from PIL import Image
2import requests
3import torch
4from transformers import AutoModelForCausalLM, AutoTokenizer
5from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch
6
7tokenizer = AutoTokenizer.from_pretrained("BAAI/Emu2-Chat")
8
9with init_empty_weights():
10 model = AutoModelForCausalLM.from_pretrained(
11 "BAAI/Emu2-Chat",
12 torch_dtype=torch.bfloat16,
13 low_cpu_mem_usage=True,
14 trust_remote_code=True)
15
16device_map = infer_auto_device_map(model, max_memory={0:'38GiB',1:'38GiB',}, no_split_module_classes=['Block','LlamaDecoderLayer'])
17# input and output logits should be on same device
18device_map["model.decoder.lm.lm_head"] = 0
19
20model = load_checkpoint_and_dispatch(
21 model,
22 'local/path/to/hf/version/Emu2-Chat/model',
23 device_map=device_map).eval()
24
25# `[<IMG_PLH>]` is the image placeholder which will be replaced by image embeddings.
26# the number of `[<IMG_PLH>]` should be equal to the number of input images
27query = "[<IMG_PLH>][red, white, 3, bottom left].[<IMG_PLH>][yellow, white, 2, top left].[<IMG_PLH>][green, black, 4, bottom right][<IMG_PLH>]"
28
29images = [
30 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/red_white_3_bottom_left.jpg?raw=true',stream=True).raw).convert('RGB'),
31 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/yellow_white_2_top_right.jpg?raw=true',stream=True).raw).convert('RGB'),
32 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/green_black_4_bottom_right.jpg?raw=true',stream=True).raw).convert('RGB'),
33 Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/blue_black_1_top_left.jpg?raw=true',stream=True).raw).convert('RGB'),
34]
35
36inputs = model.build_input_ids(
37 text=[query],
38 tokenizer=tokenizer,
39 image=images
40
41)
42
43with torch.no_grad():
44 outputs = model.generate(
45 input_ids=inputs["input_ids"],
46 attention_mask=inputs["attention_mask"],
47 image=inputs["image"].to(torch.bfloat16),
48 max_new_tokens=64,
49 length_penalty=-1)
50
51output_text = tokenizer.batch_decode(outputs, skip_special_tokens=True)1from PIL import Image
2import requests
3import torch
4from transformers import AutoModelForCausalLM, AutoTokenizer
5
6
7tokenizer = AutoTokenizer.from_pretrained("BAAI/Emu2-Chat")
8
9model = AutoModelForCausalLM.from_pretrained(
10 "BAAI/Emu2-Chat",
11 load_in_4bit=True,
12 trust_remote_code=True,
13 bnb_4bit_compute_dtype=torch.float16).eval()
14
15query = '[<IMG_PLH>]Describe the image in details:'
16image = Image.open(requests.get('https://github.com/baaivision/Emu/Emu2/examples/blue_black_1_top_left.jpg?raw=true',stream=True).raw).convert('RGB')
17
18inputs = model.build_input_ids(
19 text=[query],
20 tokenizer=tokenizer,
21 image=[image]
22
23)
24
25with torch.no_grad():
26 outputs = model.generate(
27 input_ids=inputs["input_ids"],
28 attention_mask=inputs["attention_mask"],
29 image=inputs["image"].to(torch.float16), # should be torch.float16
30 max_new_tokens=64,
31 length_penalty=-1)
32
33output_text = tokenizer.batch_decode(outputs, skip_special_tokens=True)@article{Emu2,
title={Generative Multimodal Models are In-Context Learners},
author={Quan Sun and Yufeng Cui and Xiaosong Zhang and Fan Zhang and Qiying Yu and Zhengxiong Luo and Yueze Wang and Yongming Rao and Jingjing Liu and Tiejun Huang and Xinlong Wang},
publisher={arXiv preprint arXiv:2312.13286},
year={2023},
}