Views
No views yet
1from modeling_vision import MM_LLMs, MM_LLMs_Config
2from transformers import AutoTokenizer, AutoProcessor
3from PIL import Image
4import requests
5
6model = MM_LLMs.from_pretrained(
7 'mesolitica/malaysian-tinyllama-1.1b-siglip-large-384-vision',
8 flash_attention = True,
9 dtype = torch.bfloat16,
10 torch_dtype = torch.bfloat16
11)
12_ = model.cuda()
13
14image_processor = AutoProcessor.from_pretrained('google/siglip-large-patch16-384')
15tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-siglip-large-384-vision')
16
17def prepare_dataset(messages, images: List[str] = None):
18 if images is not None:
19 images = [Image.open(f).convert('RGB') for f in images]
20 image_output = image_processor(images=images, return_tensors='pt')['pixel_values']
21 else:
22 image_output = None
23
24 prompt = tokenizer.apply_chat_template(messages, tokenize = False)
25 outputs = tokenizer(
26 prompt,
27 return_tensors='pt',
28 return_overflowing_tokens=False,
29 return_length=False)
30
31 outputs['images'] = image_output
32 outputs['image_index'] = torch.tensor([0] * len(outputs['images']))
33 outputs['image_starts'] = torch.tensor([tokenizer.convert_tokens_to_ids('<image>')] * len(outputs['images']))
34 return outputs
35
36with open('Persian-cat-breed.jpg', 'wb') as fopen:
37 fopen.write(requests.get('https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg').content)
38
39with open('nasi-goreng-1-23.jpg', 'wb') as fopen:
40 fopen.write(requests.get('https://www.jocooks.com/wp-content/uploads/2023/09/nasi-goreng-1-23.jpg').content)
41
42messages = [
43 {'role': 'user', 'content': '<image> </image> ini gambar apa'},
44]
45outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg'])
46outputs['images'] = outputs['images'].type(model.dtype)
47for k in outputs.keys():
48 if outputs[k] is not None:
49 outputs[k] = outputs[k].cuda()
50
51with torch.no_grad():
52 model_inputs = model.prepare_inputs_for_generation(**outputs)
53r = model_inputs.pop('input_ids', None)
54
55generate_kwargs = dict(
56 model_inputs,
57 max_new_tokens=300,
58 top_p=0.95,
59 top_k=50,
60 temperature=0.1,
61 do_sample=True,
62 num_beams=1,
63)
64
65r = model.llm.generate(**generate_kwargs)
66print(tokenizer.decode(r[0]))<s>Imej itu menunjukkan seekor kucing putih yang comel duduk di atas sofa hitam.</s>1messages = [
2 {'role': 'user', 'content': '<image> </image> <image> </image> apa kaitan 2 gambar ni'},
3]
4outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg', 'nasi-goreng-1-23.jpg'])
5outputs['images'] = outputs['images'].type(model.dtype)
6for k in outputs.keys():
7 if outputs[k] is not None:
8 outputs[k] = outputs[k].cuda()
9
10with torch.no_grad():
11 model_inputs = model.prepare_inputs_for_generation(**outputs)
12r = model_inputs.pop('input_ids', None)
13
14generate_kwargs = dict(
15 model_inputs,
16 max_new_tokens=300,
17 top_p=0.95,
18 top_k=50,
19 temperature=0.1,
20 do_sample=True,
21 num_beams=1,
22)
23
24r = model.llm.generate(**generate_kwargs)
25print(tokenizer.decode(r[0]))<s>Tiada hubungan yang jelas antara gambar 1 (anak kucing putih duduk di atas sofa) dan gambar 2 (foto penutup mangkuk mi telur dengan nasi dan cili). Gambar pertama ialah imej haiwan, manakala gambar kedua ialah imej makanan. Mereka tergolong dalam kategori yang berbeza dan tidak mempunyai hubungan antara satu sama lain.</s>