Views
No views yet
1from modeling_vision import MM_LLMs, MM_LLMs_Config
2from transformers import AutoTokenizer, AutoProcessor
3from PIL import Image
4import requests
5
6model = MM_LLMs.from_pretrained(
7 'mesolitica/malaysian-Qwen1.5-0.5B-siglip-base-384-vision',
8 flash_attention = True,
9 dtype = torch.bfloat16,
10 torch_dtype = torch.bfloat16
11)
12_ = model.cuda()
13
14image_processor = AutoProcessor.from_pretrained('google/siglip-base-patch16-384')
15tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-Qwen1.5-0.5B-siglip-base-384-vision')
16model.llm.generation_config.eos_token_id = tokenizer.eos_token_id
17
18def prepare_dataset(messages, images: List[str] = None):
19 if images is not None:
20 images = [Image.open(f).convert('RGB') for f in images]
21 image_output = image_processor(images=images, return_tensors='pt')['pixel_values']
22 else:
23 image_output = None
24
25 prompt = tokenizer.apply_chat_template(messages, tokenize = False)
26 outputs = tokenizer(
27 prompt,
28 return_tensors='pt',
29 return_overflowing_tokens=False,
30 return_length=False)
31
32 outputs['images'] = image_output
33 outputs['image_index'] = torch.tensor([0] * len(outputs['images']))
34 outputs['image_starts'] = torch.tensor([tokenizer.convert_tokens_to_ids('<image>')] * len(outputs['images']))
35 return outputs
36
37with open('Persian-cat-breed.jpg', 'wb') as fopen:
38 fopen.write(requests.get('https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg').content)
39
40with open('nasi-goreng-1-23.jpg', 'wb') as fopen:
41 fopen.write(requests.get('https://www.jocooks.com/wp-content/uploads/2023/09/nasi-goreng-1-23.jpg').content)
42
43messages = [
44 {'role': 'user', 'content': '<image> </image> ini gambar apa'},
45]
46outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg'])
47outputs['images'] = outputs['images'].type(model.dtype)
48for k in outputs.keys():
49 if outputs[k] is not None:
50 outputs[k] = outputs[k].cuda()
51
52with torch.no_grad():
53 model_inputs = model.prepare_inputs_for_generation(**outputs)
54r = model_inputs.pop('input_ids', None)
55
56generate_kwargs = dict(
57 model_inputs,
58 max_new_tokens=300,
59 top_p=0.95,
60 top_k=50,
61 temperature=0.1,
62 do_sample=True,
63 num_beams=1,
64)
65
66r = model.llm.generate(**generate_kwargs)
67print(tokenizer.decode(r[0]))<|endoftext|><|im_start|>assistant
Ini adalah gambar kucing putih yang duduk di atas sofa hitam.<|im_end|>1messages = [
2 {'role': 'user', 'content': '<image> </image> <image> </image> apa kaitan 2 gambar ni'},
3]
4outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg', 'nasi-goreng-1-23.jpg'])
5outputs['images'] = outputs['images'].type(model.dtype)
6for k in outputs.keys():
7 if outputs[k] is not None:
8 outputs[k] = outputs[k].cuda()
9
10with torch.no_grad():
11 model_inputs = model.prepare_inputs_for_generation(**outputs)
12r = model_inputs.pop('input_ids', None)
13
14generate_kwargs = dict(
15 model_inputs,
16 max_new_tokens=300,
17 top_p=0.95,
18 top_k=50,
19 temperature=0.1,
20 do_sample=True,
21 num_beams=1,
22)
23
24r = model.llm.generate(**generate_kwargs)
25print(tokenizer.decode(r[0]))<|endoftext|><|im_start|>assistant
Tiada hubungan langsung antara gambar 1 dan gambar 2. Gambar 1 ialah imej kucing putih dengan bulu putih, manakala gambar 2 ialah gambar mangkuk makan tengah hari kacang hitam dan lobak merah yang dicincang, dengan garpu diletakkan di sebelahnya. Kedua-duanya tidak berkaitan dari segi kandungan.<|im_end|>