Views
No views yet
ifmain/blip-image2promt-stable-diffusion-base is a model based on Salesforce/blip-image-captioning-base, trained on the Ar4ikov/civitai-sd-337k dataset (2K images). This model is designed to generate text descriptions of images in the style of prompts for use with Stable Diffusion models.1import torch
2import requests
3from PIL import Image
4from transformers import BlipProcessor, BlipForConditionalGeneration
5import re
6
7def prepare(text):
8 text = text.replace('. ','.').replace(' .','.')
9 text = text.replace('( ','(').replace(' (','(')
10 text = text.replace(') ',')').replace(' )',')')
11 text = text.replace(': ',':').replace(' :',':')
12 text = text.replace('_ ','_').replace(' _','_')
13 text = text.replace(',(())','').replace('(()),','')
14 for i in range(10):
15 text = text.replace(')))','))').replace('(((','((')
16 text = re.sub(r'<[^>]*>', '', text)
17 return text
18
19path_to_model = "ifmain/blip-image2promt-stable-diffusion-base"
20
21processor = BlipProcessor.from_pretrained(path_to_model)
22model = BlipForConditionalGeneration.from_pretrained(path_to_model, torch_dtype=torch.float16).to("cuda")
23
24img_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg'
25raw_image = Image.open(requests.get(img_url, stream=True).raw).convert('RGB')
26
27# unconditional image captioning
28inputs = processor(raw_image, return_tensors="pt").to("cuda", torch.float16)
29
30out = model.generate(**inputs, max_new_tokens=100)
31
32out_txt = processor.decode(out[0], skip_special_tokens=True)
33
34print(prepare(out_txt)) # woman sitting on the beach at sunset, rear view,((happy)),((happy)),((dog)),((mixed)),(()),((