If you want more details on how to generate your own blip cpationed dataset see this
colab
Training was done using a slightly modified version of Hugging-Face's text to image training
example script
The
main folder contains a .ckpt and a .yaml file to be put in
stable-diffusion-webui "stable-diffusion-webui/models/Stable-diffusion" folder and used to generate images
1from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler
2import torch
3
4# this will substitute the default PNDM scheduler for K-LMS
5lms = LMSDiscreteScheduler(
6 beta_start=0.00085,
7 beta_end=0.012,
8 beta_schedule="scaled_linear"
9)
10
11guidance_scale=8.5
12seed=777
13steps=50
14
15cartoon_model_path = "Norod78/sd2-simpsons-blip"
16cartoon_pipe = StableDiffusionPipeline.from_pretrained(cartoon_model_path, scheduler=lms, torch_dtype=torch.float16)
17cartoon_pipe.to("cuda")
18
19def generate(prompt, file_prefix ,samples):
20 torch.manual_seed(seed)
21 prompt += ", Very detailed, clean, high quality, sharp image"
22 cartoon_images = cartoon_pipe([prompt] * samples, num_inference_steps=steps, guidance_scale=guidance_scale)["images"]
23 for idx, image in enumerate(cartoon_images):
24 image.save(f"{file_prefix}-{idx}-{seed}-sd2-simpsons-blip.jpg")
25
26generate("An oil painting of Snoop Dogg as a simpsons character", "01_SnoopDog", 4)
27generate("Gal Gadot, cartoon", "02_GalGadot", 4)
28generate("A cartoony Simpsons town", "03_SimpsonsTown", 4)
29generate("Pikachu with the Simpsons, Eric Wallis", "04_PikachuSimpsons", 4)
Finetuned for 10,000 iterations upon
stabilityai/stable-diffusion-2-base on
BLIP captioned Simpsons images using 1xA5000 GPU on my home desktop computer