1pipe = StableDiffusionPipeline(...)
2# llm_model = load_quant('/GuanacoOnConsumerHardware', 'guanaco7b-4bit-128g.pt', 4, 128, 0)
3llm_tokenizer = LlamaTokenizer.from_pretrained("JosephusCheung/Guanaco",use_fast=False,torch_dtype=torch.float16)
4llm_model = LlamaForCausalLM.from_pretrained("JosephusCheung/Guanaco",device_map="auto",torch_dtype=torch.float16)
5
6class LLMToCLIP(nn.Module):
7 def __init__(self):
8 super(LLMToCLIP, self).__init__()
9 self.proj = nn.Linear(4096, 4096, bias=False)
10 self.deproj = nn.Linear(4096, 768, bias=False)
11
12 def forward(self, x):
13 a = self.proj(x)
14 b = self.deproj(a)
15 return b
16
17llm_to_clip=LLMToCLIP()
18llm_to_clip.load_state_dict(torch.load("toclip.pth"))
19
20llm_embeddings = llm_model(input_ids=input_ids, output_hidden_states=True).hidden_states[-1]
21
22image = pipe(prompt_embeds=llm_to_clip(llm_embeddings)).images[0]
