Views
No views yet
1from safetensors.torch import load_file
2from huggingface_hub import hf_hub_download
3
4
5model = AutoModelForCausalLM.from_pretrained(f"{REPO_NAME}-lora", device_map="auto")
6tokenizer = AutoTokenizer.from_pretrained(f"{REPO_NAME}-lora")
7tokenizer.pad_token = tokenizer.eos_token
8tokenizer.padding_side = "left"
9
10apply_peft_to_module(model, LinearWithLoRA, r=8, alpha=16, target_submodules=["k_proj", "v_proj"])
11model = model.to(DEVICE)
12
13path = hf_hub_download(f"{REPO_NAME}-lora", "model.safetensors")
14state_dict = load_file(path)
15
16model.load_state_dict(state_dict, strict=False)
17
18messages = [{"role": "system", "content": "Choose which word best describes the text below: positive, neutral or negative."}{"role": "user", "content": "<YOUR_TWEET_HERE>"}]
19text = tokenizer.apply_chat_template(messages, tokenize=False)
20model_inputs = tokenizer([text], return_tensors="pt").to(DEVICE)
21
22generated_ids = model.generate(model_inputs.input_ids, max_new_tokens=256, do_sample=True)
23response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
24
25print(response)