Views
No views yet
1from peft import AutoPeftModelForCausalLM
2from transformers import AutoTokenizer
3
4tokenizer = AutoTokenizer.from_pretrained("nicce/sexbot")
5model = AutoPeftModelForCausalLM.from_pretrained("nicce/sexbot").to("cuda") # if you get a CUDA out of memory error, try load_in_8bit=True
6
7messages = [
8 {"role": "system", "content": "You are a helpful assistant"},
9 {"role": "user", "content": "Hi, can you please explain machine learning to me?"}
10]
11
12encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt").to("cuda")
13generated_ids = model.generate(input_ids=model_inputs, min_new_tokens=10, max_new_tokens=300, do_sample=True, temperature=0.9, top_p=0.8)
14decoded = tokenizer.batch_decode(generated_ids)
15
16print(decoded[0])