Views
No views yet
1!pip install -q git+https://github.com/huggingface/peft.git
2!pip install transformers
3!pip install -U accelerate
4!pip install accelerate
5!pip install bitsandbytes # Instal bits and bytes for inference of the model1# You need a huggingface token that can access llama2
2!huggingface-cli login1import torch
2from peft import PeftModel, PeftConfig
3from transformers import AutoModelForCausalLM, AutoTokenizer
4
5peft_model_id = "danjie/Chadgpt-Llama2-13b"
6config = PeftConfig.from_pretrained(peft_model_id)
7model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, return_dict=True, load_in_8bit=True, device_map='auto')
8tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
9
10# Load the Lora model
11model = PeftModel.from_pretrained(model, peft_model_id)1def talk_with_llm(tweet: str) -> str:
2 # Encode and move tensor into cuda if applicable.
3 encoded_input = tokenizer(tweet, return_tensors='pt')
4 encoded_input = {k: v.to("cuda") for k, v in encoded_input.items()}
5
6 output = model.generate(**encoded_input, max_new_tokens=64)
7 response = tokenizer.decode(output[0], skip_special_tokens=True)
8 return response
9
10talk_with_llm("<User> Your sentence \n<Assistant>")