Views
No views yet
1
2from transformers import LlamaForCausalLM, LlamaTokenizer,GenerationConfig
3from peft import PeftModel
4
5
6device_map = "auto"
7
8tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
9model = LlamaForCausalLM.from_pretrained(
10 "decapoda-research/llama-7b-hf",
11 load_in_8bit=True,
12 device_map="auto",
13)
14
15### load model after fine tuned on alpaca datasets
16model = PeftModel.from_pretrained(model, "Nelsonlin0321/alpaca-lora-7b-tuned-on-hk-cvs-fqa")
17
18tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
19tokenizer.pad_token_id = 0
20
21
22def generate_prompt_eval(instruction):
23 template = f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
24### Instruction:
25{instruction}
26### Response:"""
27 return template
28
29eval_generation_config = GenerationConfig(
30 temperature=0.1,
31 top_p=0.75,
32 num_beams=4,
33)
34
35
36def generate_answer(instruction):
37 prompt = generate_prompt_eval(instruction)
38 inputs = tokenizer(prompt, return_tensors="pt")
39 input_ids = inputs["input_ids"].cuda()
40 generation_output = model.generate(
41 input_ids=input_ids,
42 generation_config=eval_generation_config,
43 return_dict_in_generate=True,
44 output_scores=True,
45 max_new_tokens=256
46 )
47 for s in generation_output.sequences:
48 output = tokenizer.decode(s)
49 # print(output)
50 print("Response:", output.split("### Response:")[1].strip())
51
52
53question = "Who are eligible to be disbursed with the first-instalment voucher of $1,500 on 16 April?"
54
55generate_answer(question)
56>> Response: All eligible people who have successfully registered under 2022 CVS and met the relevant eligibility criteria will be disbursed with the first-instalment voucher of $1,500 on 16 April.
57
58
59