This repository contains a Hugging Face model that has been fine-tuned on a Tamil dataset. The model uses the peft library for generating responses.
1from peft import PeftModel
2from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig
1tokenizer = LlamaTokenizer.from_pretrained("yahma/llama-7b-hf")
2model = LlamaForCausalLM.from_pretrained(
3 "yahma/llama-7b-hf",
4 load_in_8bit=True,
5 device_map="auto",
6)
1def generate_prompt(instruction, input=None):
2 if input:
3 return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
4
5### Instruction:
6{instruction}
7
8### Input:
9{input}
10
11### Response:"""
12 else:
13 return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
14
15### Instruction:
16{instruction}
17
18### Response:"""
1generation_config = GenerationConfig(
2 temperature=0.1,
3 top_p=0.75,
4 num_beams=4,
5)
6
7def evaluate(model, instruction, input=None):
8 prompt = generate_prompt(instruction, input)
9 inputs = tokenizer(prompt, return_tensors="pt")
10 input_ids = inputs["input_ids"].cuda()
11 generation_output = model.generate(
12 input_ids=input_ids,
13 generation_config=generation_config,
14 return_dict_in_generate=True,
15 output_scores=True,
16 max_new_tokens=256
17 )
18 for s in generation_output.sequences:
19 output = tokenizer.decode(s)
20 print("Response:", output.split("### Response:")[1].strip())
21
22instruct =input("Instruction: ")
23evaluate(model, instruct)
1instruct = "Write a response that appropriately completes the request."
2input = "This is a sample input."
3evaluate(model, instruct, input)
This will output a response that completes the request.
