Views
No views yet
1
2import torch
3from transformers import AutoModelForCausalLM, AutoTokenizer
4
5# Set the device
6device = "cuda" if torch.cuda.is_available() else "cpu"
7
8# Load the fine-tuned model and tokenizer
9model_path = "MISHANM/Hindi_textgeneration_eng_hindi_Llama-3-8B-Instruct"
10model = AutoModelForCausalLM.from_pretrained(model_path)
11
12# Wrap the model with DataParallel if multiple GPUs are available
13if torch.cuda.device_count() > 1:
14 print(f"Using {torch.cuda.device_count()} GPUs")
15 model = torch.nn.DataParallel(model)
16
17# Move the model to the appropriate device
18model.to(device)
19
20tokenizer = AutoTokenizer.from_pretrained(model_path)
21
22# Function to generate text
23def generate_text(prompt, max_length=1000, temperature=0.9):
24 # Format the prompt according to the chat template
25 messages = [
26 {
27 "role": "system",
28 "content": "You are a Hindi language expert and linguist, with same knowledge give answers in Hindi language. ",
29 },
30 {"role": "user", "content": prompt}
31 ]
32
33 # Apply the chat template
34 formatted_prompt = f"<|system|>{messages[0]['content']}<|user|>{messages[1]['content']}<|assistant|>"
35
36 # Tokenize and generate output
37 inputs = tokenizer(formatted_prompt, return_tensors="pt").to(device)
38 output = model.module.generate( # Use model.module for DataParallel
39 **inputs, max_new_tokens=max_length, temperature=temperature, do_sample=True
40 )
41 return tokenizer.decode(output[0], skip_special_tokens=True)
42
43# Example usage
44prompt = """What is NLP."""
45translated_text = generate_text(prompt)
46print(translated_text)
47
48@misc{MISHANM/Hindi_textgeneration_eng_hindi_Llama-3-8B-Instruct,
author = {Mishan Maurya},
title = {Introducing Fine Tuned LLM for Hindi Language},
year = {2024},
publisher = {Hugging Face},
journal = {Hugging Face repository},
}