Views
No views yet
1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3from transformers import GenerationConfig, TextStreamer , TextIteratorStreamer
4
5model = AutoModelForCausalLM.from_pretrained("Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1", torch_dtype=torch.bfloat16).to("cuda")
6tokenizer = AutoTokenizer.from_pretrained("Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1", trust_remote_code=True)
7
8# Existing messages list
9messages = [
10 {"role": "system", "content": " You are Gaja, an AI assistant created by Cognitivelab and trained on top of Llama 3 Large language model (LLM), proficient in English and Hindi. You can respond in both languages based on the user's request."},
11 {"role": "user", "content": "Who are you"}
12]
13
14input_ids = tokenizer.apply_chat_template(
15 messages,
16 add_generation_prompt=True,
17 # tokenize=False,
18 return_tensors="pt"
19).to("cuda")
20
21outputs = model.generate(
22 input_ids,
23 max_new_tokens=256,
24 eos_token_id=tokenizer.convert_tokens_to_ids("<|eot_id|>"),
25 do_sample=True,
26 temperature=0.6,
27 top_p=0.9,
28)
29response = outputs[0][input_ids.shape[-1]:]
30print(tokenizer.decode(response, skip_special_tokens=True))1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3from transformers import GenerationConfig, TextStreamer , TextIteratorStreamer
4
5model = AutoModelForCausalLM.from_pretrained("Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1", torch_dtype=torch.bfloat16).to("cuda")
6tokenizer = AutoTokenizer.from_pretrained("Cognitive-Lab/LLama3-Gaja-Hindi-8B-v0.1", trust_remote_code=True)
7
8# Existing messages list
9messages = [
10 {"role": "system", "content": " You are Gaja, an AI assistant created by Cognitivelab and trained on top of Llama 3 Large language model (LLM), proficient in English and Hindi. You can respond in both languages based on the user's request."},
11]
12
13# Function to add user input and generate response
14def process_user_input(user_input):
15 global messages
16 # Add user's input to messages list
17 messages.append({"role": "user", "content": user_input})
18
19 # Prepare the prompt for generation
20 prompt_formatted_message = tokenizer.apply_chat_template(
21 messages,
22 add_generation_prompt=True,
23 tokenize=False
24 )
25
26 # Configure generation parameters
27 generation_config = GenerationConfig(
28 repetition_penalty=1.2,
29 max_new_tokens=8000,
30 temperature=0.2,
31 top_p=0.95,
32 top_k=40,
33 bos_token_id=tokenizer.bos_token_id,
34 eos_token_id=tokenizer.convert_tokens_to_ids("<|eot_id|>"),
35 pad_token_id=tokenizer.pad_token_id,
36 do_sample=True,
37 use_cache=True,
38 return_dict_in_generate=True,
39 output_attentions=False,
40 output_hidden_states=False,
41 output_scores=False,
42 )
43
44 streamer = TextStreamer(tokenizer)
45 batch = tokenizer(str(prompt_formatted_message.strip()), return_tensors="pt")
46 print("\033[32mResponse: \033[0m") # Print an empty response
47 # Generate response
48 generated = model.generate(
49 inputs=batch["input_ids"].to("cuda"),
50 generation_config=generation_config,
51 streamer=streamer,
52
53 )
54
55 # Extract and format assistant's response
56 # print(tokenizer.decode(generated["sequences"].cpu().tolist()[0]))
57 assistant_response = tokenizer.decode(generated["sequences"].cpu().tolist()[0])
58 # Find the last occurrence of "assistant" and empty string ("")
59 assistant_start_index = assistant_response.rfind("<|start_header_id|>assistant<|end_header_id|>")
60 empty_string_index = assistant_response.rfind("<|eot_id|>")
61
62 # Extract the text between the last "assistant" and ""
63 if assistant_start_index != -1 and empty_string_index != -1:
64 final_response = assistant_response[assistant_start_index + len("<|start_header_id|>assistant<|end_header_id|>") : empty_string_index]
65 else:
66 # final_response = assistant_response # If indices not found, use the whole response
67 assert "Filed to generate multi turn prompt formate"
68
69 # Append the extracted response to the messages list
70 messages.append({"role": "assistant", "content": final_response})
71 # messages.append({"role": "assistant", "content": assistant_response})
72
73 # Print assistant's response
74 # print(f"Assistant: {assistant_response}")
75
76# Main interaction loop
77while True:
78 print("=================================================================================")
79 user_input = input("Input: ") # Prompt user for input
80
81 # Check if user_input is empty
82 if not user_input.strip(): # .strip() removes any leading or trailing whitespace
83 break # Break out of the loop if input is empty
84 # Print response placeholder
85 process_user_input(user_input) # Process user's input and generate response
86You are Gaja, an AI assistant created by Cognitivelab and trained on top of Llama 3 Large language model(LLM), proficient in English and Hindi. You can respond in both languages based on the users request.<|begin_of_text|><|start_header_id|>system<|end_header_id|>
{{ system_prompt }}<|eot_id|><|start_header_id|>user<|end_header_id|>
{{ user_message_1 }}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
{{ model_answer_1 }}<|eot_id|><|start_header_id|>user<|end_header_id|>
{{ user_message_2 }}<|eot_id|><|start_header_id|>assistant<|end_header_id|>