1from unsloth import FastLanguageModel
2import torch
3max_seq_length = 2048 # Choose any! RoPE Scaling supported internally!
4dtype = None # Auto detection (Float16 for T4/V100, Bfloat16 for Ampere+)
5load_in_4bit = True # Use 4-bit quantization to optimize memory usage
6model, tokenizer = FastLanguageModel.from_pretrained(
7 model_name = "saishshinde15/Clyrai_Valhala",
8 max_seq_length = max_seq_length,
9 dtype = dtype,
10 load_in_4bit = load_in_4bit
11)
12
13FastLanguageModel.for_inference(model)
14instruction = """You are an advanced AI assistant. Provide answers in a clear manner."""
15
16messages = [
17 {"role": "system", "content": instruction},
18 {"role": "user", "content": "Who created you?"}
19]
20
21prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
22inputs = tokenizer(prompt, return_tensors='pt', padding=True, truncation=True).to("cuda")
23outputs = model.generate(**inputs, max_new_tokens=1500, num_return_sequences=1)
24text = tokenizer.decode(outputs[0], skip_special_tokens=True)
25
26assistant_start = text.find("assistant")
27response = text[assistant_start + len("assistant"):].strip() if assistant_start != -1 else text
28
29print(response)
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4model_name = "saishshinde15/Clyrai_Valhala"
5tokenizer = AutoTokenizer.from_pretrained(model_name)
6model = AutoModelForCausalLM.from_pretrained(model_name)
7
8device = "cuda" if torch.cuda.is_available() else "cpu"
9model.to(device)
10
11instruction = """You are an advanced AI assistant. Provide answers in a clear manner."""
12
13messages = [
14 {"role": "system", "content": instruction},
15 {"role": "user", "content": "Who created you?"}
16]
17
18prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
19inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True).to(device)
20
21output_ids = model.generate(
22 **inputs,
23 max_new_tokens=1500,
24 temperature=0.8,
25 top_p=0.95,
26 do_sample=True,
27)
28
29response = tokenizer.decode(output_ids[0], skip_special_tokens=True)
30assistant_start = response.find("assistant")
31response = response[assistant_start + len("assistant"):].strip() if assistant_start != -1 else response
32
33print(response)