Views
No views yet
1max_seq_length = 2048 # Maximum context window
2dtype = None # Auto-detect optimal precision
3load_in_4bit = True # 4-bit quantization for memory efficiency1r = 16 # LoRA rank
2lora_alpha = 16 # LoRA scaling factor
3lora_dropout = 0 # No dropout
4target_modules = [ # Modules to adapt
5 "q_proj", "k_proj", "v_proj", "o_proj",
6 "gate_proj", "up_proj", "down_proj"
7]1per_device_train_batch_size = 2
2gradient_accumulation_steps = 4
3learning_rate = 2e-4
4max_steps = 60
5warmup_steps = 5
6weight_decay = 0.01
7optimizer = "adamw_8bit"pip install unsloth transformers torch1from unsloth import FastLanguageModel
2from unsloth.chat_templates import get_chat_template
3
4# Load model with exact training parameters
5model, tokenizer = FastLanguageModel.from_pretrained(
6 model_name = "your-username/bro-chatbot",
7 max_seq_length = 2048, # IMPORTANT: Use same as training
8 dtype = None,
9 load_in_4bit = True, # IMPORTANT: Use same as training
10)
11
12# Setup chat template
13tokenizer = get_chat_template(tokenizer, chat_template = "llama-3.1")
14
15# Enable fast inference
16FastLanguageModel.for_inference(model)1def chat_with_bro(message):
2 messages = [{"role": "user", "content": message}]
3 inputs = tokenizer.apply_chat_template(
4 messages,
5 tokenize=True,
6 add_generation_prompt=True,
7 return_tensors="pt"
8 ).to("cuda")
9
10 # Generate response
11 outputs = model.generate(
12 input_ids=inputs,
13 max_new_tokens=128,
14 use_cache=True,
15 temperature=0.7,
16 min_p=0.1
17 )
18
19 # Extract only the new response
20 input_length = inputs.shape[1]
21 response = tokenizer.decode(outputs[0][input_length:], skip_special_tokens=True)
22 return response
23
24# Example usage
25response = chat_with_bro("How do I learn Python?")
26print(response)1from transformers import TextStreamer
2
3def chat_with_bro_streaming(message):
4 messages = [{"role": "user", "content": message}]
5 inputs = tokenizer.apply_chat_template(
6 messages,
7 tokenize=True,
8 add_generation_prompt=True,
9 return_tensors="pt"
10 ).to("cuda")
11
12 # Stream response in real-time
13 text_streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
14 model.generate(
15 input_ids=inputs,
16 streamer=text_streamer,
17 max_new_tokens=128,
18 use_cache=True,
19 temperature=0.7,
20 min_p=0.1
21 )
22
23# Example usage
24chat_with_bro_streaming("What's the meaning of life?")1@misc{bro-chatbot-2024,
2 title={Bro Chatbot: A Casual Conversational AI},
3 author={Your Name},
4 year={2024},
5 howpublished={HuggingFace Model Hub},
6 url={https://huggingface.co/your-username/bro-chatbot}
7}