Views
No views yet
pip install torch transformers peft accelerate bitsandbytes1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
3from peft import PeftModel, prepare_model_for_kbit_training
4from accelerate import PartialState
5
6# Configure 4-bit quantization for memory efficiency
7bnb_config = BitsAndBytesConfig(
8 load_in_4bit=True,
9 bnb_4bit_quant_type="nf4",
10 bnb_4bit_compute_dtype=torch.float16,
11 bnb_4bit_use_double_quant=False,
12)
13
14# Load tokenizer
15tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct", use_fast=False)
16if tokenizer.pad_token is None:
17 tokenizer.pad_token = tokenizer.eos_token
18
19# Load base model with quantization
20base_model = AutoModelForCausalLM.from_pretrained(
21 "Qwen/Qwen2.5-7B-Instruct",
22 quantization_config=bnb_config,
23 device_map={"": PartialState().process_index},
24 torch_dtype=torch.float16,
25)
26
27# Prepare for LoRA
28base_model = prepare_model_for_kbit_training(base_model)
29
30# Load LoRA adapter
31model = PeftModel.from_pretrained(base_model, "YOUR_USERNAME/YOUR_REPO_NAME")
32
33print("✅ Model loaded with 4-bit quantization!")1def generate_text(model, tokenizer, prompt, max_new_tokens=500, temperature=0.7):
2 inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
3
4 with torch.no_grad():
5 outputs = model.generate(
6 **inputs,
7 max_new_tokens=max_new_tokens,
8 temperature=temperature,
9 do_sample=True,
10 pad_token_id=tokenizer.eos_token_id,
11 )
12
13 return tokenizer.decode(
14 outputs[0][inputs["input_ids"].shape[1]:],
15 skip_special_tokens=True
16 )
17
18# Interactive mode
19print("🤖 Interactive Chat (type 'quit' to exit)")
20while True:
21 prompt = input("\nPrompt: ").strip()
22 if prompt.lower() in ['quit', 'exit', 'q']:
23 break
24 if prompt:
25 response = generate_text(model, tokenizer, prompt)
26 print(f"Response: {response}")