Views
No views yet
bitsandbytes>=0.41.0pip install torch transformers accelerate bitsandbytes1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
3
4# Model configuration
5model_name = "AkameLLC/DeepHermes-3-Mistral-24B-Preview-BNB-NF4"
6
7# Quantization config (already applied, but needed for loading)
8quantization_config = BitsAndBytesConfig(
9 load_in_4bit=True,
10 bnb_4bit_quant_type="nf4",
11 bnb_4bit_compute_dtype=torch.bfloat16,
12 bnb_4bit_use_double_quant=True
13)
14
15# Load model and tokenizer
16model = AutoModelForCausalLM.from_pretrained(
17 model_name,
18 quantization_config=quantization_config,
19 device_map="auto",
20 torch_dtype=torch.bfloat16,
21 trust_remote_code=True
22)
23
24tokenizer = AutoTokenizer.from_pretrained(
25 model_name,
26 trust_remote_code=True
27)
28
29# Generate response
30prompt = "Explain the concept of machine learning in simple terms."
31inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
32
33with torch.no_grad():
34 outputs = model.generate(
35 **inputs,
36 max_new_tokens=512,
37 do_sample=True,
38 temperature=0.7,
39 top_p=0.9,
40 pad_token_id=tokenizer.eos_token_id
41 )
42
43response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
44print(response)1def format_chat_prompt(message: str) -> str:
2 return f"<|user|>\n{message}\n<|assistant|>\n"
3
4# Example usage
5user_message = "What are the benefits of renewable energy?"
6formatted_prompt = format_chat_prompt(user_message)
7
8inputs = tokenizer(formatted_prompt, return_tensors="pt").to(model.device)
9# ... (generation code as above)| Model Version | VRAM Usage | Model Size | Reduction |
|---|---|---|---|
| Original FP16 | ~48GB | ~48GB | - |
| BNB NF4 | ~12-15GB | ~12-15GB | 70% |
| Hardware | Tokens/Second | Batch Size |
|---|---|---|
| NVIDIA L4 | 3-5 | 1 |
| RTX 4090 | 4-6 | 1 |
| A10G | 3-5 | 1 |
1quantization_config = BitsAndBytesConfig(
2 load_in_4bit=True,
3 bnb_4bit_quant_type="nf4",
4 bnb_4bit_compute_dtype=torch.bfloat16,
5 bnb_4bit_use_double_quant=True,
6 llm_int8_enable_fp32_cpu_offload=True
7)model-*.safetensors: Quantized model weightsconfig.json: Model configurationtokenizer.json: Tokenizer configurationgeneration_config.json: Generation parametersaws_deployment_config.json: AWS deployment settings1# Reduce memory usage
2model = AutoModelForCausalLM.from_pretrained(
3 model_name,
4 quantization_config=quantization_config,
5 device_map="auto",
6 max_memory={"0": "22GB"} # Adjust based on your GPU
7)1# Enable low CPU memory usage
2model = AutoModelForCausalLM.from_pretrained(
3 model_name,
4 quantization_config=quantization_config,
5 device_map="auto",
6 low_cpu_mem_usage=True
7)1from peft import LoraConfig, get_peft_model
2
3# LoRA configuration
4lora_config = LoraConfig(
5 r=16,
6 lora_alpha=32,
7 target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
8 lora_dropout=0.1
9)
10
11# Apply LoRA
12model = get_peft_model(model, lora_config)1@misc{deephermes3-bnb-nf4,
2 title={DeepHermes-3-Mistral-24B-Preview-BNB-NF4},
3 author={AkameLLC},
4 year={2025},
5 publisher={Hugging Face},
6 url={https://huggingface.co/AkameLLC/DeepHermes-3-Mistral-24B-Preview-BNB-NF4}
7}
8
9@misc{dettmers2022gpt3int8,
10 title={GPT3.int8(): 8-bit Matrix Multiplication for Transformers at Scale},
11 author={Tim Dettmers and Mike Lewis and Sam Shleifer and Luke Zettlemoyer},
12 year={2022},
13 eprint={2208.07339},
14 archivePrefix={arXiv}
15}