Views
No views yet
Llama-2-13b-hf-4bit-64rank, is obtained from LLAMA-2-13b.
The backbone is under LoftQ/Llama-2-13b-hf-4bit-64rank and LoRA adapters are under the subfolder='loftq_init'.torch.bfloat161import torch
2from transformers import AutoModelForCausalLM, BitsAndBytesConfig
3from peft import PeftModel
4
5MODEL_ID = "LoftQ/Llama-2-13b-hf-4bit-64rank"
6
7base_model = AutoModelForCausalLM.from_pretrained(
8 MODEL_ID,
9 torch_dtype=torch.bfloat16, # you may change it with different models
10 quantization_config=BitsAndBytesConfig(
11 load_in_4bit=True,
12 bnb_4bit_compute_dtype=torch.bfloat16, # bfloat16 is recommended
13 bnb_4bit_use_double_quant=False,
14 bnb_4bit_quant_type='nf4',
15 ),
16)
17peft_model = PeftModel.from_pretrained(
18 base_model,
19 MODEL_ID,
20 subfolder="loftq_init",
21 is_trainable=True,
22)
23
24# Do training with peft_model ...| Model | Bits | Rank | LoRA Initial | GSM8K | WikiText-2 |
|---|---|---|---|---|---|
| LLAMA-2-13b | 16 | 64 | Gaussian + 0 | 45.3 | 5.12 |
| LLAMA-2-13b | 4 | 64 | Gaussian + 0 (QLoRA) | 39.9 | 5.22 |
| LLAMA-2-13b | 4 | 64 | LoftQ | 45.0 | 5.16 |
1import torch
2from transformers import AutoModelForCausalLM, BitsAndBytesConfig
3from peft import PeftModel
4
5MODEL_ID = "LoftQ/Llama-2-13b-hf-4bit-64rank"
6
7base_model = AutoModelForCausalLM.from_pretrained(
8 MODEL_ID,
9 torch_dtype=torch.bfloat16, # you may change it with different models
10 quantization_config=BitsAndBytesConfig(
11 load_in_4bit=True,
12 bnb_4bit_compute_dtype=torch.bfloat16, # bfloat16 is recommended
13 bnb_4bit_use_double_quant=False,
14 bnb_4bit_quant_type='nf4',
15 ),
16)
17peft_model = PeftModel.from_pretrained(
18 base_model,
19 MODEL_ID,
20 subfolder="gsm8k",
21 is_trainable=True,
22)
23
24# Do inference with peft_model ...1@article{li2023loftq,
2 title={Loftq: Lora-fine-tuning-aware quantization for large language models},
3 author={Li, Yixiao and Yu, Yifan and Liang, Chen and He, Pengcheng and Karampatziakis, Nikos and Chen, Weizhu and Zhao, Tuo},
4 journal={arXiv preprint arXiv:2310.08659},
5 year={2023}
6}