Views
No views yet
pip install bitsandbytes1
2# Activate 4-bit precision base model loading (bool)
3load_in_4bit = True
4
5# Activate nested quantization for 4-bit base models (double quantization) (bool)
6bnb_4bit_use_double_quant = True
7
8# Quantization type (fp4 or nf4) (string)
9bnb_4bit_quant_type = "nf4"
10
11# Compute data type for 4-bit base models
12bnb_4bit_compute_dtype = torch.bfloat16
13
14
15def load_model_tokenizer(model_name: str, bnb_config: BitsAndBytesConfig) -> Tuple[AutoModelForCausalLM, AutoTokenizer]:
16 """
17 Load the model and tokenizer from the HuggingFace model hub using quantization.
18
19 Args:
20 model_name (str): The name of the model.
21 bnb_config (BitsAndBytesConfig): The quantization configuration of BitsAndBytes.
22
23 Returns:
24 Tuple[AutoModelForCausalLM, AutoTokenizer]: The model and tokenizer.
25 """
26
27
28 model = AutoModelForCausalLM.from_pretrained(
29 model_name,
30 quantization_config = bnb_config,
31 # device_map = "auto",
32 torch_dtype="auto",
33 trust_remote_code=True
34 )
35
36 tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token = True, trust_remote_code=True)
37
38 tokenizer.pad_token = tokenizer.eos_token
39
40 return model, tokenizer
41
42
43bnb_config = BitsAndBytesConfig(
44 load_in_4bit = load_in_4bit,
45 bnb_4bit_use_double_quant = bnb_4bit_use_double_quant,
46 bnb_4bit_quant_type = bnb_4bit_quant_type,
47 bnb_4bit_compute_dtype = bnb_4bit_compute_dtype,
48 )
49
50model, tokenizer = load_model_tokenizer(model_name, bnb_config)
511
2new_model = "YuvrajSingh9886/medicinal-QnA-phi2-custom"
3
4prompt = "I have been feeling more and more down for over a month. I have started having trouble sleeping due to panic attacks, but they are almost never triggered by something that I know of."
5
6tokens = tokenizer(f"### Question: {prompt}", return_tensors='pt').to('cuda')
7tokenizer.pad_token = tokenizer.eos_token
8outputs = model.generate(**tokens, max_new_tokens=1024, num_beams=5,
9 no_repeat_ngram_size=2,
10 early_stopping=True
11 )
12print(tokenizer.batch_decode(outputs,skip_special_tokens=True)[0])
131
2def format_phi2(row):
3 question = row['Context']
4 answer = row['Response']
5
6# text = f"[INST] {question} [/INST] {answer}".replace('\xa0', ' ')
7 text = f"### Question: {question}\n ### Answer: {answer}"
8
9 return text1# LoRA attention dimension (int)
2lora_r = 64
3
4# Alpha parameter for LoRA scaling (int)
5lora_alpha = 16
6
7# Dropout probability for LoRA layers (float)
8lora_dropout = 0.05
9
10# Bias (string)
11bias = "none"
12
13# Task type (string)
14task_type = "CAUSAL_LM"
15
16# Random seed (int)
17seed = 331# Batch size per GPU for training (int)
2per_device_train_batch_size = 6
3
4# Number of update steps to accumulate the gradients for (int)
5gradient_accumulation_steps = 2
6
7# Initial learning rate (AdamW optimizer) (float)
8learning_rate = 2e-4
9
10# Optimizer to use (string)
11optim = "paged_adamw_8bit"
12
13# Number of training epochs (int)
14num_train_epochs = 4
15
16# Linear warmup steps from 0 to learning_rate (int)
17warmup_steps = 10
18
19# Enable fp16/bf16 training (set bf16 to True with an A100) (bool)
20fp16 = True
21
22# Log every X updates steps (int)
23logging_steps = 100
24
25#L2 regularization(prevents overfitting)
26weight_decay=0.0
27
28#Checkpoint saves
29save_strategy="epoch"1# Activate 4-bit precision base model loading (bool)
2load_in_4bit = True
3
4# Activate nested quantization for 4-bit base models (double quantization) (bool)
5bnb_4bit_use_double_quant = True
6
7# Quantization type (fp4 or nf4) (string)
8bnb_4bit_quant_type = "nf4"
9
10# Compute data type for 4-bit base models
11bnb_4bit_compute_dtype = torch.bfloat16
12