1%%capture
2!pip install pip3-autoremove
3!pip-autoremove torch torchvision torchaudio -y
4!pip install torch torchvision torchaudio xformers --index-url https://download.pytorch.org/whl/cu121
5!pip install unsloth
6
7---------------------------------------------------------------------------------------------
8
9from kaggle_secrets import UserSecretsClient
10user_secrets = UserSecretsClient() # from kaggle_secrets import UserSecretsClient
11hugging_face_token = user_secrets.get_secret("HF-Token")
12
13# Login to Hugging Face
14from huggingface_hub import login # Lets you login to API
15login(hugging_face_token) # from huggingface_hub import login
16
17---------------------------------------------------------------------------------------------
18
19from unsloth import FastLanguageModel
20import torch
21max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
22dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
23load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
24
25model, tokenizer = FastLanguageModel.from_pretrained(
26 model_name = "SaintHoney/PersonalManV1.0",
27 max_seq_length = max_seq_length,
28 dtype = dtype,
29 load_in_4bit = load_in_4bit,
30)
31
32---------------------------------------------------------------------------------------------
33
34model = FastLanguageModel.get_peft_model(
35 model,
36 r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
37 target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
38 "gate_proj", "up_proj", "down_proj",],
39 lora_alpha = 16,
40 lora_dropout = 0, # Supports any, but = 0 is optimized
41 bias = "none", # Supports any, but = "none" is optimized
42 # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
43 use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
44 random_state = 3407,
45 use_rslora = False, # We support rank stabilized LoRA
46 loftq_config = None, # And LoftQ
47)
48
49---------------------------------------------------------------------------------------------
50
51alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
52
53### Instruction:
54{}
55
56### Input:
57{}
58
59### Response:
60{}"""
61
62EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN
63def formatting_prompts_func(examples):
64 instructions = examples["instruction"]
65 inputs = examples["input"]
66 outputs = examples["output"]
67 texts = []
68 for instruction, input, output in zip(instructions, inputs, outputs):
69 # Must add EOS_TOKEN, otherwise your generation will go on forever!
70 text = alpaca_prompt.format(instruction, input, output) + EOS_TOKEN
71 texts.append(text)
72 return { "text" : texts, }
73pass
74
75from datasets import load_dataset
76dataset = load_dataset("HashTag766/SMART-Goals-Validation", split = "train") # specify here the number of examples from dataset
77dataset = dataset.map(formatting_prompts_func, batched = True,)
78
79---------------------------------------------------------------------------------------------
80
81from trl import SFTTrainer
82from transformers import TrainingArguments, DataCollatorForSeq2Seq
83from unsloth import is_bfloat16_supported
84
85trainer = SFTTrainer(
86 model = model,
87 tokenizer = tokenizer,
88 train_dataset = dataset,
89 dataset_text_field = "text",
90 max_seq_length = max_seq_length,
91 data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer),
92 dataset_num_proc = 2,
93 packing = False, # Can make training 5x faster for short sequences.
94 args = TrainingArguments(
95 per_device_train_batch_size = 2,
96 gradient_accumulation_steps = 4,
97 warmup_steps = 5,
98 num_train_epochs = 3, # Set this for 1 full training run.
99 # max_steps = 60,
100 learning_rate = 2e-4,
101 fp16 = not is_bfloat16_supported(),
102 bf16 = is_bfloat16_supported(),
103 logging_steps = 1,
104 optim = "adamw_8bit",
105 weight_decay = 0.01,
106 lr_scheduler_type = "linear",
107 seed = 3407,
108 output_dir = "outputs",
109 report_to = "none", # Use this for WandB etc
110 ),
111)
112
113trainer_stats = trainer.train()
114---------------------------------------------------------------------------------------------
115
116model.push_to_hub("hf/model...", token = "...") # Online saving
117tokenizer.push_to_hub("hf/model...", token = "...") # Online saving
118
