Views
No views yet
>>97868993Temp 1
rep pen 1.15-1.2 # Model is fucking fried. Good luck.
rep pen range 2048 # "
top-k 20
minp 0.01 # Try without first.---
/lmg/ - Local Models General
---
Anonymous No.61958
/lmg/ - a general dedicated to the discussion and development of local language models.
Previous threads:►News
►Recent Highlights1import os
2from unsloth import FastLanguageModel
3from unsloth import is_bfloat16_supported
4import torch
5from unsloth import is_bfloat16_supported
6from unsloth import UnslothTrainer, UnslothTrainingArguments
7from datasets import load_dataset
8
9os.environ["CUDA_VISIBLE_DEVICES"] = "0"
10
11
12max_seq_length = 4096 # Supports RoPE Scaling interally, so choose any!
13
14
15model, tokenizer = FastLanguageModel.from_pretrained(
16 model_name = "T:\models\Mistral-Nemo-Base-2407",
17 max_seq_length = max_seq_length,
18 dtype = None,
19 load_in_4bit = True,
20)
21
22# Do model patching and add fast LoRA weights
23model = FastLanguageModel.get_peft_model(
24 model,
25 r = 64,
26 lora_alpha = 64,
27 lora_dropout = 0.0, # Supports any, but = 0 is optimized
28 target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
29 "gate_proj", "up_proj", "down_proj", "embed_tokens", "lm_head",],
30 bias = "none", # Supports any, but = "none" is optimized
31 # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
32 use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
33 random_state = 1337,
34 max_seq_length = max_seq_length,
35 use_rslora = False, # We support rank stabilized LoRA
36 loftq_config = None, # And LoftQ
37)
38
39train_dataset = load_dataset(path="datasets", data_files = {"train" : "lmg-threads-cleaned-2-nodate-train-shuffled.json"}, split="train")
40eval_dataset = load_dataset(path="datasets", data_files = {"train" : "lmg-threads-cleaned-2-nodate-eval.json"}, split="train")
41# print out 5 rows of the dataset
42for row in train_dataset[:5]["text"]:
43 print("=========================")
44 print(row)
45
46
47trainer = UnslothTrainer(
48 model = model,
49 tokenizer = tokenizer,
50 train_dataset = train_dataset,
51 eval_dataset = eval_dataset,
52 dataset_text_field = "text",
53 max_seq_length = max_seq_length,
54 dataset_num_proc = 1,
55
56 args = UnslothTrainingArguments(
57 per_device_train_batch_size = 1,
58 gradient_accumulation_steps = 1,
59
60 per_device_eval_batch_size = 1,
61 eval_accumulation_steps = 4,
62 fp16_full_eval = True,
63
64 warmup_ratio = 0,
65 #max_steps=20,
66 num_train_epochs = 1,
67
68 learning_rate = 1e-4,
69 embedding_learning_rate = 1e-5,
70
71 eval_strategy = "steps",
72 eval_steps = 50,
73 do_eval = True,
74
75 fp16 = False,
76 bf16 = True,
77 logging_steps = 1,
78 optim = "adamw_8bit",
79 weight_decay = 0.01,
80 lr_scheduler_type = "constant",
81 seed = 3407,
82
83 save_strategy = "epoch",
84 output_dir = "outputs",
85 report_to = "tensorboard", # wandb, tensorboard, whatever
86 ),
87)
88
89# @title Show current memory stats
90gpu_stats = torch.cuda.get_device_properties(0)
91start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
92max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
93print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
94print(f"{start_gpu_memory} GB of memory reserved.")
95
96trainer_stats = trainer.train()
97
98
99if False: model.save_pretrained_gguf("model", tokenizer, quantization_method = "f16")
100if True: model.save_pretrained_merged("model", tokenizer, save_method = "merged_16bit",)