Views
No views yet
| Parameter | Description | Value |
|---|---|---|
max_seq_length | Maximum sequence length for the model | 4096 |
load_in_4bit | Whether to load the model in 4-bit precision | False |
model_name | Pre-trained model name from Hugging Face | meta-llama/Meta-Llama-3.1-8B |
r | Rank of the LoRA adapter | 128 |
lora_alpha | Alpha value for the LoRA module | 32 |
lora_dropout | Dropout rate for LoRA layers | 0 |
bias | Bias type for LoRA adapters | none |
use_gradient_checkpointing | Whether to use gradient checkpointing | unsloth |
train_batch_size | Per device training batch size | 8 |
gradient_accumulation_steps | Number of gradient accumulation steps | 8 |
warmup_ratio | Warmup steps as a fraction of total steps | 0.1 |
num_train_epochs | Number of training epochs | 1 |
learning_rate | Learning rate for the model | 5e-5 |
embedding_learning_rate | Learning rate for embeddings | 1e-5 |
optim | Optimizer used for training | adamw_8bit |
weight_decay | Weight decay to prevent overfitting | 0.01 |
lr_scheduler_type | Type of learning rate scheduler | linear |
1# -*- coding: utf-8 -*-
2import os
3
4from typing import (
5 Dict,
6)
7
8from datasets import load_dataset
9from unsloth import (
10 FastLanguageModel,
11 is_bfloat16_supported,
12 UnslothTrainer,
13 UnslothTrainingArguments,
14)
15
16max_seq_length = 4096
17dtype = None
18load_in_4bit = False
19
20model, tokenizer = FastLanguageModel.from_pretrained(
21 model_name="meta-llama/Meta-Llama-3.1-8B",
22 max_seq_length=max_seq_length,
23 dtype=dtype,
24 load_in_4bit=load_in_4bit,
25 token="hf_token",
26)
27
28model = FastLanguageModel.get_peft_model(
29 model,
30 r=128,
31 target_modules=[
32 "q_proj",
33 "k_proj",
34 "v_proj",
35 "o_proj",
36 "gate_proj",
37 "up_proj",
38 "down_proj",
39 "embed_tokens",
40 "lm_head",
41 ],
42 lora_alpha=32,
43 lora_dropout=0,
44 bias="none",
45 use_gradient_checkpointing="unsloth",
46 random_state=3407,
47 use_rslora=True,
48 loftq_config=None,
49)
50
51prompt = """### Référence :
52{}
53### Contenu :
54{}"""
55
56EOS_TOKEN = tokenizer.eos_token
57
58def formatting_prompts_func(examples):
59 """
60 Format input examples into prompts for a language model.
61
62 This function takes a dictionary of examples containing titles and texts,
63 combines them into formatted prompts, and appends an end-of-sequence token.
64
65 Parameters
66 ----------
67 examples : dict
68 A dictionary containing two keys:
69 - 'title': A list of titles.
70 - 'text': A list of corresponding text content.
71
72 Returns
73 -------
74 dict
75 A dictionary with a single key 'text', containing a list of formatted prompts.
76
77 Notes
78 -----
79 - The function assumes the existence of a global `prompt` variable, which is a
80 formatting string used to combine the title and text.
81 - The function also assumes the existence of a global `EOS_TOKEN` variable,
82 which is appended to the end of each formatted prompt.
83 - The input lists 'title' and 'text' are expected to have the same length.
84
85 Examples
86 --------
87 >>> examples = {
88 ... 'title': ['Title 1', 'Title 2'],
89 ... 'text': ['Content 1', 'Content 2']
90 ... }
91 >>> formatting_cpt_prompts_func(examples)
92 {'text': ['<formatted_prompt_1><EOS>', '<formatted_prompt_2><EOS>']}
93 """
94 refs = examples["ref"]
95 texts = examples["texte"]
96 outputs = []
97
98 for ref, text in zip(refs, texts):
99 text = prompt.format(ref, text) + EOS_TOKEN
100 outputs.append(text)
101
102 return {
103 "text": outputs,
104 }
105
106
107cpt_dataset = load_dataset(
108 "louisbrulenaudet/Romulus-cpt-fr",
109 split="train",
110 token="hf_token",
111)
112
113cpt_dataset = cpt_dataset.map(
114 formatting_prompts_func,
115 batched=True,
116)
117
118trainer = UnslothTrainer(
119 model=model,
120 tokenizer=tokenizer,
121 train_dataset=cpt_dataset,
122 dataset_text_field="text",
123 max_seq_length=max_seq_length,
124 dataset_num_proc=2,
125 args=UnslothTrainingArguments(
126 per_device_train_batch_size=8,
127 gradient_accumulation_steps=8,
128 warmup_ratio=0.1,
129 num_train_epochs=1,
130 learning_rate=5e-5,
131 embedding_learning_rate=1e-5,
132 fp16=not is_bfloat16_supported(),
133 bf16=is_bfloat16_supported(),
134 logging_steps=1,
135 report_to="wandb",
136 save_steps=350,
137 run_name="romulus-cpt",
138 optim="adamw_8bit",
139 weight_decay=0.01,
140 lr_scheduler_type="linear",
141 seed=3407,
142 output_dir="outputs",
143 ),
144)
145
146trainer_stats = trainer.train()
1@misc{louisbrulenaudet2024,
2 author = {Louis Brulé Naudet},
3 title = {Romulus, continually pre-trained models for French law},
4 year = {2024}
5 howpublished = {\url{https://huggingface.co/datasets/louisbrulenaudet/Romulus-cpt-fr}},
6}