Parmi la liste d'intentions suivantes :  "audio_volume_other, play_music, iot_hue_lighton, general_greet, calendar_set, audio_volume_down, social_query, audio_volume_mute, iot_wemo_on, iot_hue_lightup, audio_volume_up, iot_coffee, takeaway_query, qa_maths, play_game, cooking_query, iot_hue_lightdim, iot_wemo_off, music_settings, weather_query, news_query, alarm_remove, social_post, recommendation_events, transport_taxi, takeaway_order, music_query, calendar_query, lists_query, qa_currency, recommendation_movies, general_joke, recommendation_locations, email_querycontact, lists_remove, play_audiobook, email_addcontact, lists_createoradd, play_radio, qa_stock, alarm_query, email_sendemail, general_quirky, music_likeness, cooking_recipe, email_query, datetime_query, transport_traffic, play_podcasts, iot_hue_lightchange, calendar_remove, transport_query, transport_ticket, qa_factoid, iot_cleaning, alarm_set, datetime_convert, iot_hue_lightoff, qa_definition, music_dislikeness",\n indiquer celle présente dans le texte : quel jour de la semaine est le quinze août ? 

import os
from datasets import load_dataset
import torch
import accelerate
from transformers import AutoTokenizer, MistralForCausalLM, BitsAndBytesConfig, Trainer, TrainingArguments, DataCollatorForLanguageModeling, DataCollatorWithPadding

from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training

os.environ["WANDB_PROJECT"] = "mistral-7B-FR-Instruct-LORA"


# Load tokenizer and data
model_name = "mistralai/Mistral-7B-v0.1"
max_length=1024

tokenizer = AutoTokenizer.from_pretrained(
    model_name,
    model_max_length=max_length,
    padding_side="left", # inportant for causality
    add_eos_token=True)

tokenizer.pad_token = tokenizer.eos_token

def preprocess_data(x):

    inputs = x["inputs"]
    targets = x["targets"]

    prompts = [inputs[i] + " " + targets[i] for i in range(len(inputs))]

    inputs = tokenizer(
        prompts,
        truncation=True,
        max_length=max_length,
        padding=False
    )

    return inputs

# Load and tokenize data

train_dataset = load_dataset("CATIE-AQ/DFP", split="train", num_proc=16)
valid_dataset = load_dataset("CATIE-AQ/DFP", split="validation", num_proc=16)

# Sample a random subset
train_dataset = train_dataset.shuffle().select(range(1280000))
valid_dataset = valid_dataset.shuffle().select(range(500))

tokenized_train_dataset = train_dataset.map(preprocess_data, remove_columns=train_dataset.column_names, batched=True, batch_size=20)
tokenized_val_dataset = valid_dataset.map(preprocess_data, remove_columns=valid_dataset.column_names, batched=True, batch_size=20)

tokenized_train_dataset = tokenized_train_dataset.with_format("torch")
tokenized_val_dataset = tokenized_val_dataset.with_format("torch")

# Load model

# Optionnal quantization for QLoRA
'''bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)'''

# Flash Attention is only available on Ampere architectures (A100)!
model = MistralForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, use_flash_attention_2=True)

# Prepare LoRA

config = LoraConfig(
    r=8,
    lora_alpha=16,
    target_modules=[
        "q_proj",
        "k_proj",
        "v_proj",
        "o_proj",
        "gate_proj",
        "up_proj",
        "down_proj",
        "lm_head",
    ],
    bias="none",
    lora_dropout=0.05,
    task_type="CAUSAL_LM",
)

model = get_peft_model(model, config)

training_args = TrainingArguments(
        output_dir="mistral7B-FR-Instruct",
        remove_unused_columns=True,
        warmup_steps=1000,
        per_device_train_batch_size=4,
        gradient_accumulation_steps=32,
        max_steps=10000,
        learning_rate=5e-5,
        lr_scheduler_type="linear",
        logging_steps=50,
        fp16=True,
        optim="adamw_torch", # paged_adamw_8bit for QLoRA
        logging_dir="./logs",
        save_strategy="steps",
        save_steps=1000,
        evaluation_strategy="steps",
        eval_steps=500,
        do_eval=True,
        report_to="wandb"
    )

class DynamicDataCollator:

    def __init__(self, tokenizer):

        self.tokenizer = tokenizer

    def __call__(self, features):

        batch = self.tokenizer.pad(
            features,
            padding="longest",
            max_length=max_length,
            pad_to_multiple_of=8
        )

        labels = batch["input_ids"].clone()
        labels[labels == self.tokenizer.pad_token_id] = -100 # ignore padding indices for the loss
        labels[:, -1] = self.tokenizer.eos_token_id  # except final eos
        batch["labels"] = labels

        return batch

trainer = Trainer(
    model=model,
    train_dataset=tokenized_train_dataset,
    eval_dataset=tokenized_val_dataset,
    args=training_args,
    data_collator=DynamicDataCollator(tokenizer)
)

model.config.use_cache = False  # silence the warnings. Please re-enable for inference!
trainer.train()

@Misc{peft,
  title =        {PEFT: State-of-the-art Parameter-Efficient Fine-Tuning methods},
  author =       {Sourab Mangrulkar and Sylvain Gugger and Lysandre Debut and Younes Belkada and Sayak Paul and Benjamin Bossan},
  howpublished = {\url{https://github.com/huggingface/peft}},
  year =         {2022}
}

@misc{jiang2023mistral,
      title={Mistral 7B}, 
      author={Albert Q. Jiang and Alexandre Sablayrolles and Arthur Mensch and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Florian Bressand and Gianna Lengyel and Guillaume Lample and Lucile Saulnier and Lélio Renard Lavaud and Marie-Anne Lachaux and Pierre Stock and Teven Le Scao and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
      year={2023},
      eprint={2310.06825},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc {centre_aquitain_des_technologies_de_l'information_et_electroniques_2023,
	author       = { {Centre Aquitain des Technologies de l'Information et Electroniques} },
	title        = { DFP (Revision 1d24c09) },
	year         = 2023,
	url          = { https://huggingface.co/datasets/CATIE-AQ/DFP },
	doi          = { 10.57967/hf/1200 },
	publisher    = { Hugging Face }
}

@misc{hu2021lora,
      title={LoRA: Low-Rank Adaptation of Large Language Models}, 
      author={Edward J. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
      year={2021},
      eprint={2106.09685},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}