You can deploy this model and start earning money today!

GwendalTsang/
Camembert-fine-tuned

Name: Camembert-fine-tuned
Availability: InStock

safetensors

camembert

Views

No views yet

Version très légèrement fine-tunée avec le notebook qui suit :

cell

python

import kagglehub
import os
os.environ["TOKENIZED_CACHE_DIR"] = kagglehub.dataset_download("gwendaltsang/tokenized-dataset/versions/3")

Using Colab cache for faster access to the 'tokenized-dataset' dataset.

cell

python

import os
import time
import warnings
from pathlib import Path

os.environ.setdefault("PJRT_DEVICE", "TPU")
os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("MKL_NUM_THREADS", "1")

warnings.filterwarnings(
    "ignore",
    message=r"Transparent hugepages are not enabled\..*",
    category=UserWarning,
    module=r"jax\._src\.cloud_tpu_init",
)

SCRIPT_START = time.perf_counter()
BACKUP_AFTER_SECONDS = 6590
EPOCHS = 1

import torch
from datasets import load_from_disk
from torch.utils.data import DataLoader
from transformers import CamembertForMaskedLM, CamembertTokenizerFast

import torch_xla
import torch_xla.core.xla_model as xm
import torch_xla.distributed.parallel_loader as pl
from torch_xla.amp import syncfree


def env_int(name: str, default: int) -> int:
    return int(os.getenv(name, default))


def env_float(name: str, default: float) -> float:
    return float(os.getenv(name, default))


def env_str(name: str, default: str) -> str:
    return os.getenv(name, default)


MODEL_NAME = env_str("MODEL_NAME", "camembert-base")
TOKENIZED_CACHE_DIR = env_str("TOKENIZED_CACHE_DIR", "")
SAVE_DIR = "/content/drive/MyDrive"

BATCH_SIZE = env_int("BATCH_SIZE", 256)
LEARNING_RATE = env_float("LEARNING_RATE", 5e-5)
WEIGHT_DECAY = env_float("WEIGHT_DECAY", 0.01)
WARMUP_RATIO = env_float("WARMUP_RATIO", 0.01)
MAX_GRAD_NORM = env_float("MAX_GRAD_NORM", 1.0)
MLM_PROBABILITY = env_float("MLM_PROBABILITY", 0.15)

DATALOADER_WORKERS = env_int("DATALOADER_NUM_WORKERS", 16)
PREFETCH_FACTOR = env_int("PREFETCH_FACTOR", 8)
TORCH_NUM_THREADS = env_int("TORCH_NUM_THREADS", 4)

XLA_LOADER_PREFETCH_SIZE = env_int("XLA_LOADER_PREFETCH_SIZE", 32)
XLA_DEVICE_PREFETCH_SIZE = env_int("XLA_DEVICE_PREFETCH_SIZE", 16)
XLA_TRANSFER_THREADS = env_int("XLA_TRANSFER_THREADS", 4)


def make_optimizer(model: torch.nn.Module) -> torch.optim.Optimizer:
    no_decay = ("bias", "LayerNorm.weight", "layer_norm.weight")

    decay_params = [
        p
        for n, p in model.named_parameters()
        if p.requires_grad and not any(nd in n for nd in no_decay)
    ]
    nodecay_params = [
        p
        for n, p in model.named_parameters()
        if p.requires_grad and any(nd in n for nd in no_decay)
    ]

    return syncfree.AdamW(
        [
            {"params": decay_params, "weight_decay": WEIGHT_DECAY},
            {"params": nodecay_params, "weight_decay": 0.0},
        ],
        lr=LEARNING_RATE,
        betas=(0.9, 0.999),
        eps=1e-6,
    )


def make_scheduler(
    optimizer: torch.optim.Optimizer,
    total_steps: int,
) -> torch.optim.lr_scheduler.LambdaLR:
    warmup_steps = int(total_steps * WARMUP_RATIO)

    def lr_lambda(step: int) -> float:
        warmup = step / max(1, warmup_steps)
        decay = (total_steps - step) / max(1, total_steps - warmup_steps)
        return min(warmup, max(0.0, decay))

    return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)


def make_mlm_batch(
    input_ids: torch.Tensor,
    special_tokens_mask: torch.Tensor,
    mask_token_id: int,
    vocab_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
    rand_mask = torch.rand(input_ids.shape, device=input_ids.device)
    masked = (rand_mask < MLM_PROBABILITY) & ~special_tokens_mask.bool()

    labels = torch.where(masked, input_ids, torch.full_like(input_ids, -100))

    replace_rand = torch.rand(input_ids.shape, device=input_ids.device)
    replace_with_mask = masked & (replace_rand < 0.8)
    replace_with_random = masked & (replace_rand >= 0.8) & (replace_rand < 0.9)

    random_words = torch.randint(
        low=0,
        high=vocab_size,
        size=input_ids.shape,
        device=input_ids.device,
        dtype=input_ids.dtype,
    )

    masked_input_ids = torch.where(
        replace_with_mask,
        torch.full_like(input_ids, mask_token_id),
        input_ids,
    )
    masked_input_ids = torch.where(
        replace_with_random,
        random_words,
        masked_input_ids,
    )

    return masked_input_ids, labels


def save_model_weights_checkpoint(
    model: torch.nn.Module,
    tokenizer: CamembertTokenizerFast,
    checkpoint_dir: str | Path,
) -> None:
    checkpoint_dir = Path(checkpoint_dir)
    checkpoint_dir.mkdir(parents=True, exist_ok=True)

    torch_xla.sync()

    model_to_save = model.module if hasattr(model, "module") else model
    cpu_state_dict = {
        name: tensor.detach().cpu()
        for name, tensor in model_to_save.state_dict().items()
    }

    model_to_save.save_pretrained(
        str(checkpoint_dir),
        state_dict=cpu_state_dict,
        safe_serialization=True,
    )
    tokenizer.save_pretrained(str(checkpoint_dir))

    del cpu_state_dict


def train() -> None:
    torch.set_num_threads(TORCH_NUM_THREADS)

    device = torch_xla.device()

    tokenizer = CamembertTokenizerFast.from_pretrained(MODEL_NAME)
    mask_token_id = int(tokenizer.mask_token_id)
    vocab_size = int(tokenizer.vocab_size)

    dataset = load_from_disk(str(Path(TOKENIZED_CACHE_DIR)))
    dataset.set_format(
        type="torch",
        columns=["input_ids", "attention_mask", "special_tokens_mask"],
    )

    train_loader = DataLoader(
        dataset,
        batch_size=BATCH_SIZE,
        shuffle=True,
        drop_last=True,
        num_workers=DATALOADER_WORKERS,
        persistent_workers=True,
        prefetch_factor=PREFETCH_FACTOR,
    )

    model = CamembertForMaskedLM.from_pretrained(
        MODEL_NAME,
        use_safetensors=True,
    ).to(device)
    model.train()

    optimizer = make_optimizer(model)

    steps_per_epoch = len(train_loader)
    scheduler = make_scheduler(optimizer, steps_per_epoch * EPOCHS)

    xla_loader_kwargs = {
        "loader_prefetch_size": XLA_LOADER_PREFETCH_SIZE,
        "device_prefetch_size": XLA_DEVICE_PREFETCH_SIZE,
        "host_to_device_transfer_threads": XLA_TRANSFER_THREADS,
    }

    optimizer.zero_grad(set_to_none=True)

    for _ in range(EPOCHS):
        device_loader = pl.MpDeviceLoader(
            train_loader,
            device,
            **xla_loader_kwargs,
        )

        for step, batch in enumerate(device_loader, start=1):
            input_ids, labels = make_mlm_batch(
                input_ids=batch["input_ids"],
                special_tokens_mask=batch["special_tokens_mask"],
                mask_token_id=mask_token_id,
                vocab_size=vocab_size,
            )

            with torch.autocast("xla", dtype=torch.bfloat16):
                loss = model(
                    input_ids=input_ids,
                    attention_mask=batch["attention_mask"],
                    labels=labels,
                ).loss

            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM)

            xm.optimizer_step(optimizer)
            scheduler.step()
            optimizer.zero_grad(set_to_none=True)

            if time.perf_counter() - SCRIPT_START >= BACKUP_AFTER_SECONDS:
                save_model_weights_checkpoint(
                    model=model,
                    tokenizer=tokenizer,
                    checkpoint_dir=SAVE_DIR,
                )
                print(
                    f"[Backup 1h50] saved_dir={SAVE_DIR} "
                    f"step={step}/{steps_per_epoch} "
                    f"loss={float(loss.detach().cpu()):.6f}",
                    flush=True,
                )
                return

    save_model_weights_checkpoint(
        model=model,
        tokenizer=tokenizer,
        checkpoint_dir=SAVE_DIR,
    )
    print(f"[Final] saved_dir={SAVE_DIR} step={steps_per_epoch}/{steps_per_epoch}", flush=True)


def main() -> None:
    train()


if __name__ == "__main__":
    main()

output

Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]sentencepiece.bpe.model:   0%|          | 0.00/811k [00:00<?, ?B/s]tokenizer.json:   0%|          | 0.00/1.40M [00:00<?, ?B/s]config.json:   0%|          | 0.00/508 [00:00<?, ?B/s]model.safetensors:   0%|          | 0.00/445M [00:00<?, ?B/s]Loading weights:   0%|          | 0/202 [00:00<?, ?it/s][transformers] CamembertForMaskedLM LOAD REPORT from: camembert-base
Key                         | Status     |  | 
----------------------------+------------+--+-
roberta.pooler.dense.bias   | UNEXPECTED |  | 
roberta.pooler.dense.weight | UNEXPECTED |  | 

Notes:
- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s][Backup 1h50] saved_dir=/content/drive/MyDrive step=13375/17516 loss=1.800222

GwendalTsang/Camembert-fine-tuned

GwendalTsang/
Camembert-fine-tuned