You can deploy this model and start earning money today!

domci/
ColBERTv2-mmarco-de-0.1

Name: ColBERTv2-mmarco-de-0.1
Availability: InStock

safetensors

unicamp-dl/mmarco

mit

Views

No views yet

ColBERTv2-mmarco-de-0.1

This is a German ColBERT implementation based on colbert-ir/colbertv2.0

Base Model: dbmdz/bert-base-german-cased
Training Data: unicamp-dl/mmarco --> 10Mio random sample
Framework used for training RAGatouille Thanks a ton @bclavie !

As I'm limited on GPU Training did not go through all the way. "Only" 10 checkpoints were trained.

Code

My code is probably a mess, but YOLO!

data prep

python

from datasets import load_dataset
from ragatouille import RAGTrainer
from tqdm import tqdm
import pickle
from concurrent.futures import ThreadPoolExecutor
from tqdm.notebook import tqdm
import concurrent

SAMPLE_SIZE = -1



def int_to_string(number):
    if number < 0:
        return "full"
    elif number < 1000:
        return str(number)
    elif number < 1000000:
        return f"{number // 1000}K"
    elif number >= 1000000:
        return f"{number // 1000000}M"

def process_chunk(chunk):
    return [list(item) for item in zip(chunk["query"], chunk["positive"], chunk["negative"])]

def chunked_iterable(iterable, chunk_size):
    """Yield successive chunks from iterable."""
    for i in range(0, len(iterable), chunk_size):
        yield iterable[i:i + chunk_size]

def process_dataset_concurrently(dataset, chunksize=1000):
    with ThreadPoolExecutor() as executor:
        # Wrap the dataset with tqdm for real-time updates
        wrapped_dataset = tqdm(chunked_iterable(dataset, chunksize), total=(len(dataset) + chunksize - 1) // chunksize)
        # Submit each chunk to the executor
        futures = [executor.submit(process_chunk, chunk) for chunk in wrapped_dataset]
        results = []
        for future in concurrent.futures.as_completed(futures):
            results.extend(future.result())
        return results

dataset = load_dataset('unicamp-dl/mmarco', 'german', trust_remote_code=True)


# Shuffle the dataset and seed for reproducibility if needed
shuffled_dataset = dataset['train'].shuffle(seed=42)


if SAMPLE_SIZE > 0:
    sampled_dataset = shuffled_dataset.select(range(SAMPLE_SIZE))
else:
    sampled_dataset = shuffled_dataset


triplets = process_dataset_concurrently(sampled_dataset, chunksize=10000)
trainer = RAGTrainer(model_name=f"ColBERT-mmacro-de-{int_to_string(SAMPLE_SIZE)}", pretrained_model_name="dbmdz/bert-base-german-cased", language_code="de",)
trainer.prepare_training_data(raw_data=triplets, mine_hard_negatives=False)

Training