Views
No views yet
$TRUNCATED$ is considered a manual ending flag and doesn't affect completeness scoring.1from transformers import AutoTokenizer, AutoModelForSequenceClassification
2import torch
3
4# Load the model and tokenizer
5model_name = "opendatalab/meta-rater-cleanliness-rating"
6tokenizer = AutoTokenizer.from_pretrained(model_name)
7model = AutoModelForSequenceClassification.from_pretrained(model_name)
8
9# Example text
10text = "This is a well-formatted article about renewable energy. It contains complete sentences and proper structure."
11
12# Tokenize and predict
13inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=4096)
14with torch.no_grad():
15 outputs = model(**inputs)
16 score = outputs.logits.squeeze().argmax(dim=0)
17
18print(f"Cleanliness Score: {score:.2f}")1@article{zhuang2025meta,
2 title={Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models},
3 author={Zhuang, Xinlin and Peng, Jiahui and Ma, Ren and Wang, Yinfan and Bai, Tianyi and Wei, Xingjian and Qiu, Jiantao and Zhang, Chi and Qian, Ying and He, Conghui},
4 journal={arXiv preprint arXiv:2504.14194},
5 year={2025}
6}