This model is a fine-tuned version of
dbmdz/bert-base-turkish-cased for binary toxicity classification in
Turkish text. It was trained using a cleaned and preprocessed version of the
Overfit-GM/turkish-toxic-language dataset.
1def clean_corrected_text(text):
2 text = text.lower()
3 text = re.sub(r"http\S+|www\S+|https\S+", '', text, flags=re.MULTILINE) # URL removal
4 text = re.sub(r"@\w+", '', text) # remove @mentions
5 text = re.sub(r"[^\w\s.,!?-]", '', text) # remove special characters (e.g., emojis)
6 text = re.sub(r"\s+", ' ', text).strip() # normalize whitespaces
7 return text
1slang_words = ["kanka", "lan", "knk", "bro", "la", "birader", "kanki"]
2
3def remove_slang(text):
4 for word in slang_words:
5 text = text.replace(word, "")
6 return text.strip()
1from transformers import AutoTokenizer, AutoModelForSequenceClassification
2import torch
3
4tokenizer = AutoTokenizer.from_pretrained("fc63/turkish_toxic_language_detection_model")
5model = AutoModelForSequenceClassification.from_pretrained("fc63/turkish_toxic_language_detection_model")
6
7def predict_toxicity(text):
8 inputs = tokenizer(text, return_tensors="pt", truncation=True, padding="max_length", max_length=128)
9 outputs = model(**inputs)
10 predicted = torch.argmax(outputs.logits, dim=1).item()
11 return "Toxic" if predicted == 1 else "Non-Toxic"
Dataset used:
Overfit-GM/turkish-toxic-language
Final dataset size after preprocessing and balancing: 54068 samples
