Views
No views yet
facebook/bart-large-mnlifacebook/bart-large-mnli model for zero-shot text classification based on natural language inference (NLI).facebook/bart-large-mnliyahoo_answers_topics dataset from Hugging Face for evaluation. It contains questions categorized into 10 topics.1from datasets import load_dataset
2
3dataset = load_dataset("yahoo_answers_topics")1from transformers import pipeline
2
3classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
4
5sequence = "The team played well and won the championship."
6labels = ["sports", "politics", "education", "technology"]
7
8result = classifier(sequence, candidate_labels=labels)
9print(result)1from sklearn.metrics import accuracy_score
2
3def evaluate_zero_shot(dataset, labels):
4 correct = 0
5 total = 0
6 for example in dataset:
7 result = classifier(example["question_content"], candidate_labels=labels)
8 predicted = result["labels"][0]
9 true = labels[example["topic"]]
10 correct += int(predicted == true)
11 total += 1
12 return correct / total
13
14labels = ["Society & Culture", "Science & Mathematics", "Health", "Education",
15 "Computers & Internet", "Sports", "Business & Finance", "Entertainment & Music",
16 "Family & Relationships", "Politics & Government"]
17
18acc = evaluate_zero_shot(dataset["test"].select(range(100)), labels)
19print(f"Accuracy: {acc:.2%}")