Views
No views yet
Supernova-teraillm dataset:| Tokenizer | Tokens per Word | Efficiency |
|---|---|---|
| Supernova-Nepali (Ultra) | 3.79 | 2.20x Better |
| GPT-2 (Standard) | 8.21 | Baseline |
1import time
2from transformers import AutoTokenizer
3
4# Load the dedicated Nepali tokenizer (Pure Tokenizer Repository)
5model_id = "Supernova11c/Supernova-Nepali-Tokenizer"
6print(f"Loading tokenizer for: {model_id}")
7
8# Use clean_up_tokenization_spaces=False for BPE tokenizers to prevent warnings/corruption
9tokenizer = AutoTokenizer.from_pretrained(model_id, clean_up_tokenization_spaces=False)
10
11def stress_test_tokenizer(tokenizer):
12 print(f"\n--- Running Tokenizer Stress Test ---")
13 print(f"Tokenizer Class: {type(tokenizer).__name__}\n")
14
15 # 1. Edge Cases & Special Characters Test
16 edge_cases = [
17 "Hello, world! 🌍🚀", # Emojis & punctuation
18 " Multiple spaces and\nnewlines\t", # Whitespace handling
19 "The quick brown fox jumps over the lazy dog." * 50, # Repetition
20 "1234567890 -+*/=<>@#$%^&*()_[]{}|\\:;\"'.,?", # Symbols & Numbers
21 "नमस्ते संसार 🌟 नेपाल 🌍", # Nepali / Multi-lingual
22 "", # Empty string
23 ]
24
25 print("1. Edge Case Testing:")
26 for i, text in enumerate(edge_cases):
27 try:
28 encoded = tokenizer.encode(text)
29 decoded = tokenizer.decode(encoded, skip_special_tokens=True)
30 match = "✓" if (text.strip() == decoded.strip() or not text) else "⚠️ (Whitespace diff)"
31 print(f" Test {i+1}: {match} | Length: {len(text)} chars -> {len(encoded)} tokens")
32 except Exception as e:
33 print(f" Test {i+1}: ❌ FAILED with error: {e}")
34
35 # 2. Throughput / Speed Test
36 print("\n2. Throughput Performance Test:")
37 sample_text = (
38 "नेपाल एक सुन्दर देश हो। यहाँ विभिन्न जातजाति र भाषाभाषीका मानिसहरू बसोबास गर्छन्। "
39 ) * 500 # ~35,000 characters
40
41 num_iterations = 100
42
43 # Warmup
44 _ = tokenizer.encode(sample_text)
45
46 start_time = time.time()
47 for _ in range(num_iterations):
48 _ = tokenizer.encode(sample_text)
49 end_time = time.time()
50
51 total_time = end_time - start_time
52 total_chars = len(sample_text) * num_iterations
53 total_tokens = len(tokenizer.encode(sample_text)) * num_iterations
54
55 print(f" Processed {total_chars:,} characters in {total_time:.4f} seconds.")
56 print(f" Speed: {total_chars / total_time:,.2f} chars/sec")
57 print(f" Speed: {total_tokens / total_time:,.2f} tokens/sec")
58
59 # 3. Vocabulary & Configuration Check
60 print("\n3. Vocabulary & Configuration Check:")
61 print(f" Vocabulary Size: {len(tokenizer):,}")
62 print(f" Model Max Length: {getattr(tokenizer, 'model_max_length', 'N/A')}")
63 print(f" Pad Token: {tokenizer.pad_token} (ID: {tokenizer.pad_token_id})")
64 print(f" EOS Token: {tokenizer.eos_token} (ID: {tokenizer.eos_token_id})")
65 print("\n--- Stress Test Complete ---")
66
67# Execute the test
68stress_test_tokenizer(tokenizer)
69[PAD], [UNK], [BOS], [EOS]