This repository contains the best performing FastText and Word2Vec models for Tatar, selected through comprehensive evaluation of 57 different model configurations. These embeddings significantly outperform pre-trained Meta models and are ideal for NLP tasks in Tatar.
FastText models with subword information support.
Classical Word2Vec embeddings optimized for Tatar.
1from huggingface_hub import snapshot_download
2from gensim.models import FastText
3import os
4
5# Download and load the best model
6model_dir = snapshot_download(
7 repo_id="arabovs-ai-lab/Tatar2Vec",
8 allow_patterns="fasttext/ft_dim100_win5_min5_ngram3-6_sg.epoch1/*"
9)
10
11model_path = os.path.join(model_dir, "fasttext/ft_dim100_win5_min5_ngram3-6_sg.epoch1/ft_dim100_win5_min5_ngram3-6_sg.epoch1.model")
12model = FastText.load(model_path)
13
14# Find similar words
15similar_words = model.wv.most_similar('мәктәп', topn=5) # school
16print(similar_words)
1from huggingface_hub import snapshot_download
2from gensim.models import Word2Vec
3import os
4
5# Download and load Word2Vec model
6model_dir = snapshot_download(
7 repo_id="arabovs-ai-lab/Tatar2Vec",
8 allow_patterns="word2vec/w2v_dim200_win5_min5_sg.epoch4/*"
9)
10
11model_path = os.path.join(model_dir, "word2vec/w2v_dim200_win5_min5_sg.epoch4/w2v_dim200_win5_min5_sg.epoch4.model")
12model = Word2Vec.load(model_path)
13
14# Find similar words
15similar_words = model.wv.most_similar('китап', topn=5) # book
16print(similar_words)
1# Education related words
2education_words = model.wv.most_similar('укыту', topn=5) # teaching
3science_words = model.wv.most_similar('фән', topn=5) # science
4student_words = model.wv.most_similar('студент', topn=5) # student
5
6print("Education:", education_words)
7print("Science:", science_words)
8print("Student:", student_words)
1# Nature related words
2nature_words = model.wv.most_similar('табигать', topn=5) # nature
3water_words = model.wv.most_similar('су', topn=5) # water
4tree_words = model.wv.most_similar('агач', topn=5) # tree
5
6print("Nature:", nature_words)
7print("Water:", water_words)
8print("Tree:", tree_words)
1# Culture and arts
2culture_words = model.wv.most_similar('мәдәният', topn=5) # culture
3music_words = model.wv.most_similar('музыка', topn=5) # music
4art_words = model.wv.most_similar('сәнгать', topn=5) # art
5
6print("Culture:", culture_words)
7print("Music:", music_words)
8print("Art:", art_words)
1# Everyday words
2family_words = model.wv.most_similar('гаилә', topn=5) # family
3work_words = model.wv.most_similar('эш', topn=5) # work
4home_words = model.wv.most_similar('өй', topn=5) # home
5
6print("Family:", family_words)
7print("Work:", work_words)
8print("Home:", home_words)
1# Food related words
2food_words = model.wv.most_similar('ашамлык', topn=5) # food
3bread_words = model.wv.most_similar('икмәк', topn=5) # bread
4meal_words = model.wv.most_similar('аш', topn=5) # meal
5
6print("Food:", food_words)
7print("Bread:", bread_words)
8print("Meal:", meal_words)
1# Technology words
2tech_words = model.wv.most_similar('компьютер', topn=5) # computer
3internet_words = model.wv.most_similar('интернет', topn=5) # internet
4phone_words = model.wv.most_similar('телефон', topn=5) # phone
5
6print("Computer:", tech_words)
7print("Internet:", internet_words)
8print("Phone:", phone_words)
1# Emotional words
2happy_words = model.wv.most_similar('бәхетле', topn=5) # happy
3sad_words = model.wv.most_similar('кайгы', topn=5) # sadness
4love_words = model.wv.most_similar('мәхәббәт', topn=5) # love
5
6print("Happy:", happy_words)
7print("Sadness:", sad_words)
8print("Love:", love_words)
1# Word analogies (like king - man + woman = queen)
2# doctor - man + woman = ?
3analogy = model.wv.most_similar(
4 positive=['табиб', 'хатын'], # doctor, woman
5 negative=['ир'], # man
6 topn=3
7)
8print("Doctor - man + woman =", analogy)
9
10# Paris - France + Germany = Berlin?
11analogy = model.wv.most_similar(
12 positive=['Париж', 'Алмания'], # Paris, Germany
13 negative=['Франция'], # France
14 topn=3
15)
16print("Paris - France + Germany =", analogy)
1# FastText can handle words not in the training vocabulary
2if hasattr(model, 'wv') and hasattr(model.wv, 'get_vector'):
3 # Try some made-up or rare words
4 oov_words = ['технологияләштерү', 'цифрлаштыру', 'виртуальлаштыру']
5 for word in oov_words:
6 try:
7 vector = model.wv[word]
8 similar = model.wv.most_similar(word, topn=3)
9 print(f"OOV word '{word}': {similar}")
10 except Exception as e:
11 print(f"Couldn't process '{word}': {e}")
1from huggingface_hub import snapshot_download
2from gensim.models import FastText, Word2Vec
3import os
4
5def load_tatar2vec_model(model_name, model_type="fasttext"):
6 """Load any Tatar2Vec model easily"""
7 model_dir = snapshot_download(
8 repo_id="arabovs-ai-lab/Tatar2Vec",
9 allow_patterns=f"{model_type}/{model_name}/*"
10 )
11
12 model_path = os.path.join(model_dir, model_type, model_name, f"{model_name}.model")
13
14 if model_type == "fasttext":
15 return FastText.load(model_path)
16 else:
17 return Word2Vec.load(model_path)
18
19# Test different models with various topics
20test_words = ['тел', 'мәктәп', 'китап', 'фән', 'табигать']
21
22models_to_test = [
23 ("ft_dim100_win5_min5_ngram3-6_sg.epoch1", "fasttext", "🥇 Best FastText"),
24 ("ft_dim100_win5_min5_ngram3-6_sg.epoch3", "fasttext", "🥈 Alternative FastText"),
25 ("w2v_dim200_win5_min5_sg.epoch4", "word2vec", "🥇 Best Word2Vec"),
26 ("w2v_dim100_win5_min5_sg", "word2vec", "🥈 Compact Word2Vec")
27]
28
29for model_name, model_type, description in models_to_test:
30 print(f"\n{description}: {model_name}")
31 print("=" * 50)
32
33 model = load_tatar2vec_model(model_name, model_type)
34
35 for word in test_words:
36 try:
37 similar = model.wv.most_similar(word, topn=2)
38 print(f" '{word}': {[f'{w}({s:.3f})' for w, s in similar]}")
39 except KeyError:
40 print(f" '{word}': not in vocabulary")
These examples showcase the models' understanding of various domains in the Tatar language! 🌟
1@misc{Tatar2Vec_20251109,
2 title = {Tatar2Vec: Tatar Word Embeddings},
3 author = {Arabovs AI Lab},
4 year = 2025,
5 publisher = {Hugging Face},
6 url = {https://huggingface.co/arabovs-ai-lab/Tatar2Vec}
7}
