1def get_weight(char):
2 if '\u4e00' <= char <= '\u9faf': # kanji
3 return 6
4 elif '\u3040' <= char <= '\u309F': # hiragana
5 return 2
6 elif '\u30A0' <= char <= '\u30FF': # katakana
7 return 2
8 elif 'a' <= char <= 'z' or 'A' <= char <= 'Z':
9 return 1
10 elif '0' <= char <= '9':
11 return 1
12 else:
13 return 0
1 - `'日本語'` (kanji) → `6+6+6=18`.
2 - `'あいう'` (hiragana) → `2+2+2=6`.
3 - `'abc'` (Latin) → `1+1+1=3`.
-
Direct Trigrams (status_dic[tid] == True):
Automatically included with full weight (1.0) in the feature matrix.
No score calculation needed; treated as high-confidence seeds.
-
Aggregated (Gray) Trigrams
Included only if:
score < score_threshold (default: 18) and- They appear in a pentagram (5-trigram window) containing at least one direct (white) trigram.
(Example: A window like
[white, gray, gray] or [gray, white, gray] or [gray, gray, white] → all trigrams in the window are included, even if gray trigrams individually fail the score ≥ 18 threshold.)
1{
2 "model": <PyTorch quantized model>,
3 "metadata": {
4 "trigram2col": { <trigram_id>: <column_index> },
5 "idx2name": { <category_id>: <category_name> }
6 },
7 "agrs": {
8 "status_dic": { <trigram_id>: <is_white> },
9 "pairs_list": [ (<trigram_id>, <pentagram_id>), ... ],
10 "ngrm_to_tri": { <pentagram_id>: [<trigram1>, <trigram2>, ...] },
11 "score_dic": { <trigram_id>: <score> }
12 },
13 "config": {
14 "input_dim": 2033473,
15 "latent_size": 1024,
16 "linker_size": 512,
17 "num_categories": 11883
18 }
19}
1import torch
2
3checkpoint = torch.load("scyth_5_cpu_int8.pth", map_location="cpu")
4model = checkpoint["model"]
5model.eval()
6conf = checkpoint["config"]
7meta = checkpoint["metadata"]
1def generate_embedding(
2 text, model_name=MODEL_NAME, output_embedding_file=NEW_VECTOR_NAME):
3
4 if not text or not text.strip():
5 return None
6
7 text = text.lower()
8 model, conf, meta, checkpoint = load_quantized_model(model_name)
9 (
10 status_dic,
11 _,
12 ngrm_to_tri,
13 _,
14 text_to_id_aux,
15 _,
16 score_dic,
17 tri_to_ngrm
18 ) = load_dictionaries(checkpoint)
19 input_vec = torch.zeros(
20 (1, conf["input_dim"]),
21 dtype=torch.float32
22 )
23 total_trigrams_in_text = max(
24 0,
25 len(text) - 2
26 )
27 seen_white_tris = set()
28 text_trigrams_ids = []
29 # =====================================================
30 # TRIGRAMS
31 # =====================================================
32 for i in range(total_trigrams_in_text):
33 tid = text_to_id_aux.get(
34 text[i:i+3]
35 )
36 if tid is not None:
37 text_trigrams_ids.append(tid)
38
39 counts_in_text = Counter(
40 text_trigrams_ids
41 )
42 # =====================================================
43 # DIRECT
44 # =====================================================
45 for tid in set(text_trigrams_ids):
46 if status_dic.get(tid) is True:
47 if tid not in seen_white_tris:
48 if tid in meta["trigram2col"]:
49 seen_white_tris.add(tid)
50 col = meta["trigram2col"][tid]
51 input_vec[0, col] = 1.0
52 # =====================================================
53 # AGR
54 # =====================================================
55 processed_input_tids = set()
56 for i in range(total_trigrams_in_text):
57 tri_str = text[i:i+3]
58 t_id = text_to_id_aux.get(tri_str)
59
60 if (t_id is not None and t_id in tri_to_ngrm and t_id not in processed_input_tids):
61
62 processed_input_tids.add(t_id)
63 h_ids = tri_to_ngrm[t_id]
64
65 if not h_ids:
66 continue
67
68 def get_ngrm_score(h_idx):
69 tris_in_ngrm = ngrm_to_tri.get(
70 h_idx,
71 []
72 )
73 return sum(
74 counts_in_text.get(s_tid, 0)
75 for s_tid in tris_in_ngrm
76 )
77 best_h_id = max(
78 h_ids,
79 key=get_ngrm_score
80 )
81 for s_tri_id in ngrm_to_tri.get(
82 best_h_id,
83 []
84 ):
85 if (
86 status_dic.get(s_tri_id) is True
87 and s_tri_id not in seen_white_tris
88 ):
89 if s_tri_id in meta["trigram2col"]:
90 seen_white_tris.add(s_tri_id)
91 col = meta["trigram2col"][s_tri_id]
92 raw_score = score_dic.get(
93 s_tri_id,
94 0.0
95 )
96 normalized_score = max(
97 0.0,
98 min(
99 1.0,
100 raw_score / 18.0
101 )
102 )
103 final_weight = (
104 normalized_score
105 * GRAY_WEIGHT
106 )
107 input_vec[0, col] = final_weight
108 # =====================================================
109 # MODEL
110 # =====================================================
111 with torch.no_grad():
112 _, _, linker_embedding = model(input_vec)
113 embedding_vec = (
114 linker_embedding
115 .squeeze(0)
116 .float()
117 .cpu()
118 .numpy()
119 )
120 return embedding_vec
This model was fine-tuned using
SCYTH Cyberia, a high-performance computing cluster with the following node specifications:
