Views
No views yet
| Özellik | Değer |
|---|---|
| Mimari | Transformer + CMN (Contextual Memory Network) |
| Parametre Sayısı | ~100-300M |
| Diller | Türkçe, İngilizce |
| Bağlam Uzunluğu | 1024 token |
| Chunk Boyutu | 128 token |
| Katman Sayısı | 24 |
| Model Boyutu (d_model) | 768 |
| Attention Head | 6 |
| Lokal Bellek Token | 16 |
| Global Bellek Token | 32 |
| Vocab Size | 32.000 |
| Lisans | Apache 2.0 |
Input → [Global Mem | Local Mem | Chunk Tokens] → Transformer → Output
↑ ↑
SmartMemoryGate MemoryGate
(cross-attention) (forget/update)1import torch
2import torch.nn.functional as F
3import re
4
5def sample_next_token(logits, temperature=0.1, top_p=0.9, repetition_penalty=1.2, input_ids=None):
6 # Tekrar cezası
7 if input_ids is not None:
8 for token_id in set(input_ids[0].tolist()):
9 logits[token_id] /= repetition_penalty
10
11 logits = logits / temperature
12 probs = F.softmax(logits, dim=-1)
13
14 sorted_probs, sorted_indices = torch.sort(probs, descending=True)
15 cumulative_probs = torch.cumsum(sorted_probs, dim=-1)
16 sorted_indices_to_remove = cumulative_probs - sorted_probs > top_p
17 sorted_probs[sorted_indices_to_remove] = 0
18 sorted_probs /= sorted_probs.sum()
19
20 next_token = torch.multinomial(sorted_probs, num_samples=1)
21 return sorted_indices[next_token]
22
23def fix_spacing(text):
24 # Noktalama önündeki boşlukları sil
25 text = re.sub(r'\s+([.,!?;:)])', r'\1', text)
26 # Fazla boşlukları tek boşluğa indir
27 text = re.sub(r'\s+', ' ', text)
28 return text.strip()
29
30prev_memory = torch.zeros(1, config.num_memory_tokens, config.d_model, device=device)
31global_memory = torch.zeros(1, config.num_global_memory_tokens, config.d_model, device=device)
32
33text = "Kullanıcı: Merhaba, nasılsın?\nAsistan:"
34inputs = tokenizer(text, return_tensors="pt").to(device)
35generated = inputs["input_ids"].clone()
36
37with torch.no_grad():
38 for _ in range(100):
39 input_chunk = generated[:, -128:]
40
41 logits, prev_memory, global_memory = model(
42 input_ids=input_chunk,
43 prev_memory=prev_memory,
44 global_memory=global_memory
45 )
46
47 next_token = sample_next_token(
48 logits[0, -1, :],
49 temperature=0.1,
50 top_p=0.9,
51 repetition_penalty=1.3,
52 input_ids=generated
53 ).unsqueeze(0)
54
55 generated = torch.cat([generated, next_token], dim=1)
56
57 if next_token.item() == tokenizer.eos_token_id:
58 break
59
60output = tokenizer.decode(generated[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)
61print(fix_spacing(output))