Views
No views yet
1@misc{xylaria2024smol,
2 title={Xylaria-1.4-smol: A Compact Efficient RNN},
3 author={[Your Name]},
4 year={2024}
5}1import torch
2import torch.nn as nn
3
4class XylariaSmolRNN(nn.Module):
5 def __init__(self, config):
6 super(XylariaSmolRNN, self).__init__()
7
8
9 self.vocab_size = config['vocab_size']
10 self.embedding_dim = config['embedding_dim']
11 self.hidden_dim = config['hidden_dim']
12 self.num_layers = config['num_layers']
13 self.char_to_idx = config['char_to_idx']
14
15
16 self.embedding = nn.Embedding(
17 num_embeddings=self.vocab_size,
18 embedding_dim=self.embedding_dim,
19 padding_idx=self.char_to_idx['<PAD>']
20 )
21
22
23 self.rnn = nn.LSTM(
24 input_size=self.embedding_dim,
25 hidden_size=self.hidden_dim,
26 num_layers=self.num_layers,
27 batch_first=True
28 )
29
30
31 self.fc = nn.Linear(self.hidden_dim, self.vocab_size)
32
33
34 self.dropout = nn.Dropout(0.3)
35
36 def forward(self, x):
37
38 embedded = self.embedding(x)
39
40
41 rnn_out, (hidden, cell) = self.rnn(embedded)
42
43
44 rnn_out = self.dropout(rnn_out)
45
46
47 output = self.fc(rnn_out)
48
49 return output, (hidden, cell)
50
51def demonstrate_xylaria_model():
52
53 model_config = {
54 "vocab_size": 108,
55 "embedding_dim": 50,
56 "hidden_dim": 128,
57 "num_layers": 2,
58 "char_to_idx": {" ": 1, "!": 2, "\"": 3, "#": 4, "$": 5, "%": 6, "&": 7, "'": 8, "(": 9, ")": 10, "*": 11, "+": 12, ",": 13, "-": 14, ".": 15, "/": 16, "0": 17, "1": 18, "2": 19, "3": 20, "4": 21, "5": 22, "6": 23, "7": 24, "8": 25, "9": 26, ":": 27, ";": 28, "<": 29, "=": 30, ">": 31, "?": 32, "A": 33, "B": 34, "C": 35, "D": 36, "E": 37, "F": 38, "G": 39, "H": 40, "I": 41, "J": 42, "K": 43, "L": 44, "M": 45, "N": 46, "O": 47, "P": 48, "Q": 49, "R": 50, "S": 51, "T": 52, "U": 53, "V": 54, "W": 55, "X": 56, "Y": 57, "Z": 58, "[": 59, "\\": 60, "]": 61, "^": 62, "_": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "}": 91, "°": 92, "²": 93, "à": 94, "á": 95, "æ": 96, "é": 97, "í": 98, "ó": 99, "ö": 100, "–": 101, "'": 102, "'": 103, """: 104, """: 105, "…": 106, "<PAD>": 0, "<UNK>": 107}
59 }
60
61
62 model = XylariaSmolRNN(model_config)
63
64
65 total_params = sum(p.numel() for p in model.parameters())
66 trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
67
68 print(f"Total Parameters: {total_params}")
69 print(f"Trainable Parameters: {trainable_params}")
70 print(f"Model Size Estimate: {total_params * 4 / 1024 / 1024:.2f} MB")
71
72
73 batch_size = 1
74 sequence_length = 20
75 x = torch.randint(0, model_config['vocab_size'], (batch_size, sequence_length))
76
77
78 with torch.no_grad():
79 output, (hidden, cell) = model(x)
80 print("Model Output Shape:", output.shape)
81 print("Hidden State Shape:", hidden.shape)
82 print("Cell State Shape:", cell.shape)
83
84
85 try:
86
87 scripted_model = torch.jit.script(model)
88 scripted_model.save("xylaria_smol_model.pt")
89 print("Model exported for deployment")
90 except Exception as e:
91 print(f"Export failed: {e}")
92
93
94 def generate_text(model, start_char, max_length=100):
95
96 current_char = torch.tensor([[model.char_to_idx.get(start_char, model.char_to_idx['<UNK>'])]])
97
98
99 hidden = None
100 generated_text = [start_char]
101
102 for _ in range(max_length - 1):
103 with torch.no_grad():
104
105 embedded = model.embedding(current_char)
106 if hidden is None:
107 rnn_out, (hidden, cell) = model.rnn(embedded)
108 else:
109 rnn_out, (hidden, cell) = model.rnn(embedded, (hidden, cell))
110
111
112 output = model.fc(rnn_out)
113
114
115 probabilities = torch.softmax(output[0, -1], dim=0)
116 next_char_idx = torch.multinomial(probabilities, 1).item()
117
118
119 idx_to_char = {idx: char for char, idx in model.char_to_idx.items()}
120 next_char = idx_to_char.get(next_char_idx, '<UNK>')
121
122 generated_text.append(next_char)
123 current_char = torch.tensor([[next_char_idx]])
124
125 if next_char == '<UNK>':
126 break
127
128 return ''.join(generated_text)
129
130
131 print("\nText Generation Example:")
132 generated = generate_text(model, 'A')
133 print(generated)
134
135if __name__ == "__main__":
136 demonstrate_xylaria_model()