Views
No views yet
1Training Setting
2Parameters：127M
3(vocab_size=32768, hidden_size=768, inter_size=1536, heads=6, layers=18)
4Optimizer: AdamW
5(lr=6e-4, betas=(0.9, 0.95), eps=1e-9, weight_decay=1e-1, warmup_steps=2000)
6batch size: 8
7accumlation: 16
8dataset: fineweb (0.5B token, 1 epoch: 976 steps)
9max length: 512
10dtype: bfloat16
git clone https://github.com/Rikka-Botan/Nova-Nox-Neural-Network.git1import torch
2from torch import nn
3import torch.nn.functional as F
4from model.N4_modeling import N4C1"""
2Args:
3hidden_size: int - model hidden size,
4inter_size: int - model mlp intermediate size,
5vocab_size : int - tokenizer vocab num,
6heads: int - heads num,
7layers: int - N4D(Decoder) layers num
8"""
9
10hidden_size = 768
11intermediate_size = 3072
12vocab_size = 32064
13heads = 6
14layers = 6
15
16model = N4C(
17 hidden_size,
18 intermediate_size,
19 vocab_size,
20 heads,
21 layers
22)
23output = model(tokenized_text)1from torch.optim import AdamW
2
3optimizer = AdamW(
4 model.parameters(),
5 lr=6.0e-4,
6 betas=(0.9, 0.95),
7 eps=1e-8,
8 weight_decay=1e-1
9)
10
11for batch in dataloader:
12 optimizer.zero_grad()
13 batch = batch.to(device)
14 loss = model.to(device)(input=batch, labels=batch)[1]
15 loss.backward()
16 optimizer.step()1# N4: Nova Nox Neural Network inference
2# coding=utf-8
3# Copyright 2025 Rikka Botan. All rights reserved
4# Licensed under the "MIT License"
5
6import torch
7from transformers import AutoTokenizer
8import os
9from model.n4_modeling import N4C
10
11model_name = "mistralai/Mistral-7B-v0.3"
12tokenizer = AutoTokenizer.from_pretrained(model_name)
13cwd=os.path.abspath('your model path')
14model = N4C(
15 vocab_size=32768,
16 hidden_size=768,
17 inter_size=1536,
18 heads=6,
19 layers=18,
20 bias=False
21)
22state_dict = torch.load(os.path.join(cwd, 'N4_test_model.bin'), weights_only=True)
23model.load_state_dict(state_dict, strict=False)
24model = model.to('cpu')
25model.eval()
26text = "Large Language Models (LLMs) are advanced artificial intelligence systems designed to"
27inputs = tokenizer(text, return_tensors='pt')
28output = model.generate_n4c(
29 input_ids=inputs["input_ids"].to('cpu'),
30 max_new_tokens=128
31 temperature: float = 0.7,
32 top_k: int = 10,
33 top_p: float = 2,
34 eos_token_id: int = 2)
35for token in inputs['input_ids']:
36 print(tokenizer.decode(token), end=" ")
37for token in output:
38 print(tokenizer.decode(token), end=" ", flush=True)
39