Views
No views yet
| Field | Value |
|---|---|
| Parameters | 0.876M |
| Vocab size | 4096 |
| Context window | 256 tokens |
| Best val loss | 3.1398 |
| Best PPL (val) | 23.10 |
| Trained steps | 4900 |
roneneldan/TinyStorieswikitext-2-raw-v1wikitext-103-raw-v1hand-crafted-conversationsHuggingFaceTB/everyday-conversations-llama3.1-2k1{
2 "data_dir": "data/processed",
3 "save_dir": "runs/AtomSLM-900K-gated",
4 "config": "AtomSLM-900K-gated",
5 "steps": 5000,
6 "eval_every": 100,
7 "save_every": 500,
8 "batch_size": 32,
9 "seq_len": 256,
10 "lr": 0.0005,
11 "lr_min": 5e-05,
12 "warmup": 1000,
13 "grad_clip": 1.0,
14 "dropout": 0.1,
15 "device": "auto",
16 "resume": null,
17 "compile": false,
18 "amp": false,
19 "core_warmup_steps": 0,
20 "gated": true
21}
1import torch
2from safetensors.torch import load_file
3from models.atomgpt import AtomSLM
4from models.config import AtomSLMConfig
5
6# Load config and weights
7import json
8cfg = AtomSLMConfig(**json.load(open('config.json')))
9state_dict = load_file('model.safetensors')
10
11model = AtomSLM(cfg)
12model.load_state_dict(state_dict)
13model.eval()