Views
No views yet
boldgpt_small_patch10.kmq
KMeansTokenizer). The training objective was to auto-regressively predict the next patch with shuffled patch order and cross-entropy loss.1from boldgpt.data import ActivityTransform
2from boldgpt.models import create_model
3from datasets import load_dataset
4
5model = create_model("boldgpt_small_patch10.kmq", pretrained=True)
6
7dataset = load_dataset("clane9/NSD-Flat", split="train")
8dataset.set_format("torch")
9
10transform = ActivityTransform()
11batch = dataset[:1]
12batch["activity"] = transform(batch["activity"])
13
14# output: (B, N + 1, K) predicted next token logits
15output, state = model(batch)1torchrun --standalone --nproc_per_node=4 \
2 scripts/train_gpt.py --out_dir results \
3 --model boldgpt_small \
4 --ps 10 --vs 1024 --vocab_state checkpoints/ps-10_vs-1024_vss-4000_seed-42/tok_state.pt \
5 --shuffle --epochs 1000 --bs 512 \
6 --workers 0 --amp --compile --wandbf9720ca52d6fa6b3eb47a34cf95f8e18a8683e4c