Views
No views yet
transformers format.1from transformers import AutoModelForMaskedLM, AutoTokenizer
2import torch
3
4repo_id = "BEE-spoke-data/neobert-100k-test"
5tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
6model = AutoModelForMaskedLM.from_pretrained(repo_id, trust_remote_code=True)
7
8# Example: Fill in masked tokens
9text = "NeoBERT is the most [MASK] model of its kind!"
10
11# Tokenize (handling Metaspace tokenizer's space tokens)
12inputs = tokenizer(text, return_tensors="pt")
13input_ids = inputs["input_ids"][0].tolist()
14
15# Remove extra space tokens before [MASK] if present (Metaspace tokenizer quirk)
16cleaned_ids = []
17for i, token_id in enumerate(input_ids):
18 if token_id == 454 and i < len(input_ids) - 1 and input_ids[i + 1] == tokenizer.mask_token_id:
19 continue
20 cleaned_ids.append(token_id)
21
22if len(cleaned_ids) != len(input_ids):
23 inputs["input_ids"] = torch.tensor([cleaned_ids])
24 inputs["attention_mask"] = torch.ones_like(inputs["input_ids"])
25
26# Get predictions
27with torch.no_grad():
28 outputs = model(**inputs)
29 mask_pos = (inputs["input_ids"] == tokenizer.mask_token_id).nonzero(as_tuple=True)[1][0]
30 predictions = outputs.logits[0, mask_pos].topk(5)
31
32# Display top predictions
33for idx, score in zip(predictions.indices, predictions.values):
34 token = tokenizer.decode([idx])
35 print(f"{token}: {score:.2f}")1from transformers import AutoModel, AutoTokenizer
2
3repo_id = "BEE-spoke-data/neobert-100k-test"
4tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
5model = AutoModel.from_pretrained(repo_id, trust_remote_code=True)
6
7# Example: Generate embeddings
8text = "NeoBERT is an efficient transformer model!"
9inputs = tokenizer(text, return_tensors="pt")
10outputs = model(**inputs)
11
12# Get CLS token embedding
13cls_embedding = outputs.last_hidden_state[:, 0, :]
14print(f"Embedding shape: {cls_embedding.shape}")1model:
2 hidden_size: 768
3 num_hidden_layers: 12
4 num_attention_heads: 12
5 intermediate_size: 3072
6 max_position_embeddings: 4096
7 vocab_size: 31999
8 rope: true
9 rms_norm: true
10 hidden_act: swiglu
11 dropout_prob: 0.05
12 norm_eps: 1.0e-05
13 embedding_init_range: 0.02
14 decoder_init_range: 0.02
15 classifier_init_range: 0.02
16 flash_attention: true
17 ngpt: false
18 base_scale: 0.03227486121839514
19 pad_token_id: 0
20dataset:
21 name: EleutherAI/SmolLM2-1.7B-stage-4-100B
22 path: ''
23 num_workers: 4
24 streaming: true
25 cache_dir: null
26 max_seq_length: 1024
27 validation_split: null
28 train_split: train
29 eval_split: train[:1%]
30 num_proc: 8
31 shuffle_buffer_size: 10000
32 pre_tokenize: false
33 pre_tokenize_output: null
34 load_all_from_disk: false
35 force_redownload: false
36 pretraining_prob: 0.3
37 min_length: 512
38tokenizer:
39 name: BEE-spoke-data/wordpiece-tokenizer-32k-en_code-msp
40 path: null
41 max_length: 1024
42 padding: max_length
43 truncation: true
44 vocab_size: 31999
45optimizer:
46 name: adamw
47 lr: 0.0001
48 weight_decay: 0.01
49 betas:
50 - 0.9
51 - 0.98
52 eps: 1.0e-08
53scheduler:
54 name: cosine
55 warmup_steps: 5000
56 total_steps: null
57 num_cycles: 0.5
58 decay_steps: 50000
59 warmup_percent: null
60 decay_percent: null
61trainer:
62 per_device_train_batch_size: 16
63 per_device_eval_batch_size: 16
64 gradient_accumulation_steps: 4
65 max_steps: 100000
66 save_steps: 10000
67 eval_steps: 5000
68 logging_steps: 25
69 output_dir: ./outputs/neobert_100m_100k
70 overwrite_output_dir: true
71 bf16: true
72 gradient_checkpointing: false
73 gradient_clipping: null
74 mixed_precision: 'no'
75 seed: 42
76 resume_from_checkpoint: false
77 disable_tqdm: false
78 dataloader_num_workers: 0
79 use_cpu: false
80 report_to:
81 - wandb
82 tf32: true
83 max_ckpt: 3
84 train_batch_size: 16
85 eval_batch_size: 32
86datacollator:
87 mlm_probability: 0.2
88 pad_to_multiple_of: 8
89wandb:
90 project: neobert-pretraining
91 entity: null
92 name: neobert-100m-100k
93 tags: []
94 mode: online
95 log_interval: 100
96 resume: never
97 dir: logs/wandb
98task: pretraining
99accelerate_config_file: null
100mixed_precision: bf16
101mteb_task_type: all
102mteb_batch_size: 32
103mteb_pooling: mean
104mteb_overwrite_results: false
105pretrained_checkpoint: latest
106use_deepspeed: true
107seed: 69
108debug: false
109