Views
No views yet
ema1{
2 "attention_probs_dropout_prob": 0.1,
3 "hidden_dropout_prob": 0.1,
4 "hidden_size": 384,
5 "intermediate_size": 1280,
6 "max_position_embeddings": 512,
7 "position_bucket_size": 32,
8 "num_attention_heads": 6,
9 "num_hidden_layers": 12,
10 "vocab_size": 8192,
11 "layer_norm_eps": 1e-05,
12 "auto_map": {
13 "AutoConfig": "configuration_gpt_bert.GPTBertConfig",
14 "AutoModel": "modeling_gpt_bert.GPTBertForMaskedLM",
15 "AutoModelForCausalLM": "modeling_gpt_bert.GPTBertForMaskedLM",
16 "AutoModelForMaskedLM": "modeling_gpt_bert.GPTBertForMaskedLM"
17 },
18 "return_dict": true,
19 "output_hidden_states": false,
20 "torchscript": false,
21 "dtype": "float32",
22 "pruned_heads": {},
23 "tie_word_embeddings": true,
24 "chunk_size_feed_forward": 0,
25 "is_encoder_decoder": false,
26 "is_decoder": false,
27 "cross_attention_hidden_size": null,
28 "add_cross_attention": false,
29 "tie_encoder_decoder": false,
30 "architectures": [
31 "GPTBertForMaskedLM"
32 ],
33 "finetuning_task": null,
34 "id2label": {
35 "0": "LABEL_0",
36 "1": "LABEL_1"
37 },
38 "label2id": {
39 "LABEL_0": 0,
40 "LABEL_1": 1
41 },
42 "task_specific_params": null,
43 "problem_type": null,
44 "tokenizer_class": null,
45 "prefix": null,
46 "bos_token_id": null,
47 "pad_token_id": null,
48 "eos_token_id": null,
49 "sep_token_id": null,
50 "decoder_start_token_id": null,
51 "max_length": 20,
52 "min_length": 0,
53 "do_sample": false,
54 "early_stopping": false,
55 "num_beams": 1,
56 "num_beam_groups": 1,
57 "diversity_penalty": 0.0,
58 "temperature": 1.0,
59 "top_k": 50,
60 "top_p": 1.0,
61 "typical_p": 1.0,
62 "repetition_penalty": 1.0,
63 "length_penalty": 1.0,
64 "no_repeat_ngram_size": 0,
65 "encoder_no_repeat_ngram_size": 0,
66 "bad_words_ids": null,
67 "num_return_sequences": 1,
68 "output_scores": false,
69 "return_dict_in_generate": false,
70 "forced_bos_token_id": null,
71 "forced_eos_token_id": null,
72 "remove_invalid_values": false,
73 "exponential_decay_length_penalty": null,
74 "suppress_tokens": null,
75 "begin_suppress_tokens": null,
76 "_name_or_path": "",
77 "transformers_version": "4.56.1",
78 "tf_legacy_loss": false,
79 "use_bfloat16": false,
80 "model_type": "gpt_bert",
81 "output_attentions": false
82}tokenizer_nor_vs8192.json1from transformers import AutoTokenizer, AutoModelForMaskedLM
2model_id = 'haznitrama/babybabellm-gpt_bert-nor-causal'
3tok = AutoTokenizer.from_pretrained(model_id)
4model = AutoModelForMaskedLM.from_pretrained(model_id, trust_remote_code=True)
5out = model(**tok('Hello world', return_tensors='pt'))1# Load EMA weights explicitly if both are present
2from safetensors.torch import load_file
3import torch
4from transformers import AutoConfig, AutoModelForMaskedLM
5model_id = 'haznitrama/babybabellm-gpt_bert-nor-causal'
6config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
7model = AutoModelForMaskedLM.from_config(config, trust_remote_code=True)
8state_dict = torch.load('pytorch_model.bin') # or load_file('model_ema.safetensors')
9model.load_state_dict(state_dict, strict=False)1from transformers import AutoTokenizer, AutoModelForCausalLM
2mid='haznitrama/babybabellm-gpt_bert-nor-causal'
3tok=AutoTokenizer.from_pretrained(mid)
4model=AutoModelForCausalLM.from_pretrained(mid, trust_remote_code=True)
5print(tok.decode(model.generate(**tok('Hello', return_tensors='pt'), max_new_tokens=20)[0], skip_special_tokens=True))pytorch_model.bin added for legacy tools.trust_remote_code=True due to custom architecture.