Views
No views yet
1import json
2import os
3from transformers import RobertaConfig, RobertaForMaskedLM, TFRobertaForMaskedLM
4
5DIRNAME = "./dummy-unknown"
6
7
8config = RobertaConfig(10, 20, 1, 1, 40)
9
10model = RobertaForMaskedLM(config)
11model.save_pretrained(DIRNAME)
12
13tf_model = TFRobertaForMaskedLM.from_pretrained(DIRNAME, from_pt=True)
14tf_model.save_pretrained(DIRNAME)
15
16# Tokenizer:
17
18vocab = [
19 "l",
20 "o",
21 "w",
22 "e",
23 "r",
24 "s",
25 "t",
26 "i",
27 "d",
28 "n",
29 "\u0120",
30 "\u0120l",
31 "\u0120n",
32 "\u0120lo",
33 "\u0120low",
34 "er",
35 "\u0120lowest",
36 "\u0120newer",
37 "\u0120wider",
38 "<unk>",
39]
40vocab_tokens = dict(zip(vocab, range(len(vocab))))
41merges = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
42
43vocab_file = os.path.join(DIRNAME, "vocab.json")
44merges_file = os.path.join(DIRNAME, "merges.txt")
45with open(vocab_file, "w", encoding="utf-8") as fp:
46 fp.write(json.dumps(vocab_tokens) + "\n")
47with open(merges_file, "w", encoding="utf-8") as fp:
48 fp.write("\n".join(merges))