Views
No views yet
we advance the current scale of language models by pre-training up to trillion parameter models on the “Colossal Clean Crawled Corpus”, and achieve a 4x speedup over the T5-XXL model.
FLAN-T5 for running fine-tuned weights or fine-tune your own MoE following this notebooktransformers - bear in mind that the model is extremely large, so you may consider using disk offload from accelerate:1# pip install accelerate
2from transformers import AutoTokenizer, SwitchTransformersForConditionalGeneration
3
4tokenizer = AutoTokenizer.from_pretrained("google/switch-c-2048")
5model = SwitchTransformersForConditionalGeneration.from_pretrained("google/switch-c-2048", device_map="auto", offload_folder=<OFFLOAD_FOLDER>)
6
7input_text = "A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>."
8input_ids = tokenizer(input_text, return_tensors="pt").input_ids
9
10outputs = model.generate(input_ids)
11print(tokenizer.decode(outputs[0]))
12>>> <pad> <extra_id_0> man<extra_id_1> beer<extra_id_2> a<extra_id_3> salt<extra_id_4>.</s>1# pip install accelerate
2from transformers import AutoTokenizer, SwitchTransformersForConditionalGeneration
3
4tokenizer = AutoTokenizer.from_pretrained("google/switch-c-2048")
5model = SwitchTransformersForConditionalGeneration.from_pretrained("google/switch-c-2048", device_map="auto", offload_folder=<OFFLOAD_FOLDER>)
6
7input_text = "A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>."
8input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to(0)
9
10outputs = model.generate(input_ids)
11print(tokenizer.decode(outputs[0]))
12>>> <pad> <extra_id_0> man<extra_id_1> beer<extra_id_2> a<extra_id_3> salt<extra_id_4>.</s>1# pip install accelerate
2from transformers import AutoTokenizer, SwitchTransformersForConditionalGeneration
3
4tokenizer = AutoTokenizer.from_pretrained("google/switch-c-2048")
5model = SwitchTransformersForConditionalGeneration.from_pretrained("google/switch-c-2048", device_map="auto", torch_dtype=torch.bfloat16, offload_folder=<OFFLOAD_FOLDER>)
6
7input_text = "A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>."
8input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to(0)
9
10outputs = model.generate(input_ids)
11print(tokenizer.decode(outputs[0]))
12>>> <pad> <extra_id_0> man<extra_id_1> beer<extra_id_2> a<extra_id_3> salt<extra_id_4>.</s>1# pip install bitsandbytes accelerate
2from transformers import AutoTokenizer, SwitchTransformersForConditionalGeneration
3
4tokenizer = AutoTokenizer.from_pretrained("google/switch-c-2048")
5model = SwitchTransformersForConditionalGeneration.from_pretrained("google/switch-c-2048", device_map="auto", offload_folder=<OFFLOAD_FOLDER>)
6
7input_text = "A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>."
8input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to(0)
9
10outputs = model.generate(input_ids)
11print(tokenizer.decode(outputs[0]))
12>>> <pad> <extra_id_0> man<extra_id_1> beer<extra_id_2> a<extra_id_3> salt<extra_id_4>.</s>T5.t5x codebase together with jax.
1@misc{https://doi.org/10.48550/arxiv.2101.03961,
2 doi = {10.48550/ARXIV.2101.03961},
3
4 url = {https://arxiv.org/abs/2101.03961},
5
6 author = {Fedus, William and Zoph, Barret and Shazeer, Noam},
7
8 keywords = {Machine Learning (cs.LG), Artificial Intelligence (cs.AI), FOS: Computer and information sciences, FOS: Computer and information sciences},
9
10 title = {Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity},
11
12 publisher = {arXiv},
13
14 year = {2021},
15
16 copyright = {arXiv.org perpetual, non-exclusive license}
17}
18