Views
No views yet
With T5, we propose reframing all NLP tasks into a unified text-to-text-format where the input and output are always text strings, in contrast to BERT-style models that can only output either a class label or a span of the input. Our text-to-text framework allows us to use the same model, loss function, and hyperparameters on any NLP task.
Our text-to-text framework allows us to use the same model, loss function, and hyperparameters on any NLP task, including machine translation, document summarization, question answering, and classification tasks (e.g., sentiment analysis). We can even apply T5 to regression tasks by training it to predict the string representation of a number instead of the number itself
1@inproceedings{kim-etal-2021-model-cross,
2 title = "A Model of Cross-Lingual Knowledge-Grounded Response Generation for Open-Domain Dialogue Systems",
3 author = "Kim, San and
4 Jang, Jin Yea and
5 Jung, Minyoung and
6 Shin, Saim",
7 booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2021",
8 month = nov,
9 year = "2021",
10 address = "Punta Cana, Dominican Republic",
11 publisher = "Association for Computational Linguistics",
12 url = "https://aclanthology.org/2021.findings-emnlp.33",
13 doi = "10.18653/v1/2021.findings-emnlp.33",
14 pages = "352--365",
15 abstract = "Research on open-domain dialogue systems that allow free topics is challenging in the field of natural language processing (NLP). The performance of the dialogue system has been improved recently by the method utilizing dialogue-related knowledge; however, non-English dialogue systems suffer from reproducing the performance of English dialogue systems because securing knowledge in the same language with the dialogue system is relatively difficult. Through experiments with a Korean dialogue system, this paper proves that the performance of a non-English dialogue system can be improved by utilizing English knowledge, highlighting the system uses cross-lingual knowledge. For the experiments, we 1) constructed a Korean version of the Wizard of Wikipedia dataset, 2) built Korean-English T5 (KE-T5), a language model pre-trained with Korean and English corpus, and 3) developed a knowledge-grounded Korean dialogue model based on KE-T5. We observed the performance improvement in the open-domain Korean dialogue model even only English knowledge was given. The experimental results showed that the knowledge inherent in cross-lingual language models can be helpful for generating responses in open dialogue systems.",
16}1@article{2020t5,
2 author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
3 title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
4 journal = {Journal of Machine Learning Research},
5 year = {2020},
6 volume = {21},
7 number = {140},
8 pages = {1-67},
9 url = {http://jmlr.org/papers/v21/20-074.html}
10}
11 1from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
2
3tokenizer = AutoTokenizer.from_pretrained("KETI-AIR/ke-t5-base")
4
5model = AutoModelForSeq2SeqLM.from_pretrained("KETI-AIR/ke-t5-base")