@inproceedings{chi2021mt,
author = {Chi, Zewen and Dong, Li and Ma, Shuming and Huang, Shaohan and Mao, Xian-Ling and Huang, Heyan and Wei, Furu},
title = {MT6: Multilingual Pretrained Text-to-Text Transformer with Translation Pairs},
booktitle = {EMNLP 2021},
year = {2021},
month = {October},
abstract = {Multilingual T5 (mT5) pretrains a sequence-to-sequence model on massive monolingual texts, which has shown promising results on many cross-lingual tasks. In this paper, we improve multilingual text-to-text transfer Transformer with translation pairs (mT6). Specifically, we explore three cross-lingual text-to-text pre-training tasks, namely, machine translation, translation pair span corruption, and translation span corruption. In addition, we propose a partially non-autoregressive objective for text-to-text pre-training. We evaluate the methods on eight multilingual benchmark datasets, including sentence classification, named entity recognition, question answering, and abstractive summarization. Experimental results show that the proposed mT6 improves cross-lingual transferability over mT5.},
url = {http://approjects.co.za/?big=en-us/research/publication/mt6-multilingual-pretrained-text-to-text-transformer-with-translation-pairs/},
}