@inproceedings{chen2023structural,
author = {Chen, Nuo and Shou (寿林钧), Linjun and Song, Tengtao and Gong (YIMING), Ming and Pei, Jian and Chang, Jianhui and Jiang (姜大昕), Daxin and Li, Jia},
title = {Structural Contrastive Pretraining for Cross-Lingual Comprehension},
booktitle = {ACL 2023},
year = {2023},
month = {July},
abstract = {Multilingual language models trained using various pre-training tasks like mask language modeling (MLM) have yielded encouraging results on a wide range of downstream tasks. Despite the promising performances, structural knowledge in cross-lingual corpus is less explored in current works, leading to the semantic misalignment. In this paper, we propose a new pre-training task named Structural Contrast Pretraining (SCP) to align the structural words in a parallel sentence, improving the models' linguistic versatility and their capacity to understand representations in multilingual languages. Concretely, SCP treats each structural word in source and target languages as a positive pair. We further propose Cross-lingual Momentum Contrast (CL-MoCo) to optimize negative pairs by maintaining a large size of the queue. CL-MoCo extends the original MoCo approach into cross-lingual training and jointly optimizes the source-to-target language and target-to-source language representations in SCP, resulting in a more suitable encoder for cross-lingual transfer learning. We conduct extensive experiments and prove the effectiveness of our resulting model, named XLM-SCP, on three cross-lingual tasks across five datasets such as MLQA, WikiAnn. Our codes are available at https://github. com/nuochenpku/SCP.},
url = {http://approjects.co.za/?big=en-us/research/publication/structural-contrastive-pretraining-for-cross-lingual-comprehension/},
}