@article{wang2022list,
author = {Wang, Yaqing and Mukherjee, Subhabrata (Subho) and Liu, Xiaodong and Gao, Jing and Awadallah, Ahmed and Gao, Jianfeng},
title = {LiST: Lite Prompted Self-training Makes Parameter-efficient Few-shot Learners},
year = {2022},
month = {April},
abstract = {We present a new method LiST for efficient fine-tuning of large pre-trained language models (PLMs) in few-shot learning settings. LiST significantly improves over recent methods that adopt prompt fine-tuning using two key techniques. The first one is the use of self-training to leverage large amounts of unlabeled data for prompt-tuning to significantly boost the model performance in few-shot settings. We use self-training in conjunction with meta-learning for re-weighting noisy pseudo-prompt labels. However, traditional self-training is expensive as it requires updating all the model parameters repetitively. Therefore, we use a second technique for light-weight fine-tuning where we introduce a small number of task-specific adapter parameters that are fine-tuned during self-training while keeping the PLM encoder frozen. This also significantly reduces the overall model footprint across several tasks that can now share a common PLM encoder as backbone for inference. Combining the above techniques, LiST not only improves the model performance for few-shot learning on target domains but also reduces the model memory footprint. We present a comprehensive study on six NLU tasks to validate the effectiveness of LiST. The results show that LiST improves by 35% over classic fine-tuning methods and 6% over prompt-tuning with 96% reduction in number of trainable parameters when fine-tuned with no more than 30 labeled examples from each target domain.},
url = {http://approjects.co.za/?big=en-us/research/publication/list-lite-self-training-makes-efficient-few-shot-learners/},
journal = {NAACL 2022},
}