@inproceedings{caccia2023multi-head,
author = {Caccia, Lucas and Ponti, Edoardo and Su, Zhan and Pereira, Matheus and Le Roux, Nicolas and Sordoni, Alessandro},
title = {Multi-Head Adapter Routing for Cross-Task Generalization},
booktitle = {NeurIPS 2023},
year = {2023},
month = {October},
abstract = {Parameter-efficient fine-tuning (PEFT) for cross-task generalization consists in pre-training adapters on a multi-task training set before few-shot adaptation to test tasks. Polytropon [Ponti et al., 2023] () jointly learns an inventory of adapters and a routing function that selects a (variable-size) subset of adapters for each task during both pre-training and few-shot adaptation. In this paper, we investigate the role that adapter routing plays in its success and design new variants based on our findings. First, we build on the intuition that finer-grained routing provides more expressivity. Hence, we propose  (Multi-Head Routing), which combines  of adapter parameters and outperforms  under a comparable parameter budget; by only fine-tuning the routing function and not the adapters (), we achieve competitive performance with extreme parameter efficiency. Second, we find that  performance is a result of better multi-task optimization, rather than modular inductive biases that facilitate adapter recombination and local adaptation, as previously hypothesized. In fact, we find that  exhibits higher gradient alignment between tasks than any other method. Since this implies that routing is only crucial during multi-task pre-training, we propose , which discards routing and fine-tunes the average of the pre-trained adapters during few-shot adaptation. This establishes  as an effective method for single-adapter fine-tuning.},
url = {http://approjects.co.za/?big=en-us/research/publication/multi-head-adapter-routing-for-cross-task-generalization/},
}