{"id":1011564,"date":"2024-03-08T16:53:36","date_gmt":"2024-03-09T00:53:36","guid":{"rendered":"https:\/\/www.microsoft.com\/en-us\/research\/?post_type=msr-blog-post&#038;p=1011564"},"modified":"2024-03-11T06:12:35","modified_gmt":"2024-03-11T13:12:35","slug":"getting-modular-with-language-models-building-and-reusing-a-library-of-experts-for-task-generalization","status":"publish","type":"msr-blog-post","link":"https:\/\/www.microsoft.com\/en-us\/research\/articles\/getting-modular-with-language-models-building-and-reusing-a-library-of-experts-for-task-generalization\/","title":{"rendered":"Getting Modular with Language Models: Building and Reusing a Library of Experts for Task Generalization"},"content":{"rendered":"\n<p class=\"has-purple-color has-text-color has-link-color wp-elements-435037414f0522392071891d0623763c\"><em>Presented by&nbsp;<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/people\/alsordon\/\">Alessandro Sordoni<\/a>&nbsp;at&nbsp;<strong>Microsoft Research Forum, March 2024<\/strong><\/em><\/p>\n\n\n\n<div class=\"wp-block-media-text has-vertical-margin-none  has-vertical-padding-none  is-stacked-on-mobile has-white-background-color has-background\" style=\"grid-template-columns:25% auto\" data-bi-an=\"media-text\"><figure class=\"wp-block-media-text__media\"><img loading=\"lazy\" decoding=\"async\" width=\"360\" height=\"360\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Alessandro-Sordoni_360x360.jpg\" alt=\"Alessandro Sordoni\" class=\"wp-image-1012587 size-full\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Alessandro-Sordoni_360x360.jpg 360w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Alessandro-Sordoni_360x360-300x300.jpg 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Alessandro-Sordoni_360x360-150x150.jpg 150w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/03\/Alessandro-Sordoni_360x360-180x180.jpg 180w\" sizes=\"(max-width: 360px) 100vw, 360px\" \/><\/figure><div class=\"wp-block-media-text__content\" data-bi-an=\"media-text\">\n<blockquote class=\"wp-block-quote is-style-spectrum is-layout-flow wp-block-quote-is-layout-flow\">\n<p>\u201cWe have witnessed basically the wide adoption of large language models, such as GPT-4, that have very broad capabilities and can be used to solve a variety of tasks. But they are, kind of, expensive to serve, and somehow, we can actually think and ask ourselves, are they really necessary for most tasks that users \u2026 might need?\u201d<\/p>\n<cite><em>\u2013<\/em> Alessandro Sordoni, Principal Researcher, Microsoft Research Montreal<\/cite><\/blockquote>\n<\/div><\/div>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"Getting Modular with Language Models: Building, Reusing a Library of Experts for Task Generalization\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube-nocookie.com\/embed\/oIAmrrAEPE8?feature=oembed&rel=0\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n\n\n\n<div class=\"annotations \" data-bi-aN=\"margin-callout\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 annotations__list--right\">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=What%20are%20expert%20language%20models?\" target=\"_blank\" aria-label=\"What are expert language models?\" data-bi-type=\"annotated-link\" data-bi-cN=\"What are expert language models?\" class=\"annotations__list-thumbnail\" >\n\t\t\t\t\t<img loading=\"lazy\" decoding=\"async\" width=\"172\" height=\"96\" src=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png\" class=\"mb-2\" alt=\"Ask Microsoft research copilot experience\" srcset=\"https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-240x135.png 240w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-300x169.png 300w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1024x576.png 1024w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-768x432.png 768w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1066x600.png 1066w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-655x368.png 655w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-343x193.png 343w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-640x360.png 640w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-960x540.png 960w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo-1280x720.png 1280w, https:\/\/www.microsoft.com\/en-us\/research\/uploads\/prod\/2024\/01\/MSR-Chat-Promo.png 1400w\" sizes=\"(max-width: 172px) 100vw, 172px\" \/>\t\t\t\t<\/a>\n\t\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Microsoft research copilot experience<\/span>\n\t\t\t<a href=\"https:\/\/msrchat.azurewebsites.net\/?askmsr=What%20are%20expert%20language%20models?\" target=\"_blank\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"What are expert language models?\" data-bi-aN=\"margin-callout\" data-bi-cN=\"What are expert language models?\">\n\t\t\t\tWhat are expert language models?&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n\n\n<div class=\"wp-block-msr-show-more\">\n\t<div class=\"bg-neutral-100 p-5\">\n\t\t<div class=\"show-more-show-less\" data-mount=\"show-more-show-less\">\n\t\t\t<div>\n\t\t\t\t<span>\n\t\t\t\t\t\n\n<h3 class=\"wp-block-heading\" id=\"transcript-lightning-talk-3\">Transcript: Lightning Talk 3<\/h3>\n\n\n\n<p><strong>Getting modular with language models: Building and reusing a library of experts for task generalization<\/strong><\/p>\n\n\n\n<p><strong>Alessandro Sordoni<\/strong>, Principal Researcher, Microsoft Research Montr\u00e9al<\/p>\n\n\n\n<p>Alessandro Sordoni shares recent efforts on building and reusing large collections of expert language models to improve zero-shot and few-shot generalization to unseen tasks.<\/p>\n\n\n\n<p>Microsoft Research Forum, March 5, 2024<\/p>\n\n\n\n\t\t\t\t<\/span>\n\t\t\t\t<span id=\"show-more-show-less-toggle-1\" class=\"show-more-show-less-toggleable-content\">\n\t\t\t\t\t\n\n\n\n<p>Hi, everyone. My name is Alessandro. I\u2019m from Montr\u00e9al. I\u2019m going to share with you a vision for building, sort of, modular language models.<\/p>\n\n\n\n<p>So we have witnessed basically the wide adoption of large language models, such as GPT-4, that have very broad capabilities and can be used to solve a variety of tasks. But they are, kind of, expensive to serve, and somehow, we can actually think and ask ourselves, are they really necessary for most tasks that users for Microsoft, for example, might need?<\/p>\n\n\n\n<p>This has basically boosted the development of small language models\u2014for example, the mighty and very powerful <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/blog\/phi-2-the-surprising-power-of-small-language-models\/\">Phi-2<\/a>\u2014that can be adapted to user tasks, right. Either with full fine-tuning, which means we change all the parameters of the model, or with parameter-efficient adaptation, for example, by training <a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/lora-low-rank-adaptation-of-large-language-models\/\">LoRA<\/a>s, which only change a small amount of the parameters of the model. And basically, if we can see these experts, basically these model adapters, as experts at their own tasks, right \u2026 and this is great because we have now a cost-effective model that can solve the tasks very effectively. But the problem now is that we have only very narrow capabilities. So the question that we ask here is that now that we have all these experts models for each user, for each task, can we actually reuse the experts models for either building small models that have broader capability or for adapting to new users and tasks more efficiently?<\/p>\n\n\n\n<p>Let me show you how this system could work. So we start from a base model, which is Phi-2, and we adapt this base model for every user or for a set of tasks, and we group these, sort of, adapters into a library. And now we, kind of, come up with a, sort of, orchestration mechanism that chooses which adapters to use based on a new user query to produce a system response. The system has, sort of, some desirable properties. One is that it enhances the base language model capabilities via such an expert composition, and this resembles, a little bit, how mixture of experts work, right. But there is a big difference here. It\u2019s that these experts are not trained with the base model itself, but they are trained a posteriori. This leads us to the second point, which is basically a, sort of, decentralized training of these LoRA experts, and this is good because somehow users \u2026 for example, this preserves privacy in the sense that we do not need\u2014we do not require\u2014all the data to be shared at once and to always retrain the base model from scratch. And second, energy efficiency. These LoRA experts are very energy efficient. And so basically, we can train it very quickly. The third point is interpretability because these experts are basically associated with the task, usually, that they can solve. And so upon seeing a new user query, we can actually inspect which expert has been activated for that user query, and we can get a little bit of a sense of which actually capabilities are required.&nbsp;<\/p>\n\n\n\n<p>So in order to build this system, we have to answer two questions. One, how do we build such an expert library? And second, how do we select the relevant experts for these new inputs? So the first basically scenario that we are dealing with is a, sort of, private scenario in the sense that we have an ensemble of datasets which are tasks <em>or<\/em> user data. And in the private scenario, we assume that we cannot share data for these tasks, OK. We cannot train all the data together. And so basically, one standard approach is to fine-tune, for example, a LoRA adapter on each dataset independently. Here in the figure, we are going to end up with a library with three experts. But let\u2019s say that we can actually share a certain amount of this data, for example, if we are dealing with public tasks or stuff like that. So the idea here around this approach is to basically form a, sort of, clustering of these tasks and just train an adapter for each cluster. How do we cluster these tasks actually? We basically do a, sort of, a private LoRA fine-tuning for a few steps at the beginning, to get just a weight for each task, right. And then we cluster the weights of each task by their similarity, and we group tasks together that have high weight similarity. And we train one adapter per task. So at the end, we are going to end up with a library of two experts. This basically relies on the intuition that the similarity in the weight space for these tasks reflects how synergistic these tasks are when adapters are trained on the joint dataset.<\/p>\n\n\n\n<p>Now that we have our great library of experts, we have to choose how to actually select them upon seeing a new input, OK. Here we assume that we do not have access to the data that\u2019s been used to train the experts. So basically, you trained your experts, you gave it to us, and we figure out how to use it. So here to do routing, we basically select which expert to use for each representation in the base model. The base model is a transformer. We have a representation for each layer and for each token. And so basically, we route by dot products between each hidden states in the transformer and an expert representation. But now we have to come up with an expert representation, and we do not know which data these experts have been trained on. Here, to do so, we leverage basically the functional form of these LoRA adapters, which basically produces a linear shift on the hidden state\u2019s representations. And so we, basically, we take the linear transform of the LoRA adapter, we decompose it into singular directions, and we take the top singular direction of that matrix. That gives us our expert representation. We stack those expert representations in the matrix, in a routing matrix. We compute the dot product, much similar to the mixture of experts, kind of, sort of, parameterization, and we choose which experts we use based on the scores obtained by that.&nbsp;<\/p>\n\n\n\n<p>Here, the idea is simple, is that that singular direction gives us a sense of how the hidden states for that expert looked like when that expert was training. So in order to test basically our system here, we assume that we have some data for tasks available, and we use, like, FLAN data, which is just natural language tasks. We evaluate our systems on a set of these 10 tasks used to evaluate Phi-2, and these tasks range from commonsense reasoning, code, BBH\u2014BIG-Bench Hard\u2014et cetera. And so basically these are some results that we obtained in the recent submission, is that we have, like, Phi-2, the first part, which gets out of the box around 64, and then we actually fine-tuned Phi-2 on our own multitask dataset, and this gets a boost of around 65.5. And basically, this approach assumes that we can train on all data, right. And then we have our first dot, which is \u201cPrivate + Arrow.\u201d And so basically we \u2026 private, as I remember, it trains experts independently\u2014256 tasks\u2014and then there is post-hoc routing. And here it was very surprising to us that we can actually get some good performance even with this method.\u00a0<\/p>\n\n\n\n<p>But if we go further, we assume some sort of selective data sharing and that we have our clustering approach and then the route on top of that, we can get even further gains. And this last method\u2014 \u201cMBC + Arrow\u201d\u2014actually adds only 22 million parameters to the model.<\/p>\n\n\n\n<p>So looking forward, I believe that an exciting direction would be really to push this to fully decentralized training and continual improvement of language models in the sense people can train their experts, they give it to the platform, and model gets better. The other point is a heterogeneous library of adapters in the sense that we can actually add different sorts of adapters into this library, each with its own inductive biases, and so we can expand even more the capabilities.<\/p>\n\n\n\n<p>Thank you very much.<\/p>\n\n\t\t\t\t<\/span>\n\t\t\t<\/div>\n\t\t\t<button\n\t\t\t\tclass=\"action-trigger glyph-prepend mt-2 mb-0 show-more-show-less-toggle\"\n\t\t\t\taria-expanded=\"false\"\n\t\t\t\tdata-show-less-text=\"Show less\"\n\t\t\t\ttype=\"button\"\n\t\t\t\taria-controls=\"show-more-show-less-toggle-1\"\n\t\t\t\taria-label=\"Show more content\"\n\t\t\t\tdata-alternate-aria-label=\"Show less content\">\n\t\t\t\tShow more\t\t\t<\/button>\n\t\t<\/div>\n\t<\/div>\n<\/div>\n\n\n\n<h2 class=\"wp-block-heading alignwide\" id=\"related-resources\">Related resources<\/h2>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-top is-layout-flex wp-container-core-columns-is-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Blog<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/blog\/phi-2-the-surprising-power-of-small-language-models\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Phi-2: The surprising power of small language models\" data-bi-aN=\"citation\" data-bi-cN=\"Phi-2: The surprising power of small language models\">\n\t\t\t\tPhi-2: The surprising power of small language models&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Download<\/span>\n\t\t\t<a href=\"https:\/\/ai.azure.com\/explore\/models\/microsoft-phi-2\/version\/4\/registry\/azureml-msr\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"Phi-2\" data-bi-aN=\"citation\" data-bi-cN=\"Phi-2\">\n\t\t\t\tPhi-2&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Publication<\/span>\n\t\t\t<a href=\"https:\/\/www.microsoft.com\/en-us\/research\/publication\/lora-low-rank-adaptation-of-large-language-models\/\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"LoRA: Low-Rank Adaptation of Large Language Models\" data-bi-aN=\"citation\" data-bi-cN=\"LoRA: Low-Rank Adaptation of Large Language Models\">\n\t\t\t\tLoRA: Low-Rank Adaptation of Large Language Models&nbsp;<span class=\"glyph-append glyph-append-chevron-right glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"annotations \" data-bi-aN=\"citation\">\n\t<ul class=\"annotations__list card depth-16 bg-body p-4 \">\n\t\t<li class=\"annotations__list-item\">\n\t\t\t\t\t\t<span class=\"annotations__type d-block text-uppercase font-weight-semibold text-neutral-300 small\">Download<\/span>\n\t\t\t<a href=\"https:\/\/github.com\/microsoft\/LoRA\" target=\"_self\" class=\"annotations__link font-weight-semibold text-decoration-none\" data-bi-type=\"annotated-link\" aria-label=\"LoRA\" data-bi-aN=\"citation\" data-bi-cN=\"LoRA\">\n\t\t\t\tLoRA&nbsp;<span class=\"glyph-append glyph-append-share glyph-append-xsmall\"><\/span>\n\t\t\t<\/a>\n\t\t\t\t\t<\/li>\n\t<\/ul>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>Alessandro Sordoni shares recent efforts on building and re-using large collections of expert language models to improve zero-shot and few-shot generalization to unseen tasks.<\/p>\n","protected":false},"author":42735,"featured_media":1012584,"template":"","meta":{"msr-url-field":"","msr-podcast-episode":"","msrModifiedDate":"","msrModifiedDateEnabled":false,"ep_exclude_from_search":false,"_classifai_error":"","msr-content-parent":1009965,"footnotes":""},"research-area":[],"msr-locale":[268875],"msr-post-option":[],"class_list":["post-1011564","msr-blog-post","type-msr-blog-post","status-publish","has-post-thumbnail","hentry","msr-locale-en_us"],"msr_assoc_parent":{"id":1009965,"type":"story"},"_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011564"}],"collection":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post"}],"about":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/types\/msr-blog-post"}],"author":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/users\/42735"}],"version-history":[{"count":7,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011564\/revisions"}],"predecessor-version":[{"id":1013793,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-blog-post\/1011564\/revisions\/1013793"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media\/1012584"}],"wp:attachment":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media?parent=1011564"}],"wp:term":[{"taxonomy":"msr-research-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/research-area?post=1011564"},{"taxonomy":"msr-locale","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-locale?post=1011564"},{"taxonomy":"msr-post-option","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-post-option?post=1011564"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}