@article{jain2024ensuring,
author = {Jain, Kunal and Mallick, Ankur and Parayil, A. and St. Amant, Renee and Wang, Rujia and Ruehle, Victor and Bansal, Chetan and Rajmohan, Saravan and Khan, Redwan Ibne Seraj and Shen, Haiying and Kulkarni, Anoop and Kofsky, Steve and Choudhary, Pankhuri and Cheng, Yue},
title = {Ensuring Fair LLM Serving Amid Diverse Applications},
year = {2024},
month = {November},
abstract = {In a multi-tenant large language model (LLM) serving platform hosting diverse applications, some users may submit an excessive number of requests, causing the service to become unavailable to other users and creating unfairness. Existing fairness approaches do not account for variations in token lengths across applications and multiple LLM calls, making them unsuitable for such platforms. To address the fairness challenge, this paper analyzes millions of requests from thousands of users on MS CoPilot, a real-world multi-tenant LLM platform hosted by Microsoft. Our analysis confirms the inadequacy of existing methods and guides the development of FairServe, a system that ensures fair LLM access across diverse applications. FairServe proposes application-characteristic aware request throttling coupled with a weighted service counter based scheduling technique to curb abusive behavior and ensure fairness. Our experimental results on real-world traces demonstrate FairServe's superior performance compared to the state-of-the-art method in ensuring fairness. We are actively working on deploying our system in production, expecting to benefit millions of customers world-wide.},
url = {http://approjects.co.za/?big=en-us/research/publication/ensuring-fair-llm-serving-amid-diverse-applications/},
journal = {ArXiv},
}