@inproceedings{li2026beyond,
author = {Li, Yueying and Chen, Yuanfan and Chen, Jiayang and Choukse, Esha and Qiu, Haoran and Suh, Edward and Fonseca, Rodrigo and Scully, Zib and Gupta, Udit},
title = {Beyond Prediction: Tail-Aware Scheduling for LLM Inference},
booktitle = {ICML 2026},
year = {2026},
month = {June},
abstract = {LLM serving exhibits extreme length variability, making size-based scheduling difficult in practice. Recent LLM schedulers approximate SJF/SRPT using predicted decode lengths or ranks and primarily report mean-centric metrics such as TTFT and TBT. We show that these prediction-driven policies can be fragile under distribution shifts, bursty arrivals, and GPU memory pressure, while offering limited control over the tail latency (P90-P99) that dominates user experience, even with perfect decode-length knowledge. We introduce a distribution-aware, prediction-free scheduling framework that replaces explicit length prediction with soft priority boosting driven by lightweight statistical signals. Our design co-optimizes scheduling and cache-aware preemption to account for memory-coupled decode dynamics across workload mixes. Evaluated on production and open-source traces, our method reduces P99 TTLT by up to 35-50% relative to SRPT with perfect length knowledge and reduces TTFT by 34-47% across workloads, including reasoning-heavy and chat-heavy tasks. These results demonstrate a robust alternative for optimizing tail latency in online LLM serving.},
url = {http://approjects.co.za/?big=en-us/research/publication/beyond-prediction-tail-aware-scheduling-for-llm-inference/},
}