@article{zhang2025ui-evol,
author = {Zhang, Ziyun and Liu, Xinyi and Zhang, Xiaoyi and Wang, Jun and Chen, Gang and Lu, Yan},
title = {UI-Evol: Automatic Knowledge Evolving for Computer Use Agents},
year = {2025},
month = {May},
abstract = {External knowledge has played a crucial role in the recent development of computer use agents. We identify a critical knowledge-execution gap: retrieved knowledge often fails to translate into effective real-world task execution. Our analysis shows even 90\% correct knowledge yields only 41\% execution success rate. To bridge this gap, we propose UI-Evol, a plug-and-play module for autonomous GUI knowledge evolution. UI-Evol consists of two stages: a Retrace Stage that extracts faithful objective action sequences from actual agent-environment interactions, and a Critique Stage that refines existing knowledge by comparing these sequences against external references. We conduct comprehensive experiments on the OSWorld benchmark with the state-of-the-art Agent S2. Our results demonstrate that UI-Evol not only significantly boosts task performance but also addresses a previously overlooked issue of high behavioral standard deviation in computer use agents, leading to superior performance on computer use tasks and substantially improved agent reliability.},
url = {http://approjects.co.za/?big=en-us/research/publication/ui-evol-automatic-knowledge-evolving-for-computer-use-agents/},
journal = {ArXiv},
volume = {abs/2505.21964},
}