@inproceedings{zhang2022a,
author = {Zhang, Shangtong and Laroche, Romain and van Seijen, Harm and Whiteson, Shimon and Tachet des Combes, Remi},
title = {A Deeper Look at Discounting Mismatch in Actor-Critic Algorithms},
booktitle = {International Conference on Autonomous Agents and Multiagent Systems (AAMAS)},
year = {2022},
month = {May},
abstract = {We investigate the discounting mismatch in actor-critic algorithm implementations from a representation learning perspective. Theoretically, actor-critic algorithms usually have discounting for both actor and critic, i.e., there is a γt term in the actor update for the transition observed at time t in a trajectory and the critic is a discounted value function. Practitioners, however, usually ignore the discounting (γt) for the actor while using a discounted critic. We investigate this mismatch in two scenarios. In the first scenario, we consider optimizing an undiscounted objective (γ=1) where γt disappears naturally (1t=1). We then propose to interpret the discounting in critic in terms of a bias-variance-representation trade-off and provide supporting empirical results. In the second scenario, we consider optimizing a discounted objective (γ<1) and propose to interpret the omission of the discounting in the actor update from an auxiliary task perspective and provide supporting empirical results.},
url = {http://approjects.co.za/?big=en-us/research/publication/a-deeper-look-at-discounting-mismatch-in-actor-critic-algorithms/},
}