@inproceedings{cutler2021crowdsourcing,
author = {Cutler, Ross and Nadari, Babak and Loide, Markus and Sootla, Sten and Saabas, Ando},
title = {Crowdsourcing Approach for Subjective Evaluation of Echo Impairment},
booktitle = {2021 International Conference on Acoustics, Speech, and Signal Processing},
year = {2021},
month = {June},
abstract = {The quality of acoustic echo cancellers (AECs) in real-time communication systems is typically evaluated using objective metrics like ERLE [1] and PESQ [2], and less commonly with lab-based subjective tests like ITU-T Rec. P.831 [3]. We will show that these objective measures are not well correlated to subjective measures. We then introduce an open-source crowdsourcing approach for subjective evaluation of echo impairment which can be used to evaluate the performance of AECs. We provide a study that shows this tool is highly reproducible. This new tool has been recently used in the ICASSP 2021 AEC Challenge [4] which made the challenge possible to do quickly and cost effectively.},
publisher = {IEEE},
url = {http://approjects.co.za/?big=en-us/research/publication/crowdsourcing-approach-for-subjective-evaluation-of-echo-impairment/},
pages = {406-410},
}