@inproceedings{sarkar2016setwise,
author = {Sarkar, Advait and Morrison, Cecily and Dorn, FJ and Bedi, R. and Steinheimer, S. and Boisvert, J. and Burggraaff, J. and D'Souza, M. and Kontschieder, P. and Bulo, S. Rota and Walsh, L. and Kamm, ChP. and Zaykov, Yordan and Sellen, Abigail and Lindley, Siân},
title = {Setwise Comparison: Consistent, Scalable, Continuum Labels for Machine Learning},
booktitle = {Proceedings of CHI 2016},
year = {2016},
month = {January},
abstract = {A growing number of domains, including affect recognition and movement analysis, require a single, real number ground truth label capturing some property of a video clip. We term this the provision of continuum labels. Unfortunately, there is often an uncacceptable trade-off between label consistency and the efficiency of the labelling process with current tools. We present a novel interaction technique, ‘setwise’ comparison, which leverages the intrinsic human capability for consistent relative judgements and the TrueSkill algorithm to solve this problem. We describe SorTable, a system demonstrating this technique. We conducted a real-world study where clinicians labelled videos of patients with multiple sclerosis for the ASSESS MS computer vision system. In assessing the efficiency-consistency trade-off of setwise versus pairwise comparison, we demonstrated that not only is setwise comparison more efficient, but it also elicits more consistent labels. We further consider how our findings relate to the interactive machine learning literature.},
url = {http://approjects.co.za/?big=en-us/research/publication/setwise-comparison-consistent-scalable-continuum-labels-machine-learning/},
}