@inproceedings{law2011the,
author = {Law, Edith and Bennett, Paul and Horvitz, Eric},
title = {The Effects of Choice in Routing Relevance Judgments},
booktitle = {Proceedings of the 34th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2011)},
year = {2011},
month = {July},
abstract = {The emergence of human computation systems, including Mechanical Turk and games with a purpose, has made it feasible to distribute relevance judgment tasks to workers over the Web. Most human computation systems assign tasks to individuals randomly, and such assignments may match workers with tasks that they may be unqualified or unmotivated to perform. We compare two groups of workers, those given a choice of queries to judge versus those who are not, in terms of their self-rated competence and their actual performance. Results show that when given a choice of task, workers choose ones for which they have greater expertise, interests, confidence, and understanding.},
publisher = {ACM},
url = {http://approjects.co.za/?big=en-us/research/publication/the-effects-of-choice-in-routing-relevance-judgments/},
edition = {Proceedings of the 34th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2011)},
}