@inproceedings{327a86db0afd43568b491bd3d14a0373,
title = "Holistic crowd-powered sorting via aiD: Optimizing for accuracies, inconsistencies, and difficulties",
abstract = "We revisit the fundamental problem of sorting objects using crowdsourced pairwise comparisons. Prior work either treats these comparisons as independent tasks-in which case the resulting judgments may end up being inconsistent, or fails to capture the accuracies of workers, or difficulties of the pairwise comparisons-in which case the resulting judgments may end up being consistent with each other, but ultimately more inaccurate. We adopt a holistic approach that constructs a graph across the set of objects respecting consistency constraints. Our key contribution is a novel method of encoding difficulty of comparisons in the form of constraints on edges. We couple that with an iterative E-M-style procedure to uncover information about latent variables and constraints, along with the graph structure. We show that our approach predicts edge directions as well as difficulty values more accurately than baselines on both real and simulated data, across graphs of various sizes.",
keywords = "Constrained optimization, Crowdsourcing, Graphs, Pairwise comparisons",
author = "Shreya Rajpal and Aditya Parameswaran",
note = "Publisher Copyright: {\textcopyright} 2018 Copyright held by the owner/author(s).; 27th ACM International Conference on Information and Knowledge Management, CIKM 2018 ; Conference date: 22-10-2018 Through 26-10-2018",
year = "2018",
month = oct,
day = "17",
doi = "10.1145/3269206.3269279",
language = "English (US)",
series = "International Conference on Information and Knowledge Management, Proceedings",
publisher = "Association for Computing Machinery",
pages = "1607--1610",
editor = "Norman Paton and Selcuk Candan and Haixun Wang and James Allan and Rakesh Agrawal and Alexandros Labrinidis and Alfredo Cuzzocrea and Mohammed Zaki and Divesh Srivastava and Andrei Broder and Assaf Schuster",
booktitle = "CIKM 2018 - Proceedings of the 27th ACM International Conference on Information and Knowledge Management",
address = "United States",
}