@inproceedings{c38d38c4b4744dc8913b287e9fb27ba9,
title = "Infrastructure support for evaluation as a service",
abstract = "How do we conduct large-scale community-wide evaluations for information retrieval if we are unable to distribute the document collection? This was the challenge we faced in organizing a task on searching tweets at the Text Retrieval Conference (TREC), since Twitter's terms of service forbid redistribution of tweets. Our solution, which we call eval- uation as a service{"}, was to provide an API through which the collection can be accessed for completing the evaluation task. This paper describes the infrastructure underlying the service and its deployment at TREC 2013. We discuss the merits of the approach and potential applicability to other evaluation scenarios.",
keywords = "TREC microblog, Tweet search",
author = "Jimmy Lin and Miles Efron",
note = "Publisher Copyright: {\textcopyright} Copyright 2014 by the International World Wide Web Conferences Steering Committee.; 23rd International Conference on World Wide Web, WWW 2014 ; Conference date: 07-04-2014 Through 11-04-2014",
year = "2014",
month = apr,
day = "7",
doi = "10.1145/2567948.2577014",
language = "English (US)",
series = "WWW 2014 Companion - Proceedings of the 23rd International Conference on World Wide Web",
publisher = "Association for Computing Machinery",
pages = "79--82",
booktitle = "WWW 2014 Companion - Proceedings of the 23rd International Conference on World Wide Web",
address = "United States",
}