@article {12424, title = {Besting the Quiz Master: Crowdsourcing Incremental Classification Games}, journal = {Empirical Methods in Natural Language Processing}, year = {2012}, month = {2012///}, abstract = {Cost-sensitive classification, where the features used in machine learning tasks have a cost, has been explored as a means of balancing knowl- edge against the expense of obtaining new fea- tures. We introduce a setting where humans engage in classification with incrementally re- vealed features: the collegiate trivia circuit. By providing the community with a web-based system to practice, we collected tens of thou- sands of implicit word-by-word ratings of how useful features are for eliciting correct answers. Observing humans{\textquoteright} classification process, we improve the performance of a state-of-the art classifier. We also use the dataset to evaluate a system to compete in the incremental classifica- tion task through a reduction of reinforcement learning to classification. Our system learns when to answer a question, performing better than baselines and most human players.}, author = {Satinoff,Brianna and Jordan Boyd-Graber} }