@article {13870, title = {Task-based evaluation of text summarization using Relevance Prediction}, journal = {Information Processing \& Management}, volume = {43}, year = {2007}, month = {2007/11//}, pages = {1482 - 1499}, abstract = {This article introduces a new task-based evaluation measure called Relevance Prediction that is a more intuitive measure of an individual{\textquoteright}s performance on a real-world task than interannotator agreement. Relevance Prediction parallels what a user does in the real world task of browsing a set of documents using standard search tools, i.e., the user judges relevance based on a short summary and then that same user{\textemdash}not an independent user{\textemdash}decides whether to open (and judge) the corresponding document. This measure is shown to be a more reliable measure of task performance than LDC Agreement, a current gold-standard based measure used in the summarization evaluation community. Our goal is to provide a stable framework within which developers of new automatic measures may make stronger statistical statements about the effectiveness of their measures in predicting summary usefulness. We demonstrate{\textemdash}as a proof-of-concept methodology for automatic metric developers{\textemdash}that a current automatic evaluation measure has a better correlation with Relevance Prediction than with LDC Agreement and that the significance level for detected differences is higher for the former than for the latter.}, keywords = {Relevance prediction, Summarization evaluation, Summary usefulness}, isbn = {0306-4573}, doi = {10.1016/j.ipm.2007.01.002}, url = {http://www.sciencedirect.com/science/article/pii/S0306457307000234}, author = {Hobson,Stacy President and Dorr, Bonnie J and Monz,Christof and Schwartz,Richard} }