@article {13771,
	title = {Extrinsic Evaluation of Automatic Metrics for Summarization},
	year = {2004},
	month = {2004/07/20/},
	institution = {Instititue for Advanced Computer Studies, Univ of Maryland, College Park},
	abstract = {This paper describes extrinsic-task evaluation of summarization. We show that it is possible to save time using summaries for relevance assessment without adversely impacting the degree of accuracy that would be possible with full documents. In addition, we demonstrate that the extrinsic task we have selected exhibits a high degree of interannotator agreement, i.e., consistent relevance decisions across subjects. We also conducted a composite experiment that better reflects the actual document selection process and found that using a surrogate improves the processing speed over reading the entire document. Finally, we have found a small yet statistically significant correlation between some of the intrinsic measures and a user{\textquoteright}s performance in an extrinsic task. The overall conclusion we can draw at this point is that ROUGE-1 does correlate with precision and to a somewhat lesser degree with accuracy, but that it remains to be investigated how stable these correlations are and how differences in ROUGE-1 translate into significant differences in human performance in an extrinsic task.},
	keywords = {*ABSTRACTS, *ACCURACY, *DATA SUMMARIZATION, *DOCUMENT SUMMARIES, *DOCUMENTS, *READING MACHINES, *SOFTWARE METRICS, *STATISTICAL ANALYSIS, ABSTRACT SELECTION, ANNOTATIONS, Automation, COMPUTER PROGRAMMING AND SOFTWARE, COMPUTER SYSTEMS, CORRELATION TECHNIQUES, DATA PROCESSING, DOCUMENT SUMMARIES, EXTRINSIC TASKS, Information retrieval, INFORMATION SCIENCE, PERFORMANCE(HUMAN), PRECISION, RELEVANCY, ROUGE-1 COMPUTER PROGRAM, STATISTICS AND PROBABILITY, TEST AND EVALUATION},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA448065},
	author = {Dorr, Bonnie J and Monz,Christof and Oard, Douglas and President,Stacy and Zajic, David and Schwartz,Richard}
}
@article {13827,
	title = {Mapping WorldNet Senses to a Lexical Database of Verbs},
	year = {2001},
	month = {2001/01//},
	institution = {Instititue for Advanced Computer Studies, Univ of Maryland, College Park},
	abstract = {This paper describes automatic techniques for mapping 9611 semantically classified English verbs to WordNet senses. The verbs were initially grouped into 491 semantic classes based on syntactic categories; they were then mapped into WordNet senses according to three pieces of information: (1) prior probability of WordNet senses; (2) semantic similarity of WordNet senses for verbs within the same category; and (3) probabilistic correlations between WordNet relationship and verb frame data. Our techniques make use of a training set of 1791 disambiguated entries representing 1442 verbs occurring in 167 of the categories. The best results achieved .58 recall and .72 precision, versus a lower bound of .38 recall and .62 precision for assigning the most frequently occurring WordNet sense, and an upper bound of .75 recall and .87 precision for human judgment.},
	keywords = {*DATA BASES, *ENGLISH LANGUAGE, *LEXICAL DATABASES, *LEXICOGRAPHY, *MAPPING, *VERBS, *WORD MAPPING, *WORDNET, AMBIGUITY, correlation, FRAMES, Frequency, INFORMATION SCIENCE, JUDGEMENT(PSYCHOLOGY), linguistics, PRECISION, probability, RECALL, semantics, syntax, WORDS(LANGUAGE), WSD(WORD SENSE DISAMBIGUATION)},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA458846},
	author = {Green,Rebecca and Pearl,Lisa and Dorr, Bonnie J}
}
