@article {13771,
	title = {Extrinsic Evaluation of Automatic Metrics for Summarization},
	year = {2004},
	month = {2004/07/20/},
	institution = {Instititue for Advanced Computer Studies, Univ of Maryland, College Park},
	abstract = {This paper describes extrinsic-task evaluation of summarization. We show that it is possible to save time using summaries for relevance assessment without adversely impacting the degree of accuracy that would be possible with full documents. In addition, we demonstrate that the extrinsic task we have selected exhibits a high degree of interannotator agreement, i.e., consistent relevance decisions across subjects. We also conducted a composite experiment that better reflects the actual document selection process and found that using a surrogate improves the processing speed over reading the entire document. Finally, we have found a small yet statistically significant correlation between some of the intrinsic measures and a user{\textquoteright}s performance in an extrinsic task. The overall conclusion we can draw at this point is that ROUGE-1 does correlate with precision and to a somewhat lesser degree with accuracy, but that it remains to be investigated how stable these correlations are and how differences in ROUGE-1 translate into significant differences in human performance in an extrinsic task.},
	keywords = {*ABSTRACTS, *ACCURACY, *DATA SUMMARIZATION, *DOCUMENT SUMMARIES, *DOCUMENTS, *READING MACHINES, *SOFTWARE METRICS, *STATISTICAL ANALYSIS, ABSTRACT SELECTION, ANNOTATIONS, Automation, COMPUTER PROGRAMMING AND SOFTWARE, COMPUTER SYSTEMS, CORRELATION TECHNIQUES, DATA PROCESSING, DOCUMENT SUMMARIES, EXTRINSIC TASKS, Information retrieval, INFORMATION SCIENCE, PERFORMANCE(HUMAN), PRECISION, RELEVANCY, ROUGE-1 COMPUTER PROGRAM, STATISTICS AND PROBABILITY, TEST AND EVALUATION},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA448065},
	author = {Dorr, Bonnie J and Monz,Christof and Oard, Douglas and President,Stacy and Zajic, David and Schwartz,Richard}
}
@article {13790,
	title = {Improved Word-Level Alignment: Injecting Knowledge about MT Divergences},
	year = {2002},
	month = {2002/02/14/},
	institution = {Instititue for Advanced Computer Studies, Univ of Maryland, College Park},
	abstract = {Word-level alignments of bilingual text (bitexts) are not an integral part of statistical machine translation models, but also useful for lexical acquisition, treebank construction. and part-of-speech tagging. The frequent occurrence of divergences, structural differences between languages, presents a great challenge to the alignment task. We resolve some of the most prevalent divergence cases by using syntactic parse information to transform the sentence structure of one language to bear a closer resemblance to that of the other language. In this paper, we show that common divergence types can be found in multiple language pairs (in particular, we focus on English-Spanish and English-Arabic) and systematically identified. We describe our techniques for modifying English parse trees to form resulting sentences that share more similarity with the sentences in the other languages; finally, we present an empirical analysis comparing the complexities of performing word-level alignments with an without divergence handling. Our results suggest that divergence-handling can improve word-level alignment.},
	keywords = {*LEXICOGRAPHY, *MACHINE TRANSLATION, *STATISTICAL ANALYSIS, *WORDS(LANGUAGE), ACQUISITION, ALIGNMENT, EXPERIMENTAL DATA, LANGUAGE, linguistics, MATHEMATICAL MODELS, STATISTICS AND PROBABILITY, TREES},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA458774},
	author = {Dorr, Bonnie J and Pearl,Lisa and Hwa,Rebecca and Habash,Nizar}
}
