@article {12162,
	title = {The Use of Empirical Studies in the Development of High End Computing Applications},
	year = {2009},
	month = {2009/12//},
	institution = {University of Maryland, College Park},
	abstract = {This report provides a description of the research and development activities towards learning much about the development and measurement of productivity in high performance computing environments. Many objectives were accomplished including the development of a methodology for measuring productivity in the parallel programming domain. This methodology was tested over 25 times at 8 universities across the United States and can be used to aid other researchers studying similar environments. The productivity measurement methodology incorporates both development time and performance into a single productivity number. An Experiment Manager tool for collecting data on the development of parallel programs, as well as a suite of tools to aid in the capture and analysis of such data was also developed. Lastly, several large scale development environments were studied in order to better understand the environment used to build large parallel programming applications. That work also included several surveys and interviews with many professional programmers in these environments.},
	keywords = {*COMPUTER PROGRAMMING, *EMPIRICAL STUDIES, *HPC(HIGH PERFORMANCE COMPUTING), *METHODOLOGY, *PARALLEL PROCESSING, *PARALLEL PROGRAMMING, *PRODUCTIVITY, *SOFTWARE ENGINEERING, *SOFTWARE METRICS, ADMINISTRATION AND MANAGEMENT, APMS(AUTOMATED PERFORMANCE MEASUREMENT SYSTEM), COMPUTER PROGRAMMING AND SOFTWARE, COMPUTER SYSTEMS MANAGEMENT AND STANDARDS, data acquisition, efficiency, ENVIRONMENTS, HIGH END COMPUTING, HPCBUGBASE, HUMAN FACTORS ENGINEERING \& MAN MACHINE SYSTEM, learning, measurement, MPI(MESSAGE PASSING INTERFACE), PARALLEL ORIENTATION, PE62303E, PROGRAMMERS, SE(SOFTWARE ENGINEERING), SUPERVISORS, TEST AND EVALUATION, TEST FACILITIES, EQUIPMENT AND METEORS, TIME, tools, United States, WUAFRLT810HECA},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA511351},
	author = {Basili, Victor R. and Zelowitz,Marvin V}
}
@article {15324,
	title = {How Do Users Find Things with PubMed? Towards Automatic Utility Evaluation with User Simulations},
	year = {2008},
	month = {2008/02//},
	institution = {Human-Computer Interaction Lab, University of Maryland, College Park},
	abstract = {In the context of document retrieval in the biomedical domain, this paper explores the complex relationship between the quality of initial query results and the overall utility of an interactive system. We demonstrate that a content-similarity browsing tool can compensate for poor retrieval results, and that the relationship between retrieval performance and overall utility is non-linear. Arguments are advanced with user simulations, which characterize the relevance of documents that a user might encounter with different browsing strategies. With broader implications to IR, this work provides a case study of how user simulations can be exploited as a formative tool for automatic utility evaluation. Simulation-based studies provide researchers with an additional evaluation tool to complement interactive and Cranfield-style experiments.},
	keywords = {*BIOMEDICAL INFORMATION SYSTEMS, *BROWSING, *INFORMATION RETRIEVAL, *PUBMED, *SIMILARITY, *SIMULATION, *USER NEEDS, DOCUMENTS, FIND-SIMILAR, HUMAN FACTORS ENGINEERING \& MAN MACHINE SYSTEM, INFORMATION SCIENCE, MAN COMPUTER INTERFACE, QUALITY, queries, searching, TEST AND EVALUATION, UTILITY},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA478703},
	author = {Jimmy Lin and Smucker,Mark D.}
}
@article {13784,
	title = {Headline Generation for Written and Broadcast News},
	year = {2005},
	month = {2005/03//},
	institution = {Instititue for Advanced Computer Studies, Univ of Maryland, College Park},
	abstract = {This technical report is an overview of work done on Headline Generation for written and broadcast news. The report covers HMM Hedge, a statistical approach based on the noisy channel model, Hedge Trimmer, a parse-and-trim approach using linguistically motivated trimming rules, and Topiary, a combination of Trimmer and Unsupervised Topic Discovery. Automatic evaluation of summaries using ROUGE and BLEU is described and used to evaluate the Headline Generation systems.},
	keywords = {*INFORMATION RETRIEVAL, *RADIO BROADCASTING, AUTOMATIC, AUTOMATIC SUMMARIZATION, HEDGE TRIMMER, INFORMATION SCIENCE, RADIO COMMUNICATIONS, STATISTICAL PROCESSES, TEST AND EVALUATION, WORDS(LANGUAGE)},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA454198},
	author = {Zajic, David and Dorr, Bonnie J and Schwartz,Richard}
}
@article {13771,
	title = {Extrinsic Evaluation of Automatic Metrics for Summarization},
	year = {2004},
	month = {2004/07/20/},
	institution = {Instititue for Advanced Computer Studies, Univ of Maryland, College Park},
	abstract = {This paper describes extrinsic-task evaluation of summarization. We show that it is possible to save time using summaries for relevance assessment without adversely impacting the degree of accuracy that would be possible with full documents. In addition, we demonstrate that the extrinsic task we have selected exhibits a high degree of interannotator agreement, i.e., consistent relevance decisions across subjects. We also conducted a composite experiment that better reflects the actual document selection process and found that using a surrogate improves the processing speed over reading the entire document. Finally, we have found a small yet statistically significant correlation between some of the intrinsic measures and a user{\textquoteright}s performance in an extrinsic task. The overall conclusion we can draw at this point is that ROUGE-1 does correlate with precision and to a somewhat lesser degree with accuracy, but that it remains to be investigated how stable these correlations are and how differences in ROUGE-1 translate into significant differences in human performance in an extrinsic task.},
	keywords = {*ABSTRACTS, *ACCURACY, *DATA SUMMARIZATION, *DOCUMENT SUMMARIES, *DOCUMENTS, *READING MACHINES, *SOFTWARE METRICS, *STATISTICAL ANALYSIS, ABSTRACT SELECTION, ANNOTATIONS, Automation, COMPUTER PROGRAMMING AND SOFTWARE, COMPUTER SYSTEMS, CORRELATION TECHNIQUES, DATA PROCESSING, DOCUMENT SUMMARIES, EXTRINSIC TASKS, Information retrieval, INFORMATION SCIENCE, PERFORMANCE(HUMAN), PRECISION, RELEVANCY, ROUGE-1 COMPUTER PROGRAM, STATISTICS AND PROBABILITY, TEST AND EVALUATION},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA448065},
	author = {Dorr, Bonnie J and Monz,Christof and Oard, Douglas and President,Stacy and Zajic, David and Schwartz,Richard}
}
@article {16718,
	title = {The Bible, Truth, and Multilingual OCR Evaluation},
	year = {1998},
	month = {1998/12//},
	institution = {University of Maryland, College Park},
	abstract = {Multilingual OCR has emerged as an important information technology, thanks to the increasing need for cross-language information access. While many research groups and companies have developed OCR algorithms for various languages, it is difficult to compare the performance of these OCR algorithms across languages. This difficulty arises because most evaluation methodologies rely on the use of a document image dataset in each of the languages and it is difficult to find document datasets in different languages that are similar in content and layout. In this paper we propose to use the Bible as a dataset for comparing OCR accuracy across languages. Besides being available in a wide range of languages, Bible translation are closely parallel in content, carefully translated, surprisingly relevant with respect to modern-day language, and quite inexpensive. A project at the University of Maryland is currently implementing this idea. We have created a scanned image dataset with groundtruth from an Arabic Bible. We have also used image degradation models to create synthetically degraded images of a French Bible. We hope to generate similar Bible datasets for other languages, and we are exploring alternative corpora such as the Koran and the Bhagavad Gita that have similar properties. Quantitative OCR evaluation based on the Arabic Bible dataset is currently in progress.},
	keywords = {*BIBLE, *CORPUS, *DATASETS, *GROUNDTRUTH, *OPTICAL CHARACTER RECOGNITION, *TEST SETS, *TRANSLATIONS, accuracy, algorithms, CYBERNETICS, DOCUMENT IMAGES, DOCUMENTS, IMAGES, INFORMATION SCIENCE, LANGUAGE, linguistics, MULTILINGUAL OCR(OPTICAL CHARACTER RECOGNITION), SYMPOSIA, TEST AND EVALUATION},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA458666},
	author = {Kanungo,Tapas and Resnik, Philip}
}
