@article {12162,
	title = {The Use of Empirical Studies in the Development of High End Computing Applications},
	year = {2009},
	month = {2009/12//},
	institution = {University of Maryland, College Park},
	abstract = {This report provides a description of the research and development activities towards learning much about the development and measurement of productivity in high performance computing environments. Many objectives were accomplished including the development of a methodology for measuring productivity in the parallel programming domain. This methodology was tested over 25 times at 8 universities across the United States and can be used to aid other researchers studying similar environments. The productivity measurement methodology incorporates both development time and performance into a single productivity number. An Experiment Manager tool for collecting data on the development of parallel programs, as well as a suite of tools to aid in the capture and analysis of such data was also developed. Lastly, several large scale development environments were studied in order to better understand the environment used to build large parallel programming applications. That work also included several surveys and interviews with many professional programmers in these environments.},
	keywords = {*COMPUTER PROGRAMMING, *EMPIRICAL STUDIES, *HPC(HIGH PERFORMANCE COMPUTING), *METHODOLOGY, *PARALLEL PROCESSING, *PARALLEL PROGRAMMING, *PRODUCTIVITY, *SOFTWARE ENGINEERING, *SOFTWARE METRICS, ADMINISTRATION AND MANAGEMENT, APMS(AUTOMATED PERFORMANCE MEASUREMENT SYSTEM), COMPUTER PROGRAMMING AND SOFTWARE, COMPUTER SYSTEMS MANAGEMENT AND STANDARDS, data acquisition, efficiency, ENVIRONMENTS, HIGH END COMPUTING, HPCBUGBASE, HUMAN FACTORS ENGINEERING \& MAN MACHINE SYSTEM, learning, measurement, MPI(MESSAGE PASSING INTERFACE), PARALLEL ORIENTATION, PE62303E, PROGRAMMERS, SE(SOFTWARE ENGINEERING), SUPERVISORS, TEST AND EVALUATION, TEST FACILITIES, EQUIPMENT AND METEORS, TIME, tools, United States, WUAFRLT810HECA},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA511351},
	author = {Basili, Victor R. and Zelowitz,Marvin V}
}
@article {15324,
	title = {How Do Users Find Things with PubMed? Towards Automatic Utility Evaluation with User Simulations},
	year = {2008},
	month = {2008/02//},
	institution = {Human-Computer Interaction Lab, University of Maryland, College Park},
	abstract = {In the context of document retrieval in the biomedical domain, this paper explores the complex relationship between the quality of initial query results and the overall utility of an interactive system. We demonstrate that a content-similarity browsing tool can compensate for poor retrieval results, and that the relationship between retrieval performance and overall utility is non-linear. Arguments are advanced with user simulations, which characterize the relevance of documents that a user might encounter with different browsing strategies. With broader implications to IR, this work provides a case study of how user simulations can be exploited as a formative tool for automatic utility evaluation. Simulation-based studies provide researchers with an additional evaluation tool to complement interactive and Cranfield-style experiments.},
	keywords = {*BIOMEDICAL INFORMATION SYSTEMS, *BROWSING, *INFORMATION RETRIEVAL, *PUBMED, *SIMILARITY, *SIMULATION, *USER NEEDS, DOCUMENTS, FIND-SIMILAR, HUMAN FACTORS ENGINEERING \& MAN MACHINE SYSTEM, INFORMATION SCIENCE, MAN COMPUTER INTERFACE, QUALITY, queries, searching, TEST AND EVALUATION, UTILITY},
	url = {http://stinet.dtic.mil/oai/oai?\&verb=getRecord\&metadataPrefix=html\&identifier=ADA478703},
	author = {Jimmy Lin and Smucker,Mark D.}
}
