@article {15476,
	title = {Reliable Effects Screening: A Distributed Continuous Quality Assurance Process for Monitoring Performance Degradation in Evolving Software Systems},
	journal = {Software Engineering, IEEE Transactions on},
	volume = {33},
	year = {2007},
	month = {2007/02//},
	pages = {124 - 141},
	abstract = {Developers of highly configurable performance-intensive software systems often use in-house performance-oriented "regression testing" to ensure that their modifications do not adversely affect their software{\textquoteright}s performance across its large configuration space. Unfortunately, time and resource constraints can limit in-house testing to a relatively small number of possible configurations, followed by unreliable extrapolation from these results to the entire configuration space. As a result, many performance bottlenecks escape detection until systems are fielded. In our earlier work, we improved the situation outlined above by developing an initial quality assurance process called "main effects screening". This process 1) executes formally designed experiments to identify an appropriate subset of configurations on which to base the performance-oriented regression testing, 2) executes benchmarks on this subset whenever the software changes, and 3) provides tool support for executing these actions on in-the-field and in-house computing resources. Our initial process had several limitations, however, since it was manually configured (which was tedious and error-prone) and relied on strong and untested assumptions for its accuracy (which made its use unacceptably risky in practice). This paper presents a new quality assurance process called "reliable effects screening" that provides three significant improvements to our earlier work. First, it allows developers to economically verify key assumptions during process execution. Second, it integrates several model-driven engineering tools to make process configuration and execution much easier and less error prone. Third, we evaluate this process via several feasibility studies of three large, widely used performance-intensive software frameworks. Our results indicate that reliable effects screening can detect performance degradation in large-scale systems more reliably and with significantly less resources than conventional t- echniques},
	keywords = {configuration subset, distributed continuous quality assurance process, evolving software systems, in house testing, main effects screening, performance bottlenecks, performance degradation monitoring, performance intensive software systems, process configuration, process execution, program testing, regression testing, reliable effects screening, software benchmarks, Software performance, software performance evaluation, Software quality, software reliability, tool support},
	isbn = {0098-5589},
	doi = {10.1109/TSE.2007.20},
	author = {Yilmaz,C. and Porter, Adam and Krishna,A. S and Memon, Atif M. and Schmidt,D. C and Gokhale,A.S. and Natarajan,B.}
}
