@article {15476,
	title = {Reliable Effects Screening: A Distributed Continuous Quality Assurance Process for Monitoring Performance Degradation in Evolving Software Systems},
	journal = {Software Engineering, IEEE Transactions on},
	volume = {33},
	year = {2007},
	month = {2007/02//},
	pages = {124 - 141},
	abstract = {Developers of highly configurable performance-intensive software systems often use in-house performance-oriented "regression testing" to ensure that their modifications do not adversely affect their software{\textquoteright}s performance across its large configuration space. Unfortunately, time and resource constraints can limit in-house testing to a relatively small number of possible configurations, followed by unreliable extrapolation from these results to the entire configuration space. As a result, many performance bottlenecks escape detection until systems are fielded. In our earlier work, we improved the situation outlined above by developing an initial quality assurance process called "main effects screening". This process 1) executes formally designed experiments to identify an appropriate subset of configurations on which to base the performance-oriented regression testing, 2) executes benchmarks on this subset whenever the software changes, and 3) provides tool support for executing these actions on in-the-field and in-house computing resources. Our initial process had several limitations, however, since it was manually configured (which was tedious and error-prone) and relied on strong and untested assumptions for its accuracy (which made its use unacceptably risky in practice). This paper presents a new quality assurance process called "reliable effects screening" that provides three significant improvements to our earlier work. First, it allows developers to economically verify key assumptions during process execution. Second, it integrates several model-driven engineering tools to make process configuration and execution much easier and less error prone. Third, we evaluate this process via several feasibility studies of three large, widely used performance-intensive software frameworks. Our results indicate that reliable effects screening can detect performance degradation in large-scale systems more reliably and with significantly less resources than conventional t- echniques},
	keywords = {configuration subset, distributed continuous quality assurance process, evolving software systems, in house testing, main effects screening, performance bottlenecks, performance degradation monitoring, performance intensive software systems, process configuration, process execution, program testing, regression testing, reliable effects screening, software benchmarks, Software performance, software performance evaluation, Software quality, software reliability, tool support},
	isbn = {0098-5589},
	doi = {10.1109/TSE.2007.20},
	author = {Yilmaz,C. and Porter, Adam and Krishna,A. S and Memon, Atif M. and Schmidt,D. C and Gokhale,A.S. and Natarajan,B.}
}
@conference {15468,
	title = {Skoll: distributed continuous quality assurance},
	booktitle = {Software Engineering, 2004. ICSE 2004. Proceedings. 26th International Conference on},
	year = {2004},
	month = {2004/05//},
	pages = {459 - 468},
	abstract = {Quality assurance (QA) tasks, such as testing, profiling, and performance evaluation, have historically been done in-house on developer-generated workloads and regression suites. Since this approach is inadequate for many systems, tools and processes are being developed to improve software quality by increasing user participation in the QA process. A limitation of these approaches is that they focus on isolated mechanisms, not on the coordination and control policies and tools needed to make the global QA process efficient, effective, and scalable. To address these issues, we have initiated the Skoll project, which is developing and validating novel software QA processes and tools that leverage the extensive computing resources of worldwide user communities in a distributed, continuous manner to significantly and rapidly improve software quality. This paper provides several contributions to the study of distributed continuous QA. First, it illustrates the structure and functionality of a generic around-the-world, around-the-clock QA process and describes several sophisticated tools that support this process. Second, it describes several QA scenarios built using these tools and process. Finally, it presents a feasibility study applying these scenarios to a 1MLOC+ software package called ACE+TAO. While much work remains to be done, the study suggests that the Skoll process and tools effectively manage and control distributed, continuous QA processes. Using Skoll we rapidly identified problems that had taken the ACE+TAO developers substantially longer to find and several of which had previously not been found. Moreover, automatic analysis of QA task results often provided developers information that quickly led them to the root cause of the problems.},
	keywords = {1MLOC+ software package, ACE+TAO, around-the-clock QA process, around-the-world QA process, distributed continuous QA, distributed continuous quality assurance, distributed programming, program verification, Quality assurance, Skoll, software performance evaluation, software profiling, Software quality, Software testing},
	doi = {10.1109/ICSE.2004.1317468},
	author = {Memon, Atif M. and Porter, Adam and Yilmaz,C. and Nagarajan,A. and Schmidt,D. and Natarajan,B.}
}
@conference {18683,
	title = {Passive replication schemes in AQuA},
	year = {2002},
	month = {2002/12//},
	pages = {125 - 130},
	abstract = {Building large-scale distributed object-oriented systems that provide multidimensional quality of service (QoS) in terms of fault tolerance, scalability, and performance is challenging. In order to meet this challenge, we need an architecture that can ensure that applications{\textquoteright} requirements can be met while providing reusable technologies and software solutions. This paper describes techniques, based on the AQuA architecture, that enhance the applications{\textquoteright} dependability and scalability by introducing two types of group members and a novel passive replication scheme. In addition, we describe how to make the management structure itself dependable by using the passive replication scheme. Finally, we provide performance measurements for the passive replication scheme.},
	keywords = {AQuA architecture, distributed object management, Fault tolerance, group members, large-scale distributed object-oriented systems, management structure, multidimensional quality of service, passive replication scheme, performance measurements, reusable technologies, scalability, software fault tolerance, software performance evaluation, software reusability, software solutions},
	doi = {10.1109/PRDC.2002.1185628},
	author = {Ren,Yansong and Rubel,P. and Seri,M. and Michel Cukier and Sanders,W. H. and Courtney,T.}
}
@conference {18302,
	title = {Smiling faces are better for face recognition},
	booktitle = {Fifth IEEE International Conference on Automatic Face and Gesture Recognition, 2002. Proceedings},
	year = {2002},
	month = {2002/05//},
	pages = {52 - 57},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {This paper investigates face recognition during facial expressions. While face expressions have been treated as an adverse factor in standard face recognition approaches, our research suggests that, if a system has a choice in the selection of faces to use in training and recognition, its best performance would be obtained on faces displaying expressions. Naturally, smiling faces are the most prevalent (among expressive faces) for both training and recognition in dynamic scenarios. We employ a measure of discrimination power that is computed from between-class and within-class scatter matrices. Two databases are used to show the performance differences on different sets of faces},
	keywords = {between-class scatter matrices, Databases, discrimination power measure, dynamic scenarios, expressive faces, face recognition, facial expressions, performance, performance differences, smiling faces, software performance evaluation, Training, visual databases, within-class scatter matrices},
	isbn = {0-7695-1602-5},
	doi = {10.1109/AFGR.2002.1004132},
	author = {Yacoob,Yaser and Davis, Larry S.}
}
@article {14805,
	title = {A tool to help tune where computation is performed},
	journal = {IEEE Transactions on Software Engineering},
	volume = {27},
	year = {2001},
	month = {2001/07//},
	pages = {618 - 629},
	abstract = {We introduce a new performance metric, called load balancing factor (LBF), to assist programmers when evaluating different tuning alternatives. The LBF metric differs from traditional performance metrics since it is intended to measure the performance implications of a specific tuning alternative rather than quantifying where time is spent in the current version of the program. A second unique aspect of the metric is that it provides guidance about moving work within a distributed or parallel program rather than reducing it. A variation of the LBF metric can also be used to predict the performance impact of changing the underlying network. The LBF metric is computed incrementally and online during the execution of the program to be tuned. We also present a case study that shows that our metric can accurately predict the actual performance gains for a test suite of six programs},
	keywords = {Computational modeling, Current measurement, Distributed computing, distributed program, distributed programming, load balancing factor, Load management, parallel program, parallel programming, Performance analysis, performance evaluation, Performance gain, performance metric, Programming profession, software metrics, software performance evaluation, Testing, Time measurement, tuning},
	isbn = {0098-5589},
	doi = {10.1109/32.935854},
	author = {Eom, Hyeonsang and Hollingsworth, Jeffrey K}
}
@conference {14784,
	title = {LBF: a performance metric for program reorganization},
	booktitle = {18th International Conference on Distributed Computing Systems, 1998. Proceedings},
	year = {1998},
	month = {1998/05/26/29},
	pages = {222 - 229},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We introduce a new performance metric, called Load Balancing Factor (LBF), to assist programmers with evaluating different tuning alternatives. The LBF metric differs from traditional performance metrics since it is intended to measure the performance implications of a specific tuning alternative rather than quantifying where time is spent in the current version of the program. A second unique aspect of the metric is that it provides guidance about moving work within a distributed or parallel program rather than reducing it. A variation of the LBF metric can also be used to predict the performance impact of changing the underlying network. The LBF metric can be computed incrementally and online during the execution of the program to be tuned. We also present a case study that shows that our metric can predict the actual performance gains accurately for a test suite of six programs},
	keywords = {case study, Computational modeling, computer network, Computer science, Debugging, distributed processing, distributed program, Educational institutions, Integrated circuit testing, LBF metric, load balancing factor, Load management, measurement, NIST, parallel program, parallel programming, performance metric, program reorganization, program tuning, Programming profession, resource allocation, software metrics, software performance evaluation, US Department of Energy},
	isbn = {0-8186-8292-2},
	doi = {10.1109/ICDCS.1998.679505},
	author = {Eom, H. and Hollingsworth, Jeffrey K}
}
@article {14816,
	title = {The Paradyn parallel performance measurement tool},
	journal = {Computer},
	volume = {28},
	year = {1995},
	month = {1995/11//},
	pages = {37 - 46},
	abstract = {Paradyn is a tool for measuring the performance of large-scale parallel programs. Our goal in designing a new performance tool was to provide detailed, flexible performance information without incurring the space (and time) overhead typically associated with trace-based tools. Paradyn achieves this goal by dynamically instrumenting the application and automatically controlling this instrumentation in search of performance problems. Dynamic instrumentation lets us defer insertion until the moment it is needed (and remove it when it is no longer needed); Paradyn{\textquoteright}s Performance Consultant decides when and where to insert instrumentation},
	keywords = {Aerodynamics, Automatic control, automatic instrumentation control, Debugging, dynamic instrumentation, flexible performance information, high level languages, insertion, Instruments, large-scale parallel program, Large-scale systems, measurement, Paradyn parallel performance measurement tool, Parallel machines, parallel programming, Performance Consultant, Programming profession, scalability, software performance evaluation, software tools},
	isbn = {0018-9162},
	doi = {10.1109/2.471178},
	author = {Miller, B. P and Callaghan, M. D and Cargille, J. M and Hollingsworth, Jeffrey K and Irvin, R. B and Karavanic, K. L and Kunchithapadam, K. and Newhall, T.}
}
