@conference {15484,
	title = {Fault Detection Probability Analysis for Coverage-Based Test Suite Reduction},
	booktitle = {Software Maintenance, 2007. ICSM 2007. IEEE International Conference on},
	year = {2007},
	month = {2007/10//},
	pages = {335 - 344},
	abstract = {Test suite reduction seeks to reduce the number of test cases in a test suite while retaining a high percentage of the original suite{\textquoteright}s fault detection effectiveness. Most approaches to this problem are based on eliminating test cases that are redundant relative to some coverage criterion. The effectiveness of applying various coverage criteria in test suite reduction is traditionally based on empirical comparison of two metrics derived from the full and reduced test suites and information about a set of known faults: (1) percentage size reduction and (2) percentage fault detection reduction, neither of which quantitatively takes test coverage data into account. Consequently, no existing measure expresses the likelihood of various coverage criteria to force coverage-based reduction to retain test cases that expose specific faults. In this paper, we develop and empirically evaluate, using a number of different coverage criteria, a new metric based on the "average expected probability of finding a fault" in a reduced test suite. Our results indicate that the average probability of detecting each fault shows promise for identifying coverage criteria that work well for test suite reduction.},
	keywords = {coverage-based test suite reduction, fault detection probability analysis, Fault diagnosis, force coverage-based reduction, percentage fault detection reduction, percentage size reduction, program testing, software reliability, statistical analysis},
	doi = {10.1109/ICSM.2007.4362646},
	author = {McMaster,S. and Memon, Atif M.}
}
@conference {15489,
	title = {Studying the Characteristics of a "Good" GUI Test Suite},
	booktitle = {Software Reliability Engineering, 2006. ISSRE {\textquoteright}06. 17th International Symposium on},
	year = {2006},
	month = {2006/11//},
	pages = {159 - 168},
	abstract = {The widespread deployment of graphical-user interfaces (GUIs) has increased the overall complexity of testing. A GUI test designer needs to perform the daunting task of adequately testing the GUI, which typically has very large input interaction spaces, while considering tradeoffs between GUI test suite characteristics such as the number of test cases (each modeled as a sequence of events), their lengths, and the event composition of each test case. There are no published empirical studies on GUI testing that a GUI test designer may reference to make decisions about these characteristics. Consequently, in practice, very few GUI testers know how to design their test suites. This paper takes the first step towards assisting in GUI test design by presenting an empirical study that evaluates the effect of these characteristics on testing cost and fault detection effectiveness. The results show that two factors significantly effect the fault-detection effectiveness of a test suite: (1) the diversity of states in which an event executes and (2) the event coverage of the suite. Test designers need to improve the diversity of states in which each event executes by developing a large number of short test cases to detect the majority of "shallow" faults, which are artifacts of modern GUI design. Additional resources should be used to develop a small number of long test cases to detect a small number of "deep" faults},
	keywords = {Fault detection, Fault diagnosis, graphical user interface testing, Graphical user interfaces, program debugging, program testing},
	doi = {10.1109/ISSRE.2006.45},
	author = {Xie,Qing and Memon, Atif M.}
}
@article {15474,
	title = {Studying the fault-detection effectiveness of GUI test cases for rapidly evolving software},
	journal = {Software Engineering, IEEE Transactions on},
	volume = {31},
	year = {2005},
	month = {2005/10//},
	pages = {884 - 896},
	abstract = {Software is increasingly being developed/maintained by multiple, often geographically distributed developers working concurrently. Consequently, rapid-feedback-based quality assurance mechanisms such as daily builds and smoke regression tests, which help to detect and eliminate defects early during software development and maintenance, have become important. This paper addresses a major weakness of current smoke regression testing techniques, i.e., their inability to automatically (re)test graphical user interfaces (GUIs). Several contributions are made to the area of GUI smoke testing. First, the requirements for GUI smoke testing are identified and a GUI smoke test is formally defined as a specialized sequence of events. Second, a GUI smoke regression testing process called daily automated regression tester (DART) that automates GUI smoke testing is presented. Third, the interplay between several characteristics of GUI smoke test suites including their size, fault detection ability, and test oracles is empirically studied. The results show that: 1) the entire smoke testing process is feasible in terms of execution time, storage space, and manual effort, 2) smoke tests cannot cover certain parts of the application code, 3) having comprehensive test oracles may make up for not having long smoke test cases, and 4) using certain oracles can make up for not having large smoke test suites.},
	keywords = {daily automated regression tester, Fault diagnosis, fault-detection, formal specification, formal verification, Graphical user interfaces, GUI test cases, program testing, quality assurance mechanism, rapidly evolving software, smoke regression testing technique, software development, software fault tolerance, Software maintenance, software prototyping, Software quality, test oracles},
	isbn = {0098-5589},
	doi = {10.1109/TSE.2005.117},
	author = {Memon, Atif M. and Xie,Q.}
}
@conference {18678,
	title = {Loki: a state-driven fault injector for distributed systems},
	year = {2000},
	month = {2000///},
	pages = {237 - 242},
	abstract = {Distributed applications can fail in subtle ways that depend on the state of multiple parts of a system. This complicates the validation of such systems via fault injection, since it suggests that faults should be injected based on the global state of the system. In Loki, fault injection is performed based on a partial view of the global state of a distributed system, i.e. faults injected in one node of the system can depend on the state of other nodes. Once faults are injected, a post-runtime analysis, using off-line clock synchronization, is used to place events and injections on a single global timeline and to determine whether the intended faults were properly injected. Finally, experiments containing successful fault injections are used to estimate the specified measures. In addition to briefly reviewing the concepts behind Loki and its organization, we detail Loki{\textquoteright}s user interface. In particular, we describe the graphical user interfaces for specifying state machines and faults, for executing a campaign and for verifying whether the faults were properly injected},
	keywords = {campaign execution, computer testing, distributed applications failure, distributed processing, distributed system validation, Fault diagnosis, fault event placement, fault specification, global system state, global timeline, Graphical user interfaces, Loki, off-line clock synchronization, post-runtime analysis, specified measures estimation, state machine specification, state-driven fault injection, Synchronisation},
	doi = {10.1109/ICDSN.2000.857544},
	author = {Chandra,R. and Lefever,R.M. and Michel Cukier and Sanders,W. H.}
}
@article {16314,
	title = {Comparing detection methods for software requirements inspections: a replicated experiment},
	journal = {IEEE Transactions on Software Engineering},
	volume = {21},
	year = {1995},
	month = {1995/06//},
	pages = {563 - 575},
	abstract = {Software requirements specifications (SRS) are often validated manually. One such process is inspection, in which several reviewers independently analyze all or part of the specification and search for faults. These faults are then collected at a meeting of the reviewers and author(s). Usually, reviewers use Ad Hoc or Checklist methods to uncover faults. These methods force all reviewers to rely on nonsystematic techniques to search for a wide variety of faults. We hypothesize that a Scenario-based method, in which each reviewer uses different, systematic techniques to search for different, specific classes of faults, will have a significantly higher success rate. We evaluated this hypothesis using a 3{\texttimes}24 partial factorial, randomized experimental design. Forty eight graduate students in computer science participated in the experiment. They were assembled into sixteen, three-person teams. Each team inspected two SRS using some combination of Ad Hoc, Checklist or Scenario methods. For each inspection we performed four measurements: (1) individual fault detection rate, (2) team fault detection rate, (3) percentage of faults first identified at the collection meeting (meeting gain rate), and (4) percentage of faults first identified by an individual, but never reported at the collection meeting (meeting loss rate). The experimental results are that (1) the Scenario method had a higher fault detection rate than either Ad Hoc or Checklist methods, (2) Scenario reviewers were more effective at detecting the faults their scenarios are designed to uncover, and were no less effective at detecting other faults than both Ad Hoc or Checklist reviewers, (3) Checklist reviewers were no more effective than Ad Hoc reviewers, and (4) Collection meetings produced no net improvement in the fault detection rate-meeting gains were offset by meeting losses},
	keywords = {Assembly, Computer science, Design for experiments, detection methods, Fault detection, fault detection rate, Fault diagnosis, formal specification, formal verification, Gain measurement, individual fault detection rate, Inspection, Loss measurement, nonsystematic techniques, performance evaluation, Performance gain, replicated experiment, scenario-based method, Software development management, software requirements inspections, software requirements specifications, team fault detection rate},
	isbn = {0098-5589},
	doi = {10.1109/32.391380},
	author = {Porter, Adam and Votta,L. G. and Basili, Victor R.}
}
@conference {16320,
	title = {Software metric classification trees help guide the maintenance of large-scale systems},
	booktitle = {, Conference on Software Maintenance, 1989., Proceedings},
	year = {1989},
	month = {1989/10/16/19},
	pages = {116 - 123},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {The 80:20 rule states that approximately 20\% of a software system is responsible for 80\% of its errors. The authors propose an automated method for generating empirically-based models of error-prone software objects. These models are intended to help localize the troublesome 20\%. The method uses a recursive algorithm to automatically generate classification trees whose nodes are multivalued functions based on software metrics. The purpose of the classification trees is to identify components that are likely to be error prone or costly, so that developers can focus their resources accordingly. A feasibility study was conducted using 16 NASA projects. On average, the classification trees correctly identified 79.3\% of the software modules that had high development effort or faults},
	keywords = {automated method, automatic programming, classification, Classification tree analysis, classification trees, Computer errors, empirically-based models, error-prone software objects, Fault diagnosis, feasibility study, high development effort, Large-scale systems, multivalued functions, NASA, NASA projects, recursive algorithm, Software algorithms, software engineering, Software maintenance, Software measurement, software metrics, software modules, Software systems, trees (mathematics)},
	isbn = {0-8186-1965-1},
	doi = {10.1109/ICSM.1989.65202},
	author = {Selby,R. W and Porter, Adam}
}
@article {16323,
	title = {Learning from examples: generation and evaluation of decision trees for software resource analysis},
	journal = {IEEE Transactions on Software Engineering},
	volume = {14},
	year = {1988},
	month = {1988/12//},
	pages = {1743 - 1757},
	abstract = {A general solution method for the automatic generation of decision (or classification) trees is investigated. The approach is to provide insights through in-depth empirical characterization and evaluation of decision trees for one problem domain, specifically, that of software resource data analysis. The purpose of the decision trees is to identify classes of objects (software modules) that had high development effort, i.e. in the uppermost quartile relative to past data. Sixteen software systems ranging from 3000 to 112000 source lines have been selected for analysis from a NASA production environment. The collection and analysis of 74 attributes (or metrics), for over 4700 objects, capture a multitude of information about the objects: development effort, faults, changes, design style, and implementation style. A total of 9600 decision trees are automatically generated and evaluated. The analysis focuses on the characterization and evaluation of decision tree accuracy, complexity, and composition. The decision trees correctly identified 79.3\% of the software modules that had high development effort or faults, on the average across all 9600 trees. The decision trees generated from the best parameter combinations correctly identified 88.4\% of the modules on the average. Visualization of the results is emphasized, and sample decision trees are included},
	keywords = {Analysis of variance, Artificial intelligence, Classification tree analysis, Data analysis, decision theory, Decision trees, Fault diagnosis, Information analysis, machine learning, metrics, NASA, production environment, software engineering, software modules, software resource analysis, Software systems, Termination of employment, trees (mathematics)},
	isbn = {0098-5589},
	doi = {10.1109/32.9061},
	author = {Selby,R. W and Porter, Adam}
}
