@article {15411,
	title = {Designing and comparing automated test oracles for GUI-based software applications},
	journal = {ACM Trans. Softw. Eng. Methodol.},
	volume = {16},
	year = {2007},
	month = {2007/02//},
	abstract = {Test designers widely believe that the overall effectiveness and cost of software testing depends largely on the type and number of test cases executed on the software. This article shows that the test oracle, a mechanism that determines whether a software is executed correctly for a test case, also significantly impacts the fault detection effectiveness and cost of a test case. Graphical user interfaces (GUIs), which have become ubiquitous for interacting with today{\textquoteright}s software, have created new challenges for test oracle development. Test designers manually {\textquotedblleft}assert{\textquotedblright} the expected values of specific properties of certain GUI widgets in each test case; during test execution, these assertions are used as test oracles to determine whether the GUI executed correctly. Since a test case for a GUI is a sequence of events, a test designer must decide: (1) what to assert; and (2) how frequently to check an assertion, for example, after each event in the test case or after the entire test case has completed execution. Variations of these two factors significantly impact the fault-detection ability and cost of a GUI test case. A technique to declaratively specify different types of automated GUI test oracles is described. Six instances of test oracles are developed and compared in an experiment on four software systems. The results show that test oracles do affect the fault detection ability of test cases in different and interesting ways: (1) Test cases significantly lose their fault detection ability when using {\textquotedblleft}weak{\textquotedblright} test oracles; (2) in many cases, invoking a {\textquotedblleft}thorough{\textquotedblright} oracle at the end of test case execution yields the best cost-benefit ratio; (3) certain test cases detect faults only if the oracle is invoked during a small {\textquotedblleft}window of opportunity{\textquotedblright} during test execution; and (4) using thorough and frequently-executing test oracles can compensate for not having long test cases.},
	keywords = {Graphical user interfaces, GUI state, GUI testing, test oracles, User interfaces, widgets},
	isbn = {1049-331X},
	doi = {10.1145/1189748.1189752},
	url = {http://doi.acm.org/10.1145/1189748.1189752},
	author = {Xie,Qing and Memon, Atif M.}
}
@article {15492,
	title = {An event-flow model of GUI-based applications for testing},
	journal = {Software Testing, Verification and Reliability},
	volume = {17},
	year = {2007},
	month = {2007///},
	pages = {137 - 157},
	abstract = {Graphical user interfaces (GUIs) are by far the most popular means used to interact with today{\textquoteright}s software. The functional correctness of a GUI is required to ensure the safety, robustness and usability of an entire software system. GUI testing techniques used in practice are resource intensive; model-based automated techniques are rarely employed. A key reason for the reluctance in the adoption of model-based solutions proposed by researchers is their limited applicability; moreover, the models are expensive to create. Over the past few years, the present author has been developing different models for various aspects of GUI testing. This paper consolidates all of the models into one scalable event-flow model and outlines algorithms to semi-automatically reverse-engineer the model from an implementation. Earlier work on model-based test-case generation, test-oracle creation, coverage evaluation, and regression testing is recast in terms of this model by defining event-space exploration strategies (ESESs) and creating an end-to-end GUI testing process. Three such ESESs are described: for checking the event-flow model, test-case generation, and test-oracle creation. Two demonstrational scenarios show the application of the model and the three ESESs for experimentation and application in GUI testing. Copyright {\textcopyright} 2007 John Wiley \& Sons, Ltd.},
	keywords = {event-driven software, event-flow graph, event-flow model, Graphical user interfaces, integration tree, Model checking, test oracles, test-case generation},
	isbn = {1099-1689},
	doi = {10.1002/stvr.364},
	url = {http://onlinelibrary.wiley.com/doi/10.1002/stvr.364/abstract},
	author = {Memon, Atif M.}
}
@article {15474,
	title = {Studying the fault-detection effectiveness of GUI test cases for rapidly evolving software},
	journal = {Software Engineering, IEEE Transactions on},
	volume = {31},
	year = {2005},
	month = {2005/10//},
	pages = {884 - 896},
	abstract = {Software is increasingly being developed/maintained by multiple, often geographically distributed developers working concurrently. Consequently, rapid-feedback-based quality assurance mechanisms such as daily builds and smoke regression tests, which help to detect and eliminate defects early during software development and maintenance, have become important. This paper addresses a major weakness of current smoke regression testing techniques, i.e., their inability to automatically (re)test graphical user interfaces (GUIs). Several contributions are made to the area of GUI smoke testing. First, the requirements for GUI smoke testing are identified and a GUI smoke test is formally defined as a specialized sequence of events. Second, a GUI smoke regression testing process called daily automated regression tester (DART) that automates GUI smoke testing is presented. Third, the interplay between several characteristics of GUI smoke test suites including their size, fault detection ability, and test oracles is empirically studied. The results show that: 1) the entire smoke testing process is feasible in terms of execution time, storage space, and manual effort, 2) smoke tests cannot cover certain parts of the application code, 3) having comprehensive test oracles may make up for not having long smoke test cases, and 4) using certain oracles can make up for not having large smoke test suites.},
	keywords = {daily automated regression tester, Fault diagnosis, fault-detection, formal specification, formal verification, Graphical user interfaces, GUI test cases, program testing, quality assurance mechanism, rapidly evolving software, smoke regression testing technique, software development, software fault tolerance, Software maintenance, software prototyping, Software quality, test oracles},
	isbn = {0098-5589},
	doi = {10.1109/TSE.2005.117},
	author = {Memon, Atif M. and Xie,Q.}
}
@conference {15480,
	title = {Empirical evaluation of the fault-detection effectiveness of smoke regression test cases for GUI-based software},
	booktitle = {Software Maintenance, 2004. Proceedings. 20th IEEE International Conference on},
	year = {2004},
	month = {2004/09//},
	pages = {8 - 17},
	abstract = {Daily builds and smoke regression tests have become popular quality assurance mechanisms to detect defects early during software development and maintenance. In previous work, we addressed a major weakness of current smoke regression testing techniques, i.e., their lack of ability to automatically (re)test graphical user interface (GUI) event interactions - we presented a GUI smoke regression testing process called daily automated regression tester (DART). We have deployed DART and have found several interesting characteristics of GUI smoke tests that we empirically demonstrate in this paper. We also combine smoke tests with different types of test oracles and present guidelines for practitioners to help them generate and execute the most effective combinations of test-case length and test oracle complexity. Our experimental subjects consist of four GUI-based applications. We generate 5000-8000 smoke tests (enough to be run in one night) for each application. Our results show that: (1) short GUI smoke tests with certain test oracles are effective at detecting a large number of faults; (2) there are classes of faults that our smoke test cannot detect; (3) short smoke tests execute a large percentage of code; and (4) the entire smoke testing process is feasible to do in terms of execution time and storage space.},
	keywords = {daily automated regression tester, daily builds, fault-detection effectiveness, graphical user interface, Graphical user interfaces, GUI-based software, program testing, Quality assurance, Regression analysis, smoke regression test cases, software development, software fault tolerance, Software maintenance, Software quality, software quality assurance, test oracle complexity, test oracles, test-case length},
	doi = {10.1109/ICSM.2004.1357785},
	author = {Memon, Atif M. and Xie,Qing}
}
