@article {15470,
	title = {Developing a Single Model and Test Prioritization Strategies for Event-Driven Software},
	journal = {Software Engineering, IEEE Transactions on},
	volume = {37},
	year = {2011},
	month = {2011///},
	pages = {48 - 64},
	abstract = {Event-Driven Software (EDS) can change state based on incoming events; common examples are GUI and Web applications. These EDSs pose a challenge to testing because there are a large number of possible event sequences that users can invoke through a user interface. While valuable contributions have been made for testing these two subclasses of EDS, such efforts have been disjoint. This work provides the first single model that is generic enough to study GUI and Web applications together. In this paper, we use the model to define generic prioritization criteria that are applicable to both GUI and Web applications. Our ultimate goal is to evolve the model and use it to develop a unified theory of how all EDS should be tested. An empirical study reveals that the GUI and Web-based applications, when recast using the new model, show similar behavior. For example, a criterion that gives priority to all pairs of event interactions did well for GUI and Web applications; another criterion that gives priority to the smallest number of parameter value settings did poorly for both. These results reinforce our belief that these two subclasses of applications should be modeled and studied together.},
	keywords = {EDS, event-driven software, graphical user interface, Graphical user interfaces, GUI testing, Internet, program testing, service-oriented architecture, test prioritization strategy, Web application testing},
	isbn = {0098-5589},
	doi = {10.1109/TSE.2010.12},
	author = {Bryce,R.C. and Sampath,S. and Memon, Atif M.}
}
@conference {18341,
	title = {GUI testing using computer vision},
	booktitle = {Proceedings of the 28th international conference on Human factors in computing systems},
	series = {CHI {\textquoteright}10},
	year = {2010},
	month = {2010///},
	pages = {1535 - 1544},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Testing a GUI{\textquoteright}s visual behavior typically requires human testers to interact with the GUI and to observe whether the expected results of interaction are presented. This paper presents a new approach to GUI testing using computer vision for testers to automate their tasks. Testers can write a visual test script that uses images to specify which GUI components to interact with and what visual feedback to be observed. Testers can also generate visual test scripts by demonstration. By recording both input events and screen images, it is possible to extract the images of components interacted with and the visual feedback seen by the demonstrator, and generate a visual test script automatically. We show that a variety of GUI behavior can be tested using this approach. Also, we show how this approach can facilitate good testing practices such as unit testing, regression testing, and test-driven development.},
	keywords = {gui automation, GUI testing, test by demonstration},
	isbn = {978-1-60558-929-9},
	doi = {10.1145/1753326.1753555},
	url = {http://doi.acm.org/10.1145/1753326.1753555},
	author = {Chang,Tsung-Hsiang and Tom Yeh and Miller,Robert C.}
}
@article {15440,
	title = {Iterative execution-feedback model-directed GUI testing},
	journal = {Information and Software Technology},
	volume = {52},
	year = {2010},
	month = {2010/05//},
	pages = {559 - 575},
	abstract = {Current fully automatic model-based test-case generation techniques for GUIs employ a static model. Therefore they are unable to leverage certain state-based relationships between GUI events (e.g., one enables the other, one alters the other{\textquoteright}s execution) that are revealed at run-time and non-trivial to infer statically. We present ALT {\textendash} a new technique to generate GUI test cases in batches. Because of its {\textquotedblleft}alternating{\textquotedblright} nature, ALT enhances the next batch by using GUI run-time information from the current batch. An empirical study on four fielded GUI-based applications demonstrated that ALT was able to detect new 4- and 5-way GUI interaction faults; in contrast, previous techniques, due to their requirement of too many test cases, were unable to even test 4- and 5-way GUI interactions.},
	keywords = {event-driven software, event-flow graphs, GUI testing, model-based testing, test-case generation},
	isbn = {0950-5849},
	doi = {10.1016/j.infsof.2009.11.009},
	url = {http://www.sciencedirect.com/science/article/pii/S0950584909002092},
	author = {Xun Yuan and Memon, Atif M.}
}
@conference {15496,
	title = {Using methods \& measures from network analysis for gui testing},
	booktitle = {Software Testing, Verification, and Validation Workshops (ICSTW), 2010 Third International Conference on},
	year = {2010},
	month = {2010/04//},
	pages = {240 - 246},
	abstract = {Graphical user interfaces (GUIs) for today{\textquoteright}s applications are extremely large. Moreover, they provide many degrees of freedom to the end-user, thus allowing the user to perform a very large number of event sequences on the GUI. The large sizes and degrees of freedom create severe problems for GUI quality assurance, including GUI testing. In this paper, we leverage methods and measures from network analysis to analyze and study GUIs, with the goal of aiding GUI testing activities. We apply these methods and measures on the event-flow graph model of GUIs. Results of a case study show that "network centrality measures" are able to identify the most important events in the GUI as well as the most important sequences of events. These events and sequences are good candidates for test prioritization. In addition, the "betweenness clustering" method is able to partition the GUI into regions that can be tested separately.},
	keywords = {betweenness clustering method, event sequences, event-flow graph model, Graphical user interfaces, GUI quality assurance, GUI testing, network analysis, network centrality measures, program testing, Software quality},
	doi = {10.1109/ICSTW.2010.61},
	author = {Elsaka,E. and Moustafa,W. E and Nguyen,Bao and Memon, Atif M.}
}
@article {15504,
	title = {Using GUI Run-Time State as Feedback for Test Automation},
	year = {2009},
	month = {2009/08/18/},
	institution = {Computer Science Research Works, University of Maryland, College Park},
	abstract = {Graphical user interfaces (GUIs) are the sole mode of interactionbetween end-users and back-end code for almost all of today{\textquoteright}s software
applications.  Because of this strategic role of GUIs, their quality
has become important. During GUI testing, test cases---modeled as
sequences of user events---sample the vast input space of all possible
sequences with the goal of detecting faults; for effective testing, it
is important to sample this space carefully.  Existing techniques
either sample manually or employ manually constructed abstract
models---the abstraction and/or subsequent test-case generation
algorithms enable sampling.
This report presents an alternative approach to GUI testing -- it{\textquoteright}s
focus is on developing a fully automatic model-driven technique
to generate GUI test cases.  The technique is novel in that it uses
feedback from the execution of a {\textquoteleft}{\textquoteleft}seed test suite{\textquoteright}{\textquoteright} on a GUI.
The seed suite is generated automatically using an existing structural
event-interaction graph (EIG) model of the GUI.  During its
execution, the run-time effect of each GUI event on all other events
pinpoints new important event-semantic interaction (ESI)
relationships between them, which are used to automatically create an
ESI graph (ESIG) model and generate new test cases.  Together
with a reverse-engineering algorithm used to obtain the EIG, seed
suite, ESIG, and new test cases, the feedback-based technique yields a
fully automatic, end-to-end GUI testing process.
},
	keywords = {GUI testing, Software testing, test automation, test-case generation},
	url = {http://drum.lib.umd.edu//handle/1903/9416},
	author = {Xun Yuan and Memon, Atif M.}
}
@conference {15497,
	title = {Relationships between Test Suites, Faults, and Fault Detection in GUI Testing},
	booktitle = {Software Testing, Verification, and Validation, 2008 1st International Conference on},
	year = {2008},
	month = {2008/04//},
	pages = {12 - 21},
	abstract = {Software-testing researchers have long sought recipes for test suites that detect faults well. In the literature, empirical studies of testing techniques abound, yet the ideal technique for detecting the desired kinds of faults in a given situation often remains unclear. This work shows how understanding the context in which testing occurs, in terms of factors likely to influence fault detection, can make evaluations of testing techniques more readily applicable to new situations. We present a methodology for discovering which factors do statistically affect fault detection, and we perform an experiment with a set of test-suite- and fault-related factors in the GUI testing of two fielded, open-source applications. Statement coverage and GUI-event coverage are found to be statistically related to the likelihood of detecting certain kinds of faults.},
	keywords = {Fault detection, fault-related factors, Graphical user interfaces, GUI testing, program testing, software-testing, test suites, test-suite-related factors},
	doi = {10.1109/ICST.2008.26},
	author = {Strecker,J. and Memon, Atif M.}
}
@conference {15460,
	title = {Automated gui testing guided by usage profiles},
	booktitle = {Proceedings of the twenty-second IEEE/ACM international conference on Automated software engineering},
	series = {ASE {\textquoteright}07},
	year = {2007},
	month = {2007///},
	pages = {333 - 342},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Most software developed in recent years has a graphical userinterface (GUI). The only way for the end-user to interact with the software application is through the GUI. Hence, acceptance and system testing of the software requires GUI testing. This paper presents a new technique for testing of GUI applications. Information on the actual usage of the application, in the form of "usage profiles," is used to ensure that a new version of the application will function correctly. Usage profiles, sequences of events that end-users execute on a GUI, are used to develop a probabilistic usage model of the application. An algorithm uses the model to generate test cases that represent events the user is most likely to execute. Reverse engineering methods are used to extract the underlying structure of the application. An empirical study on four open source GUI applications reveals that test suites generated from the probabilistic model are 0.2-22\% of the size of test suites produced directly from usage profiles. Furthermore, the test suites generated from the model detect more faults per test case than those detected directly from the usage profiles, and detect faults not detected by the original profiles},
	keywords = {event-driven software, GUI testing, usage profiles},
	isbn = {978-1-59593-882-4},
	doi = {10.1145/1321631.1321681},
	url = {http://doi.acm.org/10.1145/1321631.1321681},
	author = {Brooks,Penelope A. and Memon, Atif M.}
}
@conference {15462,
	title = {Covering array sampling of input event sequences for automated gui testing},
	booktitle = {Proceedings of the twenty-second IEEE/ACM international conference on Automated software engineering},
	series = {ASE {\textquoteright}07},
	year = {2007},
	month = {2007///},
	pages = {405 - 408},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {This paper describes a new automated technique to generate test cases for GUIs by using covering arrays (CAs). The key motivation is to generate long GUI event sequences that are systematically sampled at a particular coverage strength. CAs, to date, have not been effectively used in sampling event driven systems such as GUIs which maintain state. We leverage a "stateless" abstraction of GUIs that allows us to use CAs. Once the CAs have been generated, we reuse the abstractions to reinsert ordering relationships between GUI events, thereby creating executable test cases. A feasibility study on a well-studied GUI-based application shows that the new technique is able to detect a large number of previously undetected faults},
	keywords = {covering arrays, event driven software, GUI testing},
	isbn = {978-1-59593-882-4},
	doi = {10.1145/1321631.1321695},
	url = {http://doi.acm.org/10.1145/1321631.1321695},
	author = {Xun Yuan and Cohen,Myra and Memon, Atif M.}
}
@article {15411,
	title = {Designing and comparing automated test oracles for GUI-based software applications},
	journal = {ACM Trans. Softw. Eng. Methodol.},
	volume = {16},
	year = {2007},
	month = {2007/02//},
	abstract = {Test designers widely believe that the overall effectiveness and cost of software testing depends largely on the type and number of test cases executed on the software. This article shows that the test oracle, a mechanism that determines whether a software is executed correctly for a test case, also significantly impacts the fault detection effectiveness and cost of a test case. Graphical user interfaces (GUIs), which have become ubiquitous for interacting with today{\textquoteright}s software, have created new challenges for test oracle development. Test designers manually {\textquotedblleft}assert{\textquotedblright} the expected values of specific properties of certain GUI widgets in each test case; during test execution, these assertions are used as test oracles to determine whether the GUI executed correctly. Since a test case for a GUI is a sequence of events, a test designer must decide: (1) what to assert; and (2) how frequently to check an assertion, for example, after each event in the test case or after the entire test case has completed execution. Variations of these two factors significantly impact the fault-detection ability and cost of a GUI test case. A technique to declaratively specify different types of automated GUI test oracles is described. Six instances of test oracles are developed and compared in an experiment on four software systems. The results show that test oracles do affect the fault detection ability of test cases in different and interesting ways: (1) Test cases significantly lose their fault detection ability when using {\textquotedblleft}weak{\textquotedblright} test oracles; (2) in many cases, invoking a {\textquotedblleft}thorough{\textquotedblright} oracle at the end of test case execution yields the best cost-benefit ratio; (3) certain test cases detect faults only if the oracle is invoked during a small {\textquotedblleft}window of opportunity{\textquotedblright} during test execution; and (4) using thorough and frequently-executing test oracles can compensate for not having long test cases.},
	keywords = {Graphical user interfaces, GUI state, GUI testing, test oracles, User interfaces, widgets},
	isbn = {1049-331X},
	doi = {10.1145/1189748.1189752},
	url = {http://doi.acm.org/10.1145/1189748.1189752},
	author = {Xie,Qing and Memon, Atif M.}
}
@article {15445,
	title = {Automating regression testing for evolving GUI software},
	journal = {Journal of Software Maintenance and Evolution: Research and Practice},
	volume = {17},
	year = {2005},
	month = {2005///},
	pages = {27 - 64},
	abstract = {With the widespread deployment of broadband connections worldwide, software development and maintenance are increasingly being performed by multiple engineers, often working around-the-clock to maximize code churn rates. To ensure rapid quality assurance of such software, techniques such as {\textquoteleft}nightly/daily building and smoke testing{\textquoteright} have become widespread since they often reveal bugs early in the software development process. During these builds, a development version of the software is checked out from the source code repository tree, compiled, linked, and (re)tested with the goal of (re)validating its basic functionality. Although successful for conventional software, smoke tests are difficult to develop and automatically re-run for software that has a graphical user interface (GUI). In this paper, we describe a framework called DART (Daily Automated Regression Tester) that addresses the needs of frequent and automated re-testing of GUI software. The key to our success is automation: DART automates everything from structural GUI analysis, smoke-test-case generation, test-oracle creation, to code instrumentation, test execution, coverage evaluation, regeneration of test cases, and their re-execution. Together with the operating system{\textquoteright}s task scheduler, DART can execute frequently with little input from the developer/tester to re-test the GUI software. We provide results of experiments showing the time taken and memory required for GUI analysis, test case and test oracle generation, and test execution. We empirically compare the relative costs of employing different levels of detail in the GUI test oracle. We also show the events and statements covered by the smoke test cases. Copyright {\textcopyright} 2005 John Wiley \& Sons, Ltd.},
	keywords = {daily/nightly builds, event-flow graphs, Graphical user interfaces, GUI regression testing, GUI testing, smoke testing, Software quality},
	isbn = {1532-0618},
	doi = {10.1002/smr.305},
	url = {http://onlinelibrary.wiley.com/doi/10.1002/smr.305/abstract},
	author = {Memon, Atif M. and Nagarajan,Adithya and Xie,Qing}
}
@article {15410,
	title = {Regression testing of GUIs},
	journal = {ACM SIGSOFT Software Engineering Notes},
	volume = {28},
	year = {2003},
	month = {2003/09//},
	pages = {118 - 127},
	abstract = {Although graphical user interfaces (GUIs) constitute a large part of the software being developed today and are typically created using rapid prototyping, there are no effective regression testing techniques for GUIs. The needs of GUI regression testing differ from those of traditional software. When the structure of a GUI is modified, test cases from the original GUI are either reusable or unusable on the modified GUI. Since GUI test case generation is expensive, our goal is to make the unusable test cases usable. The idea of reusing these unusable (a.k.a. obsolete) test cases has not been explored before. In this paper, we show that for GUIs, the unusability of a large number of test cases is a serious problem. We present a novel GUI regression testing technique that first automatically determines the usable and unusable test cases from a test suite after a GUI modification. It then determines which of the unusable test cases can be repaired so they can execute on the modified GUI. The last step is to repair the test cases. Our technique is integrated into a GUI testing framework that, given a test case, automatically executes it on the GUI. We implemented our regression testing technique and demonstrate for two case studies that our approach is effective in that many of the test cases can be repaired, and is practical in terms of its time performance.},
	keywords = {call-tree, classification of events, GUI call-graph, GUI control-flow graph, GUI testing, regression testing, repairing test cases},
	isbn = {0163-5948},
	doi = {10.1145/949952.940088},
	url = {http://doi.acm.org/10.1145/949952.940088},
	author = {Memon, Atif M. and Soffa,Mary Lou}
}
@conference {15428,
	title = {What test oracle should I use for effective GUI testing?},
	booktitle = {Automated Software Engineering, 2003. Proceedings. 18th IEEE International Conference on},
	year = {2003},
	month = {2003/10//},
	pages = {164 - 173},
	abstract = {Test designers widely believe that the overall effectiveness and cost of software testing depends largely on the type and number of test cases executed on the software. In this paper we show that the test oracle used during testing also contributes significantly to test effectiveness and cost. A test oracle is a mechanism that determines whether software executed correctly for a test case. We define a test oracle to contain two essential parts: oracle information that represents expected output; and an oracle procedure that compares the oracle information with the actual output. By varying the level of detail of oracle information and changing the oracle procedure, a test designer can create different types of test oracles. We design 11 types of test oracles and empirically compare them on four software systems. We seed faults in software to create 100 faulty versions, execute 600 test cases on each version, for all 11 types of oracles. In all, we report results of 660,000 test runs on software. We show (1) the time and space requirements of the oracles, (2) that faults are detected early in the testing process when using detailed oracle information and complex oracle procedures, although at a higher cost per test case, and (3) that employing expensive oracles results in detecting a large number of faults using relatively smaller number of test cases.},
	keywords = {empirical studies, graphical user interface, Graphical user interfaces, GUI testing, oracle information, oracle procedure, oracle space requirement, oracle time requirements, program testing, software engineering, Software testing, test cost, test effectiveness, test oracle},
	doi = {10.1109/ASE.2003.1240304},
	author = {Memon, Atif M. and Banerjee,I. and Nagarajan,A.}
}
@conference {15455,
	title = {Coverage criteria for GUI testing},
	booktitle = {Proceedings of the 8th European software engineering conference held jointly with 9th ACM SIGSOFT international symposium on Foundations of software engineering},
	series = {ESEC/FSE-9},
	year = {2001},
	month = {2001///},
	pages = {256 - 267},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {A widespread recognition of the usefulness of graphical user interfaces (GUIs) has established their importance as critical components of today{\textquoteright}s software. GUIs have characteristics different from traditional software, and conventional testing techniques do not directly apply to GUIs. This paper{\textquoteright}s focus is on coverage critieria for GUIs, important rules that provide an objective measure of test quality. We present new coverage criteria to help determine whether a GUI has been adequately tested. These coverage criteria use events and event sequences to specify a measure of test adequacy. Since the total number of permutations of event sequences in any non-trivial GUI is extremely large, the GUI{\textquoteright}s hierarchical structure is exploited to identify the important event sequences to be tested. A GUI is decomposed into GUI components, each of which is used as a basic unit of testing. A representation of a GUI component, called an event-flow graph, identifies the interaction of events within a component and intra-component criteria are used to evaluate the adequacy of tests on these events. The hierarchical relationship among components is represented by an integration tree, and inter-component coverage criteria are used to evaluate the adequacy of test sequences that cross components. Algorithms are given to construct event-flow graphs and an integration tree for a given GUI, and to evaluate the coverage of a given test suite with respect to the new coverage criteria. A case study illustrates the usefulness of the coverage report to guide further testing and an important correlation between event-based coverage of a GUI and statement coverage of its software{\textquoteright}s underlying code.},
	keywords = {component testing, event-based coverage, event-flow graph, GUI test coverage, GUI testing, integration tree},
	isbn = {1-58113-390-1},
	doi = {10.1145/503209.503244},
	url = {http://doi.acm.org/10.1145/503209.503244},
	author = {Memon, Atif M. and Soffa,Mary Lou and Pollack,Martha E.}
}
@article {15464,
	title = {Automated test oracles for GUIs},
	journal = {SIGSOFT Softw. Eng. Notes},
	volume = {25},
	year = {2000},
	month = {2000/11//},
	pages = {30 - 39},
	abstract = {Graphical User Interfaces (GUIs) are critical components of today{\textquoteright}s software. Because GUIs have different characteristics than traditional software, conventional testing techniques do not apply to GUI software. In previous work, we presented an approach to generate GUI test cases, which take the form of sequences of actions. In this paper we develop a test oracle technique to determine if a GUI behaves as expected for a given test case. Our oracle uses a formal model of a GUI, expressed as sets of objects, object properties, and actions. Given the formal model and a test case, our oracle automatically derives the expected state for every action in the test case. We represent the actual state of an executing GUI in terms of objects and their properties derived from the GUI{\textquoteright}s execution. Using the actual state acquired from an execution monitor, our oracle automatically compares the expected and actual states after each action to verify the correctness of the GUI for the test case. We implemented the oracle as a component in our GUI testing system, called Planning Assisted Tester for grapHical user interface Systems (PATHS), which is based on AI planning. We experimentally evaluated the practicality and effectiveness of our oracle technique and report on the results of experiments to test and verify the behavior of our version of the Microsoft WordPad{\textquoteright}s GUI.},
	keywords = {automated oracles, GUI test oracles, GUI testing},
	isbn = {0163-5948},
	doi = {10.1145/357474.355050},
	url = {http://doi.acm.org/10.1145/357474.355050},
	author = {Memon, Atif M. and Pollack,Martha E. and Soffa,Mary Lou}
}
