@booklet {19504,
	title = {Testing Run-time Evolving Systems},
	howpublished = {DROPS-IDN/2106},
	year = {Submitted},
	keywords = {004, Dynamically Evolving Systems, Software testing},
	doi = {InProceedings},
	url = {http://drops.dagstuhl.de/opus/volltexte/2009/2106/}
}
@conference {15485,
	title = {Introducing a test suite similarity metric for event sequence-based test cases},
	booktitle = {Software Maintenance, 2009. ICSM 2009. IEEE International Conference on},
	year = {2009},
	month = {2009/09//},
	pages = {243 - 252},
	abstract = {Most of today{\textquoteright}s event driven software (EDS) systems are tested using test cases that are carefully constructed as sequences of events; they test the execution of an event in the context of its preceding events. Because sizes of these test suites can be extremely large, researchers have developed techniques, such as reduction and minimization, to obtain test suites that are ldquosimilarrdquo to the original test suite, but smaller. Existing similarity metrics mostly use code coverage; they do not consider the contextual relationships between events. Consequently, reduction based on such metrics may eliminate desirable test cases. In this paper, we present a new parameterized metric, CONTeSSi(n) which uses the context of n preceding events in test cases to develop a new context-aware notion of test suite similarity for EDS. This metric is defined and evaluated by comparing four test suites for each of four open source applications. Our results show that CONT eSSi(n) is a better indicator of the similarity of EDS test suites than existing metrics.},
	keywords = {event driven software systems, event sequence-based test cases, open source systems, program testing, public domain software, software metrics, Software testing, test suite similarity metric},
	doi = {10.1109/ICSM.2009.5306305},
	author = {Brooks,P.A. and Memon, Atif M.}
}
@article {15504,
	title = {Using GUI Run-Time State as Feedback for Test Automation},
	year = {2009},
	month = {2009/08/18/},
	institution = {Computer Science Research Works, University of Maryland, College Park},
	abstract = {Graphical user interfaces (GUIs) are the sole mode of interactionbetween end-users and back-end code for almost all of today{\textquoteright}s software
applications.  Because of this strategic role of GUIs, their quality
has become important. During GUI testing, test cases---modeled as
sequences of user events---sample the vast input space of all possible
sequences with the goal of detecting faults; for effective testing, it
is important to sample this space carefully.  Existing techniques
either sample manually or employ manually constructed abstract
models---the abstraction and/or subsequent test-case generation
algorithms enable sampling.
This report presents an alternative approach to GUI testing -- it{\textquoteright}s
focus is on developing a fully automatic model-driven technique
to generate GUI test cases.  The technique is novel in that it uses
feedback from the execution of a {\textquoteleft}{\textquoteleft}seed test suite{\textquoteright}{\textquoteright} on a GUI.
The seed suite is generated automatically using an existing structural
event-interaction graph (EIG) model of the GUI.  During its
execution, the run-time effect of each GUI event on all other events
pinpoints new important event-semantic interaction (ESI)
relationships between them, which are used to automatically create an
ESI graph (ESIG) model and generate new test cases.  Together
with a reverse-engineering algorithm used to obtain the EIG, seed
suite, ESIG, and new test cases, the feedback-based technique yields a
fully automatic, end-to-end GUI testing process.
},
	keywords = {GUI testing, Software testing, test automation, test-case generation},
	url = {http://drum.lib.umd.edu//handle/1903/9416},
	author = {Xun Yuan and Memon, Atif M.}
}
@article {12814,
	title = {High-confidence medical device software and systems},
	journal = {Computer},
	volume = {39},
	year = {2006},
	month = {2006/04//},
	pages = {33 - 38},
	abstract = {Given the shortage of caregivers and the increase in an aging US population, the future of US healthcare quality does not look promising and definitely is unlikely to be cheaper. Advances in health information systems and healthcare technology offer a tremendous opportunity for improving the quality of care while reducing costs. The development and production of medical device software and systems is a crucial issue, both for the US economy and for ensuring safe advances in healthcare delivery. As devices become increasingly smaller in physical terms but larger in software terms, the design, testing, and eventual Food and Drug Administration (FDA) device approval is becoming much more expensive for medical device manufacturers both in terms of time and cost. Furthermore, the number of devices that have recently been recalled due to software and hardware problems is increasing at an alarming rate. As medical devices are becoming increasingly networked, ensuring even the same level of health safety seems a challenge.},
	keywords = {Aging, biomedical equipment, Clinical software engineering, Costs, FDA device, Food and Drug Administration device, health and safety, health care, health information system, health safety, healthcare delivery, Healthcare technology, Information systems, medical computing, medical device manufacturing, medical device software development, medical device systems development, medical information systems, Medical services, Medical software, Medical tests, networked medical devices, Production systems, Software design, Software safety, Software systems, Software testing, US healthcare quality},
	isbn = {0018-9162},
	doi = {10.1109/MC.2006.127},
	author = {Lee,I. and Pappas,G. J and Cleaveland, Rance and Hatcliff,J. and Krogh,B. H and Lee,P. and Rubin,H. and Sha,L.}
}
@conference {15468,
	title = {Skoll: distributed continuous quality assurance},
	booktitle = {Software Engineering, 2004. ICSE 2004. Proceedings. 26th International Conference on},
	year = {2004},
	month = {2004/05//},
	pages = {459 - 468},
	abstract = {Quality assurance (QA) tasks, such as testing, profiling, and performance evaluation, have historically been done in-house on developer-generated workloads and regression suites. Since this approach is inadequate for many systems, tools and processes are being developed to improve software quality by increasing user participation in the QA process. A limitation of these approaches is that they focus on isolated mechanisms, not on the coordination and control policies and tools needed to make the global QA process efficient, effective, and scalable. To address these issues, we have initiated the Skoll project, which is developing and validating novel software QA processes and tools that leverage the extensive computing resources of worldwide user communities in a distributed, continuous manner to significantly and rapidly improve software quality. This paper provides several contributions to the study of distributed continuous QA. First, it illustrates the structure and functionality of a generic around-the-world, around-the-clock QA process and describes several sophisticated tools that support this process. Second, it describes several QA scenarios built using these tools and process. Finally, it presents a feasibility study applying these scenarios to a 1MLOC+ software package called ACE+TAO. While much work remains to be done, the study suggests that the Skoll process and tools effectively manage and control distributed, continuous QA processes. Using Skoll we rapidly identified problems that had taken the ACE+TAO developers substantially longer to find and several of which had previously not been found. Moreover, automatic analysis of QA task results often provided developers information that quickly led them to the root cause of the problems.},
	keywords = {1MLOC+ software package, ACE+TAO, around-the-clock QA process, around-the-world QA process, distributed continuous QA, distributed continuous quality assurance, distributed programming, program verification, Quality assurance, Skoll, software performance evaluation, software profiling, Software quality, Software testing},
	doi = {10.1109/ICSE.2004.1317468},
	author = {Memon, Atif M. and Porter, Adam and Yilmaz,C. and Nagarajan,A. and Schmidt,D. and Natarajan,B.}
}
@conference {15428,
	title = {What test oracle should I use for effective GUI testing?},
	booktitle = {Automated Software Engineering, 2003. Proceedings. 18th IEEE International Conference on},
	year = {2003},
	month = {2003/10//},
	pages = {164 - 173},
	abstract = {Test designers widely believe that the overall effectiveness and cost of software testing depends largely on the type and number of test cases executed on the software. In this paper we show that the test oracle used during testing also contributes significantly to test effectiveness and cost. A test oracle is a mechanism that determines whether software executed correctly for a test case. We define a test oracle to contain two essential parts: oracle information that represents expected output; and an oracle procedure that compares the oracle information with the actual output. By varying the level of detail of oracle information and changing the oracle procedure, a test designer can create different types of test oracles. We design 11 types of test oracles and empirically compare them on four software systems. We seed faults in software to create 100 faulty versions, execute 600 test cases on each version, for all 11 types of oracles. In all, we report results of 660,000 test runs on software. We show (1) the time and space requirements of the oracles, (2) that faults are detected early in the testing process when using detailed oracle information and complex oracle procedures, although at a higher cost per test case, and (3) that employing expensive oracles results in detecting a large number of faults using relatively smaller number of test cases.},
	keywords = {empirical studies, graphical user interface, Graphical user interfaces, GUI testing, oracle information, oracle procedure, oracle space requirement, oracle time requirements, program testing, software engineering, Software testing, test cost, test effectiveness, test oracle},
	doi = {10.1109/ASE.2003.1240304},
	author = {Memon, Atif M. and Banerjee,I. and Nagarajan,A.}
}
@article {17283,
	title = {Low-effort, high-payoff user interface reengineering},
	journal = {IEEE Software},
	volume = {14},
	year = {1997},
	month = {1997/08//Jul},
	pages = {66 - 72},
	abstract = {Although increasingly sophisticated design methodologies for developing new user interfaces exist, low-effort, high-payoff user interface reengineering represents a new direction-and opportunity. Yet reengineering a working system is complex and risky because of the potential disruption to users and managers, their justifiable fear of change, and the lack of guarantees that such changes will be for the better. Our largely positive experiences with the projects described here lead us to believe that user interface reengineering is a viable and important process. Low effort, high-payoff improvement recommendations can probably be made for most existing systems. Nevertheless, a narrowly focused user interface reengineering plan may be inappropriate when the major problems lie outside the scope of the user interface, such as inadequate functionalities, frequent crashes, and network problems. Attempts at improving less severe problems while ignoring deeper ones may be perceived as insensitive by the users. In such cases it is important to consider either making similar short-term improvements for other parts of the systems or postponing short-term user interface reengineering in favour of a more complete system reengineering. Similarly, the need for interface stability might outweigh the benefits of the short-term improvements if a complete reengineering is planned for the near future. But most likely these proposed diagnostic strategies and opportunities for improvement are only a prelude to the much larger task of business reengineering, which implies extensive user interface reengineering},
	keywords = {Business process re-engineering, complete system reengineering, Design methodology, Error analysis, Hardware, inadequate functionalities, interface stability, iterative methods, low-effort high-payoff user interface reengineering, short-term improvements, short-term user interface reengineering, software engineering, Software testing, System analysis and design, System testing, systems re-engineering, User centered design, user centred design, User interfaces},
	isbn = {0740-7459},
	doi = {10.1109/52.595958},
	author = {Plaisant, Catherine and Rose,A. and Shneiderman, Ben and Vanniamparampil,A. J}
}
@conference {14787,
	title = {Modeling, Evaluation, and Testing of Paradyn Instrumentation System},
	booktitle = {Proceedings of the 1996 ACM/IEEE Conference on Supercomputing, 1996},
	year = {1996},
	month = {1996///},
	pages = {18 - 18},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {This paper presents a case study of modeling, evaluating, and testing the data collection services (called an instrumentation system) of the Paradyn parallel performance measurement tool using well-known performance evaluation and experiment design techniques. The overall objective of the study is to use modeling- and simulation-based evaluation to provide feedback to the tool developers to help them choose system configurations and task scheduling policies that can significantly reduce the data collection overheads. We develop and parameterize a resource occupancy model for the Paradyn instrumentation system (IS) for an IBM SP-2 platform. This model is parameterized with a measurement-based workload characterization and subsequently used to answer several "what if" questions regarding configuration options and two policies to schedule instrumentation system tasks: collect-and-forward (CF) and batch-and-forward (BF) policies. Simulation results indicate that the BF policy can significantly reduce the overheads. Based on this feedback, the BF policy was implemented in the Paradyn IS as an option to manage the data collection. Measurement-based testing results obtained from this enhanced version of the Paradyn IS are reported in this paper and indicate more than 60\% reduction in the direct IS overheads when the BF policy is used.},
	keywords = {Distributed control, Feedback, High performance computing, Instruments, Monitoring, Real time systems, Software measurement, Software systems, Software testing, System testing},
	isbn = {0-89791-854-1},
	doi = {10.1109/SUPERC.1996.183524},
	author = {Waheed, A. and Rover, D. T and Hollingsworth, Jeffrey K}
}
@article {16751,
	title = {SEES{\textemdash}A Software testing Environment Support System},
	journal = {IEEE Transactions on Software Engineering},
	volume = {SE-11},
	year = {1985},
	month = {1985/04//},
	pages = {355 - 366},
	abstract = {SEES is a database system to support program testing. The program database is automatically created during the compilation of the program by a compiler built using the YACC compiler-compiler.},
	keywords = {Computer architecture, Database systems, Error correction, Program processors, Programming profession, Relational databases, Software testing, software tools, Workstations, Writing},
	isbn = {0098-5589},
	doi = {10.1109/TSE.1985.232225},
	author = {Roussopoulos, Nick and Yeh,R. T}
}
