@article {12476,
	title = {Online Empirical Evaluation of Tracking Algorithms},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {32},
	year = {2010},
	month = {2010/08//},
	pages = {1443 - 1458},
	abstract = {Evaluation of tracking algorithms in the absence of ground truth is a challenging problem. There exist a variety of approaches for this problem, ranging from formal model validation techniques to heuristics that look for mismatches between track properties and the observed data. However, few of these methods scale up to the task of visual tracking, where the models are usually nonlinear and complex and typically lie in a high-dimensional space. Further, scenarios that cause track failures and/or poor tracking performance are also quite diverse for the visual tracking problem. In this paper, we propose an online performance evaluation strategy for tracking systems based on particle filters using a time-reversed Markov chain. The key intuition of our proposed methodology relies on the time-reversible nature of physical motion exhibited by most objects, which in turn should be possessed by a good tracker. In the presence of tracking failures due to occlusion, low SNR, or modeling errors, this reversible nature of the tracker is violated. We use this property for detection of track failures. To evaluate the performance of the tracker at time instant t, we use the posterior of the tracking algorithm to initialize a time-reversed Markov chain. We compute the posterior density of track parameters at the starting time t = 0 by filtering back in time to the initial time instant. The distance between the posterior density of the time-reversed chain (at t = 0) and the prior density used to initialize the tracking algorithm forms the decision statistic for evaluation. It is observed that when the data are generated by the underlying models, the decision statistic takes a low value. We provide a thorough experimental analysis of the evaluation methodology. Specifically, we demonstrate the effectiveness of our approach for tackling common challenges such as occlusion, pose, and illumination changes and provide the Receiver Operating Characteristic (ROC) curves. Finally, we also s how the applicability of the core ideas of the paper to other tracking algorithms such as the Kanade-Lucas-Tomasi (KLT) feature tracker and the mean-shift tracker.},
	keywords = {Back, Biomedical imaging, Computer vision, Filtering, formal model validation techniques, formal verification, ground truth, Kanade Lucas Tomasi feature tracker, Karhunen-Loeve transforms, lighting, Markov processes, mean shift tracker, model validation., online empirical evaluation, particle filtering (numerical methods), Particle filters, Particle tracking, performance evaluation, receiver operating characteristic curves, Robustness, SNR, Statistics, Surveillance, time reversed Markov chain, tracking, tracking algorithms, visual tracking},
	isbn = {0162-8828},
	doi = {10.1109/TPAMI.2009.135},
	author = {Wu,Hao and Sankaranarayanan,A. C and Chellapa, Rama}
}
@article {15474,
	title = {Studying the fault-detection effectiveness of GUI test cases for rapidly evolving software},
	journal = {Software Engineering, IEEE Transactions on},
	volume = {31},
	year = {2005},
	month = {2005/10//},
	pages = {884 - 896},
	abstract = {Software is increasingly being developed/maintained by multiple, often geographically distributed developers working concurrently. Consequently, rapid-feedback-based quality assurance mechanisms such as daily builds and smoke regression tests, which help to detect and eliminate defects early during software development and maintenance, have become important. This paper addresses a major weakness of current smoke regression testing techniques, i.e., their inability to automatically (re)test graphical user interfaces (GUIs). Several contributions are made to the area of GUI smoke testing. First, the requirements for GUI smoke testing are identified and a GUI smoke test is formally defined as a specialized sequence of events. Second, a GUI smoke regression testing process called daily automated regression tester (DART) that automates GUI smoke testing is presented. Third, the interplay between several characteristics of GUI smoke test suites including their size, fault detection ability, and test oracles is empirically studied. The results show that: 1) the entire smoke testing process is feasible in terms of execution time, storage space, and manual effort, 2) smoke tests cannot cover certain parts of the application code, 3) having comprehensive test oracles may make up for not having long smoke test cases, and 4) using certain oracles can make up for not having large smoke test suites.},
	keywords = {daily automated regression tester, Fault diagnosis, fault-detection, formal specification, formal verification, Graphical user interfaces, GUI test cases, program testing, quality assurance mechanism, rapidly evolving software, smoke regression testing technique, software development, software fault tolerance, Software maintenance, software prototyping, Software quality, test oracles},
	isbn = {0098-5589},
	doi = {10.1109/TSE.2005.117},
	author = {Memon, Atif M. and Xie,Q.}
}
@article {15498,
	title = {Preserving distributed systems critical properties: a model-driven approach},
	journal = {Software, IEEE},
	volume = {21},
	year = {2004},
	month = {2004///},
	pages = {32 - 40},
	abstract = {The need for creating predictability in distributed systems is most often specified in terms of quality-of-service (QoS) requirements, which help define the acceptable levels of dependability with which capabilities such as processing capacity, data throughput, or service availability reach users. For longer-term properties such as scalability, maintainability, adaptability, and system security, we can similarly use persistent software attributes (PSAs) to specify how and to what degree such properties must remain intact as a network expands and evolves over time. The Skoll distributed continuous software quality assurance process helps to identify viable system and software configurations for meeting stringent QOS and PSA requirements by coordinating the use of distributed computing resources. The authors tested their process using the large, rapidly evolving ACE+TAO middleware suite.},
	keywords = {configuration management, formal verification, Middleware, middleware suite, model-driven approach, persistent software attributes, QoS requirements, Quality assurance, quality of service, quality-of-service, Skoll distributed computing resources, software configuration, Software maintenance, Software quality, software quality assurance process, system dependability},
	isbn = {0740-7459},
	doi = {10.1109/MS.2004.50},
	author = {Yilmaz,C. and Memon, Atif M. and Porter, Adam and Krishna,A. S and Schmidt,D. C and Gokhale,A. and Natarajan,B.}
}
@conference {18676,
	title = {Formal specification and verification of a group membership protocol for an intrusion-tolerant group communication system},
	year = {2002},
	month = {2002/12//},
	pages = {9 - 18},
	abstract = {We describe a group membership protocol that is part of an intrusion-tolerant group communication system, and present an effort to use formal tools to model and validate our protocol. We describe in detail the most difficult part of the validation exercise, which was the determination of the right level of abstraction of the protocol for formally specifying the protocol. The validation exercise not only formally showed that the protocol satisfies its correctness claims, but also provided information that will help us make the protocol more efficient without violating correctness.},
	keywords = {computer network reliability, distributed processing, distributed systems, fault tolerant computing, formal specification, formal verification, group membership protocol, intrusion-tolerant group communication system, PROMELA, Protocols},
	doi = {10.1109/PRDC.2002.1185613},
	author = {Ramasamy,H. V. and Michel Cukier and Sanders,W. H.}
}
@conference {18687,
	title = {Probabilistic verification of a synchronous round-based consensus protocol},
	year = {1997},
	month = {1997/10//},
	pages = {165 - 174},
	abstract = {Consensus protocols are used in a variety of reliable distributed systems, including both safety-critical and business-critical applications. The correctness of a consensus protocol is usually shown, by making assumptions about the environment in which it executes, and then proving properties about the protocol. But proofs about a protocol{\textquoteright}s behavior are only as good as the assumptions which were made to obtain them, and violation of these assumptions can lead to unpredicted and serious consequences. We present a new approach for the probabilistic verification of synchronous round based consensus protocols. In doing so, we make stochastic assumptions about the environment in which a protocol operates, and derive probabilities of proper and non proper behavior. We thus can account for the violation of assumptions made in traditional proof techniques. To obtain the desired probabilities, the approach enumerates possible states that can be reached during an execution of the protocol, and computes the probability of achieving the desired properties for a given fault and network environment. We illustrate the use of this approach via the evaluation of a simple consensus protocol operating under a realistic environment which includes performance, omission, and crash failures},
	keywords = {business-critical applications, consensus protocol correctness, crash failures, formal verification, network environment, probabilistic verification, probabilities, probability, program verification, proper behavior, protocol behavior, Protocols, realistic environment, reliable distributed systems, safety-critical applications, simple consensus protocol, software reliability, stochastic assumptions, synchronous round based consensus protocol, synchronous round based consensus protocols, traditional proof techniques},
	doi = {10.1109/RELDIS.1997.632812},
	author = {Duggal,H.S. and Michel Cukier and Sanders,W. H.}
}
@article {16314,
	title = {Comparing detection methods for software requirements inspections: a replicated experiment},
	journal = {IEEE Transactions on Software Engineering},
	volume = {21},
	year = {1995},
	month = {1995/06//},
	pages = {563 - 575},
	abstract = {Software requirements specifications (SRS) are often validated manually. One such process is inspection, in which several reviewers independently analyze all or part of the specification and search for faults. These faults are then collected at a meeting of the reviewers and author(s). Usually, reviewers use Ad Hoc or Checklist methods to uncover faults. These methods force all reviewers to rely on nonsystematic techniques to search for a wide variety of faults. We hypothesize that a Scenario-based method, in which each reviewer uses different, systematic techniques to search for different, specific classes of faults, will have a significantly higher success rate. We evaluated this hypothesis using a 3{\texttimes}24 partial factorial, randomized experimental design. Forty eight graduate students in computer science participated in the experiment. They were assembled into sixteen, three-person teams. Each team inspected two SRS using some combination of Ad Hoc, Checklist or Scenario methods. For each inspection we performed four measurements: (1) individual fault detection rate, (2) team fault detection rate, (3) percentage of faults first identified at the collection meeting (meeting gain rate), and (4) percentage of faults first identified by an individual, but never reported at the collection meeting (meeting loss rate). The experimental results are that (1) the Scenario method had a higher fault detection rate than either Ad Hoc or Checklist methods, (2) Scenario reviewers were more effective at detecting the faults their scenarios are designed to uncover, and were no less effective at detecting other faults than both Ad Hoc or Checklist reviewers, (3) Checklist reviewers were no more effective than Ad Hoc reviewers, and (4) Collection meetings produced no net improvement in the fault detection rate-meeting gains were offset by meeting losses},
	keywords = {Assembly, Computer science, Design for experiments, detection methods, Fault detection, fault detection rate, Fault diagnosis, formal specification, formal verification, Gain measurement, individual fault detection rate, Inspection, Loss measurement, nonsystematic techniques, performance evaluation, Performance gain, replicated experiment, scenario-based method, Software development management, software requirements inspections, software requirements specifications, team fault detection rate},
	isbn = {0098-5589},
	doi = {10.1109/32.391380},
	author = {Porter, Adam and Votta,L. G. and Basili, Victor R.}
}
@conference {12783,
	title = {Verifying an intelligent structural control system: a case study},
	booktitle = {Real-Time Systems Symposium, 1994., Proceedings.},
	year = {1994},
	month = {1994/12/07/9},
	pages = {271 - 275},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {Describes the formal verification of the timing properties of the design of an intelligent structural control system using the Concurrency Workbench, an automatic verification tool for finite-state processes. The high-level design of the system is first given in Modechart, a graphical specification language for real-time systems, and then translated into a temporal process algebra supported by the Workbench. The facilities provided by this tool are then used to analyze the system and ultimately show it to be correct},
	keywords = {automatic verification tool, case study, Concurrency Workbench, Distributed computing, Distributed control, distributed processing, finite automata, finite-state processes, formal verification, graphical specification language, high-level design, intelligent control, intelligent structural control system verification, Logic, Modechart, Process algebra, Real time systems, real-time systems, Specification languages, structural engineering computing, temporal logic, temporal process algebra, time-varying systems, Timing, timing properties, visual languages},
	isbn = {0-8186-6600-5},
	doi = {10.1109/REAL.1994.342708},
	author = {Elseaidy,W. M and Cleaveland, Rance and Baugh,J. W.}
}
