@article {12476,
	title = {Online Empirical Evaluation of Tracking Algorithms},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {32},
	year = {2010},
	month = {2010/08//},
	pages = {1443 - 1458},
	abstract = {Evaluation of tracking algorithms in the absence of ground truth is a challenging problem. There exist a variety of approaches for this problem, ranging from formal model validation techniques to heuristics that look for mismatches between track properties and the observed data. However, few of these methods scale up to the task of visual tracking, where the models are usually nonlinear and complex and typically lie in a high-dimensional space. Further, scenarios that cause track failures and/or poor tracking performance are also quite diverse for the visual tracking problem. In this paper, we propose an online performance evaluation strategy for tracking systems based on particle filters using a time-reversed Markov chain. The key intuition of our proposed methodology relies on the time-reversible nature of physical motion exhibited by most objects, which in turn should be possessed by a good tracker. In the presence of tracking failures due to occlusion, low SNR, or modeling errors, this reversible nature of the tracker is violated. We use this property for detection of track failures. To evaluate the performance of the tracker at time instant t, we use the posterior of the tracking algorithm to initialize a time-reversed Markov chain. We compute the posterior density of track parameters at the starting time t = 0 by filtering back in time to the initial time instant. The distance between the posterior density of the time-reversed chain (at t = 0) and the prior density used to initialize the tracking algorithm forms the decision statistic for evaluation. It is observed that when the data are generated by the underlying models, the decision statistic takes a low value. We provide a thorough experimental analysis of the evaluation methodology. Specifically, we demonstrate the effectiveness of our approach for tackling common challenges such as occlusion, pose, and illumination changes and provide the Receiver Operating Characteristic (ROC) curves. Finally, we also s how the applicability of the core ideas of the paper to other tracking algorithms such as the Kanade-Lucas-Tomasi (KLT) feature tracker and the mean-shift tracker.},
	keywords = {Back, Biomedical imaging, Computer vision, Filtering, formal model validation techniques, formal verification, ground truth, Kanade Lucas Tomasi feature tracker, Karhunen-Loeve transforms, lighting, Markov processes, mean shift tracker, model validation., online empirical evaluation, particle filtering (numerical methods), Particle filters, Particle tracking, performance evaluation, receiver operating characteristic curves, Robustness, SNR, Statistics, Surveillance, time reversed Markov chain, tracking, tracking algorithms, visual tracking},
	isbn = {0162-8828},
	doi = {10.1109/TPAMI.2009.135},
	author = {Wu,Hao and Sankaranarayanan,A. C and Chellapa, Rama}
}
@conference {13849,
	title = {Reranking for Sentence Boundary Detection in Conversational Speech},
	booktitle = {2006 IEEE International Conference on Acoustics, Speech and Signal Processing, 2006. ICASSP 2006 Proceedings},
	volume = {1},
	year = {2006},
	month = {2006/05/14/19},
	pages = {I-I - I-I},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We present a reranking approach to sentence-like unit (SU) boundary detection, one of the EARS metadata extraction tasks. Techniques for generating relatively small n-best lists with high oracle accuracy are presented. For each candidate, features are derived from a range of information sources, including the output of a number of parsers. Our approach yields significant improvements over the best performing system from the NIST RT-04F community evaluation},
	keywords = {Automatic speech recognition, conversational speech, data mining, Ear, EARS metadata extraction tasks, Feature extraction, hidden Markov models, meta data, Model driven engineering, NIST, NIST RT-04F community evaluation, oracle accuracy, performance evaluation, reranking, sentence-like unit boundary detection, Speech processing, Speech recognition, Telephony},
	isbn = {1-4244-0469-X},
	doi = {10.1109/ICASSP.2006.1660078},
	author = {Roark,B. and Liu,Yang and Harper,M. and Stewart,R. and Lease,M. and Snover,M. and Shafran,I. and Dorr, Bonnie J and Hale,J. and Krasnyanskaya,A. and Yung,L.}
}
@conference {15686,
	title = {Routing in intermittent network topologies},
	booktitle = {Proceedings of the 9th ACM international symposium on Modeling analysis and simulation of wireless and mobile systems},
	series = {MSWiM {\textquoteright}06},
	year = {2006},
	month = {2006///},
	pages = {385 - 389},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	keywords = {delay tolerant network (DTN), performance evaluation, routing algorithms},
	isbn = {1-59593-477-4},
	doi = {10.1145/1164717.1164782},
	url = {http://doi.acm.org/10.1145/1164717.1164782},
	author = {Mundur, Padma and Lee,Sookyoung and Seligman,Matthew}
}
@conference {17875,
	title = {High Performance Communication between Parallel Programs},
	booktitle = {Parallel and Distributed Processing Symposium, 2005. Proceedings. 19th IEEE International},
	year = {2005},
	month = {2005/04//},
	pages = {177b- 177b - 177b- 177b},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We present algorithms for high performance communication between message-passing parallel programs, and evaluate the algorithms as implemented in InterComm. InterComm is a framework to couple parallel programs in the presence of complex data distributions within a coupled application. Multiple parallel libraries and languages may be used in the different programs of a single coupled application. The ability to couple such programs is required in many emerging application areas, such as complex simulations that model physical phenomena at multiple scales and resolutions, and image data analysis applications. We describe the new algorithms we have developed for computing inter-program communication patterns. We present experimental results showing the performance of various algorithmic tradeoffs, and also compare performance against an earlier system.},
	keywords = {Adaptive arrays, Analytical models, Chaotic communication, Computational modeling, Computer science, Data analysis, data distribution, Educational institutions, high performance communication, image data analysis, image resolution, inter-program communication patterns, InterComm, Libraries, Message passing, parallel languages, parallel libraries, parallel programming, parallel programs, performance evaluation, Wind},
	isbn = {0-7695-2312-9},
	doi = {10.1109/IPDPS.2005.243},
	author = {Jae-Yong Lee and Sussman, Alan}
}
@conference {14811,
	title = {Automated cluster-based Web service performance tuning},
	booktitle = {13th IEEE International Symposium on High performance Distributed Computing, 2004. Proceedings},
	year = {2004},
	month = {2004/06/04/6},
	pages = {36 - 44},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {Active harmony provides a way to automate performance tuning. We apply the Active Harmony system to improve the performance of a cluster-based web service system. The performance improvement cannot easily be achieved by tuning individual components for such a system. The experimental results show that there is no single configuration for the system that performs well for all kinds of workloads. By tuning the parameters, Active Harmony helps the system adapt to different workloads and improve the performance up to 16\%. For scalability, we demonstrate how to reduce the time when tuning a large system with many tunable parameters. Finally an algorithm is proposed to automatically adjust the structure of cluster-based web systems, and the system throughput is improved up to 70\% using this technique.},
	keywords = {Active Harmony system, automated performance tuning, business, cluster-based Web service system, Clustering algorithms, Computer science, Educational institutions, electronic commerce, Internet, Middleware, performance evaluation, scalability, Throughput, Transaction databases, Web server, Web services, workstation clusters},
	isbn = {0-7695-2175-4},
	doi = {10.1109/HPDC.2004.1323484},
	author = {Chung,I. -H and Hollingsworth, Jeffrey K}
}
@article {18648,
	title = {A global-state-triggered fault injector for distributed system evaluation},
	journal = {Parallel and Distributed Systems, IEEE Transactions on},
	volume = {15},
	year = {2004},
	month = {2004/07//},
	pages = {593 - 605},
	abstract = {Validation of the dependability of distributed systems via fault injection is gaining importance because distributed systems are being increasingly used in environments with high dependability requirements. The fact that distributed systems can fail in subtle ways that depend on the state of multiple parts of the system suggests that a global-state-based fault injection mechanism should be used to validate them. However, global-state-based fault injection is challenging since it is very difficult in practice to maintain the global state of a distributed system at runtime with minimal intrusion into the system execution. We present Loki, a global-state-based fault injector, which has been designed with the goals of low intrusion, high precision, and high flexibility. Loki achieves these goals by utilizing the ideas of partial view of global state, optimistic synchronization, and offline analysis. In Loki, faults are injected based on a partial, view of the global state of the system, and a post-runtime analysis is performed to place events and injections into a single global timeline and to discard experiments with incorrect fault injections. Finally, the experiments with correct fault injections are used to estimate user-specified performance and dependability measures. A flexible measure language has been designed that facilitates the specification of a wide range of measures.},
	keywords = {distributed processing, distributed system evaluation, fault tolerant computing, global-state-based fault injection mechanism, Loki, offline clock synchronization, performance evaluation, post-runtime analysis, Synchronisation, system recovery, user-specified performance},
	isbn = {1045-9219},
	doi = {10.1109/TPDS.2004.14},
	author = {Chandra,Ramesh and Lefever,R.M. and Joshi,K.R. and Michel Cukier and Sanders,W. H.}
}
@article {14768,
	title = {Resource policing to support fine-grain cycle stealing in networks of workstations},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {15},
	year = {2004},
	month = {2004/10//},
	pages = {878 - 892},
	abstract = {We present the design, implementation, and performance evaluation of a suite of resource policing mechanisms that allow guest processes to efficiently and unobtrusively exploit otherwise idle workstation resources. Unlike traditional policies that harvest cycles only from unused machines, we employ fine-grained cycle stealing to exploit resources even from machines that have active users. We developed a suite of kernel extensions that enable these policies to operate without significantly impacting host processes: 1) a new starvation-level CPU priority for guest jobs, 2) a new page replacement policy that imposes hard bounds on physical memory usage by guest processes, and 3) a new I/O scheduling mechanism called rate windows that throttle guest processes{\textquoteright} usage of I/O and network bandwidth. We evaluate both the individual impacts of each mechanism, and their utility for our fine-grain cycle stealing.},
	keywords = {65, Application software, Bandwidth, cluster computing, Computer networks, Computer Society, Concurrent computing, cycle stealing, cycle stealing., grid computing, I/O scheduling, Intelligent networks, Kernel, network bandwidth, networks of workstations, page replacement policy, parallel computing, performance evaluation, Processor scheduling, resource allocation, resource scheduling, starvation-level CPU priority, workstation clusters, workstation resources, Workstations},
	isbn = {1045-9219},
	doi = {10.1109/TPDS.2004.58},
	author = {Ryu, K. D and Hollingsworth, Jeffrey K}
}
@conference {18656,
	title = {An experimental evaluation of correlated network partitions in the Coda distributed file system},
	year = {2003},
	month = {2003/10//},
	pages = {273 - 282},
	abstract = {Experimental evaluation is an important way to assess distributed systems, and fault injection is the dominant technique in this area for the evaluation of a system{\textquoteright}s dependability. For distributed systems, network failure is an important fault model. Physical network failures often have far-reaching effects, giving rise to multiple correlated failures as seen by higher-level protocols. This paper presents an experimental evaluation, using the Loki fault injector, which provides insight into the impact that correlated network partitions have on the Coda distributed file system. In this evaluation, Loki created a network partition between two Coda file servers, during which updates were made at each server to the same replicated data volume. Upon repair of the partition, a client requested directory resolution to converge the diverging replicas. At various stages of the resolution, Loki invoked a second correlated network partition, thus allowing us to evaluate its impact on the system{\textquoteright}s correctness, performance, and availability.},
	keywords = {client-server systems, Coda distributed file system, correlated network partitions, distributed file systems, experimental evaluation, fault model, Loki fault injector, multiple correlated failures, network failure, Network topology, performance evaluation, replicated data, replicated databases, software fault tolerance, system performance evaluation},
	doi = {10.1109/RELDIS.2003.1238077},
	author = {Lefever,R.M. and Michel Cukier and Sanders,W. H.}
}
@conference {18461,
	title = {Efficient evaluation of reverberant sound fields},
	booktitle = {Applications of Signal Processing to Audio and Acoustics, 2001 IEEE Workshop on the},
	year = {2001},
	month = {2001///},
	pages = {203 - 206},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {An image method due to Allen and Berkley (1979) is often used to simulate the effect of reverberation in rooms. This method is relatively expensive computationally. We present a fast method for conducting such simulations using multipole expansions. For M real and image sources and N evaluation points, while the image method requires O(MN) operations, our method achieves the calculations in O(M + N) operations, resulting in a substantial speedup. Applications of our technique are also expected in simulation of virtual audio},
	keywords = {AB algorithm, acoustic signal processing, architectural acoustics, audio signal processing, Computational modeling, Computer interfaces, Computer simulation, Educational institutions, image method, image sources, Impedance, Laboratories, microphone arrays, multipole expansions, Nails, performance evaluation, reverberant sound fields, reverberation, room reverberation, simulation, Simulations, speedup, virtual audio},
	isbn = {0-7803-7126-7},
	doi = {10.1109/ASPAA.2001.969578},
	author = {Duraiswami, Ramani and Gumerov, Nail A. and Zotkin,Dmitry N and Davis, Larry S.}
}
@article {14805,
	title = {A tool to help tune where computation is performed},
	journal = {IEEE Transactions on Software Engineering},
	volume = {27},
	year = {2001},
	month = {2001/07//},
	pages = {618 - 629},
	abstract = {We introduce a new performance metric, called load balancing factor (LBF), to assist programmers when evaluating different tuning alternatives. The LBF metric differs from traditional performance metrics since it is intended to measure the performance implications of a specific tuning alternative rather than quantifying where time is spent in the current version of the program. A second unique aspect of the metric is that it provides guidance about moving work within a distributed or parallel program rather than reducing it. A variation of the LBF metric can also be used to predict the performance impact of changing the underlying network. The LBF metric is computed incrementally and online during the execution of the program to be tuned. We also present a case study that shows that our metric can accurately predict the actual performance gains for a test suite of six programs},
	keywords = {Computational modeling, Current measurement, Distributed computing, distributed program, distributed programming, load balancing factor, Load management, parallel program, parallel programming, Performance analysis, performance evaluation, Performance gain, performance metric, Programming profession, software metrics, software performance evaluation, Testing, Time measurement, tuning},
	isbn = {0098-5589},
	doi = {10.1109/32.935854},
	author = {Eom, Hyeonsang and Hollingsworth, Jeffrey K}
}
@conference {15691,
	title = {Integrated admission control in hierarchical video-on-demand systems},
	booktitle = {IEEE International Conference on Multimedia Computing and Systems, 1999},
	volume = {1},
	year = {1999},
	month = {1999/07//},
	pages = {220-225 vol.1 - 220-225 vol.1},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We develop a unified model of a hierarchical video-on-demand (VoD) system by integrating the storage and the network subsystems. Rather than restricting the analysis to an isolated subsystem the performance of the VoD system is analyzed as an end-to-end system. On a system-wide basis, request handling and admission control policies are designed to minimize global performance metrics. Through our simulation, we compare different request handling policies and show that a hierarchical VoD architecture with request handling that allows retrials at more than one resource will minimize overall blocking},
	keywords = {Admission control, Bandwidth, blocking, Computer science, Design methodology, end-to-end system, hierarchical video-on-demand systems, integrated admission control, Intelligent networks, Load management, Motion pictures, Network servers, network subsystem, performance, Performance analysis, performance evaluation, quality of service, request handling, resource allocation, Resource management, simulation, storage subsystem, video on demand, video servers},
	isbn = {0-7695-0253-9},
	doi = {10.1109/MMCS.1999.779196},
	author = {Mundur, Padma and Simon,R. and Sood,A.}
}
@conference {14723,
	title = {PLANet: an active internetwork},
	booktitle = {IEEE INFOCOM {\textquoteright}99. Eighteenth Annual Joint Conference of the IEEE Computer and Communications Societies. Proceedings},
	volume = {3},
	year = {1999},
	month = {1999/03/21/25},
	pages = {1124-1133 vol.3 - 1124-1133 vol.3},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We present PLANet: an active network architecture and implementation. In addition to a standard suite of Internet-like services, PLANet has two key programmability features: (1) all packets contain programs; and (2) router functionality may be extended dynamically. Packet programs are written in our special purpose programming language PLAN, the Packet Language for Active Networks, while dynamic router extensions are written in OCaml, a dialect of ML. Currently, PLANet routers run as byte-code-interpreted Linux user-space applications, and support Ethernet and IP as link layers. PLANet achieves respectable performance on standard networking operations: on 300 MHz Pentium-II{\textquoteright}s attached to 100 Mbps Ethernet, PLANet can route 48 Mbps and switch over 5000 packets per second. We demonstrate the utility of PLANet{\textquoteright}s activeness by showing experimentally how it can nontrivially improve application and aggregate network performance in congested conditions},
	keywords = {100 Mbit/s, 300 MHz, 48 Mbit/s, active internetwork, active network architecture, active network implementation, byte-code-interpreted applications, Computer architecture, Computer languages, Computer networks, congested conditions, dynamic programming, dynamic router extensions, Ethernet, Ethernet networks, INFORMATION SCIENCE, Internet, Internet-like services, internetworking, IP, IP networks, link layers, Linux user-space applications, Local area networks, ML dialect, Network performance, networking operations, OCaml, Packet Language for Active Networks, packet programs, packet switching, Pentium-II, performance, performance evaluation, PLAN, PLANet, Planets, programmability features, programming languages, router functionality, special purpose programming language, Switches, telecommunication network routing, Transport protocols, Web and internet services},
	isbn = {0-7803-5417-6},
	doi = {10.1109/INFCOM.1999.751668},
	author = {Hicks, Michael W. and Moore,J. T and Alexander,D. S and Gunter,C. A and Nettles,S. M}
}
@conference {17535,
	title = {Predicting the CPU availability of time-shared Unix systems on the computational grid},
	booktitle = {The Eighth International Symposium on High Performance Distributed Computing, 1999. Proceedings},
	year = {1999},
	month = {1999///},
	pages = {105 - 112},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {Focuses on the problem of making short- and medium-term forecasts of CPU availability on time-shared Unix systems. We evaluate the accuracy with which availability can be measured using the Unix load average, the Unix utility {\textquotedblleft}vmstat{\textquotedblright} and the Network Weather Service (NWS) CPU sensor that uses both. We also examine the autocorrelation between successive CPU measurements to determine their degree of self-similarity. While our observations show a long-range autocorrelation dependence, we demonstrate how this dependence manifests itself in the short- and medium-term predictability of the CPU resources in our study},
	keywords = {accuracy, Application software, Autocorrelation, Availability, Central Processing Unit, computational grid, correlation methods, CPU availability prediction, CPU resources predictability, CPU sensor, Dynamic scheduling, grid computing, Load forecasting, long-range autocorrelation dependence, medium-term forecasts, network operating systems, Network Weather Service, NWS, performance evaluation, self-similarity degree, short-term forecasts, successive CPU measurements, Time measurement, Time sharing computer systems, time-shared Unix systems, time-sharing systems, Unix, Unix load average, vmstat utility, Weather forecasting},
	isbn = {0-7803-5681-0},
	doi = {10.1109/HPDC.1999.805288},
	author = {Wolski,R. and Spring, Neil and Hayes,J.}
}
@conference {14762,
	title = {Benchmarking a network of PCs running parallel applications},
	booktitle = {Performance, Computing and Communications, 1998. IPCCC {\textquoteright}98., IEEE International},
	year = {1998},
	month = {1998/02/16/18},
	pages = {1 - 7},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {Presents a benchmarking study that compares the performance of a network of four PCs connected by a 100 Mbit/s fast Ethernet running three different system software configurations: TCP/IP on Windows NT, TCP/IP on Linux and a lightweight message-passing protocol (U-Net active messages) on Linux. For each configuration, we report results for communication micro-benchmarks and the NAS (Numerical Aerodynamics Simulation) parallel benchmarks. For the NAS benchmarks, the overall running time using Linux TCP/IP was 12-500\% less than the Windows NT TCP/IP configuration. Likewise, the Linux U-Net based message-passing protocol outperformed the Linux TCP/IP version by 5-200\%+. We also show that, by using Linux U-Net, we are able to achieve 125 μs latency between two processes using PVM. Finally, we report that the default mathematical libraries supplied with NT (for both gcc and Visual C++) are substantially slower than the one supplied with Linux},
	keywords = {100 Mbit/s, 125 mus, Aerodynamics, Application software, communication micro-benchmarks, default mathematical libraries, Delay, Ethernet, Ethernet networks, gcc, latency, lightweight message-passing protocol, Linux, Local area networks, mathematics computing, Message passing, microcomputer applications, Microsoft Windows NT, NAS parallel benchmarks, network operating systems, Numerical simulation, parallel applications, PARALLEL PROCESSING, PC network benchmarking, performance comparison, performance evaluation, Personal communication networks, Protocols, PVM, running time, software libraries, System software, system software configurations, TCP/IP, TCPIP, Transport protocols, U-Net active messages, Visual C++},
	isbn = {0-7803-4468-5},
	doi = {10.1109/PCCC.1998.659876},
	author = {Hollingsworth, Jeffrey K and Guven, E. and Akinlar, C.}
}
@conference {14809,
	title = {Performance measurement using low perturbation and high precision hardware assists},
	booktitle = {, The 19th IEEE Real-Time Systems Symposium, 1998. Proceedings},
	year = {1998},
	month = {1998/12/02/4},
	pages = {379 - 388},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We present the design and implementation of MultiKron PCI, a hardware performance monitor that can be plugged into any computer with a free PCI bus slot. The monitor provides a series of high-resolution timers, and the ability to monitor the utilization of the PCI bus. We also demonstrate how the monitor can be integrated with online performance monitoring tools such as the Paradyn parallel performance measurement tools to improve the overhead of key timer operations by a factor of 25. In addition, we present a series of case studies using the MultiKron hardware performance monitor to measure and tune high-performance parallel completing applications. By using the monitor, we were able to find and correct a performance bug in a popular implementation of the MPI message passing library that caused some communication primitives to run at one half their potential speed},
	keywords = {Clocks, Computerized monitoring, Counting circuits, Debugging, Hardware, hardware performance monitor, high precision hardware assists, low perturbation, measurement, MPI message passing library, MultiKron hardware performance monitor, MultiKron PCI, NIST, online performance monitoring tools, Paradyn parallel performance measurement tools, PCI bus slot, performance bug, performance evaluation, performance measurement, program debugging, program testing, real-time systems, Runtime, Timing},
	isbn = {0-8186-9212-X},
	doi = {10.1109/REAL.1998.739771},
	author = {Mink, A. and Salamon, W. and Hollingsworth, Jeffrey K and Arunachalam, R.}
}
@article {16314,
	title = {Comparing detection methods for software requirements inspections: a replicated experiment},
	journal = {IEEE Transactions on Software Engineering},
	volume = {21},
	year = {1995},
	month = {1995/06//},
	pages = {563 - 575},
	abstract = {Software requirements specifications (SRS) are often validated manually. One such process is inspection, in which several reviewers independently analyze all or part of the specification and search for faults. These faults are then collected at a meeting of the reviewers and author(s). Usually, reviewers use Ad Hoc or Checklist methods to uncover faults. These methods force all reviewers to rely on nonsystematic techniques to search for a wide variety of faults. We hypothesize that a Scenario-based method, in which each reviewer uses different, systematic techniques to search for different, specific classes of faults, will have a significantly higher success rate. We evaluated this hypothesis using a 3{\texttimes}24 partial factorial, randomized experimental design. Forty eight graduate students in computer science participated in the experiment. They were assembled into sixteen, three-person teams. Each team inspected two SRS using some combination of Ad Hoc, Checklist or Scenario methods. For each inspection we performed four measurements: (1) individual fault detection rate, (2) team fault detection rate, (3) percentage of faults first identified at the collection meeting (meeting gain rate), and (4) percentage of faults first identified by an individual, but never reported at the collection meeting (meeting loss rate). The experimental results are that (1) the Scenario method had a higher fault detection rate than either Ad Hoc or Checklist methods, (2) Scenario reviewers were more effective at detecting the faults their scenarios are designed to uncover, and were no less effective at detecting other faults than both Ad Hoc or Checklist reviewers, (3) Checklist reviewers were no more effective than Ad Hoc reviewers, and (4) Collection meetings produced no net improvement in the fault detection rate-meeting gains were offset by meeting losses},
	keywords = {Assembly, Computer science, Design for experiments, detection methods, Fault detection, fault detection rate, Fault diagnosis, formal specification, formal verification, Gain measurement, individual fault detection rate, Inspection, Loss measurement, nonsystematic techniques, performance evaluation, Performance gain, replicated experiment, scenario-based method, Software development management, software requirements inspections, software requirements specifications, team fault detection rate},
	isbn = {0098-5589},
	doi = {10.1109/32.391380},
	author = {Porter, Adam and Votta,L. G. and Basili, Victor R.}
}
@article {17839,
	title = {An integrated runtime and compile-time approach for parallelizing structured and block structured applications},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {6},
	year = {1995},
	month = {1995/07//},
	pages = {747 - 754},
	abstract = {In compiling applications for distributed memory machines, runtime analysis is required when data to be communicated cannot be determined at compile-time. One such class of applications requiring runtime analysis is block structured codes. These codes employ multiple structured meshes, which may be nested (for multigrid codes) and/or irregularly coupled (called multiblock or irregularly coupled regular mesh problems). In this paper, we present runtime and compile-time analysis for compiling such applications on distributed memory parallel machines in an efficient and machine-independent fashion. We have designed and implemented a runtime library which supports the runtime analysis required. The library is currently implemented on several different systems. We have also developed compiler analysis for determining data access patterns at compile time and inserting calls to the appropriate runtime routines. Our methods can be used by compilers for HPF-like parallel programming languages in compiling codes in which data distribution, loop bounds and/or strides are unknown at compile-time. To demonstrate the efficacy of our approach, we have implemented our compiler analysis in the Fortran 90D/HPF compiler developed at Syracuse University. We have experimented with a multi-bloc Navier-Stokes solver template and a multigrid code. Our experimental results show that our primitives have low runtime communication overheads and the compiler parallelized codes perform within 20\% of the codes parallelized by manually inserting calls to the runtime library},
	keywords = {Bandwidth, block structured applications, block structured codes, compile-time approach, compiling applications, data access patterns, Data analysis, Delay, distributed memory machines, distributed memory systems, FORTRAN, Fortran 90D/HPF compiler, High performance computing, HPF-like parallel programming languages, integrated runtime approach, irregularly coupled regular mesh problems, multigrid code, Navier-Stokes solver template, Parallel machines, parallel programming, Pattern analysis, performance evaluation, program compilers, Program processors, Runtime library, Uninterruptible power systems},
	isbn = {1045-9219},
	doi = {10.1109/71.395403},
	author = {Agrawal,G. and Sussman, Alan and Saltz, J.}
}
@conference {13837,
	title = {On automatic filtering of multilingual texts},
	booktitle = {, 1994 IEEE International Conference on Systems, Man, and Cybernetics, 1994. {\textquoteright}Humans, Information and Technology{\textquoteright}},
	volume = {2},
	year = {1994},
	month = {1994/10/02/5},
	pages = {1645-1650 vol.2 - 1645-1650 vol.2},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {An emerging requirement to sift through the increasing flood of text information has led to the rapid development of information filtering technology in the past five years. This study introduces novel approaches for filtering texts regardless of their source language. We begin with a brief description of related developments in text filtering and multilingual information retrieval. We then present three alternative approaches to selecting texts from a multilingual information stream which represent a logical evolution from existing techniques in related disciplines. Finally, a practical automated performance evaluation technique is proposed},
	keywords = {automated performance evaluation, Business communication, Computer science, Discussion forums, Displays, Educational institutions, Floods, Government, Information filtering, Information filters, Information retrieval, multilingual information retrieval, multilingual text filtering, natural languages, performance evaluation},
	isbn = {0-7803-2129-4},
	doi = {10.1109/ICSMC.1994.400083},
	author = {Oard, Douglas and DeClaris,N. and Dorr, Bonnie J and Faloutsos,C.}
}
@article {16795,
	title = {Performance comparison of three modern DBMS architectures},
	journal = {IEEE Transactions on Software Engineering},
	volume = {19},
	year = {1993},
	month = {1993/02//},
	pages = {120 - 138},
	abstract = {The introduction of powerful workstations connected through local area networks (LANs) inspired new database management system (DBMS) architectures that offer high performance characteristics. The authors examine three such software architecture configurations: client-server (CS), the RAD-UNIFY type of DBMS (RU), and enhanced client-server (ECS). Their specific functional components and design rationales are discussed. Three simulation models are used to provide a performance comparison under different job workloads. Simulation results show that the RU almost always performs slightly better than the CS, especially under light workloads, and that ECS offers significant performance improvement over both CS and RU. Under reasonable update rates, the ECS over CS (or RU) performance ratio is almost proportional to the number of participating clients (for less than 32 clients). The authors also examine the impact of certain key parameters on the performance of the three architectures and show that ECS is more scalable that the other two},
	keywords = {client-server, Computational modeling, Computer architecture, database management systems, DBMS architectures, design rationales, functional components, Indexes, Local area networks, Military computing, Packaging, Performance analysis, performance evaluation, RAD-UNIFY type, simulation models, simulation results, Software architecture, software architecture configurations, software engineering, Throughput, Workstations},
	isbn = {0098-5589},
	doi = {10.1109/32.214830},
	author = {Delis,A. and Roussopoulos, Nick}
}
@article {14791,
	title = {IPS-2: the second generation of a parallel program measurement system},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {1},
	year = {1990},
	month = {1990/04//},
	pages = {206 - 217},
	abstract = {IPS, a performance measurement system for parallel and distributed programs, is currently running on its second implementation. IPS{\textquoteright}s model of parallel programs uses knowledge about the semantics of a program{\textquoteright}s structure to provide two important features. First, IPS provides a large amount of performance data about the execution of a parallel program, and this information is organized so that access to it is easy and intuitive. Secondly, IPS provides performance analysis techniques that help to guide the programmer automatically to the location of program bottlenecks. The first implementation of IPS was a testbed for the basic design concepts, providing experience with a hierarchical program and measurement model, interactive program analysis, and automatic guidance techniques. It was built on the Charlotte distributed operating system. The second implementation, IPS-2, extends the basic system with new instrumentation techniques, an interactive and graphical user interface, and new automatic guidance analysis techniques. This implementation runs on 4.3BSD UNIX systems, on the VAX, DECstation, Sun 4, and Sequent Symmetry multiprocessor},
	keywords = {4.3BSD UNIX systems, automatic guidance techniques, Automatic testing, Charlotte distributed operating system, CPA, DECstation, design concepts, distributed programs, graphical user interface, Graphical user interfaces, Instruments, interactive program analysis, IPS-2, measurement, message systems, network operating systems, Operating systems, parallel program measurement system, parallel programming, parallel programs, Performance analysis, performance analysis techniques, performance evaluation, performance measurement system, Power system modeling, program bottlenecks, program diagnostics, Programming profession, semantics, Sequent Symmetry multiprocessor, shared-memory systems, software tools, Springs, Sun, Sun 4, Unix, VAX},
	isbn = {1045-9219},
	doi = {10.1109/71.80132},
	author = {Miller, B. P and Clark, M. and Hollingsworth, Jeffrey K and Kierstead, S. and Lim,S. -S and Torzewski, T.}
}
