@conference {15456,
	title = {Workshop on experimental evaluation of software and systems in computer science (Evaluate 2010)},
	booktitle = {Proceedings of the ACM international conference companion on Object oriented programming systems languages and applications companion},
	series = {SPLASH {\textquoteright}10},
	year = {2010},
	month = {2010///},
	pages = {291 - 292},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {We call ourselves {\textquoteright}computer scientists{\textquoteright}, but are we scientists? If we are scientists, then we must practice the scientific method. This includes a solid experimental evaluation. In our experience, our experimental methodology is ad hoc at best, and nonexistent at worst. This workshop brings together experts from different areas of computer science to discuss, explore, and attempt to identify the principles of sound experimental evaluation},
	keywords = {Evaluation, METHODOLOGY},
	isbn = {978-1-4503-0240-1},
	doi = {10.1145/1869542.1869618},
	url = {http://doi.acm.org/10.1145/1869542.1869618},
	author = {Blackburn,Steven M. and Diwan,Amer and Hauswirth,Matthias and Memon, Atif M. and Sweeney,Peter F.}
}
@article {13104,
	title = {A Comprehensive Evaluation Framework and a Comparative Study for Human Detectors},
	journal = {Intelligent Transportation Systems, IEEE Transactions on},
	volume = {10},
	year = {2009},
	month = {2009/09//},
	pages = {417 - 427},
	abstract = {We introduce a framework for evaluating human detectors that considers the practical application of a detector on a full image using multisize sliding-window scanning. We produce detection error tradeoff (DET) curves relating the miss detection rate and the false-alarm rate computed by deploying the detector on cropped windows and whole images, using, in the latter, either image resize or feature resize. Plots for cascade classifiers are generated based on confidence scores instead of on variation of the number of layers. To assess a method{\textquoteright}s overall performance on a given test, we use the average log miss rate (ALMR) as an aggregate performance score. To analyze the significance of the obtained results, we conduct 10-fold cross-validation experiments. We applied our evaluation framework to two state-of-the-art cascade-based detectors on the standard INRIA person dataset and a local dataset of near-infrared images. We used our evaluation framework to study the differences between the two detectors on the two datasets with different evaluation methods. Our results show the utility of our framework. They also suggest that the descriptors used to represent features and the training window size are more important in predicting the detection performance than the nature of the imaging process, and that the choice between resizing images or features can have serious consequences.},
	keywords = {classification;infrared, classifier;comprehensive, curve;INRIA, dataset;average, DET, DETECTION, detection;, detector;image, error, Evaluation, framework;cropped, image;image, imaging;object, log, miss, person, rate;cascade, rate;feature, rate;multisize, resize;human, resize;miss, scanning;near-infrared, sliding-window, tradeoff;false-alarm, window;detection},
	isbn = {1524-9050},
	doi = {10.1109/TITS.2009.2026670},
	author = {Hussein,M. and Porikli, F. and Davis, Larry S.}
}
@conference {16069,
	title = {Aligning temporal data by sentinel events: discovering patterns in electronic health records},
	booktitle = {Proceedings of the twenty-sixth annual SIGCHI conference on Human factors in computing systems},
	series = {CHI {\textquoteright}08},
	year = {2008},
	month = {2008///},
	pages = {457 - 466},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Electronic Health Records (EHRs) and other temporal databases contain hidden patterns that reveal important cause-and-effect phenomena. Finding these patterns is a challenge when using traditional query languages and tabular displays. We present an interactive visual tool that complements query formulation by providing operations to align, rank and filter the results, and to visualize estimates of the intervals of validity of the data. Display of patient histories aligned on sentinel events (such as a first heart attack) enables users to spot precursor, co-occurring, and aftereffect events. A controlled study demonstrates the benefits of providing alignment (with a 61\% speed improvement for complex tasks). A qualitative study and interviews with medical professionals demonstrates that the interface can be learned quickly and seems to address their needs.},
	keywords = {electronic health record, Evaluation, Information Visualization, search, temporal data, Uncertainty},
	isbn = {978-1-60558-011-1},
	doi = {10.1145/1357054.1357129},
	url = {http://doi.acm.org/10.1145/1357054.1357129},
	author = {Wang,Taowei David and Plaisant, Catherine and Quinn,Alexander J. and Stanchak,Roman and Murphy,Shawn and Shneiderman, Ben}
}
@conference {16073,
	title = {BELIV{\textquoteright}08: Beyond time and errors: novel evaluation methods for information visualization},
	booktitle = {CHI {\textquoteright}08 extended abstracts on Human factors in computing systems},
	series = {CHI EA {\textquoteright}08},
	year = {2008},
	month = {2008///},
	pages = {3913 - 3916},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Information visualization systems allow users to produce insights, innovations and discoveries. Evaluating such tools is a challenging task and the goal of BELIV{\textquoteright}08 is to make a step ahead in the comprehension of such a complex activity. Current evaluation methods exhibit noticeable limitations and researchers in the area experiment some frustration with evaluation processes that are time consuming and too often leading to unsatisfactory results. The most used evaluation metrics such as task time completion and number of errors appear insufficient to quantify the quality of an information visualization system; thus the name of the workshop: "beyond time and errors".},
	keywords = {Evaluation, Information Visualization},
	isbn = {978-1-60558-012-8},
	doi = {10.1145/1358628.1358955},
	url = {http://doi.acm.org/10.1145/1358628.1358955},
	author = {Bertini,Enrico and Perer,Adam and Plaisant, Catherine and Santucci,Giuseppe}
}
@conference {17239,
	title = {Integrating statistics and visualization: case studies of gaining clarity during exploratory data analysis},
	booktitle = {Proceedings of the twenty-sixth annual SIGCHI conference on Human factors in computing systems},
	series = {CHI {\textquoteright}08},
	year = {2008},
	month = {2008///},
	pages = {265 - 274},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Although both statistical methods and visualizations have been used by network analysts, exploratory data analysis remains a challenge. We propose that a tight integration of these technologies in an interactive exploratory tool could dramatically speed insight development. To test the power of this integrated approach, we created a novel social network analysis tool, SocialAction, and conducted four long-term case studies with domain experts, each working on unique data sets with unique problems. The structured replicated case studies show that the integrated approach in SocialAction led to significant discoveries by a political analyst, a bibliometrician, a healthcare consultant, and a counter-terrorism researcher. Our contributions demonstrate that the tight integration of statistics and visualizations improves exploratory data analysis, and that our evaluation methodology for long-term case studies captures the research strategies of data analysts.},
	keywords = {case studies, Evaluation, exploratory data analysis, Information Visualization, social networks, Statistics},
	isbn = {978-1-60558-011-1},
	doi = {10.1145/1357054.1357101},
	url = {http://doi.acm.org/10.1145/1357054.1357101},
	author = {Perer,Adam and Shneiderman, Ben}
}
@conference {12582,
	title = {In Situ Evaluation of Tracking Algorithms Using Time Reversed Chains},
	booktitle = {Computer Vision and Pattern Recognition, 2007. CVPR {\textquoteright}07. IEEE Conference on},
	year = {2007},
	month = {2007/06//},
	pages = {1 - 8},
	abstract = {Automatic evaluation of visual tracking algorithms in the absence of ground truth is a very challenging and important problem. In the context of online appearance modeling, there is an additional ambiguity involving the correctness of the appearance model. In this paper, we propose a novel performance evaluation strategy for tracking systems based on particle filter using a time reversed Markov chain. Starting from the latest observation, the time reversed chain is propagated back till the starting time t = 0 of the tracking algorithm. The posterior density of the time reversed chain is also computed. The distance between the posterior density of the time reversed chain (at t = 0) and the prior density used to initialize the tracking algorithm forms the decision statistic for evaluation. It is postulated that when the data is generated true to the underlying models, the decision statistic takes a low value. We empirically demonstrate the performance of the algorithm against various common failure modes in the generic visual tracking problem. Finally, we derive a small frame approximation that allows for very efficient computation of the decision statistic.},
	keywords = {(numerical, algorithm;Markov, chain;tracking, decision, density;time, detection;particle, Evaluation, evaluation;object, filter;performance, Filtering, Markov, methods);tracking;visual, processes;decision, reversed, servoing;, situ, statistics;in, strategy;posterior, systems;visual, theory;object, tracking, tracking;particle},
	doi = {10.1109/CVPR.2007.382992},
	author = {Wu,Hao and Sankaranarayanan,A. C and Chellapa, Rama}
}
@article {15289,
	title = {User simulations for evaluating answers to question series},
	journal = {Information Processing \& Management},
	volume = {43},
	year = {2007},
	month = {2007/05//},
	pages = {717 - 729},
	abstract = {Recently, question series have become one focus of research in question answering. These series are comprised of individual factoid, list, and {\textquotedblleft}other{\textquotedblright} questions organized around a central topic, and represent abstractions of user{\textendash}system dialogs. Existing evaluation methodologies have yet to catch up with this richer task model, as they fail to take into account contextual dependencies and different user behaviors. This paper presents a novel simulation-based methodology for evaluating answers to question series that addresses some of these shortcomings. Using this methodology, we examine two different behavior models: a {\textquotedblleft}QA-styled{\textquotedblright} user and an {\textquotedblleft}IR-styled{\textquotedblright} user. Results suggest that an off-the-shelf document retrieval system is competitive with state-of-the-art QA systems in this task. Advantages and limitations of evaluations based on user simulations are also discussed.},
	keywords = {Evaluation, Question answering, User simulations},
	isbn = {0306-4573},
	doi = {10.1016/j.ipm.2006.06.006},
	url = {http://www.sciencedirect.com/science/article/pii/S0306457306000963},
	author = {Jimmy Lin}
}
@conference {16089,
	title = {Task taxonomy for graph visualization},
	booktitle = {Proceedings of the 2006 AVI workshop on BEyond time and errors: novel evaluation methods for information visualization},
	series = {BELIV {\textquoteright}06},
	year = {2006},
	month = {2006///},
	pages = {1 - 5},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Our goal is to define a list of tasks for graph visualization that has enough detail and specificity to be useful to: 1) designers who want to improve their system and 2) to evaluators who want to compare graph visualization systems. In this paper, we suggest a list of tasks we believe are commonly encountered while analyzing graph data. We define graph specific objects and demonstrate how all complex tasks could be seen as a series of low-level tasks performed on those objects. We believe that our taxonomy, associated with benchmark datasets and specific tasks, would help evaluators generalize results collected through a series of controlled experiments.},
	keywords = {Evaluation, graph visualization, task taxonomy},
	isbn = {1-59593-562-2},
	doi = {10.1145/1168149.1168168},
	url = {http://doi.acm.org/10.1145/1168149.1168168},
	author = {Lee,Bongshin and Plaisant, Catherine and Parr,Cynthia Sims and Fekete,Jean-Daniel and Henry,Nathalie}
}
@article {17258,
	title = {Interactive sonification of choropleth maps},
	journal = {IEEE Multimedia},
	volume = {12},
	year = {2005},
	month = {2005/06//April},
	pages = {26 - 35},
	abstract = {Auditory information is an important channel for the visually impaired. Effective sonification (the use of non-speech audio to convey information) promotes equal working opportunities for people with vision impairments by helping them explore data collections for problem solving and decision making. Interactive sonification systems can make georeferenced data accessible to people with vision impairments. The authors compare methods for using sound to encode georeferenced data patterns and for navigating maps.},
	keywords = {audio signal processing, audio user interfaces, Auditory (non-speech) feedback, auditory information, cartography, choropleth maps, data collections, decision making, Evaluation, Feedback, georeferenced data, Guidelines, handicapped aids, Hardware, HUMANS, information resources, interaction style, Interactive sonification, interactive systems, Navigation, nonspeech audio, problem solving, Problem-solving, sound, universal usability, US Government, User interfaces, vision impairments, World Wide Web},
	isbn = {1070-986X},
	doi = {10.1109/MMUL.2005.28},
	author = {Zhao,Haixia and Smith,B. K and Norman,K. and Plaisant, Catherine and Shneiderman, Ben}
}
@conference {16118,
	title = {The challenge of information visualization evaluation},
	booktitle = {Proceedings of the working conference on Advanced visual interfaces},
	series = {AVI {\textquoteright}04},
	year = {2004},
	month = {2004///},
	pages = {109 - 116},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {As the field of information visualization matures, the tools and ideas described in our research publications are reaching users. The reports of usability studies and controlled experiments are helpful to understand the potential and limitations of our tools, but we need to consider other evaluation approaches that take into account the long exploratory nature of users tasks, the value of potential discoveries or the benefits of overall awareness. We need better metrics and benchmark repositories to compare tools, and we should also seek reports of successful adoption and demonstrated utility.},
	keywords = {adoption, Evaluation, return on investment, technology transfer, usability, usefulness, User studies, Visualization},
	isbn = {1-58113-867-9},
	doi = {10.1145/989863.989880},
	url = {http://doi.acm.org/10.1145/989863.989880},
	author = {Plaisant, Catherine}
}
@conference {16121,
	title = {Les le{\c c}ons tir{\'e}es des deux comp{\'e}titions de visualisation d{\textquoteright}information},
	booktitle = {Proceedings of the 16th conference on Association Francophone d{\textquoteright}Interaction Homme-Machine},
	series = {IHM 2004},
	year = {2004},
	month = {2004///},
	pages = {7 - 12},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Information visualization needs benchmarks to carry on. A benchmark is aimed at comparing information visualization techniques or systems. A benchmark is made of a dataset, a list of tasks mostly based on finding facts about the dataset, and a list of interesting or important findings about the datasets (the nuggets to find). For the second year, we are organizing the InfoVis Contest aimed at collecting results for benchmarks. We describe here the main lessons we learned.},
	keywords = {benchmark, contest, Evaluation, Information Visualization},
	isbn = {1-58113-926-8},
	doi = {10.1145/1148613.1148616},
	url = {http://doi.acm.org/10.1145/1148613.1148616},
	author = {Fekete,Jean-Daniel and Plaisant, Catherine}
}
@conference {13623,
	title = {Performance evaluation of object detection algorithms},
	booktitle = {Pattern Recognition, 2002. Proceedings. 16th International Conference on},
	volume = {3},
	year = {2002},
	month = {2002///},
	pages = {965 - 969 vol.3 - 965 - 969 vol.3},
	abstract = {The continuous development of object detection algorithms is ushering in the need for evaluation tools to quantify algorithm performance. In this paper a set of seven metrics are proposed for quantifying different aspects of a detection algorithm{\textquoteright}s performance. The strengths and weaknesses of these metrics are described. They are implemented in the Video Performance Evaluation Resource (ViPER) system and will be used to evaluate algorithms for detecting text, faces, moving people and vehicles. Results for running two previous text-detection algorithms on a common data set are presented.},
	keywords = {algorithms;, common, data, DETECTION, detection;, Evaluation, evaluation;, image, object, performance, recognition;, resource, set;, system;, text-detection, video},
	doi = {10.1109/ICPR.2002.1048198},
	author = {Mariano,V.Y. and Min,Junghye and Park,Jin-Hyeong and Kasturi,R. and Mihalcik,D. and Huiping Li and David Doermann and Drayer,T.}
}
@conference {16174,
	title = {Excentric labeling: dynamic neighborhood labeling for data visualization},
	booktitle = {Proceedings of the SIGCHI conference on Human factors in computing systems: the CHI is the limit},
	series = {CHI {\textquoteright}99},
	year = {1999},
	month = {1999///},
	pages = {512 - 519},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {The widespread use of information visualization is hampered bythe lack of effective labeling techniques. An informal taxonomy of
labeling methods is proposed. We then describe excentric labeling,
a new dynamic technique to label a neighborhood of objects located
around the cursor. This technique does not intrude into the
existing interaction, it is not computationally intensive, and was
easily applied to several visualization applications. A pilot study
with eight subjects indicates a strong speed benefit over a zoom
interface for tasks that involve the exploration of large numbers
of objects. Observations and comments from users are presented.
},
	keywords = {dynamic labeling, Evaluation, label, Visualization},
	isbn = {0-201-48559-1},
	doi = {10.1145/302979.303148},
	url = {http://doi.acm.org/10.1145/302979.303148},
	author = {Fekete,Jean-Daniel and Plaisant, Catherine}
}
@article {18392,
	title = {Experimental validation in software engineering},
	journal = {Information and Software Technology},
	volume = {39},
	year = {1997},
	month = {1997///},
	pages = {735 - 743},
	abstract = {Although experimentation is an accepted approach toward scientific validation in most scientific disciplines, it only recently has gained acceptance within the software development community. In this paper we discuss a 12-model classification scheme for performing experimentation within the software development domain. We evaluate over 600 published papers in the computer science literature and over one hundred papers from other scientific disciplines in order to determine: (1) how well the computer science community is succeeding at validating its theories, and (2) how computer science compares to other scientific disciplines.},
	keywords = {data collection, Evaluation, experimentation, measurement},
	isbn = {0950-5849},
	doi = {10.1016/S0950-5849(97)00025-6},
	url = {http://www.sciencedirect.com/science/article/pii/S0950584997000256},
	author = {Zelkowitz, Marvin V and Wallace,Dolores}
}
@conference {13612,
	title = {OCR-based rate-distortion analysis of residual coding},
	booktitle = {Image Processing, 1997. Proceedings., International Conference on},
	volume = {3},
	year = {1997},
	month = {1997/10//},
	pages = {690 -693 vol.3 - 690 -693 vol.3},
	abstract = {Symbolic compression of document images provides access to symbols found in document images and exploits the redundancy found within them. Document images are highly structured and contain large numbers of repetitive symbols. We have shown that while symbolically compressing a document image we are able to perform compressed-domain processing. Symbolic compression forms representative prototypes for symbols and encode the image by the location of these prototypes and a residual (the difference between symbol and prototype). We analyze the rate-distortion tradeoff by varying the amount of residual used in compression for both distance- and row-order coding. A measure of distortion is based on the performance of an OCR system on the resulting image. The University of Washington document database images, ground truth, and OCR evaluation software are used for experiments},
	keywords = {analysis;redundancy;representative, character, coding;distortion, coding;image, coding;lossy, coding;row-order, coding;symbolic, compression;data, compression;document, compression;lossy, database, distortion, Evaluation, image, images;document, images;experiments;ground, measure;document, OCR, of, performance;University, processing;distance-order, processing;image, prototypes;residual, recognition;rate, representation;optical, representation;progressive, software;OCR, system, theory;, transmission;rate-distortion, truth;image, Washington;compressed-domain},
	doi = {10.1109/ICIP.1997.632215},
	author = {Kia,O. E and David Doermann}
}
@conference {15524,
	title = {Semantic query optimization for object databases},
	booktitle = {Data Engineering, 1997. Proceedings. 13th International Conference on},
	year = {1997},
	month = {1997/04//},
	pages = {444 - 453},
	abstract = {Presents a technique for semantic query optimization (SQO) for object databases. We use the ODMG-93 (Object Data Management Group) standard ODL (Object Database Language) and OQL (Object Query Language) languages. The ODL object schema and the OQL object query are translated into a DATALOG representation. Semantic knowledge about the object model and the particular application is expressed as integrity constraints. This is an extension of the ODMG-93 standard. SQO is performed in the DATALOG representation, and an equivalent logic query and (subsequently) an equivalent OQL object query are obtained. SQO is based on the residue technique of Chakravarthy et al. (1990). We show that our technique generalizes previous research on SQO for object databases. It can be applied to queries with structure constructors and method application. It utilizes integrity constraints about keys, methods and knowledge of access support relations, to produce equivalent queries, which may have more efficient evaluation plans},
	keywords = {application;object, constraints;keys;method, constructors;DATALOG;data, databases;object-oriented, databases;query, Datalog, Evaluation, integrity;data, languages;query, Logic, object, optimization;structure, plans;residue, processing;software, query, query;access, query;integrity, relations;equivalent, representation;ODL, schema;ODMG-93, standard;OQL, standards;, structures;object-oriented, Support, technique;semantic},
	doi = {10.1109/ICDE.1997.581983},
	author = {Grant,J. and Gryz,J. and Minker, Jack and Raschid, Louiqa}
}
@article {17222,
	title = {Image-browser taxonomy and guidelines for designers},
	journal = {IEEE Software},
	volume = {12},
	year = {1995},
	month = {1995/03//},
	pages = {21 - 32},
	abstract = {In many applications users must browse large images. Most designers merely use two one-dimensional scroll bars or ad hoc designs for two-dimensional scroll bars. However, the complexity of two-dimensional browsing suggests that more careful analysis, design, and evaluation might lead to significant improvements. Our exploration of existing 2D browsers has led us to identify many features and a wide variety of tasks performed with the browsers. We introduce an informal specification technique to describe 2D browsers and a task taxonomy, suggest design features and guidelines, and assess existing strategies. We focus on the tools to explore a selected image and so do not cover techniques to browse a series of images or to browse large-image databases},
	keywords = {analysis, Computer Graphics, design, designer guidelines, Equations, Europe, Evaluation, Formal specifications, Graphical user interfaces, Guidelines, IMAGE PROCESSING, image-browser taxonomy, informal specification technique, Laboratories, large image browsing, Layout, Road transportation, selected image exploration, SHAPE, Software design, task taxonomy, Taxonomy, tools, two-dimensional browsing, user interface management systems, visual databases},
	isbn = {0740-7459},
	doi = {10.1109/52.368260},
	author = {Plaisant, Catherine and Carr,D. and Shneiderman, Ben}
}
@conference {18376,
	title = {Automatic program analysis and evaluation},
	booktitle = {Proceedings of the 2nd international conference on Software engineering},
	series = {ICSE {\textquoteright}76},
	year = {1976},
	month = {1976///},
	pages = {158 - 163},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {Los Alamitos, CA, USA},
	abstract = {There is currently considerable interest in the computing community in the evaluation of computer programming. However, in order to objectively evaluate such concepts, it is necessary to undertake a thorough evaluation of the programming process itself. Most previous studies of this type have analyzed, by hand usually, a few instances of programs. This has led to some general conjectures; however, the amount of information that must be processed precludes any large scale analysis. In order to avoid this problem, an automatic data collection facility has been implemented as part of a PL/1 compiler at the University of Maryland. This system automatically collects information on each program that has been compiled - at almost no additional cost to the user of the compiler. This paper will describe the system and will evaluate some of the characteristics of some of the 25,000 programs that have been run since July, 1975.},
	keywords = {ERRORS, Evaluation, Program complexity, Program measurement, Static and dynamic analysis},
	url = {http://dl.acm.org/citation.cfm?id=800253.807668},
	author = {Zelkowitz, Marvin V}
}
