@article {18374,
	title = {An update to experimental models for validating computer technology},
	journal = {Journal of Systems and Software},
	volume = {82},
	year = {2009},
	month = {2009/03//},
	pages = {373 - 376},
	abstract = {In 1998 a survey was published on the extent to which software engineering papers validate the claims made in those papers. The survey looked at publications in 1985, 1990 and 1995. This current paper updates that survey with data from 2000 to 2005. The basic conclusion is that the situation is improving. One earlier complaint that access to data repositories was difficult is becoming less prevalent and the percentage of papers including validation is increasing.},
	keywords = {data collection, experimentation, Technology evaluation},
	isbn = {0164-1212},
	doi = {10.1016/j.jss.2008.06.040},
	url = {http://www.sciencedirect.com/science/article/pii/S0164121208001751},
	author = {Zelkowitz, Marvin V}
}
@article {18577,
	title = {In VINI veritas: realistic and controlled network experimentation},
	journal = {SIGCOMM Comput. Commun. Rev.},
	volume = {36},
	year = {2006},
	month = {2006/08//},
	pages = {3 - 14},
	abstract = {This paper describes VINI, a virtual network infrastructure that allows network researchers to evaluate their protocols and services in a realistic environment that also provides a high degree of control over network conditions. VINI allows researchers to deploy and evaluate their ideas with real routing software, traffic loads, and network events. To provide researchers flexibility in designing their experiments, VINI supports simultaneous experiments with arbitrary network topologies on a shared physical infrastructure. This paper tackles the following important design question: What set of concepts and techniques facilitate flexible, realistic, and controlled experimentation (e.g., multiple topologies and the ability to tweak routing algorithms) on a fixed physical infrastructure? We first present VINI{\textquoteright}s high-level design and the challenges of virtualizing a single network. We then present PL-VINI, an implementation of VINI on PlanetLab, running the "Internet In a Slice". Our evaluation of PL-VINI shows that it provides a realistic and controlled environment for evaluating new protocols and services.},
	keywords = {architecture, experimentation, Internet, Routing, virtualization},
	isbn = {0146-4833},
	doi = {10.1145/1151659.1159916},
	url = {http://doi.acm.org/10.1145/1151659.1159916},
	author = {Bavier,Andy and Feamster, Nick and Huang,Mark and Peterson,Larry and Rexford,Jennifer}
}
@conference {14777,
	title = {Combining self-reported and automatic data to improve programming effort measurement},
	booktitle = {Proceedings of the 10th European software engineering conference held jointly with 13th ACM SIGSOFT international symposium on Foundations of software engineering},
	series = {ESEC/FSE-13},
	year = {2005},
	month = {2005///},
	pages = {356 - 365},
	publisher = {ACM},
	organization = {ACM},
	address = {Lisbon, Portugal},
	abstract = {Measuring effort accurately and consistently across subjects in a programming experiment can be a surprisingly difficult task. In particular, measures based on self-reported data may differ significantly from measures based on data which is recorded automatically from a subject{\textquoteright}s computing environment. Since self-reports can be unreliable, and not all activities can be captured automatically, a complete measure of programming effort should incorporate both classes of data. In this paper, we show how self-reported and automatic effort can be combined to perform validation and to measure total programming effort.},
	keywords = {effort, experimentation, human factors, manual approaches, measurement, process metrics, verification},
	isbn = {1-59593-014-0},
	doi = {10.1145/1081706.1081762},
	author = {Hochstein, Lorin and Basili, Victor R. and Zelkowitz, Marvin V and Hollingsworth, Jeffrey K and Carver, Jeff}
}
@article {14758,
	title = {Recovering system specific rules from software repositories},
	journal = {SIGSOFT Softw. Eng. Notes},
	volume = {30},
	year = {2005},
	month = {2005/05//},
	pages = {1 - 5},
	abstract = {One of the most successful applications of static analysis based bug finding tools is to search the source code for violations of system-specific rules. These rules may describe how functions interact in the code, how data is to be validated or how an API is to be used. To apply these tools, the developer must encode a rule that must be followed in the source code. The difficulty is that many of these system-specific rules are undocumented and "grow" over time as the source code changes. Most research in this area relies on expert programmers to document these little-known rules. In this paper we discuss a method to automatically recover a subset of these rules, function usage patterns, by mining the software repository. We present a preliminary study that applies our work to a large open source software project.},
	keywords = {data warehouse and repository, debugging aids, design, experimentation, measurement, performance},
	isbn = {0163-5948},
	doi = {10.1145/1082983.1083144},
	author = {Williams, Chadd C and Hollingsworth, Jeffrey K}
}
@conference {14785,
	title = {Using Dynamic Tracing Sampling to Measure Long Running Programs},
	booktitle = {Proceedings of the 2005 ACM/IEEE conference on Supercomputing},
	series = {SC {\textquoteright}05},
	year = {2005},
	month = {2005///},
	pages = {59{\textendash} - 59{\textendash}},
	publisher = {IEEE Computer Society},
	organization = {IEEE Computer Society},
	abstract = {Detailed cache simulation can be useful to both system developers and application writers to understand an application{\textquoteright}s performance. However, measuring long running programs can be extremely slow. In this paper we present a technique to use dynamic sampling of trace snippets throughout an application{\textquoteright}s execution. We demonstrate that our approach improves accuracy compared to sampling a few timesteps at the beginning of execution by judiciously choosing the frequency, as well as the points in the control flow, at which samples are collected. Our approach is validated using the SIGMA tracing and simulation framework for the IBM Power family of processors.},
	keywords = {data communications, design, experimentation, measurement, performance, tracing},
	isbn = {1-59593-061-2},
	doi = {http://dx.doi.org/10.1109/SC.2005.77},
	url = {http://dx.doi.org/10.1109/SC.2005.77},
	author = {Odom, Jeffrey and Hollingsworth, Jeffrey K and DeRose,Luiz and Ekanadham, Kattamuri and Sbaraglia, Simone}
}
@article {18392,
	title = {Experimental validation in software engineering},
	journal = {Information and Software Technology},
	volume = {39},
	year = {1997},
	month = {1997///},
	pages = {735 - 743},
	abstract = {Although experimentation is an accepted approach toward scientific validation in most scientific disciplines, it only recently has gained acceptance within the software development community. In this paper we discuss a 12-model classification scheme for performing experimentation within the software development domain. We evaluate over 600 published papers in the computer science literature and over one hundred papers from other scientific disciplines in order to determine: (1) how well the computer science community is succeeding at validating its theories, and (2) how computer science compares to other scientific disciplines.},
	keywords = {data collection, Evaluation, experimentation, measurement},
	isbn = {0950-5849},
	doi = {10.1016/S0950-5849(97)00025-6},
	url = {http://www.sciencedirect.com/science/article/pii/S0950584997000256},
	author = {Zelkowitz, Marvin V and Wallace,Dolores}
}
@article {17226,
	title = {Improving the human factors aspect of database interactions},
	journal = {ACM Trans. Database Syst.},
	volume = {3},
	year = {1978},
	month = {1978/12//},
	pages = {417 - 439},
	abstract = {The widespread dissemination of computer and information systems to nontechnically trained individuals requires a new approach to the design and development of database interfaces. This paper provides the motivational background for controlled psychological experimentation in exploring the person/machine interface. Frameworks for the reductionist approach are given, research methods discussed, research issues presented, and a small experiment is offered as an example of what can be accomplished. This experiment is a comparison of natural and artificial language query facilities. Although subjects posed approximately equal numbers of valid queries with either facility, natural language users made significantly more invalid queries which could not be answered from the database that was described.},
	keywords = {Data models, Database systems, experimentation, human factors, natural language interfaces, Psychology, query languages},
	isbn = {0362-5915},
	doi = {10.1145/320289.320295},
	url = {http://doi.acm.org/10.1145/320289.320295},
	author = {Shneiderman, Ben}
}
