@article {12109,
	title = {A pilot study to compare programming effort for two parallel programming models},
	journal = {Journal of Systems and Software},
	volume = {81},
	year = {2008},
	month = {2008/11//},
	pages = {1920 - 1930},
	abstract = {ContextWriting software for the current generation of parallel systems requires significant programmer effort, and the community is seeking alternatives that reduce effort while still achieving good performance.
Objective
Measure the effect of parallel programming models (message-passing vs. PRAM-like) on programmer effort.
Design, setting, and subjects
One group of subjects implemented sparse-matrix dense-vector multiplication using message-passing (MPI), and a second group solved the same problem using a PRAM-like model (XMTC). The subjects were students in two graduate-level classes: one class was taught MPI and the other was taught XMTC.
Main outcome measures
Development time, program correctness.
Results
Mean XMTC development time was 4.8\&$\#$xa0;h less than mean MPI development time (95\% confidence interval, 2.0{\textendash}7.7), a 46\% reduction. XMTC programs were more likely to be correct, but the difference in correctness rates was not statistically significant (p\&$\#$xa0;=\&$\#$xa0;.16).
Conclusions
XMTC solutions for this particular problem required less effort than MPI equivalents, but further studies are necessary which examine different types of problems and different levels of programmer experience.
},
	keywords = {effort, empirical study, Message-passing, MPI, parallel programming, PRAM, XMT},
	isbn = {0164-1212},
	doi = {10.1016/j.jss.2007.12.798},
	url = {http://www.sciencedirect.com/science/article/pii/S0164121208000125},
	author = {Hochstein, Lorin and Basili, Victor R. and Vishkin, Uzi and Gilbert,John}
}
@conference {12115,
	title = {An empirical study to compare two parallel programming models},
	booktitle = {Proceedings of the eighteenth annual ACM symposium on Parallelism in algorithms and architectures},
	series = {SPAA {\textquoteright}06},
	year = {2006},
	month = {2006///},
	pages = {114 - 114},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {While there are strong beliefs within the community about whether one particular parallel programming model is easier to use than another, there has been little research to analyze these claims empirically. Currently, the most popular paradigm is message-passing, as implemented by the MPI library [1]. However, MPI is considered to be difficult for developing programs, because it forces the programmer to work at a very low level of abstraction. One alternative parallel programming model is the PRAM model, which supports fine-grained parallelism and has a substantial history of algorithmic theory [2]. It is not possible to program current parallel machines using the PRAM model because modern architectures are not designed to support such a model efficiently. However, current trends towards multicore chips suggest that large-scale, fine-grained uniform-memory access parallel machines may soon be feasible. XMT-C is an extension of the C language that supports parallel directives to provide a PRAM-like model to the programmer. A prototype compiler exists that generates code which runs on a simulator for an XMT architecture [3].To better understand how much benefit a PRAM-like model could provide over a message-passing model, we conducted a feasibility study in an academic setting to compare the effort required to solve a particular problem. The questions under study were: can we measure the effort in developing a program using these two programming models and can we differentiate the amount of effort for each model?.The subjects participating in the study were divided up into two groups. One group solved a problem using the MPI library in either C,C++, or Fortran, and the other group solved the problem using XMT-C. The task was to write a function to multiply a sparse matrix with a dense vector. To obtain subjects, we leveraged existing graduate-level parallel programming courses at two different universities: University of California, Santa Barbara (UCSB), and University of Maryland (UMD). At UCSB, the students solved the problem in MPI, and at UMD, the students solved the same problem in XMT-C. The focus of the UCSB class was on developing parallel programs to run on the current generation of architectures, and the students were taught MPI, as well as other models. The focus of the UMD class was parallel algorithms in the PRAM model, and the students were taught XMT-C. The students were assigned to treatment groups by class. The study was integrated into each class, as the problem was a required assignment in each class.Subjects kept track of their effort with a self-reported effort log. We also collected automatic effort data by instrumenting the compilers, which recorded data at each compile. We computed three effort measures: self-reported, instrumented, and combined. Self-reported effort measures are based entirely on effort logs, instrumented effort measures are based entirely on timestamps from the instrumented compilers, and combined effort measures are based on compiler timestamps when the subject is working on the instrumented machine, and self-reported effort when the subject is working off the instrumented machine.The results of this preliminary study answer both of our questions in the positive. In this case, on average, students required less effort to solve the problem using XMT-C compared to MPI. The reduction in mean effort was approximately 50\% for all three measures, which was statistically significant at the level of p < .05 using a t-test.This study demonstrates that the effect of programming model on effort can be directly measured through empirical studies with human subjects. While no single study can conclusively demonstrate the advantage of one programming model over another, a series of studies examining different models and different problems can provide insights into the relative strengths of parallel programming models in different contexts. The study described above is one of a series of such studies that we are currently conducting.},
	keywords = {effort, empirical study, HPC, Message-passing, MPI, parallel programming, PRAM, Productivity, XMT},
	isbn = {1-59593-452-9},
	doi = {10.1145/1148109.1148127},
	url = {http://doi.acm.org/10.1145/1148109.1148127},
	author = {Hochstein, Lorin and Basili, Victor R.}
}
@article {12125,
	title = {COTS-based software development: Processes and open issues},
	journal = {Journal of Systems and Software},
	volume = {61},
	year = {2002},
	month = {2002/04/01/},
	pages = {189 - 199},
	abstract = {The work described in this paper is an investigation of the COTS-based software development within a particular NASA environment, with an emphasis on the processes used. Fifteen projects using a COTS-based approach were studied and their actual process was documented. This process is evaluated to identify essential differences in comparison to traditional software development. The main differences, and the activities for which projects require more guidance, are requirements definition and COTS selection, high level design, integration and testing. Starting from these empirical observations, a new process and set of guidelines for COTS-based development are developed and briefly presented.},
	keywords = {Commercial off-the-shelf, Component-based, COTS, empirical study, Software development process},
	isbn = {0164-1212},
	doi = {10.1016/S0164-1212(01)00147-9},
	url = {http://www.sciencedirect.com/science/article/pii/S0164121201001479},
	author = {Morisio,M and Seaman,C.B and Basili, Victor R. and Parra,A.T and Kraft,S.E and Condon,S.E}
}
