{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,18]],"date-time":"2026-04-18T03:14:28Z","timestamp":1776482068449,"version":"3.51.2"},"reference-count":107,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2009,12,5]],"date-time":"2009-12-05T00:00:00Z","timestamp":1259971200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2010,6]]},"DOI":"10.1007\/s10664-009-9121-0","type":"journal-article","created":{"date-parts":[[2009,12,4]],"date-time":"2009-12-04T07:19:26Z","timestamp":1259911166000},"page":"250-276","source":"Crossref","is-referenced-by-count":48,"title":["Applying empirical software engineering to software architecture: challenges and lessons learned"],"prefix":"10.1007","volume":"15","author":[{"given":"Davide","family":"Falessi","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Muhammad Ali","family":"Babar","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Giovanni","family":"Cantone","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Philippe","family":"Kruchten","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2009,12,5]]},"reference":[{"key":"9121_CR1","doi-asserted-by":"crossref","unstructured":"Ali Babar M (2008) Assessment of a framework for designing and evaluating security sensitive architecture. 12th International Conference on Evaluation and Assessment in Software Engineering (EASE08). Bari, Italy","DOI":"10.14236\/ewic\/EASE2008.17"},{"key":"9121_CR2","doi-asserted-by":"crossref","unstructured":"Ali Babar M, Kitchenham B (2007a) Assessment of a framework for comparing software architecture analysis methods. 11th International Conference on Evaluation and Assessment in Software Engineering (EASE)","DOI":"10.14236\/ewic\/EASE2007.2"},{"key":"9121_CR3","doi-asserted-by":"crossref","unstructured":"Ali Babar M, Kitchenham B (2007b) The impact of group size on software architecture evaluation: A controlled experiment. Proceedings of the First International Symposium on Empirical Software Engineering and Measurement. IEEE Computer Society","DOI":"10.1109\/ESEM.2007.38"},{"key":"9121_CR4","doi-asserted-by":"crossref","unstructured":"Ali Babar M, Zhu L, Jeffery R (2004) A framework for classifying and comparing software architecture evaluation methods. Proceedings of the Australian Software Engineering Conference","DOI":"10.1109\/ASWEC.2004.1290484"},{"key":"9121_CR5","doi-asserted-by":"crossref","unstructured":"Ali Babar M, Bass L, Gorton I (2007) Factors influencing industrial practices of software architecture evaluation: An empirical investigation. Quality of Software Architectures (QoSA). Massachusetts,USA","DOI":"10.1007\/978-3-540-77619-2_6"},{"issue":"1","key":"9121_CR6","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1007\/s10664-007-9052-6","volume":"13","author":"M Ali Babar","year":"2008","unstructured":"Ali Babar M, Kitchenham B, Jeffery R (2008) Comparing distributed and face-to-face meetings for software architecture evaluation: a controlled experiment. Empir Software Eng; Int J 13(1):39\u201362","journal-title":"Empir Software Eng; Int J"},{"issue":"2","key":"9121_CR7","doi-asserted-by":"crossref","first-page":"65","DOI":"10.1109\/TSE.2007.17","volume":"33","author":"E Arisholm","year":"2007","unstructured":"Arisholm E, Gallis H, Dyb\u00e5 T, Sjoberg D (2007) Evaluating pair programming with respect to system complexity and programmer expertise. IEEE Trans Softw Eng 33(2):65\u201386","journal-title":"IEEE Trans Softw Eng"},{"key":"9121_CR8","doi-asserted-by":"crossref","unstructured":"Barbacci MR, Ellison R, Lattanze AJ, Stafford JA, Weinstock CB, Wood WG (2003) Quality attribute workshops (qaws), third edition. http:\/\/www.sei.cmu.edu\/publications\/documents\/03.reports\/03tr016.html","DOI":"10.21236\/ADA418428"},{"key":"9121_CR9","volume-title":"The role of experimentation in software engineering: past, current, and future. Proceedings of the 18th International Conference on Software Engineering","author":"VR Basili","year":"1996","unstructured":"Basili VR (1996) The role of experimentation in software engineering: past, current, and future. Proceedings of the 18th International Conference on Software Engineering. IEEE Computer Society, Berlin"},{"issue":"7","key":"9121_CR10","first-page":"11","volume":"12","author":"V Basili","year":"1986","unstructured":"Basili V, Selby R, Hutchens D (1986) Experimentation in software engineering. IEEE Trans Softw Eng 12(7):11","journal-title":"IEEE Trans Softw Eng"},{"key":"9121_CR11","unstructured":"Basili V, Caldiera G, Rombach D (1994) Goal\/question\/metric paradigm. Encyclopedia of Software Engineering 1 (John Wiley & Sons), 528\u2013532"},{"key":"9121_CR12","volume-title":"Software architecture in practice","author":"L Bass","year":"2003","unstructured":"Bass L, Clements P, Kazman R (2003) Software architecture in practice, 2nd edn. Addison-Wesley, Reading","edition":"2"},{"issue":"1\u20132","key":"9121_CR13","doi-asserted-by":"crossref","first-page":"129","DOI":"10.1016\/S0164-1212(03)00080-3","volume":"69","author":"P Bengtsson","year":"2004","unstructured":"Bengtsson P, Lassing N, Bosch J, Vliet Hv (2004) Architecture-level modifiability analysis (alma). J Syst Softw 69(1\u20132):129\u2013147","journal-title":"J Syst Softw"},{"key":"9121_CR14","unstructured":"Biffl S, Gutjahr W (2001) Influence of team size and defect detection technique on inspection effectiveness. Proceedings of the 7th International Symposium on Software Metrics. IEEE Computer Society"},{"key":"9121_CR15","doi-asserted-by":"crossref","unstructured":"Biffl S, Aurum A, Bohem B, Erdogmus H, Gr\u00fcnbacher P (2005) Value-based software engineering. Springer","DOI":"10.1007\/3-540-29263-2"},{"key":"9121_CR16","volume-title":"Software engineering economics Prentice Hall PTR Advances in computing science and technology","author":"BW Boehm","year":"1981","unstructured":"Boehm BW (1981) Software engineering economics Prentice Hall PTR Advances in computing science and technology. Prentice-Hall, Englewood Cliffs"},{"issue":"3","key":"9121_CR17","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1109\/MS.2006.86","volume":"23","author":"G Booch","year":"2006","unstructured":"Booch G (2006a) The accidental architecture. IEEE Softw 23(3):9\u201311","journal-title":"IEEE Softw"},{"issue":"6","key":"9121_CR18","doi-asserted-by":"crossref","first-page":"14","DOI":"10.1109\/MS.2006.162","volume":"23","author":"G Booch","year":"2006","unstructured":"Booch G (2006b) Goodness of fit. IEEE Softw 23(6):14\u201315","journal-title":"IEEE Softw"},{"issue":"5","key":"9121_CR19","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1109\/MS.2007.146","volume":"24","author":"G Booch","year":"2007","unstructured":"Booch G (2007a) The economics of architecture-first. IEEE Softw 24(5):18\u201320","journal-title":"IEEE Softw"},{"issue":"3","key":"9121_CR20","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1109\/MS.2007.93","volume":"24","author":"G Booch","year":"2007","unstructured":"Booch G (2007b) The irrelevance of architecture. IEEE Softw 24(3):10\u201311","journal-title":"IEEE Softw"},{"issue":"4","key":"9121_CR21","doi-asserted-by":"crossref","first-page":"571","DOI":"10.1016\/j.jss.2006.07.009","volume":"80","author":"P Brereton","year":"2007","unstructured":"Brereton P, Kitchenham BA, Budgen D, Turner M, Khalil M (2007) Lessons from applying the systematic literature review process within the software engineering domain. J Syst Softw 80(4):571\u2013583","journal-title":"J Syst Softw"},{"key":"9121_CR22","doi-asserted-by":"crossref","unstructured":"Brooks A, Roper M, Wood M, Daly J, Miller J (2008) Replication\u2019s role in software engineering. In: Guide to advanced empirical software engineering Springer","DOI":"10.1007\/978-1-84800-044-5_14"},{"key":"9121_CR23","unstructured":"Brown WJ, Malveau RC, McCormick HW, Mowbray TJ (1998) Antipatterns: Refactoring software, architectures, and projects in crisis. Wiley"},{"key":"9121_CR24","unstructured":"Cantone G, D\u2019Angi\u00f2 A, Falessi D, Lomartire A, Pesce G, Scarrone S (2008) Does an intentional architecture pay off? A controlled experiment. Technical Report 09.77, University of Rome TorVergata"},{"key":"9121_CR25","doi-asserted-by":"crossref","unstructured":"Carver J, Jaccheri L, Morasca S, Shull F (2003) Issues in using students in empirical studies in software engineering education. Proceedings of the 9th International Symposium on Software Metrics. IEEE Computer Society","DOI":"10.1109\/METRIC.2003.1232471"},{"key":"9121_CR26","doi-asserted-by":"crossref","unstructured":"Chen L, Ali Babar M, Cawley C (2009) Evaluation of variability management approaches: A systematic review. 13th International Conference on Evaluation and Assessment in Software Engineering","DOI":"10.14236\/ewic\/EASE2009.14"},{"key":"9121_CR27","volume-title":"Documenting software architectures: views and beyond","author":"P Clements","year":"2002a","unstructured":"Clements P, Bachmann F, Bass L, Garlan D, Ivers J, Little R, Nord R, Stafford J (2002a) Documenting software architectures: views and beyond. Addison-Wesley, Boston"},{"key":"9121_CR28","volume-title":"Evaluating software architecture: methods and case studies","author":"P Clements","year":"2002b","unstructured":"Clements P, Kazman R, Klein M (2002b) Evaluating software architecture: methods and case studies. Addison-Wesley, Boston"},{"key":"9121_CR29","volume-title":"Controlling software projects","author":"T Marco De","year":"1986","unstructured":"De Marco T (1986) Controlling software projects. P Hall, New York"},{"key":"9121_CR30","unstructured":"Dehnadi S, Bornat R (2006) The camel has two humps. http:\/\/www.cs.mdx.ac.uk\/research\/PhDArea\/saeed"},{"key":"9121_CR31","unstructured":"Desouza K, Dings\u00f8yr T, Awazu Y (2005) Experiences with conducting project postmortems: Reports vs. Stories and practitioner perspective. Proceedings of the Proceedings of the 38th Annual Hawaii International Conference on System Sciences (HICSS\u201905) - Track 8 - Volume 08. IEEE Computer Society"},{"issue":"7","key":"9121_CR32","doi-asserted-by":"crossref","first-page":"638","DOI":"10.1109\/TSE.2002.1019479","volume":"28","author":"L Dobrica","year":"2002","unstructured":"Dobrica L, Niemel\u00e4 E (2002) A survey on software architecture analysis methods. IEEE Trans Softw Eng 28(7):638\u2013653","journal-title":"IEEE Trans Softw Eng"},{"issue":"9\u201310","key":"9121_CR33","doi-asserted-by":"crossref","first-page":"833","DOI":"10.1016\/j.infsof.2008.01.006","volume":"50","author":"T Dyb\u00e5","year":"2008","unstructured":"Dyb\u00e5 T, Dings\u00f8yr T (2008) Empirical studies of agile software development: a systematic review. Inform Software Tech 50(9\u201310):833\u2013859","journal-title":"Inform Software Tech"},{"issue":"1","key":"9121_CR34","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1109\/MS.2005.6","volume":"22","author":"T Dyb\u00e5","year":"2005","unstructured":"Dyb\u00e5 T, Kitchenham B, J\u00f8rgensen M (2005) Evidence-based software engineering for practitioners. IEEE Softw 22(1):58\u201365","journal-title":"IEEE Softw"},{"issue":"3","key":"9121_CR35","doi-asserted-by":"crossref","first-page":"407","DOI":"10.1109\/TSE.2008.15","volume":"34","author":"W Dzidek","year":"2008","unstructured":"Dzidek W, Arisholm E, Briand L (2008) A realistic empirical evaluation of the costs and benefits of uml in software maintenance. IEEE Trans. Softw. Eng. 34(3):407\u2013432","journal-title":"IEEE Trans. Softw. Eng."},{"key":"9121_CR36","doi-asserted-by":"crossref","unstructured":"Easterbrook S, Singer J, Storey M-A, Damian D (2008) Selecting empirical methods for software engineering research. In: Shull F, Singer J, Sj\u00f8berg D (eds) Guide to advanced empirical software engineering Springer","DOI":"10.1007\/978-1-84800-044-5_11"},{"key":"9121_CR37","doi-asserted-by":"crossref","unstructured":"Eden A, Kazman R (2003) Architecture, design, implementation. Proceedings of the 25th International Conference on Software Engineering. Portland, Oregon. IEEE Computer Society","DOI":"10.1109\/ICSE.2003.1201196"},{"key":"9121_CR38","unstructured":"Eeles P (2005) Capturing architectural requirements. IBM Rational developer works. Available at: http:\/\/www.ibm.com\/developerworks\/rational\/library\/4706.html"},{"key":"9121_CR39","doi-asserted-by":"crossref","unstructured":"Eguiluz HR, Barbacci MR (2003) Interactions among techniques addressing quality attributes","DOI":"10.21236\/ADA416601"},{"key":"9121_CR40","volume-title":"An explorative analysis of user evaluation studies in information visualisation. Proceedings of the 2006 AVI workshop on BEyond time and errors: novel evaluation methods for information visualization","author":"G Ellis","year":"2006","unstructured":"Ellis G, Dix A (2006) An explorative analysis of user evaluation studies in information visualisation. Proceedings of the 2006 AVI workshop on BEyond time and errors: novel evaluation methods for information visualization. ACM, Venice"},{"key":"9121_CR41","volume-title":"Empirical evaluations of regression test selection techniques: a systematic review. Proceedings of the Second ACM-IEEE international symposium on Empirical software engineering and measurement","author":"E Engstrom","year":"2008","unstructured":"Engstrom E, Skoglund M, Runeson P (2008) Empirical evaluations of regression test selection techniques: a systematic review. Proceedings of the Second ACM-IEEE international symposium on Empirical software engineering and measurement. ACM, Kaiserslautern"},{"issue":"5","key":"9121_CR42","doi-asserted-by":"crossref","first-page":"4","DOI":"10.1109\/MS.2008.120","volume":"25","author":"H Erdogmus","year":"2008","unstructured":"Erdogmus H (2008) Must software research stand divided? IEEE Softw 25(5):4\u20136","journal-title":"IEEE Softw"},{"key":"9121_CR43","unstructured":"Falessi D, Becker M (2006) Documenting design decisions: A framework and its analysis in the ambient intelligence domain. BelAmI-Report 005.06\/E, Fraunhofer IESE"},{"key":"9121_CR44","doi-asserted-by":"crossref","unstructured":"Falessi D, Cantone G, Becker M (2006) Documenting design decision rationale to improve individual and team design decision making: An experimental evaluation. Proceedings of the 5th ACM\/IEEE International Symposium on Empirical Software Engineering. Rio de Janeiro, Brazil","DOI":"10.1145\/1159733.1159755"},{"key":"9121_CR45","doi-asserted-by":"crossref","unstructured":"Falessi D, Cantone G, Kruchten P (2007a) Do architecture design methods meet architects' needs?. Sixth Working IEEE\/IFIP Conference on Software Architecture, Mumbai, India","DOI":"10.1109\/WICSA.2007.23"},{"key":"9121_CR46","volume-title":"Issues in applying empirical software engineering to software architecture. First European conference on software architecture","author":"D Falessi","year":"2007","unstructured":"Falessi D, Kruchten P, Cantone G (2007b) Issues in applying empirical software engineering to software architecture. First European conference on software architecture. Springer, Aranjunez"},{"key":"9121_CR47","volume-title":"Value-based design decision rationale documentation: Principles and empirical feasibility study. Proceeding of the Seventh Working IEEE \/ IFIP Conference on Software Architecture (WICSA 2008)","author":"D Falessi","year":"2008a","unstructured":"Falessi D, Cantone G, Kruchten P (2008a) Value-based design decision rationale documentation: Principles and empirical feasibility study. Proceeding of the Seventh Working IEEE \/ IFIP Conference on Software Architecture (WICSA 2008). IEEE Computer Society, Vancouver"},{"key":"9121_CR48","volume-title":"A value-based approach for documenting design decisions rationale: A replicated experiment. Proceedings of the 3rd international workshop on sharing and reusing architectural knowledge","author":"D Falessi","year":"2008b","unstructured":"Falessi D, Capilla R, Cantone G (2008b) A value-based approach for documenting design decisions rationale: A replicated experiment. Proceedings of the 3rd international workshop on sharing and reusing architectural knowledge. ACM, Leipzig"},{"issue":"6","key":"9121_CR49","doi-asserted-by":"crossref","first-page":"42","DOI":"10.1109\/52.329400","volume":"11","author":"R Glass","year":"1994","unstructured":"Glass R (1994) The software-research crisis. IEEE Softw 11(6):42\u201347","journal-title":"IEEE Softw"},{"issue":"5","key":"9121_CR50","doi-asserted-by":"crossref","first-page":"96","DOI":"10.1109\/MS.2008.121","volume":"25","author":"R Glass","year":"2008","unstructured":"Glass R (2008) Negative productivity and what to do about it. IEEE Softw 25(5):96","journal-title":"IEEE Softw"},{"issue":"2","key":"9121_CR51","doi-asserted-by":"crossref","first-page":"242","DOI":"10.1109\/TSE.2008.13","volume":"34","author":"J Hannay","year":"2008","unstructured":"Hannay J, J\u00f8rgensen M (2008) The role of deliberate artificial design elements in software engineering experiments. IEEE Trans Softw Eng 34(2):242\u2013259","journal-title":"IEEE Trans Softw Eng"},{"issue":"1","key":"9121_CR52","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1023\/A:1009731704468","volume":"3","author":"W Harrison","year":"1998","unstructured":"Harrison W (1998) Sharing the wealth: accumulating and sharing lessons learned in empirical software engineering research. Empir Software Eng 3(1):7\u20138","journal-title":"Empir Software Eng"},{"issue":"1","key":"9121_CR53","doi-asserted-by":"crossref","first-page":"106","DOI":"10.1016\/j.jss.2006.05.024","volume":"80","author":"C Hofmeister","year":"2007","unstructured":"Hofmeister C, Kruchten P, Nord RL, Obbink H, Ran A, America P (2007) A general model of software architecture design derived from five industrial approaches. J Syst Softw 80(1):106\u2013126. doi: 10.1016\/j.jss.2006.05.024","journal-title":"J Syst Softw"},{"issue":"3","key":"9121_CR54","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1023\/A:1026586415054","volume":"5","author":"M Host","year":"2000","unstructured":"Host M, Regnell B, Wohlin C (2000) Using students as subjects: a comparative study of students and professionals in lead-time impact assessment. Empir Software Eng J 5(3):201\u2013214","journal-title":"Empir Software Eng J"},{"key":"9121_CR55","doi-asserted-by":"crossref","unstructured":"Houdek F (2003) External experiments: A workable paradigm for collaboration between industry and academia. In: Lecture notes on empirical software engineering World Scientific Publishing Co., Inc","DOI":"10.1142\/9789812795588_0004"},{"key":"9121_CR56","unstructured":"ISESE (2009) http:\/\/www.belami-project.org\/ , Last access: November 2009"},{"key":"9121_CR57","unstructured":"ISO\/IEC 42010 I. P (2008) Systems and software engineering\u2014architecture description"},{"key":"9121_CR58","volume-title":"Software architecture as a set of architectural design decisions. 5th Working IEEE\/IFIP Conference on Software Architecture (WICSA 5)","author":"A Jansen","year":"2005","unstructured":"Jansen A, Bosch J (2005) Software architecture as a set of architectural design decisions. 5th Working IEEE\/IFIP Conference on Software Architecture (WICSA 5). IEEE CS, Pittsburgh"},{"key":"9121_CR59","doi-asserted-by":"crossref","unstructured":"Jeffery R, Scott L (2002) Has twenty-five years of empirical software engineering made a difference? Proceedings of the Ninth Asia-Pacific Software Engineering Conference. IEEE Computer Society","DOI":"10.1109\/APSEC.2002.1183076"},{"key":"9121_CR60","volume-title":"Some lessons learned in conducting software engineering surveys in china. Proceedings of the Second ACM-IEEE international symposium on Empirical software engineering and measurement","author":"J Ji","year":"2008","unstructured":"Ji J, Li J, Conradi R, Liu C, Ma J, Chen W (2008) Some lessons learned in conducting software engineering surveys in china. Proceedings of the Second ACM-IEEE international symposium on Empirical software engineering and measurement. ACM, Kaiserslautern"},{"key":"9121_CR61","unstructured":"Jones C (1994) Assessment and control of software risks. P Hall"},{"key":"9121_CR62","unstructured":"Juristo N, Moreno AM (2006) Basics of software engineering experimentation Springer"},{"key":"9121_CR63","unstructured":"Kazman R, Klein M, Clements P (2004) Atam: Method for architecture evaluation"},{"key":"9121_CR64","doi-asserted-by":"crossref","unstructured":"Kitchenham BA (1996) Evaluating software engineering methods and tool part 1: The evaluation context and evaluation methods","DOI":"10.1145\/381790.381795"},{"key":"9121_CR65","unstructured":"Kitchenham B (2004) Procedures for performing systematic reviews. Joint Technical Report, Keele University TR\/SE-0401 and NICTA 0400011T.1"},{"issue":"2","key":"9121_CR66","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1007\/s10664-008-9061-0","volume":"13","author":"B Kitchenham","year":"2008","unstructured":"Kitchenham B (2008) The role of replications in empirical software engineering\u2014a word of warning. J Empir Softw Eng 13(2):219\u2013221","journal-title":"J Empir Softw Eng"},{"key":"9121_CR67","doi-asserted-by":"crossref","unstructured":"Kitchenham B, Dyb\u00e5 T, J\u00f8rgensen M (2004) Evidence-based software engineering. Proceedings of the 26th International Conference on Software Engineering. IEEE Computer Society","DOI":"10.1109\/ICSE.2004.1317449"},{"key":"9121_CR68","unstructured":"Kruchten P (2001) Common misconceptions about software architecture. The Rational Edge"},{"key":"9121_CR69","unstructured":"Kruchten P (2003) The rational unified process: An introduction, 3rd edn. Addison-Wesley Professional"},{"key":"9121_CR70","unstructured":"Kruchten P (2004) An ontology of architectural design decisions in software intensive systems. Proceedings of the 2nd Groningen Workshop on Software Variability"},{"issue":"2","key":"9121_CR71","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1109\/MS.2006.59","volume":"23","author":"P Kruchten","year":"2006","unstructured":"Kruchten P, Obbink H, Stafford J (2006) The past, present and future for software architecture. IEEE Softw 23(2):2\u201310","journal-title":"IEEE Softw"},{"key":"9121_CR72","doi-asserted-by":"crossref","unstructured":"Laitenberger O, Rombach D (2003) (quasi-)experimental studies in industrial settings. In: Lecture notes on empirical software engineering World Scientific Publishing Co., Inc","DOI":"10.1142\/9789812795588_0005"},{"issue":"11","key":"9121_CR73","doi-asserted-by":"crossref","first-page":"928","DOI":"10.1109\/TSE.2005.127","volume":"31","author":"V Liu","year":"2005","unstructured":"Liu V, Gorton I, Fekete A (2005) Design-level performance prediction of component-based applications. IEEE Trans Softw Eng 31(11):928\u2013941","journal-title":"IEEE Trans Softw Eng"},{"key":"9121_CR74","volume-title":"On the difficulty of replicating human subjects studies in software engineering. Proceedings of the 30th international conference on Software engineering","author":"J Lung","year":"2008","unstructured":"Lung J, Aranda J, Easterbrook S, Wilson G (2008) On the difficulty of replicating human subjects studies in software engineering. Proceedings of the 30th international conference on Software engineering. ACM, Leipzig"},{"issue":"2","key":"9121_CR75","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1109\/MS.2005.28","volume":"22","author":"JF Maranzano","year":"2005","unstructured":"Maranzano JF, Rozsypal SA, Zimmerman GH, Warnken GW, Wirth PE, Weiss DM (2005) Architecture reviews: practice and experience. IEEE Softw 22(2):34\u201343","journal-title":"IEEE Softw"},{"issue":"4","key":"9121_CR76","doi-asserted-by":"crossref","first-page":"438","DOI":"10.1109\/32.799936","volume":"25","author":"G Murphy","year":"1999","unstructured":"Murphy G, Walker R, Baniassad E (1999) Evaluating emerging software development technologies: lessons learned from assessing aspect-oriented programming. IEEE Trans Softw Eng 25(4):438\u2013455","journal-title":"IEEE Trans Softw Eng"},{"key":"9121_CR77","doi-asserted-by":"crossref","unstructured":"Oates BJ (2003) Widening the scope of evidence gathering in software engineering. Proceedings of the Eleventh Annual International Workshop on Software Technology and Engineering Practice. IEEE Computer Society","DOI":"10.1109\/STEP.2003.39"},{"key":"9121_CR78","unstructured":"Obbink H, Kruchten P, Kozaczynski W, Hilliard R., Ran A, Postema H, Lutz D, Kazman R, Tracz W, Kahane E (2002) Report on software architecture review and assessment (sara), version 1.0. At http:\/\/philippe.Kruchten.Com\/architecture\/sarav1.Pdf"},{"key":"9121_CR79","volume-title":"Empirical studies of software engineering: a roadmap. Proceedings of the Conference on the future of software engineering","author":"DE Perry","year":"2000","unstructured":"Perry DE, Porter AA, Votta LG (2000) Empirical studies of software engineering: a roadmap. Proceedings of the Conference on the future of software engineering. ACM, Limerick"},{"issue":"5","key":"9121_CR80","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1109\/52.232392","volume":"10","author":"C Potts","year":"1993","unstructured":"Potts C (1993) Software-engineering research revisited. IEEE Softw 10(5):19\u201328","journal-title":"IEEE Softw"},{"key":"9121_CR81","unstructured":"Prechelt L (2007) Optimizing return-on-investment (roi) for empirical software engineering studies working group results. In: Empirical software engineering issues. Critical assessment and future directions"},{"key":"9121_CR82","doi-asserted-by":"crossref","unstructured":"Punter T, Ciolkowski M, Freimut B, John I (2003) Conducting on-line surveys in software engineering. Proceedings of the 2003 International Symposium on Empirical Software Engineering. IEEE Computer Society","DOI":"10.1109\/ISESE.2003.1237967"},{"key":"9121_CR83","unstructured":"Reeves JW (1992) What is software design? C++ Journal 2 (2)"},{"key":"9121_CR84","unstructured":"SEI (2007) Published software architecture definitions. http:\/\/www.sei.cmu.edu\/architecture\/published_definitions.html"},{"issue":"2","key":"9121_CR85","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1109\/MS.2006.58","volume":"23","author":"M Shaw","year":"2006","unstructured":"Shaw M, Clements P (2006) The golden age of software architecture. IEEE Softw 23(2):31\u201339","journal-title":"IEEE Softw"},{"key":"9121_CR86","volume-title":"Software architecture: perspectives on an emerging discipline","author":"M Shaw","year":"1996","unstructured":"Shaw M, Garlan D (1996) Software architecture: perspectives on an emerging discipline. Prentice-Hall, Upper Saddle River"},{"key":"9121_CR87","unstructured":"Shull F, Basili V, Carver J, Maldonado J, Travassos G, Mendon M, Fabbri S (2002) Replicating software engineering experiments: Addressing the tacit knowledge problem. Proceedings of the 2002 International Symposium on Empirical Software Engineering. IEEE Computer Society"},{"key":"9121_CR88","doi-asserted-by":"crossref","unstructured":"Shull F, Carver J, Hochstein L, Basili V (2005) Empirical study design in the area of high-performance computing (hpc). International Symposium on Empirical Software Engineering, 2005","DOI":"10.1109\/ISESE.2005.1541839"},{"issue":"1","key":"9121_CR89","doi-asserted-by":"crossref","first-page":"16","DOI":"10.1109\/MS.2006.33","volume":"23","author":"F Shull","year":"2006","unstructured":"Shull F, Seaman C, Zelkowitz M (2006) Victor R. Basili\u2019s contributions to software quality. IEEE Softw 23(1):16\u201318","journal-title":"IEEE Softw"},{"key":"9121_CR90","volume-title":"The sciences of the artificial","author":"H Simon","year":"1996","unstructured":"Simon H (1996) The sciences of the artificial, 3rd edn. The MIT, Cambridge","edition":"3"},{"key":"9121_CR91","volume-title":"Conducting experiments on software evolution. Proceedings of the 4th International Workshop on Principles of Software Evolution","author":"DIK Sj\u00f8berg","year":"2001","unstructured":"Sj\u00f8berg DIK, Arisholm E, J\u00f8rgensen M (2001) Conducting experiments on software evolution. Proceedings of the 4th International Workshop on Principles of Software Evolution. ACM, Vienna"},{"key":"9121_CR92","doi-asserted-by":"crossref","unstructured":"Sj\u00f8berg DIK, Anda B, Arisholm E, Tore D, J\u00f8rgensen M, Karahasanovic A, Koren E, Marek V (2002) Conducting realistic experiments in software engineering. Proceedings of the 2002 International Symposium on Empirical Software Engineering. IEEE Computer Society","DOI":"10.1109\/ISESE.2002.1166921"},{"key":"9121_CR93","doi-asserted-by":"crossref","unstructured":"Sj\u00f8berg D, Anda B, Arisholm E, Dyb\u00e5 T, J\u00f8rgensen M, Karahasanovi\u0107 A, Vok\u00e1c M (2003) Challenges and recommendations when increasing the realism of controlled software engineering experiments. In: Empirical methods and studies in software engineering","DOI":"10.1007\/978-3-540-45143-3_3"},{"key":"9121_CR94","volume-title":"The future of empirical methods in software engineering research. Future of Software Engineering, Proceedings of the 29th International Conference on Software Engineering (ICSE 2007)","author":"DIK Sj\u00f8berg","year":"2007","unstructured":"Sj\u00f8berg DIK, Dyb\u00e5 T, J\u00f8rgensen M (2007) The future of empirical methods in software engineering research. Future of Software Engineering, Proceedings of the 29th International Conference on Software Engineering (ICSE 2007). IEEE Computer Society, Minneapolis"},{"key":"9121_CR95","doi-asserted-by":"crossref","unstructured":"Sj\u00f8berg DIK, Dyb\u00e5 T, Anda BCD, Hannay JE (2008) Building theories in software engineering. In: Guide to advanced empirical software engineering","DOI":"10.1007\/978-1-84800-044-5_12"},{"key":"9121_CR96","doi-asserted-by":"crossref","unstructured":"Smolander K (2002) Four metaphors of architecture in software organizations: Finding out the meaning of architecture in practice. International Symposium on Empirical Software Engineering (ISESE 2002). Nara, Japan","DOI":"10.1109\/ISESE.2002.1166942"},{"issue":"9","key":"9121_CR97","doi-asserted-by":"crossref","first-page":"1425","DOI":"10.1016\/j.jss.2006.09.046","volume":"80","author":"M Staples","year":"2007","unstructured":"Staples M, Niazi M (2007) Experiences using systematic review guidelines. J Syst Softw 80(9):1425\u20131437","journal-title":"J Syst Softw"},{"key":"9121_CR98","volume-title":"Using students as subjects\u2014an empirical evaluation. Proceedings of the Second ACM-IEEE International Symposium on Empirical Software Engineering and Measurement","author":"M Svahnberg","year":"2008","unstructured":"Svahnberg M, Aurum A, Wohlin C (2008) Using students as subjects\u2014an empirical evaluation. Proceedings of the Second ACM-IEEE International Symposium on Empirical Software Engineering and Measurement. ACM, Kaiserslautern"},{"issue":"12","key":"9121_CR99","doi-asserted-by":"crossref","first-page":"1792","DOI":"10.1016\/j.jss.2006.04.029","volume":"79","author":"A Tang","year":"2007","unstructured":"Tang A, Ali Babar M, Gorton I, Han J (2007) A survey of architecture design rationale. J Syst Softw 79(12):1792\u20131804","journal-title":"J Syst Softw"},{"issue":"5","key":"9121_CR100","doi-asserted-by":"crossref","first-page":"551","DOI":"10.1007\/s10664-007-9037-5","volume":"12","author":"P Tonella","year":"2007","unstructured":"Tonella P, Torchiano M, Bois B, Syst T (2007) Empirical studies in reverse engineering: state of the art and future trends. Empir Software Eng 12(5):551\u2013571","journal-title":"Empir Software Eng"},{"issue":"2","key":"9121_CR101","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1109\/MS.2005.27","volume":"22","author":"J Tyree","year":"2005","unstructured":"Tyree J, Akerman A (2005) Architecture decisions: demystifying architecture. IEEE Softw 22(2):19\u201327","journal-title":"IEEE Softw"},{"key":"9121_CR102","doi-asserted-by":"crossref","first-page":"437","DOI":"10.1007\/s10664-005-3862-1","volume":"10","author":"S Vegas","year":"2005","unstructured":"Vegas S, Basili V (2005) A characterisation schema for software testing techniques. Empir Software Eng 10:437\u2013466","journal-title":"Empir Software Eng"},{"key":"9121_CR103","volume-title":"Analysis of the influence of communication between researchers on experiment replication. Proceedings of the 2006 ACM\/IEEE international symposium on Empirical software engineering","author":"S Vegas","year":"2006","unstructured":"Vegas S, Juristo N, Moreno A, Solari M, Letelier P (2006) Analysis of the influence of communication between researchers on experiment replication. Proceedings of the 2006 ACM\/IEEE international symposium on Empirical software engineering. ACM, Rio de Janeiro"},{"issue":"3","key":"9121_CR104","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1023\/B:EMSE.0000027778.69251.1f","volume":"9","author":"M Vokac","year":"2004","unstructured":"Vokac M, Tichy W, Sjoberg D, Arisholm E, Aldrin M (2004) A controlled experiment comparing the maintainability of programs designed with and without design patterns; a replication in a real programming environment. Empir Software Eng: An Int J 9(3):149\u2013195","journal-title":"Empir Software Eng: An Int J"},{"key":"9121_CR105","volume-title":"In support of student pair-programming. Proceedings of the 32nd SIGCSE Technical Symposium on Computer Science Education","author":"L Williams","year":"2001","unstructured":"Williams L, Upchurch RL (2001) In support of student pair-programming. Proceedings of the 32nd SIGCSE Technical Symposium on Computer Science Education. ACM, Charlotte"},{"key":"9121_CR106","doi-asserted-by":"crossref","unstructured":"Wohlin C, Runeson P, H\u00f6st M, Ohlsson MC, Regnell B, Wesslen A (2000) Experimentation in software engineering: An introduction. Springer","DOI":"10.1007\/978-1-4615-4625-2"},{"issue":"5","key":"9121_CR107","doi-asserted-by":"crossref","first-page":"23","DOI":"10.1109\/2.675630","volume":"31","author":"MV Zelkowitz","year":"1998","unstructured":"Zelkowitz MV, Wallace DR (1998) Experimental models for validating technology. Comput 31(5):23\u201331","journal-title":"Comput"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-009-9121-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-009-9121-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-009-9121-0","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,24]],"date-time":"2020-05-24T02:08:11Z","timestamp":1590286091000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-009-9121-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009,12,5]]},"references-count":107,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2010,6]]}},"alternative-id":["9121"],"URL":"https:\/\/doi.org\/10.1007\/s10664-009-9121-0","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2009,12,5]]}}}