{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T22:56:33Z","timestamp":1773442593352,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2014,5,31]],"date-time":"2014-05-31T00:00:00Z","timestamp":1401494400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2014,5,31]]},"DOI":"10.1145\/2597073.2597082","type":"proceedings-article","created":{"date-parts":[[2014,5,20]],"date-time":"2014-05-20T13:48:00Z","timestamp":1400593680000},"page":"202-211","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":180,"title":["Modern code reviews in open-source projects: which problems do they fix?"],"prefix":"10.1145","author":[{"given":"Moritz","family":"Beller","sequence":"first","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"given":"Alberto","family":"Bacchelli","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"given":"Andy","family":"Zaidman","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"given":"Elmar","family":"Juergens","sequence":"additional","affiliation":[{"name":"CQSE, Germany"}]}],"member":"320","published-online":{"date-parts":[[2014,5,31]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"IEEE standard classification for software anomalies. IEEE Std 1044-2009 (Revision of IEEE Std 1044-1993) pages 1\u201323 2010.  IEEE standard classification for software anomalies. IEEE Std 1044-2009 (Revision of IEEE Std 1044-1993) pages 1\u201323 2010."},{"key":"e_1_3_2_1_2_1","volume-title":"Thomson Brooks \/Cole","author":"Abraham B.","year":"2006","unstructured":"B. Abraham and J. Ledolter . Introduction to regression modeling . Thomson Brooks \/Cole , 2006 . B. Abraham and J. Ledolter. Introduction to regression modeling. Thomson Brooks \/Cole, 2006."},{"key":"e_1_3_2_1_3_1","volume-title":"The hawthorne e ffect: A reconsideration of the methodological artifact. Journal of applied psychology, 69(2):334","author":"Adair J.","year":"1984","unstructured":"J. Adair . The hawthorne e ffect: A reconsideration of the methodological artifact. Journal of applied psychology, 69(2):334 , 1984 . J. Adair. The hawthorne e ffect: A reconsideration of the methodological artifact. Journal of applied psychology, 69(2):334, 1984."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2007.17"},{"key":"e_1_3_2_1_5_1","first-page":"721","volume-title":"Proceedings of the 2013 International Conference on Software Engineering","author":"Bacchelli A.","unstructured":"A. Bacchelli and C. Bird . Expectations, outcomes, and challenges of modern code review . In Proceedings of the 2013 International Conference on Software Engineering , pages 712\u2013 721 . IEEE Press, 2013. A. Bacchelli and C. Bird. Expectations, outcomes, and challenges of modern code review. In Proceedings of the 2013 International Conference on Software Engineering, pages 712\u2013721. IEEE Press, 2013."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/253228.253461"},{"key":"e_1_3_2_1_7_1","first-page":"131","volume-title":"Reverse Engineering (WCRE), 2013 20th Working Conference on","author":"Baysal O.","unstructured":"O. Baysal , O. Kononenko , R. Holmes , and M. Godfrey . The influence of non-technical factors on code review . In Reverse Engineering (WCRE), 2013 20th Working Conference on , pages 122\u2013 131 . IEEE, 2013. O. Baysal, O. Kononenko, R. Holmes, and M. Godfrey. The influence of non-technical factors on code review. In Reverse Engineering (WCRE), 2013 20th Working Conference on, pages 122\u2013131. IEEE, 2013."},{"key":"e_1_3_2_1_8_1","volume-title":"Technische Universit\u00e4t M\u00fcnchen, 2013","author":"Beller M.","year":"1861","unstructured":"M. Beller . Quantifying Continuous Code Reviews. Master\u2019s thesis , Technische Universit\u00e4t M\u00fcnchen, 2013 . http:\/\/dx.doi.org\/10.6084\/m9.figshare.9 1861 8. 10.6084\/m9.figshare.918618 M. Beller. Quantifying Continuous Code Reviews. Master\u2019s thesis, Technische Universit\u00e4t M\u00fcnchen, 2013. http:\/\/dx.doi.org\/10.6084\/m9.figshare.918618."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/AGILE.2010.18"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.5555\/823456.824010"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","DOI":"10.1007\/3-540-27662-9","volume-title":"Software defect reduction top 10 list. Foundations of empirical software engineering: the legacy of Victor R. Basili, page 426","author":"Boehm B.","year":"2005","unstructured":"B. Boehm and V. Basili . Software defect reduction top 10 list. Foundations of empirical software engineering: the legacy of Victor R. Basili, page 426 , 2005 . B. Boehm and V. Basili. Software defect reduction top 10 list. Foundations of empirical software engineering: the legacy of Victor R. Basili, page 426, 2005."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/966789.966803"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.177364"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/581339.581422"},{"key":"e_1_3_2_1_15_1","volume-title":"A coe \ufb03cient of agreement for nominal scales. Educational and psychological measurement, 20(1):37\u201346","author":"Cohen J.","year":"1960","unstructured":"J. Cohen . A coe \ufb03cient of agreement for nominal scales. Educational and psychological measurement, 20(1):37\u201346 , 1960 . J. Cohen. A coe \ufb03cient of agreement for nominal scales. Educational and psychological measurement, 20(1):37\u201346, 1960."},{"key":"e_1_3_2_1_16_1","first-page":"114","volume-title":"Prod. International Conf. on Dependable Systems and Networks (DSN)","author":"Duraes J.","unstructured":"J. Duraes and H. Madeira . Definition of software fault emulation operators: A field data study . In Prod. International Conf. on Dependable Systems and Networks (DSN) , pages 105\u2013 114 . IEEE, 2003. J. Duraes and H. Madeira. Definition of software fault emulation operators: A field data study. In Prod. International Conf. on Dependable Systems and Networks (DSN), pages 105\u2013114. IEEE, 2003."},{"key":"e_1_3_2_1_17_1","first-page":"554","volume-title":"Proceedings of the International Conf. on Software Engineering (ICSE)","author":"Ebert C.","unstructured":"C. Ebert , C. Parro , R. Suttels , and H. Kolarczyk . Improving validation activities in a global software development . In Proceedings of the International Conf. on Software Engineering (ICSE) , pages 545\u2013 554 . IEEE CS, 2001. C. Ebert, C. Parro, R. Suttels, and H. Kolarczyk. Improving validation activities in a global software development. In Proceedings of the International Conf. on Software Engineering (ICSE), pages 545\u2013554. IEEE CS, 2001."},{"key":"e_1_3_2_1_18_1","first-page":"333","volume-title":"Proceedings International Symp. Software Reliability Engineering (ISSRE)","author":"El Emam K.","unstructured":"K. El Emam and I. Wieczorek . The repeatability of code defect classifications . In Proceedings International Symp. Software Reliability Engineering (ISSRE) , pages 322\u2013 333 . IEEE, 1998. K. El Emam and I. Wieczorek. The repeatability of code defect classifications. In Proceedings International Symp. Software Reliability Engineering (ISSRE), pages 322\u2013333. IEEE, 1998."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1147\/sj.153.0182"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-34333-9_2"},{"key":"e_1_3_2_1_21_1","volume-title":"Statistical methods for rates and proportions","author":"Fleiss J.","year":"1981","unstructured":"J. Fleiss . Statistical methods for rates and proportions , 1981 . J. Fleiss. Statistical methods for rates and proportions, 1981."},{"key":"e_1_3_2_1_22_1","unstructured":"Gerrit. https:\/\/code.google.com\/p\/gerrit\/. Accessed 2014 \/01\/20.  Gerrit. https:\/\/code.google.com\/p\/gerrit\/. Accessed 2014 \/01\/20."},{"key":"e_1_3_2_1_23_1","unstructured":"GitHub. https:\/\/github.com\/. Accessed 2014 \/01\/14.  GitHub. https:\/\/github.com\/. Accessed 2014 \/01\/14."},{"key":"e_1_3_2_1_24_1","volume-title":"A guide to chi-squared testing","author":"Greenwood P.","year":"1996","unstructured":"P. Greenwood and M. Nikulin . A guide to chi-squared testing , volume 280 . Wiley-Interscience , 1996 . P. Greenwood and M. Nikulin. A guide to chi-squared testing, volume 280. Wiley-Interscience, 1996."},{"key":"e_1_3_2_1_25_1","first-page":"286","volume-title":"Proceedings of the Tenth International Workshop on Mining Software Repositories","author":"Guzzi A.","unstructured":"A. Guzzi , A. Bacchelli , M. Lanza , M. Pinzger , and A. v. Deursen . Communication in open source software development mailing lists . In Proceedings of the Tenth International Workshop on Mining Software Repositories , pages 277\u2013 286 . IEEE Press, 2013. A. Guzzi, A. Bacchelli, M. Lanza, M. Pinzger, and A. v. Deursen. Communication in open source software development mailing lists. In Proceedings of the Tenth International Workshop on Mining Software Repositories, pages 277\u2013286. IEEE Press, 2013."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/52.43047"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2008.100"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2009.27"},{"key":"e_1_3_2_1_29_1","unstructured":"N. Kennedy. Google Mondrian: web-based code review and storage. http:\/\/www.niallkennedy.com\/blog\/2006\/11\/ google-mondrian.html 2006. Accessed 2013 \/12\/16.  N. Kennedy. Google Mondrian: web-based code review and storage. http:\/\/www.niallkennedy.com\/blog\/2006\/11\/ google-mondrian.html 2006. Accessed 2013 \/12\/16."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.2174\/1874107X00903010015"},{"key":"e_1_3_2_1_31_1","unstructured":"C. Krebs. Ecological methodology volume 620. Benjamin \/Cummings Menlo Park California 1999.  C. Krebs. Ecological methodology volume 620. Benjamin \/Cummings Menlo Park California 1999."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2008.71"},{"key":"e_1_3_2_1_33_1","volume-title":"principles, patterns, and practices","author":"Martin R.","year":"2003","unstructured":"R. Martin . Agile software development : principles, patterns, and practices . Prentice Hall PTR , 2003 . R. Martin. Agile software development: principles, patterns, and practices. Prentice Hall PTR, 2003."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/1378727.1378744"},{"key":"e_1_3_2_1_35_1","volume-title":"Learning Gerrit Code Review","author":"Milanesio L.","year":"2013","unstructured":"L. Milanesio . Learning Gerrit Code Review . Packt Publishing Ltd , 2013 . L. Milanesio. Learning Gerrit Code Review. Packt Publishing Ltd, 2013."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1009735805377"},{"key":"e_1_3_2_1_37_1","volume-title":"Cleanroom software engineering","author":"Mills H.","year":"1987","unstructured":"H. Mills , M. Dyer , and R. Linger . Cleanroom software engineering . 1987 . H. Mills, M. Dyer, and R. Linger. Cleanroom software engineering. 1987."},{"key":"e_1_3_2_1_38_1","first-page":"31","volume-title":"Software Maintenance, 1998. Proceedings. International Conference on","author":"Munson J.","unstructured":"J. Munson and S. Elbaum . Code churn: A measure for estimating the impact of code change . In Software Maintenance, 1998. Proceedings. International Conference on , pages 24\u2013 31 . IEEE, 1998. J. Munson and S. Elbaum. Code churn: A measure for estimating the impact of code change. In Software Maintenance, 1998. Proceedings. International Conference on, pages 24\u201331. IEEE, 1998."},{"key":"e_1_3_2_1_39_1","unstructured":"Mylyn Reviews. http:\/\/www.eclipse.org\/reviews\/. Accessed 2013 \/12\/16.  Mylyn Reviews. http:\/\/www.eclipse.org\/reviews\/. Accessed 2013 \/12\/16."},{"key":"e_1_3_2_1_40_1","first-page":"347","volume-title":"IEEE Conference on Software Maintenance, Reengineering, and Reverse Engineering (CSMR-WCRE)","author":"Osman H.","unstructured":"H. Osman , M. Lungu , and O. Nierstrasz . Mining frequent bug-fix code changes . In IEEE Conference on Software Maintenance, Reengineering, and Reverse Engineering (CSMR-WCRE) , pages 343\u2013 347 . IEEE CS, 2014. H. Osman, M. Lungu, and O. Nierstrasz. Mining frequent bug-fix code changes. In IEEE Conference on Software Maintenance, Reengineering, and Reverse Engineering (CSMR-WCRE), pages 343\u2013347. IEEE CS, 2014."},{"key":"e_1_3_2_1_41_1","unstructured":"Phabricator. http:\/\/phabricator.org\/. Accessed 2014 \/04.  Phabricator. http:\/\/phabricator.org\/. Accessed 2014 \/04."},{"key":"e_1_3_2_1_42_1","first-page":"112","volume-title":"Proceedings International Conf. on Software Engineering (ICSE)","author":"Porter A.","unstructured":"A. Porter and L. Votta . An experiment to assess di fferent defect detection methods for software requirements inspections . In Proceedings International Conf. on Software Engineering (ICSE) , pages 103\u2013 112 . IEEE CS, 1994. A. Porter and L. Votta. An experiment to assess di fferent defect detection methods for software requirements inspections. In Proceedings International Conf. on Software Engineering (ICSE), pages 103\u2013112. IEEE CS, 1994."},{"key":"e_1_3_2_1_43_1","first-page":"21","volume-title":"Java for Embedded Systems Workshop","volume":"1","author":"Ritzau T.","year":"2000","unstructured":"T. Ritzau and J. Andersson . Dynamic deployment of java applications . In Java for Embedded Systems Workshop , volume 1 , page 21 , 2000 . T. Ritzau and J. Andersson. Dynamic deployment of java applications. In Java for Embedded Systems Workshop, volume 1, page 21, 2000."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2006.89"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2004.03.004"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.825763"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/1117309.1117357"},{"issue":"5","key":"e_1_3_2_1_48_1","first-page":"360","article-title":"Understanding interobserver agreement: the kappa statistic","volume":"37","author":"Viera A.","year":"2005","unstructured":"A. Viera and J. Garrett . Understanding interobserver agreement: the kappa statistic . Fam Med , 37 ( 5 ): 360 \u2013 363 , 2005 . A. Viera and J. Garrett. Understanding interobserver agreement: the kappa statistic. Fam Med, 37(5):360\u2013363, 2005.","journal-title":"Fam Med"},{"key":"e_1_3_2_1_49_1","volume-title":"Whenever i apply a change to the eclipse platform project without additional review, i feel a bit scared. https: \/\/twitter.com\/vogella\/status\/423799752833003521","author":"Vogel L.","year":"2014","unstructured":"L. Vogel . Whenever i apply a change to the eclipse platform project without additional review, i feel a bit scared. https: \/\/twitter.com\/vogella\/status\/423799752833003521 , 2014 . L. Vogel. Whenever i apply a change to the eclipse platform project without additional review, i feel a bit scared. https: \/\/twitter.com\/vogella\/status\/423799752833003521, 2014."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/167049.167070"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390817.1390829"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1007\/11430230_4"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352322.1352171"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/267895.267915"},{"key":"e_1_3_2_1_55_1","volume-title":"Experiments: planning, analysis, and optimization","author":"Wu C.","year":"2011","unstructured":"C. Wu and M. Hamada . Experiments: planning, analysis, and optimization , volume 552 . John Wiley & Sons , 2011 . C. Wu and M. Hamada. Experiments: planning, analysis, and optimization, volume 552. John Wiley & Sons, 2011."}],"event":{"name":"ICSE '14: 36th International Conference on Software Engineering","location":"Hyderabad India","acronym":"ICSE '14","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","TCSE IEEE Computer Society's Tech. Council on Software Engin."]},"container-title":["Proceedings of the 11th Working Conference on Mining Software Repositories"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2597073.2597082","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2597073.2597082","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T06:56:03Z","timestamp":1750229763000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2597073.2597082"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,5,31]]},"references-count":55,"alternative-id":["10.1145\/2597073.2597082","10.1145\/2597073"],"URL":"https:\/\/doi.org\/10.1145\/2597073.2597082","relation":{},"subject":[],"published":{"date-parts":[[2014,5,31]]},"assertion":[{"value":"2014-05-31","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}