{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T12:16:24Z","timestamp":1778933784947,"version":"3.51.4"},"reference-count":34,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2021,7,17]],"date-time":"2021-07-17T00:00:00Z","timestamp":1626480000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,7,17]],"date-time":"2021-07-17T00:00:00Z","timestamp":1626480000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2021,9]]},"DOI":"10.1007\/s10664-021-10005-5","type":"journal-article","created":{"date-parts":[[2021,7,17]],"date-time":"2021-07-17T10:02:28Z","timestamp":1626516148000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Understanding peer review of software engineering papers"],"prefix":"10.1007","volume":"26","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5992-2366","authenticated-orcid":false,"given":"Neil A.","family":"Ernst","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7824-9151","authenticated-orcid":false,"given":"Jeffrey C.","family":"Carver","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0619-6027","authenticated-orcid":false,"given":"Daniel","family":"Mendez","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5328-368X","authenticated-orcid":false,"given":"Marco","family":"Torchiano","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,7,17]]},"reference":[{"issue":"5","key":"10005_CR1","doi-asserted-by":"publisher","first-page":"525","DOI":"10.1091\/mbc.e11-01-0002","volume":"22","author":"DG Drubin","year":"2011","unstructured":"Drubin DG (2011) Any jackass can trash a manuscript, but it takes good scholarship to create one (how MBoC promotes civil and constructive peer review). Mol Biol Cell 22(5):525\u2013527. https:\/\/doi.org\/10.1091\/mbc.e11-01-0002","journal-title":"Mol Biol Cell"},{"issue":"1","key":"10005_CR2","doi-asserted-by":"publisher","first-page":"257","DOI":"10.1007\/s11192-014-1331-6","volume":"101","author":"JM Fernandes","year":"2014","unstructured":"Fernandes JM (2014) Authorship trends in software engineering. Scientometrics 101(1):257\u2013271. https:\/\/doi.org\/10.1007\/s11192-014-1331-6","journal-title":"Scientometrics"},{"issue":"1","key":"10005_CR3","doi-asserted-by":"publisher","first-page":"339","DOI":"10.1007\/s11192-018-2969-2","volume":"118","author":"SPJM Horbach","year":"2018","unstructured":"Horbach SPJM, Halffman W (2018) The ability of different peer review procedures to ag problematic publications. Scientometrics 118(1):339\u2013373. https:\/\/doi.org\/10.1007\/s11192-018-2969-2","journal-title":"Scientometrics"},{"issue":"5","key":"10005_CR4","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1145\/571681.571686","volume":"27","author":"B Kitchenham","year":"2002","unstructured":"Kitchenham B, Peeger SL (2002) Principles of survey research: part 1-6. ACM SIGSOFT Softw Eng Notes 27(5):17\u201320","journal-title":"ACM SIGSOFT Softw Eng Notes"},{"key":"10005_CR5","doi-asserted-by":"publisher","unstructured":"MacAuley D (2012) The role of the manuscript assessor. In: How to write a paper, chap. 16. https:\/\/doi.org\/10.1002\/9781118488713.ch16. Wiley, pp 102\u2013114","DOI":"10.1002\/9781118488713.ch16"},{"key":"10005_CR6","doi-asserted-by":"publisher","unstructured":"Mathew G, Agrawal A, Menzies T (2019) Finding trends in software research. IEEE Trans Softw Eng :1\u20131. https:\/\/doi.org\/10.1109\/tse.2018.2870388","DOI":"10.1109\/tse.2018.2870388"},{"key":"10005_CR7","doi-asserted-by":"crossref","unstructured":"Mendez D, Graziotin D, Wagner S, Seibold H (2020) Open science in software engineering. In: Felderer M, Travassos G-H (eds) Contemporary empirical methods in software engineering. arXiv:1908.05899. Springer","DOI":"10.1007\/978-3-030-32489-6_17"},{"key":"10005_CR8","unstructured":"Nierstrasz O (1998) Identify the champion. Pattern Languages of Programming (PLoP). WUCS-98-25"},{"issue":"1","key":"10005_CR9","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1109\/MS.2020.3028681","volume":"38","author":"I Ozkaya","year":"2021","unstructured":"Ozkaya I (2021) Protecting the health and longevity of the peer-review process in the software engineering community. IEEE Softw 38(1):3\u20136. https:\/\/doi.org\/10.1109\/ms.2020.3028681","journal-title":"IEEE Softw"},{"key":"10005_CR10","doi-asserted-by":"publisher","unstructured":"Peterson DAM (2020) Dear reviewer 2: Go F\u2019 Yourself. Soc Sci Q. https:\/\/doi.org\/10.1111\/ssqu.12824","DOI":"10.1111\/ssqu.12824"},{"key":"10005_CR11","doi-asserted-by":"publisher","unstructured":"Petre M, et al. (2020) Reviewing computing education papers. In: Proceedings of the 2020 ACM conference on innovation and technology in computer science education. https:\/\/doi.org\/10.1145\/3341525.3394994. ACM","DOI":"10.1145\/3341525.3394994"},{"key":"10005_CR12","doi-asserted-by":"publisher","first-page":"75","DOI":"10.1016\/j.infsof.2017.10.019","volume":"95","author":"L Prechelt","year":"2018","unstructured":"Prechelt L, Graziotin D, Mendez D (2018) A community\u2019s perspective on the status and future of peer review in software engineering. Inf Softw Technol 95:75\u201385. https:\/\/doi.org\/10.1016\/j.infsof.2017.10.019","journal-title":"Inf Softw Technol"},{"key":"10005_CR13","unstructured":"Price E (2014) The NIPS Experiment. http:\/\/blog.mrtz.org\/2014\/12\/15\/thenipsexperiment.html. Online; accessed August 2020"},{"issue":"2","key":"10005_CR14","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1007\/s11192-013-1002-z","volume":"97","author":"A Ragone","year":"2013","unstructured":"Ragone A, Mirylenka K, Casati F, Marchese M (2013) On peer review in computer science: analysis of its effectiveness and suggestions for improvement. Scientometrics 97(2):317\u2013 356. https:\/\/doi.org\/10.1007\/s11192-013-1002-z","journal-title":"Scientometrics"},{"issue":"2","key":"10005_CR15","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1145\/3385678.3385681","volume":"45","author":"P Ralph","year":"2020","unstructured":"Ralph P, Robbes R (2020) The ACM SIGSOFT paper and peer review quality initiative. ACM SIGSOFT Softw Eng Notes 45(2):17\u201318. https:\/\/doi.org\/10.1145\/3385678.3385681","journal-title":"ACM SIGSOFT Softw Eng Notes"},{"key":"10005_CR16","volume-title":"Writing science: how to write papers that get cited and proposals that get funded","author":"J Schimel","year":"2011","unstructured":"Schimel J (2011) Writing science: how to write papers that get cited and proposals that get funded. Oxford University Press, Oxford"},{"issue":"1","key":"10005_CR17","doi-asserted-by":"publisher","first-page":"567","DOI":"10.1007\/s11192-017-2264-7","volume":"113","author":"M Seeber","year":"2017","unstructured":"Seeber M, Bacchelli A (2017) Does single blind peer review hinder newcomers?. Scientometrics 113(1):567\u2013585. https:\/\/doi.org\/10.1007\/s11192-017-2264-7","journal-title":"Scientometrics"},{"key":"10005_CR18","doi-asserted-by":"crossref","unstructured":"Shaw M (2003) Writing good software engineering research papers: minitutorial. In: Proceedings of the ACM\/IEEE international conference on software engineering, pp 726\u2013736","DOI":"10.1109\/ICSE.2003.1201262"},{"key":"10005_CR19","doi-asserted-by":"publisher","first-page":"120","DOI":"10.1016\/j.infsof.2018.01.006","volume":"99","author":"M Shepperd","year":"2018","unstructured":"Shepperd M, Ajienka N, Counsell S (2018) The role and value of replication in empirical software engineering results. Inf Softw Technol 99:120\u2013132. https:\/\/doi.org\/10.1016\/j.infsof.2018.01.006","journal-title":"Inf Softw Technol"},{"issue":"4","key":"10005_CR20","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1109\/2.55470","volume":"23","author":"AJ Smith","year":"1990","unstructured":"Smith AJ (1990) The task of the referee. Computer 23(4):65\u2013 71","journal-title":"Computer"},{"key":"10005_CR21","doi-asserted-by":"crossref","unstructured":"Smith E, Loftin R, Murphy-Hill E, Bird C, Zimmermann T (2013) Improving developer participation rates in surveys. In: CHASE workshop at ICSE","DOI":"10.1109\/CHASE.2013.6614738"},{"issue":"1","key":"10005_CR22","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1145\/3375572.3375575","volume":"45","author":"J Soldani","year":"2020","unstructured":"Soldani J, Kuhrmann M, Pfahl D (2020) Pains and gains of peer-reviewing in software engineering. ACM SIGSOFT Softw Eng Notes 45(1):12\u201313. https:\/\/doi.org\/10.1145\/3375572.3375575","journal-title":"ACM SIGSOFT Softw Eng Notes"},{"issue":"8","key":"10005_CR23","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1016\/S0167-7799(02)01985-6","volume":"20","author":"R Spier","year":"2002","unstructured":"Spier R (2002) The history of the peer-review process. Trends Biotechnol 20 (8):357\u2013358. https:\/\/doi.org\/10.1016\/s0167-7799(02)01985-6","journal-title":"Trends Biotechnol"},{"issue":"1","key":"10005_CR24","doi-asserted-by":"publisher","first-page":"501","DOI":"10.1007\/s11192-017-2518-4","volume":"113","author":"F Squazzoni","year":"2017","unstructured":"Squazzoni F, Brezis E, Maru\u0161i\u0107 A (2017) Scientometrics of peer review. Scientometrics 113(1):501\u2013502. https:\/\/doi.org\/10.1007\/s11192-017-2518-4.","journal-title":"Scientometrics"},{"key":"10005_CR25","doi-asserted-by":"crossref","unstructured":"Storey M-A, Ernst NA, Williams C, Kalliamvakou E (2019) The who, what, how of software engineering research: a socio-technical framework. arXiv:1905.1284","DOI":"10.1007\/s10664-020-09858-z"},{"key":"10005_CR26","doi-asserted-by":"publisher","unstructured":"Tennant JP, et al. (2017) A multi-disciplinary perspective on emergent and future innovations in peer review. In: F1000Research. https:\/\/doi.org\/10.12688\/f1000research.12037.3, vol 6, p 1151","DOI":"10.12688\/f1000research.12037.3"},{"key":"10005_CR27","doi-asserted-by":"publisher","first-page":"e111","DOI":"10.7717\/peerj-cs.111","volume":"3","author":"J Terrell","year":"2017","unstructured":"Terrell J, Kofink A, Middleton J, Rainear C, Murphy-Hill E, Parnin C, Stallings J (2017) Gender differences and bias in open source: pull request acceptance of women versus men. Peer J Comput Sci 3:e111. https:\/\/doi.org\/10.7717\/peerj-cs.111","journal-title":"Peer J Comput Sci"},{"key":"10005_CR28","doi-asserted-by":"publisher","unstructured":"Theisen C, Dunaiski M, Williams L, Visser W (2017) Writing good software engineering research papers: revisited. In: Proceedings of the ACM\/IEEE international conference on software engineering. https:\/\/doi.org\/10.1109\/icse-c.2017.51","DOI":"10.1109\/icse-c.2017.51"},{"issue":"48","key":"10005_CR29","doi-asserted-by":"publisher","first-page":"12708","DOI":"10.1073\/pnas.1707323114","volume":"114","author":"A Tomkins","year":"2017","unstructured":"Tomkins A, Zhang M, Heavlin WD (2017) Reviewer bias in single- versus double-blind peer review. Proc Natl Acad Sci 114(48):12708\u201312713. https:\/\/doi.org\/10.1073\/pnas.1707323114","journal-title":"Proc Natl Acad Sci"},{"issue":"3","key":"10005_CR30","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1145\/1168092.1168093","volume":"35","author":"AKH Tung","year":"2006","unstructured":"Tung AKH (2006) Impact of double blind reviewing on SIGMOD publication. ACM SIGMOD Record 35 (3):6\u20137. https:\/\/doi.org\/10.1145\/1168092.1168093","journal-title":"ACM SIGMOD Record"},{"key":"10005_CR31","doi-asserted-by":"crossref","unstructured":"Winslett M, Braganholo V (2016) H V Jagadish speaks out on PVLDB, CoRR and data-driven research. In: SIGMOD Record 42.2","DOI":"10.1145\/3003665.3003676"},{"issue":"2","key":"10005_CR32","doi-asserted-by":"publisher","first-page":"1033","DOI":"10.1007\/s11192-020-03488-4","volume":"125","author":"D Wolfram","year":"2020","unstructured":"Wolfram D, Wang P, Hembree A, Park H (2020) Open peer review: promoting transparency in open science. Scientometrics 125(2):1033\u20131051. https:\/\/doi.org\/10.1007\/s11192-020-03488-4","journal-title":"Scientometrics"},{"issue":"2","key":"10005_CR33","doi-asserted-by":"publisher","first-page":"250","DOI":"10.5465\/amr.1987.4307820","volume":"12","author":"WJ Zerbe","year":"1987","unstructured":"Zerbe WJ, Paulhus DL (1987) Socially desirable responding in organizational behavior: a reconception. Acad Manag Rev 12(2):250. https:\/\/doi.org\/10.2307\/258533","journal-title":"Acad Manag Rev"},{"issue":"1","key":"10005_CR34","doi-asserted-by":"publisher","first-page":"607","DOI":"10.1007\/s11192-020-03545-y","volume":"125","author":"Q Zong","year":"2020","unstructured":"Zong Q, Xie Y, Liang J (2020) Does open peer review improve citation count? Evidence from a propensity score matching analysis of PeerJ. Scientometrics 125 (1):607\u2013623. https:\/\/doi.org\/10.1007\/s11192-020-03545-y.","journal-title":"Scientometrics"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-021-10005-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-021-10005-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-021-10005-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,8,27]],"date-time":"2021-08-27T19:52:07Z","timestamp":1630093927000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-021-10005-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,17]]},"references-count":34,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2021,9]]}},"alternative-id":["10005"],"URL":"https:\/\/doi.org\/10.1007\/s10664-021-10005-5","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,7,17]]},"assertion":[{"value":"16 June 2021","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 July 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"103"}}