{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T22:56:35Z","timestamp":1773442595791,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,6,21]],"date-time":"2021-06-21T00:00:00Z","timestamp":1624233600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,6,21]]},"DOI":"10.1145\/3463274.3463336","type":"proceedings-article","created":{"date-parts":[[2021,6,18]],"date-time":"2021-06-18T23:43:11Z","timestamp":1624059791000},"page":"181-190","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":11,"title":["Detection and Elimination of Systematic Labeling Bias in Code Reviewer Recommendation Systems"],"prefix":"10.1145","author":[{"given":"K. Ayberk","family":"Tecimer","sequence":"first","affiliation":[{"name":"Technical University of Munich, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Eray","family":"T\u00fcz\u00fcn","sequence":"additional","affiliation":[{"name":"Bilkent University, Turkey"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hamdi","family":"Dibeklioglu","sequence":"additional","affiliation":[{"name":"Bilkent University, Turkey"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hakan","family":"Erdogmus","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2021,6,21]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2019.00019"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606617"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2017.265100500"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.5555\/2486788.2486915"},{"key":"#cr-split#-e_1_3_2_1_5_1.1","doi-asserted-by":"crossref","unstructured":"Christian Bird Adrian Bachmann Eirik Aune John Duffy Abraham Bernstein Vladimir Filkov and Premkumar Devanbu. 2009. Fair and Balanced? Bias in Bug-Fix Datasets. 121-130. https:\/\/doi.org\/10.1145\/1595696.1595716 10.1145\/1595696.1595716","DOI":"10.1145\/1595696.1595716"},{"key":"#cr-split#-e_1_3_2_1_5_1.2","doi-asserted-by":"crossref","unstructured":"Christian Bird Adrian Bachmann Eirik Aune John Duffy Abraham Bernstein Vladimir Filkov and Premkumar Devanbu. 2009. Fair and Balanced? Bias in Bug-Fix Datasets. 121-130. https:\/\/doi.org\/10.1145\/1595696.1595716","DOI":"10.1145\/1595696.1595716"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2015.21"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPR.2014.756"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2597073.2597108"},{"key":"#cr-split#-e_1_3_2_1_9_1.1","doi-asserted-by":"crossref","unstructured":"Emre Do\u011fan Eray T\u00fcz\u00fcn K.\u00a0Ayberk Tecimer and H.\u00a0Altay G\u00fcvenir. 2019. Investigating the Validity of Ground Truth in Code Reviewer Recommendation Studies. In 2019 ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM). 1-6. https:\/\/doi.org\/10.1109\/ESEM.2019.8870190 10.1109\/ESEM.2019.8870190","DOI":"10.1109\/ESEM.2019.8870190"},{"key":"#cr-split#-e_1_3_2_1_9_1.2","doi-asserted-by":"crossref","unstructured":"Emre Do\u011fan Eray T\u00fcz\u00fcn K.\u00a0Ayberk Tecimer and H.\u00a0Altay G\u00fcvenir. 2019. Investigating the Validity of Ground Truth in Code Reviewer Recommendation Studies. In 2019 ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM). 1-6. https:\/\/doi.org\/10.1109\/ESEM.2019.8870190","DOI":"10.1109\/ESEM.2019.8870190"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1147\/sj.153.0182"},{"key":"#cr-split#-e_1_3_2_1_11_1.1","doi-asserted-by":"crossref","unstructured":"Miko\u0142aj Fejzer Piotr Przymus and Krzysztof Stencel. 2018. Profile based recommendation of code reviewers. Journal of Intelligent Information Systems(2018). https:\/\/doi.org\/10.1007\/s10844-017-0484-1 10.1007\/s10844-017-0484-1","DOI":"10.1007\/s10844-017-0484-1"},{"key":"#cr-split#-e_1_3_2_1_11_1.2","doi-asserted-by":"crossref","unstructured":"Miko\u0142aj Fejzer Piotr Przymus and Krzysztof Stencel. 2018. Profile based recommendation of code reviewers. Journal of Intelligent Information Systems(2018). https:\/\/doi.org\/10.1007\/s10844-017-0484-1","DOI":"10.1007\/s10844-017-0484-1"},{"key":"#cr-split#-e_1_3_2_1_12_1.1","doi-asserted-by":"crossref","unstructured":"Alberto Fern\u00e1ndez Salvador Garc\u00eda Mikel Galar Ronaldo Prati Bartosz Krawczyk and Francisco Herrera. 2018. Learning from Imbalanced Data Sets. https:\/\/doi.org\/10.1007\/978-3-319-98074-4 10.1007\/978-3-319-98074-4","DOI":"10.1007\/978-3-319-98074-4"},{"key":"#cr-split#-e_1_3_2_1_12_1.2","doi-asserted-by":"crossref","unstructured":"Alberto Fern\u00e1ndez Salvador Garc\u00eda Mikel Galar Ronaldo Prati Bartosz Krawczyk and Francisco Herrera. 2018. Learning from Imbalanced Data Sets. https:\/\/doi.org\/10.1007\/978-3-319-98074-4","DOI":"10.1007\/978-3-319-98074-4"},{"key":"e_1_3_2_1_13_1","unstructured":"Google. 2020. Code Review Developer Guide. https:\/\/github.com\/google\/eng-practices\/blob\/master\/review\/index.md Google. 2020. Code Review Developer Guide. https:\/\/github.com\/google\/eng-practices\/blob\/master\/review\/index.md"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.5555\/2487085.2487113"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11390-015-1577-3"},{"key":"#cr-split#-e_1_3_2_1_16_1.1","doi-asserted-by":"crossref","unstructured":"Vladimir Kovalenko Nava Tintarev Evgeny Pasynkov Christian Bird and Alberto Bacchelli. 2018. Does reviewer recommendation help developers?IEEE Transactions on Software Engineering(2018) 1. https:\/\/doi.org\/10.1109\/TSE.2018.2868367 10.1109\/TSE.2018.2868367","DOI":"10.1109\/TSE.2018.2868367"},{"key":"#cr-split#-e_1_3_2_1_16_1.2","doi-asserted-by":"crossref","unstructured":"Vladimir Kovalenko Nava Tintarev Evgeny Pasynkov Christian Bird and Alberto Bacchelli. 2018. Does reviewer recommendation help developers?IEEE Transactions on Software Engineering(2018) 1. https:\/\/doi.org\/10.1109\/TSE.2018.2868367","DOI":"10.1109\/TSE.2018.2868367"},{"key":"e_1_3_2_1_17_1","volume-title":"Patch Reviewer Recommendation in OSS Projects. 2013 20th Asia-Pacific Software Engineering Conference (APSEC) 2 (2013","author":"Lee John\u00a0Boaz","year":"2013","unstructured":"John\u00a0Boaz Lee , A. Ihara , A. Monden , and K. Matsumoto . 2013 . Patch Reviewer Recommendation in OSS Projects. 2013 20th Asia-Pacific Software Engineering Conference (APSEC) 2 (2013 ), 1\u20136. https:\/\/doi.org\/10.1109\/APSEC. 2013 .103 10.1109\/APSEC.2013.103 John\u00a0Boaz Lee, A. Ihara, A. Monden, and K. Matsumoto. 2013. Patch Reviewer Recommendation in OSS Projects. 2013 20th Asia-Pacific Software Engineering Conference (APSEC) 2 (2013), 1\u20136. https:\/\/doi.org\/10.1109\/APSEC.2013.103"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA.2018.00068"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2877759"},{"key":"e_1_3_2_1_20_1","volume-title":"A Case Study of Bias in Bug-Fix Datasets","author":"Nguyen Thanh H\u00a0D","year":"2014","unstructured":"Thanh H\u00a0D Nguyen , Bram Adams , and Ahmed\u00a0 E Hassan . 2010. A Case Study of Bias in Bug-Fix Datasets . April 2014 (2010). https:\/\/doi.org\/10.1109\/WCRE.2010.37 10.1109\/WCRE.2010.37 Thanh H\u00a0D Nguyen, Bram Adams, and Ahmed\u00a0E Hassan. 2010. A Case Study of Bias in Bug-Fix Datasets. April 2014 (2010). https:\/\/doi.org\/10.1109\/WCRE.2010.37"},{"key":"e_1_3_2_1_21_1","volume-title":"Search-Based Peer Reviewers Recommendation in Modern Code Review. 2016 IEEE International Conference on Software Maintenance and Evolution (ICSME) (2016","author":"Ouni Ali","year":"2016","unstructured":"Ali Ouni , Raula\u00a0Gaikovina Kula , and Katsuro Inoue . 2016 . Search-Based Peer Reviewers Recommendation in Modern Code Review. 2016 IEEE International Conference on Software Maintenance and Evolution (ICSME) (2016 ), 367\u2013377. https:\/\/doi.org\/10.1109\/ICSME.2016.65 10.1109\/ICSME.2016.65 Ali Ouni, Raula\u00a0Gaikovina Kula, and Katsuro Inoue. 2016. Search-Based Peer Reviewers Recommendation in Modern Code Review. 2016 IEEE International Conference on Software Maintenance and Evolution (ICSME) (2016), 367\u2013377. https:\/\/doi.org\/10.1109\/ICSME.2016.65"},{"key":"e_1_3_2_1_22_1","series-title":"Lecture Notes in Business Information Processing, Vol.\u00a093. 92\u2013105. https:\/\/doi.org\/10.1007\/978-3-642-25676-9_8","volume-title":"Toward a Theory of Debiasing Software Development","author":"Ralph Paul","unstructured":"Paul Ralph . 2010. Toward a Theory of Debiasing Software Development . In Lecture Notes in Business Information Processing, Vol.\u00a093. 92\u2013105. https:\/\/doi.org\/10.1007\/978-3-642-25676-9_8 10.1007\/978-3-642-25676-9_8 Paul Ralph. 2010. Toward a Theory of Debiasing Software Development. In Lecture Notes in Business Information Processing, Vol.\u00a093. 92\u2013105. https:\/\/doi.org\/10.1007\/978-3-642-25676-9_8"},{"key":"e_1_3_2_1_23_1","volume-title":"The SEOSS 33 dataset \u2014 Requirements, bug reports, code history, and trace links for entire projects. Data in Brief","author":"Rath Michael","year":"2019","unstructured":"Michael Rath and Patrick M\u00e4der . 2019. The SEOSS 33 dataset \u2014 Requirements, bug reports, code history, and trace links for entire projects. Data in Brief ( 2019 ). https:\/\/doi.org\/10.1016\/j.dib.2019.104005 10.1016\/j.dib.2019.104005 Michael Rath and Patrick M\u00e4der. 2019. The SEOSS 33 dataset \u2014 Requirements, bug reports, code history, and trace links for entire projects. Data in Brief (2019). https:\/\/doi.org\/10.1016\/j.dib.2019.104005"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9102-8"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3183519.3183525"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","first-page":"130","DOI":"10.1002\/sys.20138","article-title":"Attribute Substitution in Systems Engineering","volume":"13","author":"Smith Eric","year":"2009","unstructured":"Eric Smith and A\u00a0Terry Bahill . 2009 . Attribute Substitution in Systems Engineering . Systems Engineering 13 (2009), 130 \u2013 148 . https:\/\/doi.org\/10.1002\/sys.20138 10.1002\/sys.20138 Eric Smith and A\u00a0Terry Bahill. 2009. Attribute Substitution in Systems Engineering. Systems Engineering 13(2009), 130\u2013148. https:\/\/doi.org\/10.1002\/sys.20138","journal-title":"Systems Engineering"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of COLING 2014, the 25th International Conference on Computational Linguistics: Tutorial Abstracts.","author":"S\u00f8gaard Anders","year":"2014","unstructured":"Anders S\u00f8gaard , Barbara Plank , and Dirk Hovy . 2014 . Selection Bias, Label Bias, and Bias in Ground Truth . In Proceedings of COLING 2014, the 25th International Conference on Computational Linguistics: Tutorial Abstracts. Anders S\u00f8gaard, Barbara Plank, and Dirk Hovy. 2014. Selection Bias, Label Bias, and Bias in Ground Truth. In Proceedings of COLING 2014, the 25th International Conference on Computational Linguistics: Tutorial Abstracts."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/203241.203256"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3345629.3345637"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106455"},{"key":"e_1_3_2_1_31_1","volume-title":"Who Should Review My Code ?2015 IEEE 22nd International Conference on Software Analysis, Evolution, and Reengineering (SANER)(2015), 141\u2013150. https:\/\/doi.org\/10.1109\/SANER.2015.7081824","author":"Thongtanunam Patanamon","unstructured":"Patanamon Thongtanunam , Chakkrit Tantithamthavorn , Raula\u00a0Gaikovina Kula , Norihiro Yoshida , Hajimu Iida , and Ken-ichi Matsumoto. 2015. Who Should Review My Code ?2015 IEEE 22nd International Conference on Software Analysis, Evolution, and Reengineering (SANER)(2015), 141\u2013150. https:\/\/doi.org\/10.1109\/SANER.2015.7081824 10.1109\/SANER.2015.7081824 Patanamon Thongtanunam, Chakkrit Tantithamthavorn, Raula\u00a0Gaikovina Kula, Norihiro Yoshida, Hajimu Iida, and Ken-ichi Matsumoto. 2015. Who Should Review My Code ?2015 IEEE 22nd International Conference on Software Analysis, Evolution, and Reengineering (SANER)(2015), 141\u2013150. https:\/\/doi.org\/10.1109\/SANER.2015.7081824"},{"key":"#cr-split#-e_1_3_2_1_32_1.1","doi-asserted-by":"crossref","unstructured":"Xin Xia David Lo Emad Shihab Xinyu Wang and Bo Zhou. 2014. Automatic high accuracy prediction of reopened bugs. Automated Software Engineering(2014). https:\/\/doi.org\/10.1007\/s10515-014-0162-2 10.1007\/s10515-014-0162-2","DOI":"10.1007\/s10515-014-0162-2"},{"key":"#cr-split#-e_1_3_2_1_32_1.2","doi-asserted-by":"crossref","unstructured":"Xin Xia David Lo Emad Shihab Xinyu Wang and Bo Zhou. 2014. Automatic high accuracy prediction of reopened bugs. Automated Software Engineering(2014). https:\/\/doi.org\/10.1007\/s10515-014-0162-2","DOI":"10.1007\/s10515-014-0162-2"},{"key":"e_1_3_2_1_33_1","volume-title":"Who Should Review This Change ?2015 IEEE International Conference on Software Maintenance and Evolution (ICSME)","author":"Xia Xin","year":"2015","unstructured":"Xin Xia , David Lo , Xinyu Wang , and Xiaohu Yang . 2015. Who Should Review This Change ?2015 IEEE International Conference on Software Maintenance and Evolution (ICSME) ( 2015 ), 261\u2013270. https:\/\/doi.org\/10.1109\/ICSM.2015.7332472 10.1109\/ICSM.2015.7332472 Xin Xia, David Lo, Xinyu Wang, and Xiaohu Yang. 2015. Who Should Review This Change ?2015 IEEE International Conference on Software Maintenance and Evolution (ICSME) (2015), 261\u2013270. https:\/\/doi.org\/10.1109\/ICSM.2015.7332472"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/SOFTWAREMINING.2017.8100850"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2500238"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.scico.2021.102652"}],"event":{"name":"EASE 2021: Evaluation and Assessment in Software Engineering","location":"Trondheim Norway","acronym":"EASE 2021"},"container-title":["Evaluation and Assessment in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3463274.3463336","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3463274.3463336","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:31:27Z","timestamp":1750195887000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3463274.3463336"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6,21]]},"references-count":42,"alternative-id":["10.1145\/3463274.3463336","10.1145\/3463274"],"URL":"https:\/\/doi.org\/10.1145\/3463274.3463336","relation":{},"subject":[],"published":{"date-parts":[[2021,6,21]]},"assertion":[{"value":"2021-06-21","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}