{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,14]],"date-time":"2026-04-14T02:06:46Z","timestamp":1776132406064,"version":"3.50.1"},"reference-count":51,"publisher":"Association for Computing Machinery (ACM)","issue":"6","license":[{"start":{"date-parts":[[2022,5,20]],"date-time":"2022-05-20T00:00:00Z","timestamp":1653004800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CAREER 1942124"],"award-info":[{"award-number":["CAREER 1942124"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Commun. ACM"],"published-print":{"date-parts":[[2022,6]]},"abstract":"<jats:p>Improving the peer review process in a scientific manner shows promise.<\/jats:p>","DOI":"10.1145\/3528086","type":"journal-article","created":{"date-parts":[[2022,5,20]],"date-time":"2022-05-20T14:56:52Z","timestamp":1653058612000},"page":"76-87","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":47,"title":["Challenges, experiments, and computational solutions in peer review"],"prefix":"10.1145","volume":"65","author":[{"given":"Nihar B.","family":"Shah","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA"}]}],"member":"320","published-online":{"date-parts":[[2022,5,20]]},"reference":[{"key":"e_1_2_1_1_1","volume-title":"Proceedings of Conf. on Theoretical Aspects of Rationality and Knowledge","author":"Alon N.","year":"2011","unstructured":"Alon , N. , Fischer , F. , Procaccia , A. , and Tennenholtz , M . Sum of us: Strategy proof selection from the selectors . In Proceedings of Conf. on Theoretical Aspects of Rationality and Knowledge , ( 2011 ). Alon, N., Fischer, F., Procaccia, A., and Tennenholtz, M. Sum of us: Strategy proof selection from the selectors. In Proceedings of Conf. on Theoretical Aspects of Rationality and Knowledge, (2011)."},{"key":"e_1_2_1_2_1","volume-title":"T. Conference reviewing considered harmful. ACM SIGOPS Operating Systems Rev.","author":"Anderson","year":"2009","unstructured":"Anderson , T. Conference reviewing considered harmful. ACM SIGOPS Operating Systems Rev. , ( 2009 ). Anderson, T. Conference reviewing considered harmful. ACM SIGOPS Operating Systems Rev., (2009)."},{"key":"e_1_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1603723113"},{"key":"e_1_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1152\/advan.00104.2006"},{"key":"e_1_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2005.02.002"},{"key":"e_1_2_1_6_1","volume-title":"Sense About Science","author":"Brown T.","year":"2004","unstructured":"Brown , T. Peer review and the acceptance of new scientific ideas: Discussion paper from a working party on equipping the public with an understanding of peer peview: November 2002--May 2004 . Sense About Science , ( 2004 ). Brown, T. Peer review and the acceptance of new scientific ideas: Discussion paper from a working party on equipping the public with an understanding of peer peview: November 2002--May 2004. Sense About Science, (2004)."},{"key":"e_1_2_1_7_1","author":"Cabanac G.","year":"2013","unstructured":"Cabanac , G. and Preuss , T. Capitalizing on order effects in the bids of peer-reviewed conferences to secure reviews by expert referees. J. Assoc. Information Science and Tech. ( 2013 ). Cabanac, G. and Preuss, T. Capitalizing on order effects in the bids of peer-reviewed conferences to secure reviews by expert referees. J. Assoc. Information Science and Tech. (2013).","journal-title":"J. Assoc. Information Science and Tech. ("},{"key":"e_1_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.annemergmed.2010.07.027"},{"key":"e_1_2_1_9_1","volume-title":"Proceedings of ICML Workshop on Peer Reviewing and Publishing Models","author":"Charlin L.","year":"2013","unstructured":"Charlin , L. and Zemel , R . The Toronto Paper Matching System: An automated paper-reviewer assignment system . In Proceedings of ICML Workshop on Peer Reviewing and Publishing Models , ( 2013 ). Charlin, L. and Zemel, R. The Toronto Paper Matching System: An automated paper-reviewer assignment system. In Proceedings of ICML Workshop on Peer Reviewing and Publishing Models, (2013)."},{"key":"e_1_2_1_10_1","volume-title":"Proceedings of Conf. Uncertainty in Artificial Intelligence","author":"Fiez T.","year":"2020","unstructured":"Fiez , T. , Shah , N. , and Ratliff , L . A SUPER* algorithm to optimize paper bidding in peer review . In Proceedings of Conf. Uncertainty in Artificial Intelligence , ( 2020 ). Fiez, T., Shah, N., and Ratliff, L. A SUPER* algorithm to optimize paper bidding in peer review. In Proceedings of Conf. Uncertainty in Artificial Intelligence, (2020)."},{"key":"e_1_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1809400.1809413"},{"key":"e_1_2_1_12_1","volume-title":"Machine Learning Research","author":"Freund Y.","year":"2003","unstructured":"Freund , Y. , Iyer , R. , Schapire , R. , and Singer , Y . An efficient boosting algorithm for combining preferences. J . Machine Learning Research , ( 2003 ). Freund, Y., Iyer, R., Schapire, R., and Singer, Y. An efficient boosting algorithm for combining preferences. J. Machine Learning Research, (2003)."},{"key":"e_1_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00453-009-9386-0"},{"key":"e_1_2_1_14_1","volume-title":"Proceedings of ICNC","author":"Guo L.","year":"2013","unstructured":"Guo , L. , Wu , J. , Chang , W. , Wu , J. , and Li , J . K-loop free assignment in conference review systems . In Proceedings of ICNC , ( 2013 ). Guo, L., Wu, J., Chang, W., Wu, J., and Li, J. K-loop free assignment in conference review systems. In Proceedings of ICNC, (2013)."},{"key":"e_1_2_1_15_1","volume-title":"Science","author":"Hand E.","year":"2016","unstructured":"Hand , E. No pressure : NSF test finds eliminating deadlines halves number of grant proposals . Science , ( 2016 ). Hand, E. No pressure: NSF test finds eliminating deadlines halves number of grant proposals. Science, (2016)."},{"key":"e_1_2_1_16_1","volume-title":"NeurIPS","author":"Jecmen S.","year":"2020","unstructured":"Jecmen , S. , Zhang , H. , Liu , R. , Shah , N. , Conitzer , V. , and Fang , F . Mitigating manipulation in peer review via randomized reviewer assignments . NeurIPS , ( 2020 ). Jecmen, S., Zhang, H., Liu, R., Shah, N., Conitzer, V., and Fang, F. Mitigating manipulation in peer review via randomized reviewer assignments. NeurIPS, (2020)."},{"key":"e_1_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.287.21.2784"},{"key":"e_1_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330899"},{"key":"e_1_2_1_19_1","volume-title":"NIH Extramural Nexus","author":"Lauer M.","year":"2020","unstructured":"Lauer , M. Case study in review integrity: Asking for favorable treatment . NIH Extramural Nexus , ( 2020 ). Lauer, M. Case study in review integrity: Asking for favorable treatment. NIH Extramural Nexus, (2020)."},{"key":"e_1_2_1_20_1","volume-title":"Commun. ACM","author":"Le Goues C.","year":"2013","unstructured":"Le Goues , C. , Brun , Y. , Apel , S. , Berger , E. , Khurshid , S. , and Smaragdakis , Y . Effectiveness of anonymization in double-blind review . Commun. ACM , ( 2013 ). Le Goues, C., Brun, Y., Apel, S., Berger, E., Khurshid, S., and Smaragdakis, Y. Effectiveness of anonymization in double-blind review. Commun. ACM, (2013)."},{"key":"e_1_2_1_21_1","volume-title":"Philosophy of Science","author":"Lee C.","year":"2015","unstructured":"Lee , C. Commensuration bias in peer review . Philosophy of Science , ( 2015 ). Lee, C. Commensuration bias in peer review. Philosophy of Science, (2015)."},{"key":"e_1_2_1_22_1","author":"Lee C.","year":"2013","unstructured":"Lee , C. , Sugimoto , C. , Zhang , G. , and Cronin , B. Bias in peer review. J. Assoc. Information Science and Technology, ( 2013 ). Lee, C., Sugimoto, C., Zhang, G., and Cronin, B. Bias in peer review. J. Assoc. Information Science and Technology, (2013).","journal-title":"J. Assoc. Information Science and Technology, ("},{"key":"e_1_2_1_23_1","unstructured":"Leyton-Brown K. and Mausam. AAAI 2021---Introduction; https:\/\/bit.ly\/3r2L3Rr; (min. 3 onward).  Leyton-Brown K. and Mausam. AAAI 2021---Introduction; https:\/\/bit.ly\/3r2L3Rr; (min. 3 onward)."},{"key":"e_1_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3429776"},{"key":"e_1_2_1_25_1","volume-title":"Publication prejudices: an experimental study of confirmatory bias in the peer review system. Cognitive Therapy and Research","author":"Mahoney M.","year":"1977","unstructured":"Mahoney , M. Publication prejudices: an experimental study of confirmatory bias in the peer review system. Cognitive Therapy and Research ( 1977 ). Mahoney, M. Publication prejudices: an experimental study of confirmatory bias in the peer review system. Cognitive Therapy and Research (1977)."},{"key":"e_1_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i6.16608"},{"key":"e_1_2_1_27_1","volume-title":"Proceedings of USENIX Security Symp.","author":"Markwood I.","year":"2017","unstructured":"Markwood , I. , Shen , D. , Liu , Y. , and Lu , Z . Mirage: Content masking attack against information-based online services . In Proceedings of USENIX Security Symp. , ( 2017 ). Markwood, I., Shen, D., Liu, Y., and Lu, Z. Mirage: Content masking attack against information-based online services. In Proceedings of USENIX Security Symp., (2017)."},{"key":"e_1_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-41575-3_20"},{"key":"e_1_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1281192.1281247"},{"key":"e_1_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/Allerton.2011.6120296"},{"key":"e_1_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1083-6101.2006.00025.x"},{"key":"e_1_2_1_32_1","unstructured":"Neubig G. Wieting J. McCarthy A. Stent A. Schluter N. and Cohn T. ACL reviewer matching code; https:\/\/github.com\/acl-org\/reviewerpaper-matching.  Neubig G. Wieting J. McCarthy A. Stent A. Schluter N. and Cohn T. ACL reviewer matching code; https:\/\/github.com\/acl-org\/reviewerpaper-matching."},{"key":"e_1_2_1_33_1","volume-title":"Artificial Intelligence Research","author":"Noothigattu R.","year":"2021","unstructured":"Noothigattu , R. , Shah , N. , and Procaccia , A . Loss functions, axioms, and peer review. J . Artificial Intelligence Research , ( 2021 ). Noothigattu, R., Shah, N., and Procaccia, A. Loss functions, axioms, and peer review. J. Artificial Intelligence Research, (2021)."},{"key":"e_1_2_1_34_1","volume-title":"Your comments are meaner than your score: Score calibration talk influences intra-and inter-panel variability during scientific grant peer review. Research Evaluation","author":"Pier E.","year":"2017","unstructured":"Pier , E. , Raclaw , J. , Kaatz , A. , Brauer , M. , Carnes , M. , Nathan , M. , and Ford , C . Your comments are meaner than your score: Score calibration talk influences intra-and inter-panel variability during scientific grant peer review. Research Evaluation ( 2017 ). Pier, E., Raclaw, J., Kaatz, A., Brauer, M., Carnes, M., Nathan, M., and Ford, C. Your comments are meaner than your score: Score calibration talk influences intra-and inter-panel variability during scientific grant peer review. Research Evaluation (2017)."},{"key":"e_1_2_1_35_1","volume-title":"Nature","author":"Rennie D.","year":"2016","unstructured":"Rennie , D. Let's make peer review scientific . Nature , ( 2016 ). Rennie, D. Let's make peer review scientific. Nature, (2016)."},{"key":"e_1_2_1_36_1","volume-title":"Mapping the bid behavior of conference referees. J. Informetrics","author":"Rodriguez M.","year":"2007","unstructured":"Rodriguez , M. , Bollen , J. , and Van de Sompel , H. Mapping the bid behavior of conference referees. J. Informetrics ( 2007 ). Rodriguez, M., Bollen, J., and Van de Sompel, H. Mapping the bid behavior of conference referees. J. Informetrics (2007)."},{"key":"e_1_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v25i1.7847"},{"key":"e_1_2_1_38_1","volume-title":"JMLR","author":"Shah N.","year":"2018","unstructured":"Shah , N. , Tabibian , B. , Muandet , K. , Guyon , I. , and Von Luxburg , U. Design and analysis of the NIPS 2016 review process . JMLR , ( 2018 ). Shah, N., Tabibian, B., Muandet, K., Guyon, I., and Von Luxburg, U. Design and analysis of the NIPS 2016 review process. JMLR, (2018)."},{"key":"e_1_2_1_39_1","volume-title":"Radiology","author":"Siegelman S.","year":"1991","unstructured":"Siegelman , S. Assassins and zealots: Variations in peer review . Radiology , ( 1991 ). Siegelman, S. Assassins and zealots: Variations in peer review. Radiology, (1991)."},{"key":"e_1_2_1_40_1","doi-asserted-by":"crossref","unstructured":"Smith R. Peer review: Reform or revolution? Time to open up the black box of peer review. (1997).  Smith R. Peer review: Reform or revolution? Time to open up the black box of peer review. (1997).","DOI":"10.1136\/bmj.315.7111.759"},{"key":"e_1_2_1_41_1","unstructured":"Soergel D. Saunders A. and McCallum A. Open scholarship and peer review: A time for experimentation (2013).  Soergel D. Saunders A. and McCallum A. Open scholarship and peer review: A time for experimentation (2013)."},{"key":"e_1_2_1_42_1","volume-title":"JMLR","author":"Stelmakh I.","year":"2021","unstructured":"Stelmakh , I. , Shah , N. , and Singh , A . PeerReview4All: Fair and accurate reviewer assignment in peer review . JMLR , ( 2021 ). Stelmakh, I., Shah, N., and Singh, A. PeerReview4All: Fair and accurate reviewer assignment in peer review. JMLR, (2021)."},{"key":"e_1_2_1_43_1","volume-title":"Experiments with the ICML 2020 peer-review process","author":"Stelmakh I.","unstructured":"Stelmakh , I. , Shah , N. , Singh , A. , Daum\u00e9 III, H. , and Rastogi , C . Experiments with the ICML 2020 peer-review process , (2020); https:\/\/blog.ml.cmu.edu\/2020\/12\/01\/icml2020exp\/ Stelmakh, I., Shah, N., Singh, A., Daum\u00e9 III, H., and Rastogi, C. Experiments with the ICML 2020 peer-review process, (2020); https:\/\/blog.ml.cmu.edu\/2020\/12\/01\/icml2020exp\/"},{"key":"e_1_2_1_44_1","unstructured":"Teplitskiy M. Ranub H. Grayb G. Meniettid M. Guinan E. and Lakhani K. Social influence among experts: Field experimental evidence from peer review (2019).  Teplitskiy M. Ranub H. Grayb G. Meniettid M. Guinan E. and Lakhani K. Social influence among experts: Field experimental evidence from peer review (2019)."},{"key":"e_1_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1707323114"},{"key":"e_1_2_1_46_1","unstructured":"Vijaykumar T. Potential organized fraud in ACM\/IEEE computer architecture conferences (2020); https:\/\/bit.ly\/3o2Zjb3.  Vijaykumar T. Potential organized fraud in ACM\/IEEE computer architecture conferences (2020); https:\/\/bit.ly\/3o2Zjb3."},{"key":"e_1_2_1_47_1","volume-title":"Proceedings of AAMAS","author":"Wang J.","year":"2019","unstructured":"Wang , J. and Shah , N . Your 2 is my 1, your 3 is my 9: Handling arbitrary miscalibrations in ratings . In Proceedings of AAMAS , ( 2019 ). Wang, J. and Shah, N. Your 2 is my 1, your 3 is my 9: Handling arbitrary miscalibrations in ratings. In Proceedings of AAMAS, (2019)."},{"key":"e_1_2_1_48_1","volume-title":"Publishing research consortium peer review survey","author":"Ware M.","year":"2015","unstructured":"Ware , M. Publishing research consortium peer review survey 2015 . Publishing Research Consortium , (2016). Ware, M. Publishing research consortium peer review survey 2015. Publishing Research Consortium, (2016)."},{"key":"e_1_2_1_49_1","volume-title":"Reviewing peer review. Communications of the ACM","author":"Wing J.","year":"2011","unstructured":"Wing , J. and Chi , E . Reviewing peer review. Communications of the ACM ( 2011 ). Wing, J. and Chi, E. Reviewing peer review. Communications of the ACM (2011)."},{"key":"e_1_2_1_50_1","unstructured":"Wu R. Guo C. Wu F. Kidambi R. van der Maaten L. and Weinberger K. Making paper reviewing robust to bid manipulation attacks (2021); arXiv:2102.06020.  Wu R. Guo C. Wu F. Kidambi R. van der Maaten L. and Weinberger K. Making paper reviewing robust to bid manipulation attacks (2021); arXiv:2102.06020."},{"key":"e_1_2_1_51_1","volume-title":"Proceedings of IJCAI","author":"Xu Y.","year":"2019","unstructured":"Xu , Y. , Zhao , H. , Shi , X. , and Shah , N . On strategyproof conference review . In Proceedings of IJCAI , ( 2019 ). Xu, Y., Zhao, H., Shi, X., and Shah, N. On strategyproof conference review. In Proceedings of IJCAI, (2019)."}],"container-title":["Communications of the ACM"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3528086","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3528086","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3528086","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:25Z","timestamp":1750186945000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3528086"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,20]]},"references-count":51,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2022,6]]}},"alternative-id":["10.1145\/3528086"],"URL":"https:\/\/doi.org\/10.1145\/3528086","relation":{},"ISSN":["0001-0782","1557-7317"],"issn-type":[{"value":"0001-0782","type":"print"},{"value":"1557-7317","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,5,20]]},"assertion":[{"value":"2022-05-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}