{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T14:20:23Z","timestamp":1766067623002,"version":"3.37.3"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2022,11,9]],"date-time":"2022-11-09T00:00:00Z","timestamp":1667952000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,11,9]],"date-time":"2022-11-09T00:00:00Z","timestamp":1667952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["2072082","U1811261"],"award-info":[{"award-number":["2072082","U1811261"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["2072082","U1811261"],"award-info":[{"award-number":["2072082","U1811261"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["2072082","U1811261"],"award-info":[{"award-number":["2072082","U1811261"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["2072082","U1811261"],"award-info":[{"award-number":["2072082","U1811261"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["World Wide Web"],"published-print":{"date-parts":[[2023,3]]},"DOI":"10.1007\/s11280-022-01109-z","type":"journal-article","created":{"date-parts":[[2022,11,9]],"date-time":"2022-11-09T10:03:24Z","timestamp":1667988204000},"page":"683-708","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["What have we learned from OpenReview?"],"prefix":"10.1007","volume":"26","author":[{"given":"Gang","family":"Wang","sequence":"first","affiliation":[]},{"given":"Qi","family":"Peng","sequence":"additional","affiliation":[]},{"given":"Yanfeng","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Mingyang","family":"Zhang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,11,9]]},"reference":[{"issue":"3","key":"1109_CR1","doi-asserted-by":"publisher","first-page":"70","DOI":"10.1145\/2979672","volume":"60","author":"S Price","year":"2017","unstructured":"Price, S., Flach, P.A.: Computational support for academic peer review: A perspective from artificial intelligence. Commun. ACM 60(3), 70\u201379 (2017)","journal-title":"Commun. ACM"},{"key":"1109_CR2","doi-asserted-by":"crossref","unstructured":"Kang, D., Ammar, W., Dalvi, B., van Zuylen, M., Kohlmeier, S., Hovy, E.H., Schwartz, R.: A dataset of peer reviews (peerread): Collection, insights and NLP applications. In: North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp 1647\u20131661 (2018)","DOI":"10.18653\/v1\/N18-1149"},{"key":"1109_CR3","doi-asserted-by":"crossref","unstructured":"Wang, K., Wan, X.: Sentiment analysis of peer review texts for scholarly papers. In: International ACM SIGIR Conference, pp 175\u2013184 (2018)","DOI":"10.1145\/3209978.3210056"},{"key":"1109_CR4","doi-asserted-by":"crossref","unstructured":"Li, S., Zhao, W.X., Yin, E.J., Wen, J.: A neural citation count prediction model based on peer review text. In: Natural Language Processing, pp 4913\u20134923 (2019)","DOI":"10.18653\/v1\/D19-1497"},{"key":"1109_CR5","doi-asserted-by":"crossref","unstructured":"Ghosal, T., Verma, R., Ekbal, A., Bhattacharyya, P.: Deepsentipeer: Harnessing sentiment in review texts to recommend peer review decisions. In: Association for Computational Linguistics, pp 1120\u20131130 (2019)","DOI":"10.18653\/v1\/P19-1106"},{"key":"1109_CR6","unstructured":"Stelmakh, I., Shah, N.B., Singh, A.: On testing for biases in peer review. In: NeurIPS, pp 5287\u20135297 (2019)"},{"key":"1109_CR7","unstructured":"He, H.: Some metadata for those curious about their #ICLR2020. https:\/\/twitter.com\/cHHillee\/status\/1191823707100131329"},{"key":"1109_CR8","first-page":"2579","volume":"9","author":"Lvd Maaten","year":"2008","unstructured":"Maaten, L.v.d., Hinton, G.: Visualizing data using t-sne. J. Mach. Learn. Res. 9, 2579\u20132605 (2008)","journal-title":"J. Mach. Learn. Res."},{"key":"1109_CR9","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Pavlu, V.: Query hardness estimation using jensen-shannon divergence among multiple scoring functions. In: European Conference on Information Retrieval, pp 198\u2013209 (2007)","DOI":"10.1007\/978-3-540-71496-5_20"},{"key":"1109_CR10","unstructured":"Clark, K., Luong, M.-T., Le, Q.V., Manning, C.D.: Electra: Pre-training text encoders as discriminators rather than generators. In: International Conference on Learning Representations (2020)"},{"issue":"2","key":"1109_CR11","doi-asserted-by":"publisher","first-page":"180","DOI":"10.1007\/s41019-020-00122-4","volume":"5","author":"T Ito","year":"2020","unstructured":"Ito, T., Tsubouchi, K., Sakaji, H., Yamashita, T., Izumi, K.: Contextual sentiment neural network for document sentiment analysis. Data Sci. Eng. 5(2), 180\u2013192 (2020)","journal-title":"Data Sci. Eng."},{"key":"1109_CR12","first-page":"167","volume-title":"Causal Inference using Regression on the Treatment Variable. Analytical Methods for Social Research","author":"A Gelman","year":"2006","unstructured":"Gelman, A., Hill, J.: Causal Inference using Regression on the Treatment Variable. Analytical Methods for Social Research, pp 167\u2013198. Cambridge University Press, Cambridge (2006)"},{"key":"1109_CR13","unstructured":"Allison, P.D.: Multiple Regression: A Primer. Pine Forge Press, Pine Forge Press (1999)"},{"key":"1109_CR14","unstructured":"jiqizhixin: Artificial-Intelligence-Terminology. https:\/\/github.com\/jiqizhixin\/Artificial-Intelligence-Terminology"},{"issue":"301","key":"1109_CR15","doi-asserted-by":"publisher","first-page":"236","DOI":"10.1080\/01621459.1963.10500845","volume":"58","author":"H Joe","year":"1963","unstructured":"Joe, H., Ward, J.: Hierarchical grouping to optimize an objective function. J. Am. Stat. Assoc. 58(301), 236\u2013244 (1963)","journal-title":"J. Am. Stat. Assoc."},{"key":"1109_CR16","unstructured":"Trivedi: Causality or causal inference or conditions for causal inference. https:\/\/conceptshacked.com\/causal-inference\/ (2020)"},{"issue":"1","key":"1109_CR17","first-page":"10","volume":"80","author":"AM Alsaqr","year":"2021","unstructured":"Alsaqr, A.M.: Remarks on the use of pearson\u2019s and spearman\u2019s correlation coefficients in assessing relationships in ophthalmic data. Afr. Vision Eye Health 80 (1), 10 (2021)","journal-title":"Afr. Vision Eye Health"},{"key":"1109_CR18","unstructured":"Devlin, J., Chang, M.-W., Lee, K., Toutanova, K.: Bert: Pre-training of deep bidirectional transformers for language understanding. In: North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp 4171\u20134186 (2019)"},{"key":"1109_CR19","unstructured":"Charlin, L., Zemel, R.: The toronto paper matching system: an automated paper-reviewer assignment system (2013)"},{"issue":"9","key":"1109_CR20","doi-asserted-by":"publisher","first-page":"0184711","DOI":"10.1371\/journal.pone.0184711","volume":"12","author":"MJ Mrowinski","year":"2017","unstructured":"Mrowinski, M.J., Fronczak, P., Fronczak, A., Ausloos, M., Nedic, O.: Artificial intelligence in peer review: How can evolutionary computation support journal editors? PloS ONE 12(9), 0184711 (2017)","journal-title":"PloS ONE"},{"key":"1109_CR21","doi-asserted-by":"crossref","unstructured":"Roos, M., Rothe, J., Scheuermann, B.: How to calibrate the scores of biased reviewers by quadratic programming. In: AAAI Conference on Artificial Intelligence, pp 255\u2013260 (2011)","DOI":"10.1609\/aaai.v25i1.7847"},{"key":"1109_CR22","unstructured":"Stelmakh, I., Shah, N., Singh, A.: On testing for biases in peer review. Adv. Neural Inf. Process. Syst. 32 (2019)"},{"key":"1109_CR23","unstructured":"Shah, N.B.: An overview of challenges, experiments, and computational solutions in peer review. Communications of the ACM (2021)"},{"issue":"1","key":"1109_CR24","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1177\/01430343211070165","volume":"43","author":"L Fallon","year":"2022","unstructured":"Fallon, L., Grapin, S., Newman, D.S., Noltemeyer, A.: Promoting equity and social justice in the peer review process: Tips for reviewers. Sch. Psychol. Int. 43(1), 12\u201317 (2022)","journal-title":"Sch. Psychol. Int."},{"key":"1109_CR25","unstructured":"Manzoor, E., Shah, N.B.: Uncovering latent biases in text: Method and application to peer review. arXiv e-prints 2010 (2020)"},{"issue":"4","key":"1109_CR26","doi-asserted-by":"publisher","first-page":"843","DOI":"10.1007\/s11948-014-9580-6","volume":"21","author":"JT Mecca","year":"2015","unstructured":"Mecca, J.T., Gibson, C., Giorgini, V., Medeiros, K.E., Mumford, M.D., Connelly, S.: Researcher perspectives on conflicts of interest: A qualitative analysis of views from academia. Sci. Eng. Ethics 21(4), 843\u2013855 (2015)","journal-title":"Sci. Eng. Ethics"},{"key":"1109_CR27","doi-asserted-by":"crossref","unstructured":"Wu, S., U, L.H., Bhowmick, S.S., Gatterbauer, W.: Pistis: A conflict of interest declaration and detection system for peer review management. In: Proceedings of the 2018 International Conference on Management of Data, pp 1713\u20131716 (2018)","DOI":"10.1145\/3183713.3193552"},{"key":"1109_CR28","doi-asserted-by":"crossref","unstructured":"Aleman-Meza, B., Nagarajan, M., Ramakrishnan, C., Ding, L., Kolari, P., Sheth, A.P., Arpinar, I.B., Joshi, A., Finin, T.: Semantic analytics on social networks: experiences in addressing the problem of conflict of interest detection, pp 407\u2013416 (2006)","DOI":"10.1145\/1135777.1135838"},{"key":"1109_CR29","doi-asserted-by":"crossref","unstructured":"Wu, S., U, L.H., Bhowmick, S.S., Gatterbauer, W.: Conflict of interest declaration and detection system in heterogeneous networks, pp 2383\u20132386 (2017)","DOI":"10.1145\/3132847.3133134"},{"key":"1109_CR30","unstructured":"CLOSET: ConfLict Of IntereSt DEtection & ManagemenT System. https:\/\/personal.ntu.edu.sg\/assourav\/research\/DARE\/closet.html"},{"key":"1109_CR31","doi-asserted-by":"publisher","first-page":"56","DOI":"10.3389\/fncom.2011.00056","volume":"5","author":"A Birukou","year":"2011","unstructured":"Birukou, A., Wakeling, J.R., Bartolini, C., Casati, F., Marchese, M., Mirylenka, K., Osman, N., Ragone, A., Sierra, C., Wassef, A.: Alternatives to peer review: novel approaches for research evaluation. Front. Comput. Neurosci. 5, 56 (2011)","journal-title":"Front. Comput. Neurosci."},{"key":"1109_CR32","doi-asserted-by":"crossref","unstructured":"Gao, Y., Eger, S., Kuznetsov, I., Gurevych, I., Miyao, Y.: Does my rebuttal matter? insights from a major NLP conference. In: North American Chapter of the Association for Computational Linguistics, pp 1274\u20131290 (2019)","DOI":"10.18653\/v1\/N19-1129"},{"issue":"4","key":"1109_CR33","doi-asserted-by":"publisher","first-page":"100","DOI":"10.1145\/1519103.1519122","volume":"37","author":"G Cormode","year":"2009","unstructured":"Cormode, G.: How not to review a paper: The tools and techniques of the adversarial reviewer. ACM SIGMOD Rec. 37(4), 100\u2013104 (2009)","journal-title":"ACM SIGMOD Rec."},{"key":"1109_CR34","doi-asserted-by":"crossref","unstructured":"Stelmakh, I.: Experiments with the ICML 2020 Peer-Review Process. https:\/\/blog.ml.cmu.edu\/2020\/12\/01\/icml2020exp\/","DOI":"10.51737\/2766-4813.2020.008"},{"key":"1109_CR35","first-page":"1","volume":"19","author":"NB Shah","year":"2018","unstructured":"Shah, N.B., Tabibian, B., Muandet, K., Guyon, I., Von Luxburg, U.: Design and analysis of the nips 2016 review process. J Mach. Learn. Res. 19, 1\u201334 (2018)","journal-title":"J Mach. Learn. Res."},{"key":"1109_CR36","unstructured":"Bharadhwaj, H., Turpin, D., Garg, A., Anderson, A.: De-anonymization of authors through arxiv submissions during double-blind review. arXiv:2007.00177 (2020)"},{"key":"1109_CR37","unstructured":"Tran, D., Valtchanov, A., Ganapathy, K., Feng, R., Slud, E., Goldblum, M., Goldstein, T.: An open review of openreview: A critical analysis of the machine learning conference review process. arXiv:2010.05137 (2020)"},{"key":"1109_CR38","doi-asserted-by":"crossref","unstructured":"Stelmakh, I., Rastogi, C., Liu, R., Chawla, S., Echenique, F., Shah, N.B.: Cite-seeing and reviewing: A study on citation bias in peer review. arXiv:2203.17239 (2022)","DOI":"10.1371\/journal.pone.0283980"},{"key":"1109_CR39","doi-asserted-by":"crossref","unstructured":"Kohli, G.S., Kaur, P., Singh, M., Ghosal, T., Rana, P.S.: Arguably@ ai debater-nlpcc 2021 task 3: Argument pair extraction from peer review and rebuttals. In: Natural Language Processing and Chinese Computing, pp 590\u2013602 (2021)","DOI":"10.1007\/978-3-030-88483-3_48"},{"key":"1109_CR40","doi-asserted-by":"crossref","unstructured":"Cheng, L., Bing, L., Yu, Q., Lu, W., Si, L.: Ape: argument pair extraction from peer review and rebuttal via multi-task learning. In: Empirical Methods in Natural Language Processing, pp 7000\u20137011 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.569"}],"container-title":["World Wide Web"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11280-022-01109-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11280-022-01109-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11280-022-01109-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T13:51:41Z","timestamp":1701352301000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11280-022-01109-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,9]]},"references-count":40,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2023,3]]}},"alternative-id":["1109"],"URL":"https:\/\/doi.org\/10.1007\/s11280-022-01109-z","relation":{},"ISSN":["1386-145X","1573-1413"],"issn-type":[{"type":"print","value":"1386-145X"},{"type":"electronic","value":"1573-1413"}],"subject":[],"published":{"date-parts":[[2022,11,9]]},"assertion":[{"value":"14 April 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 August 2022","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 September 2022","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 November 2022","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"This article does not contain any studies involving human participants and\/or animals by any of the authors.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"All authors have read and agreed to the published version of the manuscript.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Consent for publication"}},{"value":"All authors have agreed to participate in the research described in this manuscript.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"The authors declare no conflict of interest.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Conflict of interests"}},{"value":"Not applicable","order":6,"name":"Ethics","group":{"name":"EthicsHeading","label":"Human and animal ethics"}}]}}