{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,12]],"date-time":"2026-02-12T21:07:57Z","timestamp":1770930477537,"version":"3.50.1"},"reference-count":26,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,2,12]],"date-time":"2026-02-12T00:00:00Z","timestamp":1770854400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,2,12]],"date-time":"2026-02-12T00:00:00Z","timestamp":1770854400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Data Sci Anal"],"published-print":{"date-parts":[[2026,12]]},"DOI":"10.1007\/s41060-026-01047-3","type":"journal-article","created":{"date-parts":[[2026,2,12]],"date-time":"2026-02-12T20:17:59Z","timestamp":1770927479000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["ConsistentPeer: reviewers through GraphRAG-driven counterfactuals to measure consistency in peer review"],"prefix":"10.1007","volume":"22","author":[{"given":"Prabhat Kumar","family":"Bharti","sequence":"first","affiliation":[]},{"given":"Mihir","family":"Panchal","sequence":"additional","affiliation":[]},{"given":"Viral","family":"Dalal","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2026,2,12]]},"reference":[{"issue":"3","key":"1047_CR1","doi-asserted-by":"publisher","first-page":"70","DOI":"10.1145\/2979672","volume":"60","author":"S Price","year":"2017","unstructured":"Price, S., Flach, P.A.: Computational support for academic peer review: a perspective from artificial intelligence. Commun. ACM 60(3), 70\u201379 (2017)","journal-title":"Commun. ACM"},{"issue":"2","key":"1047_CR2","doi-asserted-by":"publisher","first-page":"124","DOI":"10.1087\/20100207","volume":"23","author":"L Bornmann","year":"2010","unstructured":"Bornmann, L., Daniel, H.-D.: Reliability of reviewers\u2019 ratings when using public peer review: a case study. Learned Publ. 23(2), 124\u2013131 (2010)","journal-title":"Learned Publ."},{"issue":"48","key":"1047_CR3","doi-asserted-by":"publisher","first-page":"12708","DOI":"10.1073\/pnas.1707323114","volume":"114","author":"A Tomkins","year":"2017","unstructured":"Tomkins, A., Zhang, M., Heavlin, W.D.: Reviewer bias in single-versus double-blind peer review. Proc. Natl. Acad. Sci. 114(48), 12708\u201312713 (2017)","journal-title":"Proc. Natl. Acad. Sci."},{"key":"1047_CR4","unstructured":"Cortes, C., Lawrence, N.D.: Inconsistency in conference peer review: Revisiting the 2014 neurips experiment. (2021). arXiv preprint arXiv:2109.09774"},{"issue":"11","key":"1047_CR5","doi-asserted-by":"publisher","first-page":"1205","DOI":"10.1016\/j.jclinepi.2009.12.011","volume":"63","author":"D Chavalarias","year":"2010","unstructured":"Chavalarias, D., Ioannidis, J.P.: Science mapping analysis characterizes 235 biases in biomedical research. J. Clin. Epidemiol. 63(11), 1205\u20131215 (2010)","journal-title":"J. Clin. Epidemiol."},{"issue":"1","key":"1047_CR6","doi-asserted-by":"publisher","first-page":"393","DOI":"10.1007\/s11192-020-03348-1","volume":"123","author":"ES Brezis","year":"2020","unstructured":"Brezis, E.S., Birukou, A.: Arbitrariness in the peer review process. Scientometrics 123(1), 393\u2013411 (2020)","journal-title":"Scientometrics"},{"key":"1047_CR7","doi-asserted-by":"crossref","unstructured":"Bharti, P.K., Ghosal, T., Agrawal, M., Ekbal, A.: How confident was your reviewer? estimating reviewer confidence from peer review texts. In: International Workshop on Document Analysis Systems, pp. 126\u2013139 (2022). Springer","DOI":"10.1007\/978-3-031-06555-2_9"},{"issue":"7","key":"1047_CR8","doi-asserted-by":"publisher","first-page":"4109","DOI":"10.1007\/s11192-024-05070-8","volume":"129","author":"W Wu","year":"2024","unstructured":"Wu, W., Xi, H., Zhang, C.: Are the confidence scores of reviewers consistent with the review content? Evidence from top conference proceedings in ai. Scientometrics 129(7), 4109\u20134135 (2024)","journal-title":"Scientometrics"},{"issue":"6","key":"1047_CR9","doi-asserted-by":"publisher","first-page":"11344","DOI":"10.1371\/journal.pone.0011344","volume":"5","author":"L Bornmann","year":"2010","unstructured":"Bornmann, L., Daniel, H.-D.: The usefulness of peer review for selecting manuscripts for publication: a utility analysis taking as an example a high-impact journal. PLoS ONE 5(6), 11344 (2010)","journal-title":"PLoS ONE"},{"key":"1047_CR10","doi-asserted-by":"crossref","unstructured":"Pei, J., Jurgens, D.: Measuring sentence-level and aspect-level (un) certainty in science communications. (2021). arXiv preprint arXiv:2109.14776","DOI":"10.18653\/v1\/2021.emnlp-main.784"},{"key":"1047_CR11","doi-asserted-by":"crossref","unstructured":"Rogers, A., Augenstein, I.: What can we do to improve peer review in NLP? (2020). arXiv preprint arXiv:2010.03863","DOI":"10.18653\/v1\/2020.findings-emnlp.112"},{"key":"1047_CR12","unstructured":"Bharti, P., Ghosal, T., Agarwal, M., Ekbal, A.: A method for automatically estimating the informativeness of peer reviews. In: Proceedings of the 19th International Conference on Natural Language Processing (ICON), pp. 280\u2013289 (2022)"},{"issue":"6","key":"1047_CR13","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1145\/3528086","volume":"65","author":"NB Shah","year":"2022","unstructured":"Shah, N.B.: Challenges, experiments, and computational solutions in peer review. Commun. ACM 65(6), 76\u201387 (2022)","journal-title":"Commun. ACM"},{"key":"1047_CR14","doi-asserted-by":"crossref","unstructured":"Ghosal, T., Varanasi, K.K., Kordoni, V.: Hedgepeer: a dataset for uncertainty detection in peer reviews. In: Proceedings of the 22nd ACM\/IEEE Joint Conference on Digital Libraries, pp. 1\u20135 (2022)","DOI":"10.1145\/3529372.3533300"},{"key":"1047_CR15","doi-asserted-by":"crossref","unstructured":"Al\u00a0Serhali, A., Niehren, J.: Earliest query answering for deterministic stepwise hedge automata. In: International Conference on Implementation and Application of Automata, pp. 53\u201365. Springer (2023)","DOI":"10.1007\/978-3-031-40247-0_3"},{"issue":"2","key":"1047_CR16","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1007\/s00799-023-00382-1","volume":"25","author":"GL Fernandes","year":"2024","unstructured":"Fernandes, G.L., Vaz-de-Melo, P.O.: Enhancing the examination of obstacles in an automated peer review system. Int. J. Digit. Libr. 25(2), 341\u2013364 (2024)","journal-title":"Int. J. Digit. Libr."},{"key":"1047_CR17","unstructured":"Peng, B., Zhu, Y., Liu, Y., Bo, X., Shi, H., Hong, C., Zhang, Y., Tang, S.: Graph retrieval-augmented generation: A survey. (2024). arXiv preprint arXiv:2408.08921"},{"key":"1047_CR18","doi-asserted-by":"crossref","unstructured":"Sarmah, B., Mehta, D., Hall, B., Rao, R., Patel, S., Pasquali, S.: Hybridrag: Integrating knowledge graphs and vector retrieval augmented generation for efficient information extraction. In: Proceedings of the 5th ACM International Conference on AI in Finance, pp. 608\u2013616 (2024)","DOI":"10.1145\/3677052.3698671"},{"key":"1047_CR19","doi-asserted-by":"crossref","unstructured":"Huang, H., Chen, C., He, C., Li, Y., Jiang, J., Zhang, W.: Can LLMs be good graph judger for knowledge graph construction? (2024). arXiv preprint arXiv:2411.17388","DOI":"10.18653\/v1\/2025.emnlp-main.554"},{"key":"1047_CR20","doi-asserted-by":"crossref","unstructured":"Su, A., Sun, J., Dong, K., Ling, W.: Parameter recommendation system enhanced by rag based on process knowledge. In: 2024 3rd International Conference on Electronics and Information Technology (EIT), pp. 467\u2013471. IEEE (2024)","DOI":"10.1109\/EIT63098.2024.10762220"},{"issue":"2","key":"1047_CR21","doi-asserted-by":"publisher","first-page":"178","DOI":"10.1007\/s12109-023-09943-3","volume":"39","author":"M Lauria","year":"2023","unstructured":"Lauria, M.: Reviewing peer review: a flawed system: With immense potential. Publ. Res. Q. 39(2), 178\u2013190 (2023)","journal-title":"Publ. Res. Q."},{"key":"1047_CR22","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1016\/j.ijar.2014.11.003","volume":"57","author":"S Ferson","year":"2015","unstructured":"Ferson, S., O\u2019Rawe, J., Antonenko, A., Siegrist, J., Mickley, J., Luhmann, C.C., Sentz, K., Finkel, A.M.: Natural language of uncertainty: numeric hedge words. Int. J. Approx. Reason. 57, 19\u201339 (2015)","journal-title":"Int. J. Approx. Reason."},{"issue":"1","key":"1047_CR23","doi-asserted-by":"publisher","first-page":"0259238","DOI":"10.1371\/journal.pone.0259238","volume":"17","author":"T Ghosal","year":"2022","unstructured":"Ghosal, T., Kumar, S., Bharti, P.K., Ekbal, A.: Peer review analyze: a novel benchmark resource for computational analysis of peer reviews. PLoS ONE 17(1), 0259238 (2022)","journal-title":"PLoS ONE"},{"issue":"2","key":"1047_CR24","doi-asserted-by":"publisher","first-page":"5711","DOI":"10.1002\/cpe.5711","volume":"33","author":"N Van Han","year":"2021","unstructured":"Van Han, N., Cong Vinh, P.: Reasoning with words: a hedge algebra linguistic cognitive map approach. Concurr. Comput. Pract. Exp. 33(2), 5711 (2021)","journal-title":"Concurr. Comput. Pract. Exp."},{"issue":"4","key":"1047_CR25","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1093\/applin\/17.4.433","volume":"17","author":"K Hyland","year":"1996","unstructured":"Hyland, K.: Writing without conviction? hedging in science research articles. Appl. Linguis. 17(4), 433\u2013454 (1996)","journal-title":"Appl. Linguis."},{"issue":"5","key":"1047_CR26","doi-asserted-by":"publisher","first-page":"853","DOI":"10.1080\/07294360.2015.1011086","volume":"34","author":"A Dobele","year":"2015","unstructured":"Dobele, A.: Assessing the quality of feedback in the peer-review process. High. Educ. Res. Dev. 34(5), 853\u2013868 (2015)","journal-title":"High. Educ. Res. Dev."}],"container-title":["International Journal of Data Science and Analytics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s41060-026-01047-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s41060-026-01047-3","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s41060-026-01047-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,12]],"date-time":"2026-02-12T20:18:01Z","timestamp":1770927481000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s41060-026-01047-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,2,12]]},"references-count":26,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2026,12]]}},"alternative-id":["1047"],"URL":"https:\/\/doi.org\/10.1007\/s41060-026-01047-3","relation":{},"ISSN":["2364-415X","2364-4168"],"issn-type":[{"value":"2364-415X","type":"print"},{"value":"2364-4168","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,2,12]]},"assertion":[{"value":"29 July 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 January 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 February 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"64"}}