{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T04:09:01Z","timestamp":1774411741137,"version":"3.50.1"},"reference-count":77,"publisher":"Elsevier BV","issue":"6","license":[{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/100010097","name":"China Association for Science and Technology","doi-asserted-by":"publisher","award":["YESS20240562"],"award-info":[{"award-number":["YESS20240562"]}],"id":[{"id":"10.13039\/100010097","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["72204087"],"award-info":[{"award-number":["72204087"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003395","name":"Shanghai Municipal Education Commission","doi-asserted-by":"publisher","award":["23CGA28"],"award-info":[{"award-number":["23CGA28"]}],"id":[{"id":"10.13039\/501100003395","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100014991","name":"Shanghai Planning Office of Philosophy and Social Science","doi-asserted-by":"publisher","award":["2022ETQ001"],"award-info":[{"award-number":["2022ETQ001"]}],"id":[{"id":"10.13039\/501100014991","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003024","name":"Shanghai Education Development Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003024","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012226","name":"Fundamental Research Funds for the Central Universities","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100012226","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information Processing & Management"],"published-print":{"date-parts":[[2026,9]]},"DOI":"10.1016\/j.ipm.2026.104704","type":"journal-article","created":{"date-parts":[[2026,3,6]],"date-time":"2026-03-06T11:23:32Z","timestamp":1772796212000},"page":"104704","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"title":["Toward better pragmatic tagging of peer review: Enhancing benchmark datasets via human-in-the-loop multi-agent collaboration"],"prefix":"10.1016","volume":"63","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1419-7495","authenticated-orcid":false,"given":"Guoxiu","family":"He","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3857-9307","authenticated-orcid":false,"given":"Tiancheng","family":"Su","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0000-3805-5064","authenticated-orcid":false,"given":"Meicong","family":"Zhang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1080-5893","authenticated-orcid":false,"given":"Jia","family":"Yuan","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8562-8347","authenticated-orcid":false,"given":"Zhuoren","family":"Jiang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2507-4586","authenticated-orcid":false,"given":"Li","family":"Ma","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0001-2666-2665","authenticated-orcid":false,"given":"Bo","family":"Xiao","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1301-7840","authenticated-orcid":false,"given":"Xiaomin","family":"Zhu","sequence":"additional","affiliation":[]}],"member":"78","reference":[{"key":"10.1016\/j.ipm.2026.104704_bib0001","doi-asserted-by":"crossref","DOI":"10.1016\/j.eswa.2022.118487","article-title":"An end-to-end framework for information extraction from Italian resumes","volume":"210","author":"Barducci","year":"2022","journal-title":"Expert Systems with Applications"},{"key":"10.1016\/j.ipm.2026.104704_bib0002","series-title":"Cognitive bias","first-page":"1","author":"Blanco","year":"2017"},{"key":"10.1016\/j.ipm.2026.104704_bib0003","unstructured":"Cai, W., Wang, C., Yan, J., Huang, J., & Fang, X. (2025). Reasoning with omnithought: A large cot dataset with verbosity and cognitive difficulty annotations. arXiv preprint arXiv: 2505.10937."},{"issue":"2","key":"10.1016\/j.ipm.2026.104704_bib0004","doi-asserted-by":"crossref","first-page":"354","DOI":"10.1002\/asi.23180","article-title":"Understanding review helpfulness as a function of reviewer reputation, review rating, and review depth","volume":"66","author":"Chua","year":"2015","journal-title":"Journal of the Association for Information Science and Technology"},{"key":"10.1016\/j.ipm.2026.104704_bib0005","unstructured":"D\u2019Arcy, M., Hope, T., Birnbaum, L., & Downey, D. (2024a). Marg: Multi-agent review generation for scientific papers. arXiv preprint arXiv: 2401.04259."},{"key":"10.1016\/j.ipm.2026.104704_bib0006","series-title":"Proceedings of the 62nd annual meeting of the association for computational linguistics (volume 1: Long papers)","first-page":"6985","article-title":"ARIES: A corpus of scientific paper edits made in response to peer reviews","author":"D\u2019Arcy","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0007","doi-asserted-by":"crossref","DOI":"10.1016\/j.eswa.2024.125533","article-title":"Integrated sentiment analysis with BERT for enhanced hybrid recommendation systems","volume":"261","author":"Darraz","year":"2025","journal-title":"Expert Systems With Applications"},{"key":"10.1016\/j.ipm.2026.104704_bib0008","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing","first-page":"5081","article-title":"LLMs assist NLP researchers: Critique paper (meta-)reviewing","author":"Du","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0009","series-title":"Proceedings of the 41st international conference on machine learning","article-title":"Improving factuality and reasoning in language models through multiagent debate","author":"Du","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0010","series-title":"Proceedings of the 10th workshop on argument mining","first-page":"187","article-title":"Overview of PragTag-2023: Low-resource multi-domain pragmatic tagging of peer reviews","author":"Dycke","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0011","unstructured":"Gao, Z., Brantley, K., & Joachims, T. (2024). Reviewer2: Optimizing review generation through prompt generation. arXiv preprint arXiv: 2402.10886."},{"issue":"12","key":"10.1016\/j.ipm.2026.104704_bib0012","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2023.47607","article-title":"Prevalence of short peer reviews in 3 leading general medical journals","volume":"6","author":"Geldsetzer","year":"2023","journal-title":"JAMA Network Open"},{"key":"10.1016\/j.ipm.2026.104704_bib0013","series-title":"Proceedings of the international AAAI conference on web and social media","first-page":"653","article-title":"Human and LLM biases in hate speech annotations: A socio-demographic analysis of annotators and targets","volume":"vol. 19","author":"Giorgi","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0014","series-title":"Proceedings of the 10th workshop on argument mining","first-page":"212","article-title":"Nus-ids at pragtag-2023: Improving pragmatic tagging of peer reviews through unlabeled data","author":"Gollapalli","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0015","unstructured":"Han, S., Zhang, Q., Yao, Y., Jin, W., Xu, Z., & He, C. (2024). LLM multi-agent systems: Challenges and open problems. arXiv preprint arXiv: 2402.03578."},{"issue":"2","key":"10.1016\/j.ipm.2026.104704_bib0016","doi-asserted-by":"crossref","DOI":"10.1016\/j.joi.2025.101645","article-title":"Sequential citation counts prediction enhanced by dynamic contents","volume":"19","author":"He","year":"2025","journal-title":"Journal of Informetrics"},{"issue":"4","key":"10.1016\/j.ipm.2026.104704_bib0017","doi-asserted-by":"crossref","DOI":"10.1016\/j.joi.2024.101596","article-title":"Predicting the emergence of disruptive technologies by comparing with references via soft prompt-aware shared BERT","volume":"18","author":"He","year":"2024","journal-title":"Journal of Informetrics"},{"key":"10.1016\/j.ipm.2026.104704_bib0018","series-title":"Proceedings of the 2024 conference of the North American chapter of the association for computational linguistics: Human language technologies (volume 6: Industry track)","first-page":"165","article-title":"AnnoLLM: Making large language models to be better crowdsourced annotators","author":"He","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0019","doi-asserted-by":"crossref","DOI":"10.1016\/j.neucom.2024.128015","article-title":"An overview: Attention mechanisms in multi-agent reinforcement learning","volume":"598","author":"Hu","year":"2024","journal-title":"Neurocomputing"},{"issue":"3","key":"10.1016\/j.ipm.2026.104704_bib0020","doi-asserted-by":"crossref","DOI":"10.1016\/j.joi.2023.101427","article-title":"What makes a successful rebuttal in computer science conferences?: A perspective on social interaction","volume":"17","author":"Huang","year":"2023","journal-title":"Journal of Informetrics"},{"issue":"6","key":"10.1016\/j.ipm.2026.104704_bib0021","doi-asserted-by":"crossref","DOI":"10.1016\/j.ipm.2025.104225","article-title":"Papereval: A universal, quantitative, and explainable paper evaluation method powered by a multi-agent system","volume":"62","author":"Huang","year":"2025","journal-title":"Information Processing & Management"},{"key":"10.1016\/j.ipm.2026.104704_bib0022","unstructured":"Huang, X., Liu, W., Chen, X., Wang, X., Wang, H., Lian, D., Wang, Y., Tang, R., & Chen, E. (2024). Understanding the planning of LLM agents: A survey. arXiv preprint arXiv: 2402.02716."},{"key":"10.1016\/j.ipm.2026.104704_bib0023","series-title":"Proceedings of the 62nd annual meeting of the association for computational linguistics (volume 1: Long papers)","first-page":"4912","article-title":"MapCoder: Multi-agent code generation for competitive problem solving","author":"Islam","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0024","series-title":"Proceedings of the ACM web conference 2023","first-page":"3816","article-title":"A dataset on malicious paper bidding in peer review","author":"Jecmen","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0025","series-title":"Companion proceedings of the ACM on web conference 2025","first-page":"2494","article-title":"Apeer: Automatic prompt engineering enhances large language model reranking","author":"Jin","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0026","series-title":"Proceedings of the 2024 joint international conference on computational linguistics, language resources and evaluation (LREC-COLING 2024)","first-page":"2883","article-title":"CASIMIR: A corpus of scientific articles enhanced with multiple author-integrated revisions","author":"Jourdan","year":"2024"},{"issue":"2","key":"10.1016\/j.ipm.2026.104704_bib0027","doi-asserted-by":"crossref","first-page":"93","DOI":"10.3325\/cmj.2024.65.93","article-title":"Exploring ChatGPT\u2019s abilities in medical article writing and peer review","volume":"65","author":"Kadi","year":"2024","journal-title":"Croatian Medical Journal"},{"key":"10.1016\/j.ipm.2026.104704_bib0028","series-title":"Proceedings of the 2018 conference of the North American chapter of the association for computational linguistics: Human language technologies, volume 1 (long papers)","first-page":"1647","article-title":"A dataset of peer reviews (PeerRead): Collection, insights and NLP applications","author":"Kang","year":"2018"},{"key":"10.1016\/j.ipm.2026.104704_bib0029","series-title":"Proceedings of the 31st international conference on computational linguistics","first-page":"11422","article-title":"A dataset for expert reviewer recommendation with large language models as zero-shot rankers","author":"Karan","year":"2025"},{"issue":"3","key":"10.1016\/j.ipm.2026.104704_bib0030","first-page":"227","article-title":"Peer review in scientific publications: Benefits, critiques, & a survival guide","volume":"25","author":"Kelly","year":"2014","journal-title":"Ejifcc"},{"key":"10.1016\/j.ipm.2026.104704_bib0031","series-title":"Proceedings of the 2022 conference of the North American chapter of the association for computational linguistics: human language technologies","first-page":"1234","article-title":"DISAPERE: A dataset for discourse structure in peer review discussions","author":"Kennard","year":"2022"},{"key":"10.1016\/j.ipm.2026.104704_bib0032","series-title":"Proceedings of the 18th conference of the European chapter of the association for computational linguistics: System demonstrations","first-page":"168","article-title":"MEGAnno+: A human-LLM collaborative annotation system","author":"Kim","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0033","series-title":"3rd International conference on learning representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, conference track proceedings","article-title":"Adam: A method for stochastic optimization","author":"Kingma","year":"2015"},{"key":"10.1016\/j.ipm.2026.104704_bib0034","first-page":"22199","article-title":"Large language models are zero-shot reasoners","volume":"35","author":"Kojima","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.ipm.2026.104704_bib0035","article-title":"Assessing the societal influence of academic research with chatGPT: Impact case study evaluations","author":"Kousha","year":"2024","journal-title":"Journal of the Association for Information Science and Technology"},{"key":"10.1016\/j.ipm.2026.104704_bib0036","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing","first-page":"22663","article-title":"\u2018Quis custodiet ipsos custodes?\u2019 Who will watch the watchmen? on detecting AI-generated peer-reviews","author":"Kumar","year":"2024"},{"issue":"1","key":"10.1016\/j.ipm.2026.104704_bib0037","doi-asserted-by":"crossref","first-page":"159","DOI":"10.2307\/2529310","article-title":"The measurement of observer agreement for categorical data","volume":"33","author":"Landis","year":"1977","journal-title":"Biometrics"},{"issue":"1","key":"10.1016\/j.ipm.2026.104704_bib0038","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1002\/asi.22784","article-title":"Bias in peer review","volume":"64","author":"Lee","year":"2013","journal-title":"Journal of the American Society for information Science and Technology"},{"key":"10.1016\/j.ipm.2026.104704_bib0039","series-title":"Advances in neural information processing systems","first-page":"53418","article-title":"MACM: Utilizing a multi-agent system for condition mining in solving complex mathematical problems","volume":"vol. 37","author":"Lei","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0040","series-title":"Proceedings of the 2023 conference on empirical methods in natural language processing","first-page":"1487","article-title":"CoAnnotating: Uncertainty-guided work allocation between human and large language models for data annotation","author":"Li","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0041","unstructured":"Lin, M., Chen, Z., Liu, Y., Zhao, X., Wu, Z., Wang, J., Zhang, X., Wang, S., & Chen, H. (2024). Decoding time series with LLMs: A multi-agent framework for cross-domain annotation. arXiv preprint arXiv: 2410.17462."},{"issue":"2","key":"10.1016\/j.ipm.2026.104704_bib0042","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3704999","article-title":"Understanding before recommendation: Semantic aspect-aware review exploitation via large language models","volume":"43","author":"Liu","year":"2025","journal-title":"ACM Transactions on Information Systems"},{"key":"10.1016\/j.ipm.2026.104704_bib0043","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., & Stoyanov, V. (2019). RoBERTa: A robustly optimized bert pretraining approach. arXiv preprint arXiv: 1907.11692."},{"key":"10.1016\/j.ipm.2026.104704_bib0044","series-title":"Proceedings of the 5th annual ACM web science conference","first-page":"234","article-title":"Experiences surveying the crowd: Reflections on methods, participation, and reliability","author":"Marshall","year":"2013"},{"key":"10.1016\/j.ipm.2026.104704_bib0045","series-title":"International conference on data intelligence and cognitive informatics","first-page":"387","article-title":"Prompt engineering in large language models","author":"Marvin","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0046","series-title":"Proceedings on \u201cI can\u2019t believe it\u2019s not better: Failure modes in the age of foundation models\u201d at neurIPS 2023 workshops","first-page":"38","article-title":"Are large language models good annotators?","volume":"vol. 239","author":"Mohta","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0047","doi-asserted-by":"crossref","unstructured":"Nahum, O., Calderon, N., Keller, O., Szpektor, I., & Reichart, R. (2024). Are LLMs better than reported? Detecting label errors and mitigating their effect on model performance. arXiv preprint arXiv: 2410.18889.","DOI":"10.18653\/v1\/2025.emnlp-main.1360"},{"key":"10.1016\/j.ipm.2026.104704_bib0048","series-title":"2020 19th IEEE international conference on machine learning and applications (ICMLA)","first-page":"314","article-title":"Automatic generation of reviews of scientific papers","author":"Nikiforovskaya","year":"2020"},{"key":"10.1016\/j.ipm.2026.104704_bib0049","series-title":"Proceedings of the 32nd ACM conference on hypertext and social media","first-page":"165","article-title":"What is unclear? Computational assessment of task clarity in crowdsourcing","author":"Nouri","year":"2021"},{"key":"10.1016\/j.ipm.2026.104704_bib0050","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing","first-page":"16658","article-title":"Small LLMs are weak tool learners: A multi-LLM agent","author":"Shen","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0051","series-title":"Proceedings of the fourth workshop on scholarly document processing (SDP 2024)","first-page":"257","article-title":"An analysis of tasks and datasets in peer reviewing","author":"Staudinger","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0052","unstructured":"Stelmakh, I., Wieting, J., Neubig, G., & Shah, N. B. (2023). A gold standard dataset for the reviewer assignment problem. arXiv preprint arXiv: 2303.16750."},{"key":"10.1016\/j.ipm.2026.104704_bib0053","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing","first-page":"930","article-title":"Large language models for data annotation and synthesis: A survey","author":"Tan","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0054","series-title":"Proceedings of the 30th international conference on intelligent user interfaces","first-page":"1398","article-title":"Text-to-SQL domain adaptation via human-LLM collaborative data annotation","author":"Tian","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0055","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing","first-page":"17086","article-title":"Foundational autoraters: Taming large language models for better automatic evaluation","author":"Vu","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0056","doi-asserted-by":"crossref","unstructured":"Wang, K., Zhang, G., Zhou, Z., Wu, J., Yu, M., Zhao, S., Yin, C., Fu, J., Yan, Y., Luo, H. et al. (2025). A comprehensive survey in LLM (-agent) full stack safety: Data, training and deployment. arXiv preprint arXiv: 2504.15585.","DOI":"10.36227\/techrxiv.175322671.12208812\/v1"},{"key":"10.1016\/j.ipm.2026.104704_bib0057","series-title":"Findings of the association for computational linguistics: EMNLP 2021","first-page":"4195","article-title":"Want to reduce labeling cost? GPT-3 can help","author":"Wang","year":"2021"},{"key":"10.1016\/j.ipm.2026.104704_bib0058","series-title":"AAAI 2025 workshop on advancing LLM-based multi-agent collaboration","article-title":"Don\u2019t just demo, teach me the principles: A principle-based multi-agent prompting strategy for text classification","author":"Wei","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0059","series-title":"Amia joint summits on translational science proceedings","first-page":"614","article-title":"Enhancing disease detection in radiology reports through fine-tuning lightweight LLM on weak labels","author":"Wei","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0060","series-title":"Icassp 2025-2025 IEEE international conference on acoustics, speech and signal processing (icassp)","first-page":"1","article-title":"Enhancing text annotation through rationale-driven collaborative few-shot prompting","author":"Wu","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0061","doi-asserted-by":"crossref","DOI":"10.1093\/nsr\/nwaf169","article-title":"Automated literature research and review generation method based on large language models","author":"Wu","year":"2025","journal-title":"National Science Review"},{"issue":"7","key":"10.1016\/j.ipm.2026.104704_bib0062","doi-asserted-by":"crossref","first-page":"4109","DOI":"10.1007\/s11192-024-05070-8","article-title":"Are the confidence scores of reviewers consistent with the review content? Evidence from top conference proceedings in AI","volume":"129","author":"Wu","year":"2024","journal-title":"Scientometrics"},{"key":"10.1016\/j.ipm.2026.104704_bib0063","doi-asserted-by":"crossref","DOI":"10.1002\/asi.70005","article-title":"Automated novelty evaluation of academic paper: A collaborative approach integrating human and large language model knowledge","author":"Wu","year":"2025","journal-title":"Journal of the Association for Information Science and Technology"},{"issue":"5","key":"10.1016\/j.ipm.2026.104704_bib0064","doi-asserted-by":"crossref","first-page":"392","DOI":"10.3390\/info16050392","article-title":"Syncode: Synergistic human\u2013LLM collaboration for enhanced data annotation in stack overflow","volume":"16","author":"Xia","year":"2025","journal-title":"Information"},{"issue":"16","key":"10.1016\/j.ipm.2026.104704_bib0065","doi-asserted-by":"crossref","DOI":"10.1073\/pnas.2409752122","article-title":"Evaluating interdisciplinary research: Disparate outcomes for topic and knowledge base","volume":"122","author":"Xiang","year":"2025","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"10.1016\/j.ipm.2026.104704_bib0066","series-title":"Proceedings of the 31st international conference on computational linguistics: Industry track","first-page":"511","article-title":"A simple yet efficient prompt compression method for text classification data annotation using LLM","author":"Xie","year":"2025"},{"key":"10.1016\/j.ipm.2026.104704_bib0067","series-title":"Proceedings of the 33rd ACM international conference on information and knowledge management","first-page":"2764","article-title":"Predicting scientific impact through diffusion, conformity, and contribution disentanglement","author":"Xue","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0068","unstructured":"Yang, A., Yang, B., Hui, B., Zheng, B., Yu, B., Zhou, C., Li, C., Li, C., Liu, D., Huang, F. et al. (2024). Qwen2 technical report. arXiv preprint arXiv: 2407.10671."},{"key":"10.1016\/j.ipm.2026.104704_bib0069","unstructured":"Ye, R., Pang, X., Chai, J., Chen, J., Yin, Z., Xiang, Z., Dong, X., Shao, J., & Chen, S. (2024). Are we there yet? revealing the risks of utilizing large language models in scholarly peer review. arXiv preprint arXiv: 2412.01708."},{"key":"10.1016\/j.ipm.2026.104704_bib0070","series-title":"Findings of the association for computational linguistics: EMNLP 2024","first-page":"10164","article-title":"Automated peer reviewing in paper SEA: Standardization, evaluation, and analysis","author":"Yu","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0071","series-title":"Pluralistic alignment workshop at neurIPS 2024","article-title":"Diverging preferences: When do annotators disagree and do models know?","author":"Zhang","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0072","series-title":"Findings of the association for computational linguistics: ACL 2023","first-page":"1284","article-title":"Causal matching with text embeddings: A case study in estimating the causal effects of peer review policies","author":"Zhang","year":"2023"},{"key":"10.1016\/j.ipm.2026.104704_bib0073","unstructured":"Zhang, Y., Zhang, H., Ji, W., Hua, T., Haber, N., Cao, H., & Liang, W. (2025). From replication to redesign: Exploring pairwise comparisons for LLM-based peer review. arXiv preprint arXiv: 2506.11343."},{"key":"10.1016\/j.ipm.2026.104704_bib0074","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing","first-page":"2712","article-title":"An electoral approach to diversify LLM-based multi-agent collective decision-making","author":"Zhao","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0075","series-title":"Proceedings of the 2024 joint international conference on computational linguistics, language resources and evaluation (LREC-COLING 2024)","first-page":"9340","article-title":"Is LLM a reliable reviewer? A comprehensive evaluation of LLM on automatic paper reviewing tasks","author":"Zhou","year":"2024"},{"key":"10.1016\/j.ipm.2026.104704_bib0076","series-title":"Proceedings of the 63rd annual meeting of the association for computational linguistics (volume 1: Long papers)","first-page":"29330","article-title":"DeepReview: Improving LLM-based paper review with human-like deep thinking process","author":"Zhu","year":"2025"},{"issue":"1","key":"10.1016\/j.ipm.2026.104704_bib0077","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1162\/coli_a_00502","article-title":"Can large language models transform computational social science?","volume":"50","author":"Ziems","year":"2024","journal-title":"Computational Linguistics"}],"container-title":["Information Processing & Management"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0306457326000956?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0306457326000956?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T03:25:54Z","timestamp":1774409154000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0306457326000956"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,9]]},"references-count":77,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2026,9]]}},"alternative-id":["S0306457326000956"],"URL":"https:\/\/doi.org\/10.1016\/j.ipm.2026.104704","relation":{},"ISSN":["0306-4573"],"issn-type":[{"value":"0306-4573","type":"print"}],"subject":[],"published":{"date-parts":[[2026,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Toward better pragmatic tagging of peer review: Enhancing benchmark datasets via human-in-the-loop multi-agent collaboration","name":"articletitle","label":"Article Title"},{"value":"Information Processing & Management","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.ipm.2026.104704","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Ltd. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"104704"}}