{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T01:23:50Z","timestamp":1777339430524,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,8]],"date-time":"2024-07-08T00:00:00Z","timestamp":1720396800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"United States National Science Foundation","award":["2313137"],"award-info":[{"award-number":["2313137"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372007"],"award-info":[{"award-number":["62372007"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,8]]},"DOI":"10.1145\/3670865.3673532","type":"proceedings-article","created":{"date-parts":[[2024,12,17]],"date-time":"2024-12-17T20:13:33Z","timestamp":1734466413000},"page":"582-612","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Eliciting Informative Text Evaluations with Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3935-7352","authenticated-orcid":false,"given":"Yuxuan","family":"Lu","sequence":"first","affiliation":[{"name":"CFCS, School of CS, Peking University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8058-0820","authenticated-orcid":false,"given":"Shengwei","family":"Xu","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, Michigan, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5188-7543","authenticated-orcid":false,"given":"Yichi","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, Michigan, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5901-3004","authenticated-orcid":false,"given":"Yuqing","family":"Kong","sequence":"additional","affiliation":[{"name":"CFCS, School of CS, Peking University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6878-0670","authenticated-orcid":false,"given":"Grant","family":"Schoenebeck","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, Michigan, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,12,17]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al. Gpt-4 technical report. arXiv preprint arXiv:2303.08774","author":"Achiam Josh","year":"2023","unstructured":"Josh Achiam, Steven Adler, Sandhini Agarwal, Lama Ahmad, Ilge Akkaya, Florencia Leoni Aleman, Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al. Gpt-4 technical report. arXiv preprint arXiv:2303.08774, 2023."},{"key":"e_1_3_2_1_2_1","volume-title":"The Eleventh International Conference on Learning Representations","author":"Arora Simran","year":"2022","unstructured":"Simran Arora, Avanika Narayan, Mayee F Chen, Laurel Orr, Neel Guha, Kush Bhatia, Ines Chami, and Christopher Re. Ask me anything: A simple strategy for prompting language models. In The Eleventh International Conference on Learning Representations, 2022."},{"key":"e_1_3_2_1_3_1","volume-title":"Language models are few-shot learners. Advances in neural information processing systems, 33:1877--1901","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al. Language models are few-shot learners. Advances in neural information processing systems, 33:1877--1901, 2020."},{"key":"e_1_3_2_1_4_1","volume-title":"Measurement integrity in peer prediction: A peer assessment case study. arXiv preprint arXiv:2108.05521","author":"Burrell Noah","year":"2021","unstructured":"Noah Burrell and Grant Schoenebeck. Measurement integrity in peer prediction: A peer assessment case study. arXiv preprint arXiv:2108.05521, 2021."},{"key":"e_1_3_2_1_5_1","volume-title":"The wisdom of the crowd and higher-order beliefs. arXiv preprint arXiv:2102.02666","author":"Chen Yi-Chun","year":"2021","unstructured":"Yi-Chun Chen, Manuel Mueller-Frank, and Mallesh M Pai. The wisdom of the crowd and higher-order beliefs. arXiv preprint arXiv:2102.02666, 2021."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1093\/oso\/9780195064650.001.0001"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.2307\/1911240"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488417"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.26"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219166.3219222"},{"key":"e_1_3_2_1_11_1","volume-title":"Incentivizing evaluation via limited access to ground truth: Peerprediction makes things worse. arXiv preprint arXiv:1606.07042","author":"Gao Alice","year":"2016","unstructured":"Alice Gao, James R Wright, and Kevin Leyton-Brown. Incentivizing evaluation via limited access to ground truth: Peerprediction makes things worse. arXiv preprint arXiv:1606.07042, 2016."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-020-00257-z"},{"key":"e_1_3_2_1_13_1","volume-title":"Strictly proper scoring rules, prediction, and estimation. Journal of the American statistical Association, 102(477):359--378","author":"Gneiting Tilmann","year":"2007","unstructured":"Tilmann Gneiting and Adrian E Raftery. Strictly proper scoring rules, prediction, and estimation. Journal of the American statistical Association, 102(477):359--378, 2007."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611975994.147"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3638239"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219166.3219172"},{"key":"e_1_3_2_1_17_1","volume-title":"An information theoretic framework for designing information elicitation mechanisms that reward truth-telling. ACM Transactions on Economics and Computation (TEAC), 7(1):1--33","author":"Kong Yuqing","year":"2019","unstructured":"Yuqing Kong and Grant Schoenebeck. An information theoretic framework for designing information elicitation mechanisms that reward truth-telling. ACM Transactions on Economics and Computation (TEAC), 7(1):1--33, 2019."},{"key":"e_1_3_2_1_18_1","volume-title":"Can large language models provide useful feedback on research papers? a large-scale empirical analysis","author":"Liang Weixin","year":"2023","unstructured":"Weixin Liang, Yuhui Zhang, Hancheng Cao, Binglu Wang, Daisy Ding, Xinyu Yang, Kailas Vodrahalli, Siyu He, Daniel Smith, Yian Yin, Daniel McFarland, and James Zou. Can large language models provide useful feedback on research papers? a large-scale empirical analysis, 2023."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3565559"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1050.0379"},{"key":"e_1_3_2_1_21_1","volume-title":"A bayesian truth serum for subjective data. science, 306(5695):462--466","author":"Prelec Drazen","year":"2004","unstructured":"Drazen Prelec. A bayesian truth serum for subjective data. science, 306(5695):462--466, 2004."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v28i1.8797"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/355112.355122"},{"key":"e_1_3_2_1_24_1","volume-title":"Learning and strongly truthful multi-task peer prediction: A variational approach. arXiv preprint arXiv:2009.14730","author":"Schoenebeck Grant","year":"2020","unstructured":"Grant Schoenebeck and Fang-Yi Yu. Learning and strongly truthful multi-task peer prediction: A variational approach. arXiv preprint arXiv:2009.14730, 2020."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3565560"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1009957816843"},{"key":"e_1_3_2_1_27_1","volume-title":"A mathematical theory of communication. The Bell system technical journal, 27(3):379--423","author":"Shannon Claude Elwood","year":"1948","unstructured":"Claude Elwood Shannon. A mathematical theory of communication. The Bell system technical journal, 27(3):379--423, 1948."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2940716.2940790"},{"key":"e_1_3_2_1_29_1","volume-title":"ArXivorg","author":"Srinivasan Siddarth","year":"2021","unstructured":"Siddarth Srinivasan and Jamie Morgenstern. Auctions and peer prediction for scientific peer review. ArXivorg, 2021."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1146\/annurev-economics-080315-015325"},{"key":"e_1_3_2_1_31_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric Michael Smith Ranjan Subramanian Xiaoqing Ellen Tan Binh Tang Ross Taylor Adina Williams Jian Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. Llama 2: Open foundation and fine-tuned chat models 2023."},{"key":"e_1_3_2_1_32_1","first-page":"1492","volume-title":"Proceedings of the Twenty-Sixth AAAI Conference on Artificial Intelligence, AAAI'12","author":"Witkowski Jens","year":"2012","unstructured":"Jens Witkowski and David C. Parkes. A robust bayesian truth serum for small populations. In Proceedings of the Twenty-Sixth AAAI Conference on Artificial Intelligence, AAAI'12, page 1492--1498. AAAI Press, 2012."},{"key":"e_1_3_2_1_33_1","volume-title":"Elicitationgpt: Text elicitation mechanisms via language models. arXiv preprint arXiv:2406.09363","author":"Wu Yifan","year":"2024","unstructured":"Yifan Wu and Jason Hartline. Elicitationgpt: Text elicitation mechanisms via language models. arXiv preprint arXiv:2406.09363, 2024."},{"key":"e_1_3_2_1_34_1","volume-title":"Spot check equivalence: an interpretable metric for information elicitation mechanisms. arXiv preprint arXiv:2402.13567","author":"Xu Shengwei","year":"2024","unstructured":"Shengwei Xu, Yichi Zhang, Paul Resnick, and Grant Schoenebeck. Spot check equivalence: an interpretable metric for information elicitation mechanisms. arXiv preprint arXiv:2402.13567, 2024."},{"key":"e_1_3_2_1_35_1","volume-title":"The Eleventh International Conference on Learning Representations (ICLR)","author":"Zeng Aohan","year":"2023","unstructured":"Aohan Zeng, Xiao Liu, Zhengxiao Du, Zihan Wang, Hanyu Lai, Ming Ding, Zhuoyi Yang, Yifan Xu, Wendi Zheng, Xiao Xia, Weng Lam Tam, Zixuan Ma, Yufei Xue, Jidong Zhai, Wenguang Chen, Zhiyuan Liu, Peng Zhang, Yuxiao Dong, and Jie Tang. GLM-130b: An open bilingual pre-trained model. In The Eleventh International Conference on Learning Representations (ICLR), 2023. URL https:\/\/openreview.net\/forum?id=-Aw0rrrPUF."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3543507.3583292"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3543507.3583334"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.858"}],"event":{"name":"EC '24: 25th ACM Conference on Economics and Computation","location":"New Haven CT USA","acronym":"EC '24","sponsor":["SIGecom Special Interest Group on Economics and Computation"]},"container-title":["Proceedings of the 25th ACM Conference on Economics and Computation"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3670865.3673532","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3670865.3673532","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:10:00Z","timestamp":1750295400000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3670865.3673532"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,8]]},"references-count":38,"alternative-id":["10.1145\/3670865.3673532","10.1145\/3670865"],"URL":"https:\/\/doi.org\/10.1145\/3670865.3673532","relation":{},"subject":[],"published":{"date-parts":[[2024,7,8]]},"assertion":[{"value":"2024-12-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}