{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T02:03:16Z","timestamp":1777341796227,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":32,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,13]],"date-time":"2024-05-13T00:00:00Z","timestamp":1715558400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2313137"],"award-info":[{"award-number":["2313137"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Science Foundation","award":["2007256"],"award-info":[{"award-number":["2007256"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,13]]},"DOI":"10.1145\/3589334.3645679","type":"proceedings-article","created":{"date-parts":[[2024,5,8]],"date-time":"2024-05-08T07:08:13Z","timestamp":1715152093000},"page":"276-287","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Spot Check Equivalence: An Interpretable Metric for Information Elicitation Mechanisms"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8058-0820","authenticated-orcid":false,"given":"Shengwei","family":"Xu","sequence":"first","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5188-7543","authenticated-orcid":false,"given":"Yichi","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8368-0600","authenticated-orcid":false,"given":"Paul","family":"Resnick","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6878-0670","authenticated-orcid":false,"given":"Grant","family":"Schoenebeck","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,5,13]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3033274.3085127"},{"key":"e_1_3_2_2_2_1","volume-title":"Measurement Integrity in Peer Prediction: A Peer Assessment Case Study. arXiv preprint arXiv:2108.05521","author":"Burrell Noah","year":"2021","unstructured":"Noah Burrell and Grant Schoenebeck. 2021. Measurement Integrity in Peer Prediction: A Peer Assessment Case Study. arXiv preprint arXiv:2108.05521 (2021)."},{"key":"e_1_3_2_2_3_1","first-page":"1","article-title":"Opportunities for crowdsourcing research on amazon mechanical turk","volume":"5","author":"Chen Jenny J","year":"2011","unstructured":"Jenny J Chen, Natala J Menezes, Adam D Bradley, and T North. 2011. Opportunities for crowdsourcing research on amazon mechanical turk. Interfaces 5, 3 (2011), 1.","journal-title":"Interfaces"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488417"},{"key":"e_1_3_2_2_5_1","volume-title":"Peer truth serum: incentives for crowdsourcing measurements and opinions. arXiv preprint arXiv:1704.05269","author":"Faltings Boi","year":"2017","unstructured":"Boi Faltings, Radu Jurca, and Goran Radanovic. 2017. Peer truth serum: incentives for crowdsourcing measurements and opinions. arXiv preprint arXiv:1704.05269 (2017)."},{"key":"e_1_3_2_2_6_1","volume-title":"Incentivizing evaluation via limited access to ground truth: Peer-prediction makes things worse. arXiv preprint arXiv:1606.07042","author":"Gao Alice","year":"2016","unstructured":"Alice Gao, James R Wright, and Kevin Leyton-Brown. 2016. Incentivizing evaluation via limited access to ground truth: Peer-prediction makes things worse. arXiv preprint arXiv:1606.07042 (2016)."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2019.03.004"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33011996"},{"key":"e_1_3_2_2_9_1","volume-title":"Optimal Scoring Rules for Multi-dimensional Effort. arXiv preprint arXiv:2211.03302","author":"Hartline Jason D","year":"2022","unstructured":"Jason D Hartline, Liren Shan, Yingkai Li, and Yifan Wu. 2022. Optimal Scoring Rules for Multi-dimensional Effort. arXiv preprint arXiv:2211.03302 (2022)."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/11600930_26"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","unstructured":"Yuqing Kong. [n. d.]. Dominantly Truthful Multi-task Peer Prediction with a Constant Number of Tasks. 2398--2411. https:\/\/doi.org\/10.1137\/1.9781611975994.147 arXiv:https:\/\/epubs.siam.org\/doi\/pdf\/10.1137\/1.9781611975994.147","DOI":"10.1137\/1.9781611975994.147"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611975994.147"},{"key":"e_1_3_2_2_13_1","first-page":"13329","article-title":"Eliciting thinking hierarchy without a prior","volume":"35","author":"Kong Yuqing","year":"2022","unstructured":"Yuqing Kong, Yunqi Li, Yubo Zhang, Zhihuan Huang, and Jinzhao Wu. 2022. Eliciting thinking hierarchy without a prior. Advances in Neural Information Processing Systems 35 (2022), 13329--13341.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219166.3219172"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219166.3219194"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3296670"},{"key":"e_1_3_2_2_17_1","unstructured":"Nathan Lambert. 2023. Llama 2: an incredible open LLM. https:\/\/www. interconnects.ai\/p\/llama-2-from-meta'sd=pf"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490486.3538338"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1050.0379"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3465456.3467639"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2856102"},{"key":"e_1_3_2_2_22_1","volume-title":"Learning and strongly truthful multitask peer prediction: A variational approach. arXiv preprint arXiv:2009.14730","author":"Schoenebeck Grant","year":"2020","unstructured":"Grant Schoenebeck and Fang-Yi Yu. 2020. Learning and strongly truthful multitask peer prediction: A variational approach. arXiv preprint arXiv:2009.14730 (2020)."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449840"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2940716.2940790"},{"key":"e_1_3_2_2_25_1","unstructured":"MATTEO VENANZI William Teacy Alexander Rogers and Nicholas Jennings. 2015. Weather Sentiment - Amazon Mechanical Turk dataset. https:\/\/eprints. soton.ac.uk\/376543\/"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1378704.1378719"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v2i1.13151"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677278"},{"key":"e_1_3_2_2_30_1","volume-title":"Report-sensitive spotchecking in peer-grading systems. arXiv preprint arXiv:1906.05884","author":"Zarkoob Hedayat","year":"2019","unstructured":"Hedayat Zarkoob, Hu Fu, and Kevin Leyton-Brown. 2019. Report-sensitive spotchecking in peer-grading systems. arXiv preprint arXiv:1906.05884 (2019)."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3543507.3583334"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3543507.3583292"}],"event":{"name":"WWW '24: The ACM Web Conference 2024","location":"Singapore Singapore","acronym":"WWW '24","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM Web Conference 2024"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589334.3645679","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3589334.3645679","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T00:27:13Z","timestamp":1755822433000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589334.3645679"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,13]]},"references-count":32,"alternative-id":["10.1145\/3589334.3645679","10.1145\/3589334"],"URL":"https:\/\/doi.org\/10.1145\/3589334.3645679","relation":{},"subject":[],"published":{"date-parts":[[2024,5,13]]},"assertion":[{"value":"2024-05-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}