{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:12:22Z","timestamp":1750219942879,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,7]],"date-time":"2023-07-07T00:00:00Z","timestamp":1688688000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NSF","award":["2007256"],"award-info":[{"award-number":["2007256"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,9]]},"DOI":"10.1145\/3580507.3597744","type":"proceedings-article","created":{"date-parts":[[2023,7,7]],"date-time":"2023-07-07T14:19:22Z","timestamp":1688739562000},"page":"369-389","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Measurement Integrity in Peer Prediction: A Peer Assessment Case Study"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3448-080X","authenticated-orcid":false,"given":"Noah","family":"Burrell","sequence":"first","affiliation":[{"name":"Computer Science & Engineering, University of Michigan, Ann Arbor, United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6878-0670","authenticated-orcid":false,"given":"Grant","family":"Schoenebeck","sequence":"additional","affiliation":[{"name":"School of Information, University of Michigan, Ann Arbor, United States of America"}]}],"member":"320","published-online":{"date-parts":[[2023,7,7]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"R\u00e9mi Bachelet Drissa Zongo and Aline Bourelle. 2015. Does peer grading work? How to implement and improve it?  R\u00e9mi Bachelet Drissa Zongo and Aline Bourelle. 2015. Does peer grading work? How to implement and improve it?"},{"key":"e_1_3_2_1_2_1","unstructured":"Comparing instructor and peer assessment in MOOC GdP. In European MOOCs Stakeholders Summit 2015 (Proceedings of the Research Track). Mons Belgium. https:\/\/halshs.archives-ouvertes.fr\/halshs-01146710  Comparing instructor and peer assessment in MOOC GdP. In European MOOCs Stakeholders Summit 2015 (Proceedings of the Research Track). Mons Belgium. https:\/\/halshs.archives-ouvertes.fr\/halshs-01146710"},{"key":"e_1_3_2_1_3_1","unstructured":"Anujit Chakraborty Jatin Jindal and Swaprava Nath. 2020. Incentivizing Effort and Precision in Peer Grading. (2020). arXiv:1807.11657v6 [cs.GT] Working Paper.  Anujit Chakraborty Jatin Jindal and Swaprava Nath. 2020. Incentivizing Effort and Precision in Peer Grading. (2020). arXiv:1807.11657v6 [cs.GT] Working Paper."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488417"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2538862.2538900"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jet.2020.105065"},{"key":"e_1_3_2_1_7_1","unstructured":"Boi Faltings Radu Jurca and Goran Radanovic. 2017a. Peer Truth Serum: Incentives for Crowdsourcing Measurements and Opinions. arXiv:1704.05269  Boi Faltings Radu Jurca and Goran Radanovic. 2017a. Peer Truth Serum: Incentives for Crowdsourcing Measurements and Opinions. arXiv:1704.05269"},{"volume-title":"Game Theory for Data Science: Eliciting Truthful Information","author":"Faltings Boi","key":"e_1_3_2_1_8_1","unstructured":"Boi Faltings , Goran Radanovic , and Ronald Brachman . 2017b. Game Theory for Data Science: Eliciting Truthful Information . Morgan & Claypool Publishers . Boi Faltings, Goran Radanovic, and Ronald Brachman. 2017b. Game Theory for Data Science: Eliciting Truthful Information. Morgan & Claypool Publishers."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600057.2602865"},{"key":"e_1_3_2_1_10_1","volume-title":"Workshop on Algorithmic Game Theory and Data Science at the 17th ACM Conference on Electronic Commerce. arXiv:1606","author":"Gao Xi Alice","year":"2016","unstructured":"Xi Alice Gao , James R. Wright , and Kevin Leyton-Brown . 2016 . Incentivizing Evaluation via Limited Access to Ground Truth: Peer-Prediction Makes Things Worse . Workshop on Algorithmic Game Theory and Data Science at the 17th ACM Conference on Electronic Commerce. arXiv:1606 .07042 Xi Alice Gao, James R. Wright, and Kevin Leyton-Brown. 2016. Incentivizing Evaluation via Limited Access to Ground Truth: Peer-Prediction Makes Things Worse. Workshop on Algorithmic Game Theory and Data Science at the 17th ACM Conference on Electronic Commerce. arXiv:1606.07042"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33011996"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611975994.147"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3296670"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490486.3538338"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3391403.3399488"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1050.0379"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3465456.3467639"},{"key":"e_1_3_2_1_18_1","volume-title":"Tuned Models of Peer Assessment in MOOCs. In 6th International Conference on Educational Data Mining","author":"Piech Chris","year":"2013","unstructured":"Chris Piech , Jonathan Huang , Zhenghao Chen , Chuong Do , Andrew Ng , and Daphne Koller . 2013 . Tuned Models of Peer Assessment in MOOCs. In 6th International Conference on Educational Data Mining . Memphis, TN. arXiv:1307.2579 Chris Piech, Jonathan Huang, Zhenghao Chen, Chuong Do, Andrew Ng, and Daphne Koller. 2013. Tuned Models of Peer Assessment in MOOCs. In 6th International Conference on Educational Data Mining. Memphis, TN. arXiv:1307.2579"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2856102"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 17th International Conference on Multiple Criteria Decision Making, Whistler, BC Canada. 6--11","author":"Saaty Thomas L","year":"2004","unstructured":"Thomas L Saaty . 2004 . Scales from measurement-not measurement from scales . In Proceedings of the 17th International Conference on Multiple Criteria Decision Making, Whistler, BC Canada. 6--11 . Thomas L Saaty. 2004. Scales from measurement-not measurement from scales. In Proceedings of the 17th International Conference on Multiple Criteria Decision Making, Whistler, BC Canada. 6--11."},{"key":"e_1_3_2_1_21_1","volume-title":"Learning and Strongly Truthful Multi-Task Peer Prediction: A Variational Approach. In 12th Innovations in Theoretical Computer Science Conference (ITCS 2021","author":"Schoenebeck Grant","year":"2021","unstructured":"Grant Schoenebeck and Fang-Yi Yu . 2021 . Learning and Strongly Truthful Multi-Task Peer Prediction: A Variational Approach. In 12th Innovations in Theoretical Computer Science Conference (ITCS 2021 ). arXiv:2009.14730 Grant Schoenebeck and Fang-Yi Yu. 2021. Learning and Strongly Truthful Multi-Task Peer Prediction: A Variational Approach. In 12th Innovations in Theoretical Computer Science Conference (ITCS 2021). arXiv:2009.14730"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2940716.2940790"},{"key":"e_1_3_2_1_23_1","volume-title":"Measuring Performance of Peer Prediction Mechanisms Using Replicator Dynamics. In 25th International Joint Conference on Artificial Intelligence (IJCAI 2016","author":"Shnayder Victor","year":"2091","unstructured":"Victor Shnayder , Rafael M. Frongillo , and David C. Parkes . 2016b . Measuring Performance of Peer Prediction Mechanisms Using Replicator Dynamics. In 25th International Joint Conference on Artificial Intelligence (IJCAI 2016 ). New York, NY. http:\/\/nrs.harvard.edu\/urn-3:HUL.InstRepos:322 2091 6 Victor Shnayder, Rafael M. Frongillo, and David C. Parkes. 2016b. Measuring Performance of Peer Prediction Mechanisms Using Replicator Dynamics. In 25th International Joint Conference on Artificial Intelligence (IJCAI 2016). New York, NY. http:\/\/nrs.harvard.edu\/urn-3:HUL.InstRepos:32220916"},{"volume-title":"Practical Peer Prediction for Peer Assessment. In Fourth AAAI Conference on Human Computation and Crowdsourcing","author":"Shnayder Victor","key":"e_1_3_2_1_24_1","unstructured":"Victor Shnayder and David C. Parkes . 2016 . Practical Peer Prediction for Peer Assessment. In Fourth AAAI Conference on Human Computation and Crowdsourcing . Austin, TX. http:\/\/nrs.harvard.edu\/urn-3:HUL.InstRepos:34732142 Victor Shnayder and David C. Parkes. 2016. Practical Peer Prediction for Peer Assessment. In Fourth AAAI Conference on Human Computation and Crowdsourcing. Austin, TX. http:\/\/nrs.harvard.edu\/urn-3:HUL.InstRepos:34732142"},{"key":"e_1_3_2_1_25_1","volume-title":"On the Theory of Scales of Measurement. Science 103, 2684","author":"Stevens S. S.","year":"1946","unstructured":"S. S. Stevens . 1946. On the Theory of Scales of Measurement. Science 103, 2684 ( 1946 ), 677--680. http:\/\/www.jstor.org\/stable\/1671815 S. S. Stevens. 1946. On the Theory of Scales of Measurement. Science 103, 2684 (1946), 677--680. http:\/\/www.jstor.org\/stable\/1671815"},{"key":"e_1_3_2_1_26_1","volume-title":"Learning the Prior in Minimal Peer Prediction. In 3rd Workshop on Social Computing and User Generated Content (SC","author":"Witkowski Jens","year":"2013","unstructured":"Jens Witkowski and David C. Parkes . 2013 . Learning the Prior in Minimal Peer Prediction. In 3rd Workshop on Social Computing and User Generated Content (SC 2013 ). Philadelphia, PA, 39:1--39:12. http:\/\/nrs.harvard.edu\/urn-3:HUL. InstRepos:34222829 Jens Witkowski and David C. Parkes. 2013. Learning the Prior in Minimal Peer Prediction. In 3rd Workshop on Social Computing and User Generated Content (SC 2013). Philadelphia, PA, 39:1--39:12. http:\/\/nrs.harvard.edu\/urn-3:HUL. InstRepos:34222829"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677278"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340631.3394878"},{"key":"e_1_3_2_1_29_1","unstructured":"Hedayat Zarkoob Farzad Abdolhosseini and Kevin Leyton-Brown. 2021. Mechanical TA 2: A System for Peer Grading with TA Support. arXiv:2101.10078 [cs.CY]  Hedayat Zarkoob Farzad Abdolhosseini and Kevin Leyton-Brown. 2021. Mechanical TA 2: A System for Peer Grading with TA Support. arXiv:2101.10078 [cs.CY]"},{"key":"e_1_3_2_1_30_1","unstructured":"Hedayat Zarkoob Greg d'Eon Lena Podina and Kevin Leyton-Brown. 2022. Better Peer Grading through Bayesian Inference. arXiv:2209.01242 [cs.AI]  Hedayat Zarkoob Greg d'Eon Lena Podina and Kevin Leyton-Brown. 2022. Better Peer Grading through Bayesian Inference. arXiv:2209.01242 [cs.AI]"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.5555\/3398761.3398944"}],"event":{"name":"EC '23: 24th ACM Conference on Economics and Computation","sponsor":["SIGecom Special Interest Group on Economics and Computation"],"location":"London United Kingdom","acronym":"EC '23"},"container-title":["Proceedings of the 24th ACM Conference on Economics and Computation"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3580507.3597744","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3580507.3597744","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:48:56Z","timestamp":1750182536000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3580507.3597744"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,7]]},"references-count":31,"alternative-id":["10.1145\/3580507.3597744","10.1145\/3580507"],"URL":"https:\/\/doi.org\/10.1145\/3580507.3597744","relation":{},"subject":[],"published":{"date-parts":[[2023,7,7]]},"assertion":[{"value":"2023-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}