{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,15]],"date-time":"2026-08-15T03:48:12Z","timestamp":1786765692510,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":60,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,3]],"date-time":"2024-06-03T00:00:00Z","timestamp":1717372800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,3]]},"DOI":"10.1145\/3630106.3659012","type":"proceedings-article","created":{"date-parts":[[2024,6,5]],"date-time":"2024-06-05T13:14:21Z","timestamp":1717593261000},"page":"1875-1884","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":21,"title":["AI as a Sport: On the Competitive Epistemologies of Benchmarking"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6720-5794","authenticated-orcid":false,"given":"Will","family":"Orr","sequence":"first","affiliation":[{"name":"University of Southern California, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4919-1832","authenticated-orcid":false,"given":"Edward B.","family":"Kang","sequence":"additional","affiliation":[{"name":"New York University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,6,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Zeeshan Ahmed. 2023. Harry Potter by Balenciaga: AI-generated Future of Entertainment? | The Express Tribune. https:\/\/tribune.com.pk\/story\/2409540\/harry-potter-by-balenciaga-is-this-the-ai-generated-future-of-entertainment."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","unstructured":"Syeda\u00a0Nahida Akter Zichun Yu Aashiq Muhamed Tianyue Ou Alex B\u00e4uerle \u00c1ngel\u00a0Alexander Cabrera Krish Dholakia Chenyan Xiong and Graham Neubig. 2023. An In-depth Look at Gemini\u2019s Language Abilities. https:\/\/doi.org\/10.48550\/arXiv.2312.11444 arxiv:2312.11444\u00a0[cs]","DOI":"10.48550\/arXiv.2312.11444"},{"key":"e_1_3_2_1_3_1","volume-title":"Netflix Recommendations: Beyond the 5 Stars (Part 1). https:\/\/netflixtechblog.com\/netflix-recommendations-beyond-the-5-stars-part-1-55838468f429.","author":"Amatriain Xavier","year":"2012","unstructured":"Xavier Amatriain and Justin Basilico. 2012. Netflix Recommendations: Beyond the 5 Stars (Part 1). https:\/\/netflixtechblog.com\/netflix-recommendations-beyond-the-5-stars-part-1-55838468f429."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Robert Bell Jim Bennett Yehuda Koren and Chris Volinsky. 2009. The Million Dollar Programming Prize. https:\/\/spectrum.ieee.org\/the-million-dollar-programming-prize.","DOI":"10.1109\/MSPEC.2009.4907383"},{"key":"e_1_3_2_1_5_1","first-page":"6","volume-title":"Proceedings of the KDD Cup Workshop","author":"Bennett J.","year":"2007","unstructured":"J. Bennett and S. Lanning. 2007. The Netflix Prize. In Proceedings of the KDD Cup Workshop 2007, page 3\u20136. New York, ACM. Association for Computing Machinery, New York, NY, USA, 3\u20136."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEC.1961.5219162"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1460299.1460326"},{"key":"e_1_3_2_1_8_1","volume-title":"Bowker and Susan\u00a0Leigh Star","author":"C.","year":"1999","unstructured":"Geoffrey\u00a0C. Bowker and Susan\u00a0Leigh Star. 1999. Sorting Things Out: Classification and Its Consequences. MIT Press, Cambridge, MA."},{"key":"e_1_3_2_1_9_1","volume-title":"BellKor\u2019s Pragmatic Chaos Wins $1 Million Netflix Prize by Mere Minutes. Wired","author":"Buskirk Eliot\u00a0Van","year":"2009","unstructured":"Eliot\u00a0Van Buskirk. 2009. BellKor\u2019s Pragmatic Chaos Wins $1 Million Netflix Prize by Mere Minutes. Wired (2009)."},{"key":"e_1_3_2_1_10_1","unstructured":"Mita Chaturvedi. 2021. How Useful Was The Netflix Prize Really?https:\/\/analyticsindiamag.com\/how-useful-was-the-netflix-prize-really\/."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1017\/S1351324917000389"},{"key":"e_1_3_2_1_12_1","volume-title":"Excavating AI: The Politics of Training Sets for Machine Learning. https:\/\/excavating.ai.","author":"Crawford Kate","year":"2019","unstructured":"Kate Crawford and Trevor Paglen. 2019. Excavating AI: The Politics of Training Sets for Machine Learning. https:\/\/excavating.ai."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517211035955"},{"key":"e_1_3_2_1_15_1","volume-title":"Discipline and Punish: The Birth of the Prison (reprint ed.)","author":"Foucault Michel","unstructured":"Michel Foucault. 1991. Discipline and Punish: The Birth of the Prison (reprint ed.). Penguin Books, London."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","unstructured":"Zishan Guo Renren Jin Chuang Liu Yufei Huang Dan Shi Supryadi Linhao Yu Yan Liu Jiaxuan Li Bojian Xiong and Deyi Xiong. 2023. Evaluating Large Language Models: A Comprehensive Survey. https:\/\/doi.org\/10.48550\/arXiv.2310.19736 arxiv:2310.19736\u00a0[cs]","DOI":"10.48550\/arXiv.2310.19736"},{"key":"e_1_3_2_1_17_1","unstructured":"Ed Haas. 2023. The Great Hogwarts Rave of 1996 | Facebook. https:\/\/www.facebook.com\/groups\/cursedaiwtf\/permalink\/1407296606545485."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1177\/1461444814538646"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"Dan Hendrycks Collin Burns Steven Basart Andy Zou Mantas Mazeika Dawn Song and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. https:\/\/doi.org\/10.48550\/arXiv.2009.03300 arxiv:2009.03300\u00a0[cs]","DOI":"10.48550\/arXiv.2009.03300"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1002\/j.1538-7305.1962.tb02426.x"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/JRPROC.1962.288194"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/PGEC.1963.263427"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1457838.1457894"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEC.1960.5219829"},{"key":"e_1_3_2_1_25_1","unstructured":"Dan Jackson. 2017. The Netflix Prize: How a $1 Million Contest Changed Binge-Watching Forever. https:\/\/www.thrillist.com\/entertainment\/nation\/the-netflix-prize."},{"key":"e_1_3_2_1_26_1","volume-title":"Programming, Formulating","author":"Jaton Florian","unstructured":"Florian Jaton. 2021. The Constitution of Algorithms: Ground-Truthing, Programming, Formulating. The MIT Press, Cambridge, Massachusetts."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517221146122"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594011"},{"key":"e_1_3_2_1_29_1","unstructured":"Jules King Perpetual Baffour Scott Crossley Ryan Holbrook and Maggie Demkin. 2023. LLM - Detect AI Generated Text. https:\/\/kaggle.com\/competitions\/llm-detect-ai-generated-text."},{"key":"e_1_3_2_1_30_1","volume-title":"Thirty-Fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2). Association for Computing Machinery","author":"Koch Bernard","year":"2021","unstructured":"Bernard Koch, Emily Denton, Alex Hanna, and Jacob\u00a0Gates Foster. 2021. Reduced, Reused and Recycled: The Life of a Dataset in Machine Learning Research. In Thirty-Fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2). Association for Computing Machinery, New York, NY."},{"key":"e_1_3_2_1_31_1","unstructured":"Mark Liberman. 2015. Reproducible Research and the Common Task Method. https:\/\/www.simonsfoundation.org\/event\/reproducible-research-and-the-common-task-method\/."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v41i2.5297"},{"key":"e_1_3_2_1_33_1","unstructured":"James Lighthill. 1972. Lighthill Report. http:\/\/www.chilton-computing.org.uk\/inf\/literature\/reports\/lighthill_report\/p001.htm."},{"key":"e_1_3_2_1_34_1","unstructured":"Livemint. 2023. 90% in Bar Exam 88% in LSAT: GPT-4 Beats 90% of Humans in World\u2019s Toughest Exam. https:\/\/www.livemint.com\/news\/world\/90-in-bar-exam-88-in-lsat-gpt-4-beats-90-of-humans-in-world-s-toughest-exam-11678882508873.html."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.15849\/IJASCA.230320.01"},{"key":"e_1_3_2_1_36_1","volume-title":"The Metric Society: On the Quantification of the Social","author":"Mau Steffen","unstructured":"Steffen Mau. 2019. The Metric Society: On the Quantification of the Social. Polity Press, Cambridge, UK : Medford, MA."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-56286-1_2"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/11765.001.0001"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","unstructured":"Arvind Narayanan and Vitaly Shmatikov. 2007. How To Break Anonymity of the Netflix Prize Dataset. https:\/\/doi.org\/10.48550\/arXiv.cs\/0610105 arxiv:cs\/0610105","DOI":"10.48550\/arXiv.cs"},{"key":"e_1_3_2_1_40_1","unstructured":"Ryan Northrup. 2023. Voldemort DJs The Great Hogwarts Rave In Harry Potter Art. https:\/\/screenrant.com\/harry-potter-voldemort-dj-hogwarts-rave-art\/."},{"key":"e_1_3_2_1_41_1","volume-title":"The Social Construction of Datasets: On the Practices, Processes and Challenges of Dataset Creation for Machine Learning. SocArXiv","author":"Orr Will","year":"2023","unstructured":"Will Orr and Kate Crawford. 2023. The Social Construction of Datasets: On the Practices, Processes and Challenges of Dataset Creation for Machine Learning. SocArXiv (2023)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2021.100336"},{"key":"e_1_3_2_1_43_1","volume-title":"Exclusive: The $2 Per Hour Workers Who Made ChatGPT Safer. https:\/\/time.com\/6247678\/openai-chatgpt-kenya-workers\/.","author":"Perrigo Billy","year":"2023","unstructured":"Billy Perrigo. 2023. Exclusive: The $2 Per Hour Workers Who Made ChatGPT Safer. https:\/\/time.com\/6247678\/openai-chatgpt-kenya-workers\/."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1121\/1.1911801"},{"key":"e_1_3_2_1_45_1","volume-title":"Trust in Numbers: The Pursuit of Objectivity in Science and Public Life (2. print., and 1. paperback printing ed.)","author":"Porter M.","unstructured":"Theodore\u00a0M. Porter. 1996. Trust in Numbers: The Pursuit of Objectivity in Science and Public Life (2. print., and 1. paperback printing ed.). Princeton Univ.Press, Princeton, N.J."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1086\/667828"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","unstructured":"Colin Raffel Noam Shazeer Adam Roberts Katherine Lee Sharan Narang Michael Matena Yanqi Zhou Wei Li and Peter\u00a0J. Liu. 2020. Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer. https:\/\/doi.org\/10.48550\/arXiv.1910.10683 arxiv:1910.10683\u00a0[cs stat]","DOI":"10.48550\/arXiv.1910.10683"},{"key":"e_1_3_2_1_48_1","series-title":"Round 2","volume-title":"Thirty-Fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track","author":"Raji Inioluwa\u00a0Deborah","unstructured":"Inioluwa\u00a0Deborah Raji, Emily Denton, Emily\u00a0M. Bender, Alex Hanna, and Amandalynne Paullada. 2021. AI and the Everything in the Whole Wide World Benchmark. In Thirty-Fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2). Association for Computing Machinery, New York, NY."},{"key":"e_1_3_2_1_49_1","unstructured":"Ben Recht. 2023. Revisiting Highleyman\u2019s Data. https:\/\/www.argmin.net\/p\/revisiting-highleymans-data."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","unstructured":"Siddharth Samsi Dan Zhao Joseph McDonald Baolin Li Adam Michaleas Michael Jones William Bergeron Jeremy Kepner Devesh Tiwari and Vijay Gadepally. 2023. From Words to Watts: Benchmarking the Energy Costs of Large Language Model Inference. https:\/\/doi.org\/10.48550\/arXiv.2310.03003 arxiv:2310.03003\u00a0[cs]","DOI":"10.48550\/arXiv.2310.03003"},{"key":"e_1_3_2_1_51_1","volume-title":"Leviathan and the Air-Pump: Hobbes, Boyle, and the Experimental Life (1. paperback pr. with corr ed.). Princeton Univ","author":"Shapin Steven","unstructured":"Steven Shapin and Simon Schaffer. 1989. Leviathan and the Air-Pump: Hobbes, Boyle, and the Experimental Life (1. paperback pr. with corr ed.). Princeton Univ. Pr, Princeton, N.J."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i09.7123"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2312.11805"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cpet.2021.07.001"},{"key":"e_1_3_2_1_55_1","unstructured":"Jon Victor. 2023. How Google Got Back on Its Feet in AI Race. https:\/\/www.theinformation.com\/articles\/how-google-got-back-on-its-feet-in-ai-race."},{"key":"e_1_3_2_1_56_1","volume-title":"Beyond Measure: The Hidden History of Measurement","author":"Vincent James","year":"2022","unstructured":"James Vincent. 2022. Beyond Measure: The Hidden History of Measurement. Faber, London."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-5446"},{"key":"e_1_3_2_1_58_1","volume-title":"The Exploited Labor Behind Artificial Intelligence. Noema (Oct","author":"Williams Adrienne","year":"2022","unstructured":"Adrienne Williams, Milagros Miceli, and Timnit Gebru. 2022. The Exploited Labor Behind Artificial Intelligence. Noema (Oct. 2022)."},{"key":"e_1_3_2_1_59_1","unstructured":"Matteo Wong. 2023. AI Doomerism Is a Decoy."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1088\/1742-6596"}],"event":{"name":"FAccT '24: The 2024 ACM Conference on Fairness, Accountability, and Transparency","location":"Rio de Janeiro Brazil","acronym":"FAccT '24"},"container-title":["The 2024 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3630106.3659012","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3630106.3659012","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:57:07Z","timestamp":1750291027000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3630106.3659012"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,3]]},"references-count":60,"alternative-id":["10.1145\/3630106.3659012","10.1145\/3630106"],"URL":"https:\/\/doi.org\/10.1145\/3630106.3659012","relation":{},"subject":[],"published":{"date-parts":[[2024,6,3]]},"assertion":[{"value":"2024-06-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}