{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,29]],"date-time":"2026-06-29T18:47:10Z","timestamp":1782758830079,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":67,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T00:00:00Z","timestamp":1782345600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,6,25]]},"DOI":"10.1145\/3805689.3812240","type":"proceedings-article","created":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T16:20:39Z","timestamp":1782231639000},"page":"840-872","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Unsteady Metrics and Benchmarking Cultures of AI Model Builders"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2464-7699","authenticated-orcid":false,"given":"Stefan","family":"Baack","sequence":"first","affiliation":[{"name":"Independent Researcher, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8683-3692","authenticated-orcid":false,"given":"Maty","family":"Bohacek","sequence":"additional","affiliation":[{"name":"Stanford University, Palo Alto, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4096-7465","authenticated-orcid":false,"given":"Christo","family":"Buschek","sequence":"additional","affiliation":[{"name":"Independent Researcher, Vienna, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,6,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462563"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.744"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"Dario Amodei Chris Olah Jacob Steinhardt Paul Christiano John Schulman and Dan Man\u00e9. 2016. Concrete Problems in AI Safety. arXiv:1606.06565 [cs] doi:10.48550\/arXiv.1606.06565","DOI":"10.48550\/arXiv.1606.06565"},{"key":"e_1_3_2_1_4_1","unstructured":"Anthropic. 2025. Claude 3.7 Sonnet System Card."},{"key":"e_1_3_2_1_5_1","volume-title":"Angelika Romanou, Franziska Sofia Hafner, Harry Mayne, Jan Batzner, Negar Foroutan, Chris Schmitz, Karolina Korgul, Hunar Batra, et al.","author":"Bean Andrew M","year":"2025","unstructured":"Andrew M Bean, Ryan Othniel Kearns, Angelika Romanou, Franziska Sofia Hafner, Harry Mayne, Jan Batzner, Negar Foroutan, Chris Schmitz, Karolina Korgul, Hunar Batra, et al. 2025. Measuring what Matters: Construct Validity in Large Language Model Benchmarks. arXiv preprint arXiv:2511.04703 (2025)."},{"key":"e_1_3_2_1_6_1","volume-title":"Forty-second International Conference on Machine Learning Position Paper Track.","author":"Blili-Hamelin Borhane","unstructured":"Borhane Blili-Hamelin, Christopher Graziul, Leif Hancox-Li, Hananel Hazan, El-Mahdi El-Mhamdi, Avijit Ghosh, Katherine A Heller, Jacob Metcalf, Fabricio Murai, Eryk Salvaggio, et al. [n. d.]. Position: Stop treating AGI as the north-star goal of AI research. In Forty-second International Conference on Machine Learning Position Paper Track."},{"key":"e_1_3_2_1_7_1","volume-title":"Uncovering Competency Gaps in Large Language Models and Their Benchmarks. arXiv preprint arXiv:2512.20638","author":"Bohacek Matyas","year":"2025","unstructured":"Matyas Bohacek, Nino Scherrer, Nicholas Dufour, Thomas Leung, Christoph Bregler, and Stephanie CY Chan. 2025. Uncovering Competency Gaps in Large Language Models and Their Benchmarks. arXiv preprint arXiv:2512.20638 (2025)."},{"key":"e_1_3_2_1_8_1","volume-title":"On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258","author":"Bommasani Rishi","year":"2021","unstructured":"Rishi Bommasani. 2021. On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258 (2021)."},{"key":"e_1_3_2_1_9_1","volume-title":"The 2024 Foundation Model Transparency Index. arXiv preprint arXiv:2407.12929","author":"Bommasani Rishi","year":"2024","unstructured":"Rishi Bommasani, Kevin Klyman, Sayash Kapoor, Shayne Longpre, Betty Xiong, Nestor Maslej, and Percy Liang. 2024. The 2024 Foundation Model Transparency Index. arXiv preprint arXiv:2407.12929 (2024)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1111\/nyas.15007"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.385"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/s44206-025-00190-x"},{"key":"e_1_3_2_1_13_1","volume-title":"Francisca Gauna Selasco, Luca Nicol\u00e1s Forziati Gangi, Matheo Sandleris Musa, Lola Ramos Pereyra, Mario Leiva, Juan Gustavo Corvalan, Mar\u00eda Vanina Martinez, and Gerardo Simari.","author":"Carro Mar\u00eda Victoria","year":"2025","unstructured":"Mar\u00eda Victoria Carro, Denise Alejandra Mester, Francisca Gauna Selasco, Luca Nicol\u00e1s Forziati Gangi, Matheo Sandleris Musa, Lola Ramos Pereyra, Mario Leiva, Juan Gustavo Corvalan, Mar\u00eda Vanina Martinez, and Gerardo Simari. 2025. A Conceptual Framework for AI Capability Evaluations. arXiv preprint arXiv:2506.18213 (2025)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"Yupeng Chang Xu Wang Jindong Wang Yuan Wu Linyi Yang Kaijie Zhu Hao Chen Xiaoyuan Yi Cunxiang Wang Yidong Wang et al. 2024. A survey on evaluation of large language models. ACM transactions on intelligent systems and technology 15 3 (2024) 1\u201345.","DOI":"10.1145\/3641289"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.emnlp-main.511"},{"key":"e_1_3_2_1_16_1","volume-title":"A survey on data contamination for large language models. arXiv preprint arXiv:2502.14425","author":"Cheng Yuxing","year":"2025","unstructured":"Yuxing Cheng, Yi Chang, and Yuan Wu. 2025. A survey on data contamination for large language models. arXiv preprint arXiv:2502.14425 (2025)."},{"key":"e_1_3_2_1_17_1","volume-title":"Forty-first International Conference on Machine Learning.","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios Nikolas Angelopoulos, Tianle Li, Dacheng Li, Banghua Zhu, Hao Zhang, Michael Jordan, Joseph E Gonzalez, et al. 2024. Chatbot arena: An open platform for evaluating llms by human preference. In Forty-first International Conference on Machine Learning."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.482"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.2307\/2095101"},{"key":"e_1_3_2_1_20_1","volume-title":"Training on the test task confounds evaluation and emergence. arXiv preprint arXiv:2407.07890","author":"Dominguez-Olmedo Ricardo","year":"2024","unstructured":"Ricardo Dominguez-Olmedo, Florian E Dorner, and Moritz Hardt. 2024. Training on the test task confounds evaluation and emergence. arXiv preprint arXiv:2407.07890 (2024)."},{"key":"e_1_3_2_1_21_1","volume-title":"Can we trust AI benchmarks? an interdisciplinary review of current issues in AI evaluation. arXiv preprint arXiv:2502.06559","author":"Eriksson Maria","year":"2025","unstructured":"Maria Eriksson, Erasmo Purificato, Arman Noroozian, Joao Vinagre, Guillaume Chaslot, Emilia Gomez, and David Fernandez-Llorca. 2025. Can we trust AI benchmarks? an interdisciplinary review of current issues in AI evaluation. arXiv preprint arXiv:2502.06559 (2025)."},{"key":"e_1_3_2_1_22_1","volume-title":"Utility is in the eye of the user: A critique of NLP leaderboards. arXiv preprint arXiv:2009.13888","author":"Ethayarajh Kawin","year":"2020","unstructured":"Kawin Ethayarajh and Dan Jurafsky. 2020. Utility is in the eye of the user: A critique of NLP leaderboards. arXiv preprint arXiv:2009.13888 (2020)."},{"key":"e_1_3_2_1_23_1","volume-title":"Line goes up? inherent limitations of benchmarks for evaluating large language models. arXiv preprint arXiv:2502.14318","author":"Fodor James","year":"2025","unstructured":"James Fodor. 2025. Line goes up? inherent limitations of benchmarks for evaluating large language models. arXiv preprint arXiv:2502.14318 (2025)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.naacl-long.262"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-349-17295-5_4"},{"key":"e_1_3_2_1_26_1","volume-title":"Benchmark inflation: Revealing llm performance gaps using retro-holdouts. arXiv preprint arXiv:2410.09247","author":"Haimes Jacob","year":"2024","unstructured":"Jacob Haimes, Cenny Wenner, Kunvar Thaman, Vassil Tashev, Clement Neo, Esben Kran, and Jason Schreiber. 2024. Benchmark inflation: Revealing llm performance gaps using retro-holdouts. arXiv preprint arXiv:2410.09247 (2024)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"Dan Hendrycks Collin Burns Steven Basart Andy Zou Mantas Mazeika Dawn Song and Jacob Steinhardt. 2020. Measuring Massive Multitask Language Understanding. arXiv:2009.03300 [cs] doi:10.48550\/arXiv.2009.03300","DOI":"10.48550\/arXiv.2009.03300"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","unstructured":"Naman Jain King Han Alex Gu Wen-Ding Li Fanjia Yan Tianjun Zhang Sida Wang Armando Solar-Lezama Koushik Sen and Ion Stoica. 2024. LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code. doi:10.48550\/ARXIV.2403.07974","DOI":"10.48550\/ARXIV.2403.07974"},{"key":"e_1_3_2_1_29_1","volume-title":"Deprecating Benchmarks: Criteria and Framework. arXiv preprint arXiv:2507.06434","author":"Joaquin Ayrton San","year":"2025","unstructured":"Ayrton San Joaquin, Rokas Gipi\u0161kis, Leon Staufer, and Ariel Gil. 2025. Deprecating Benchmarks: Criteria and Framework. arXiv preprint arXiv:2507.06434 (2025)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1093\/polsoc\/puae012"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.324"},{"key":"e_1_3_2_1_32_1","volume-title":"reused and recycled: The life of a dataset in machine learning research. arXiv preprint arXiv:2112.01716","author":"Koch Bernard","year":"2021","unstructured":"Bernard Koch, Emily Denton, Alex Hanna, and Jacob G Foster. 2021. Reduced, reused and recycled: The life of a dataset in machine learning research. arXiv preprint arXiv:2112.01716 (2021)."},{"key":"e_1_3_2_1_33_1","volume-title":"Haidar Khan, Israt Jahan, Amran Bhuiyan, Chee Wei Tan, Md Rizwan Parvez, et al.","author":"Rahman Laskar Md Tahmid","year":"2024","unstructured":"Md Tahmid Rahman Laskar, Sawsan Alqahtani, M Saiful Bari, Mizanur Rahman, Mohammad Abdullah Matin Khan, Haidar Khan, Israt Jahan, Amran Bhuiyan, Chee Wei Tan, Md Rizwan Parvez, et al. 2024. A systematic survey and critical review on evaluating large language models: Challenges, limitations, and recommendations. arXiv preprint arXiv:2407.04069 (2024)."},{"key":"e_1_3_2_1_34_1","volume-title":"Avoiding data contamination in language model evaluation: Dynamic test construction with latest materials. arXiv preprint arXiv:2312.12343","author":"Li Yucheng","year":"2023","unstructured":"Yucheng Li, Frank Geurin, and Chenghua Lin. 2023. Avoiding data contamination in language model evaluation: Dynamic test construction with latest materials. arXiv preprint arXiv:2312.12343 (2023)."},{"key":"e_1_3_2_1_35_1","unstructured":"Percy Liang Rishi Bommasani Tony Lee Dimitris Tsipras Dilara Soylu Michihiro Yasunaga Yian Zhang Deepak Narayanan Yuhuai Wu Ananya Kumar et al. 2022. Holistic evaluation of language models. arXiv preprint arXiv:2211.09110 (2022)."},{"key":"e_1_3_2_1_36_1","volume-title":"Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2).","author":"Liao Thomas","year":"2021","unstructured":"Thomas Liao, Rohan Taori, Inioluwa Deborah Raji, and Ludwig Schmidt. 2021. Are we learning yet? a meta review of evaluation failures across machine learning. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1080\/21670811.2025.2522281"},{"key":"e_1_3_2_1_38_1","unstructured":"Nestor Maslej Loredana Fattorini Raymond Perrault Yolanda Gil Vanessa Parli Njenga Kariuki Emily Capstick Anka Reuel Erik Brynjolfsson John Etchemendy et al. 2025. Artificial intelligence index report 2025. arXiv preprint arXiv:2504.07139 (2025)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","unstructured":"Meredith Ringel Morris Jascha Sohl-dickstein Noah Fiedel Tris Warkentin Allan Dafoe Aleksandra Faust Clement Farabet and Shane Legg. 2024. Levels of AGI for Operationalizing Progress on the Path to AGI. arXiv:2311.02462 [cs] doi:10.48550\/arXiv.2311.02462","DOI":"10.48550\/arXiv.2311.02462"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v39i23.34678"},{"key":"e_1_3_2_1_41_1","unstructured":"Rasmus Kleis Nielsen. 2024. How News Coverage Often Uncritical Helps Build up the AI Hype. http:\/\/reutersinstitute.politics.ox.ac.uk\/news\/how-news-coverage-often-uncritical-helps-build-ai-hype"},{"key":"e_1_3_2_1_42_1","unstructured":"OpenAI. 2023. GPT-4 Research Preview: Capabilities and Limitations."},{"key":"e_1_3_2_1_43_1","unstructured":"OpenAI. 2023. GPT-4 System Card. (2023)."},{"key":"e_1_3_2_1_44_1","unstructured":"OpenAI. 2024. OpenAI o1 System Card."},{"key":"e_1_3_2_1_45_1","volume-title":"The Twelfth International Conference on Learning Representations.","author":"Oren Yonatan","year":"2023","unstructured":"Yonatan Oren, Nicole Meister, Niladri S Chatterji, Faisal Ladhak, and Tatsunori Hashimoto. 2023. Proving test set contamination in black-box language models. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-022-34591-0"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","unstructured":"Long Phan Alice Gatti Ziwen Han Nathaniel Li Josephina Hu Hugh Zhang Chen Bo Calvin Zhang Mohamed Shaaban John Ling Sean Shi Michael Choi Anish Agrawal Arnav Chopra Adam Khoja Ryan Kim Richard Ren Jason Hausenloy Oliver Zhang Mantas Mazeika Dmitry Dodonov Tung Nguyen Jaeho Lee Daron Anderson Mikhail Doroshenko Alun Cennyth Stokes Mobeen Mahmood Oleksandr Pokutnyi Oleg Iskra Jessica P. Wang John-Clark Levin Mstyslav Kazakov Fiona Feng Steven Y. Feng Haoran Zhao Michael Yu Varun Gangal Chelsea Zou Zihan Wang Serguei Popov Robert Gerbicz Geoff Galgon Johannes Schmitt Will Yeadon Yongki Lee Scott Sauers Alvaro Sanchez Fabian Giska Marc Roth S\u00f8ren Riis Saiteja Utpala Noah Burns Gashaw M. Goshu Mohinder Maheshbhai Naiya Chidozie Agu Zachary Giboney Antrell Cheatom Francesco Fournier-Facio Sarah-Jane Crowson Lennart Finke Zerui Cheng Jennifer Zampese Ryan G. Hoerr Mark Nandor Hyunwoo Park Tim Gehrunger Jiaqi Cai Ben McCarty Alexis C. Garretson Edwin Taylor Damien Sileo Qiuyu Ren Usman Qazi Lianghui Li Jungbae Nam John B. Wydallis Pavel Arkhipov Jack Wei Lun Shi Aras Bacho Chris G. Willcocks Hangrui Cao Sumeet Motwani Emily de Oliveira Santos Johannes Veith Edward Vendrow Doru Cojoc Kengo Zenitani Joshua Robinson Longke Tang Yuqi Li Joshua Vendrow Natanael Wildner Fraga Vladyslav Kuchkin Andrey Pupasov Maksimov Pierre Marion Denis Efremov Jayson Lynch Kaiqu Liang Aleksandar Mikov Andrew Gritsevskiy Julien Guillod G\u00f6zdenur Demir Dakotah Martinez Ben Pageler Kevin Zhou Saeed Soori Ori Press Henry Tang Paolo Rissone Sean R. Green Lina Br\u00fcssel Moon Twayana Aymeric Dieuleveut Joseph Marvin Imperial Ameya Prabhu Jinzhou Yang Nick Crispino Arun Rao Dimitri Zvonkine Gabriel Loiseau Mikhail Kalinin Marco Lukas Ciprian Manolescu Nate Stambaugh Subrata Mishra Tad Hogg Carlo Bosio Brian P. Coppola Julian Salazar Jaehyeok Jin Rafael Sayous Stefan Ivanov Philippe Schwaller Shaipranesh Senthilkuma Andres M. Bran Andres Algaba Kelsey Van den Houte Lynn Van Der Sypt Brecht Verbeken David Noever Alexei Kopylov Benjamin Myklebust Bikun Li Lisa Schut Evgenii Zheltonozhskii Qiaochu Yuan Derek Lim Richard Stanley Tong Yang John Maar Julian Wykowski Mart\u00ed Oller Anmol Sahu Cesare Giulio Ardito Yuzheng Hu Ariel Ghislain Kemogne Kamdoum Alvin Jin Tobias Garcia Vilchis Yuexuan Zu Martin Lackner James Koppel Gongbo Sun Daniil S. Antonenko Steffi Chern Bingchen Zhao Pierrot Arsene Joseph M. Cavanagh Daofeng Li Jiawei Shen Donato Crisostomi Wenjin Zhang Ali Dehghan Sergey Ivanov David Perrella Nurdin Kaparov Allen Zang Ilia Sucholutsky Arina Kharlamova Daniil Orel Vladislav Poritski Shalev Ben-David Zachary Berger Parker Whitfill Michael Foster Daniel Munro Linh Ho Shankar Sivarajan Dan Bar Hava Aleksey Kuchkin David Holmes Alexandra Rodriguez-Romero Frank Sommerhage Anji Zhang Richard Moat Keith Schneider Zakayo Kazibwe Don Clarke Dae Hyun Kim Felipe Meneguitti Dias Sara Fish Veit Elser Tobias Kreiman Victor Efren Guadarrama Vilchis Immo Klose Ujjwala Anantheswaran Adam Zweiger Kaivalya Rawal Jeffery Li Jeremy Nguyen Nicolas Daans Haline Heidinger Maksim Radionov V\u00e1clav Rozho\u0148 Vincent Ginis Christian Stump Niv Cohen Rafa\u0142 Po\u015bwiata Josef Tkadlec Alan Goldfarb Chenguang Wang Piotr Padlewski Stanislaw Barzowski Kyle Montgomery Ryan Stendall Jamie Tucker-Foltz Jack Stade T. Ryan Rogers Tom Goertzen Declan Grabb Abhishek Shukla Alan Givr\u00e9 John Arnold Ambay Archan Sen Muhammad Fayez Aziz Mark H. Inlow Hao He Ling Zhang Younesse Kaddar Ivar \u00c4ngquist Yanxu Chen Harrison K. Wang Kalyan Ramakrishnan Elliott Thornley Antonio Terpin Hailey Schoelkopf Eric Zheng Avishy Carmi Ethan D. L. Brown Kelin Zhu Max Bartolo Richard Wheeler Martin Stehberger Peter Bradshaw J. P. Heimonen Kaustubh Sridhar Ido Akov Jennifer Sandlin Yury Makarychev Joanna Tam Hieu Hoang David M. Cunningham Vladimir Goryachev Demosthenes Patramanis Michael Krause Andrew Redenti David Aldous Jesyin Lai Shannon Coleman Jiangnan Xu Sangwon Lee Ilias Magoulas Sandy Zhao Ning Tang Michael K. Cohen Orr Paradise Jan Hendrik Kirchner Maksym Ovchynnikov Jason O. Matos Adithya Shenoy Michael Wang Yuzhou Nie Anna Sztyber-Betley Paolo Faraboschi Robin Riblet Jonathan Crozier Shiv Halasyamani Shreyas Verma Prashant Joshi Eli Meril Ziqiao Ma J\u00e9r\u00e9my Andr\u00e9oletti Raghav Singhal Jacob Platnick Volodymyr Nevirkovets Luke Basler Alexander Ivanov Seri Khoury Nils Gustafsson Marco Piccardo Hamid Mostaghimi Qijia Chen Virendra Singh Tran Quoc Kh\u00e1nh Paul Rosu Hannah Szlyk Zachary Brown Himanshu Narayan Aline Menezes Jonathan Roberts William Alley Kunyang Sun Arkil Patel Max Lamparth Anka Reuel Linwei Xin Hanmeng Xu Jacob Loader Freddie Martin Zixuan Wang Andrea Achilleos Thomas Preu Tomek Korbak Ida Bosio Fereshteh Kazemi Ziye Chen Bir\u00f3 B\u00e1lint Eve J. Y. Lo Jiaqi Wang Maria In\u00eas S. Nunes Jeremiah Milbauer M. Saiful Bari Zihao Wang Behzad Ansarinejad Yewen Sun Stephane Durand Hossam Elgnainy Guillaume Douville Daniel Tordera George Balabanian Hew Wolff Lynna Kvistad Hsiaoyun Milliron Ahmad Sakor Murat Eron Andrew Favre D. O Shailesh Shah Xiaoxiang Zhou Firuz Kamalov Sherwin Abdoli Tim Santens Shaul Barkan Allison Tee Robin Zhang Alessandro Tomasiello G. Bruno De Luca Shi-Zhuo Looi Vinh-Kha Le Noam Kolt Jiayi Pan Emma Rodman Jacob Drori Carl J. Fossum Niklas Muennighoff Milind Jagota Ronak Pradeep Honglu Fan Jonathan Eicher Michael Chen Kushal Thaman William Merrill Moritz Firsching Carter Harris Stefan Ciob\u00e2c\u0103 Jason Gross Rohan Pandey Ilya Gusev Adam Jones Shashank Agnihotri Pavel Zhelnov Mohammadreza Mofayezi Alexander Piperski David K. Zhang Kostiantyn Dobarskyi Roman Leventov Ignat Soroko Joshua Duersch Vage Taamazyan Andrew Ho Wenjie Ma William Held Ruicheng Xian Armel Randy Zebaze Mohanad Mohamed Julian Noah Leser Michelle X. Yuan Laila Yacar Johannes Lengler Katarzyna Olszewska Claudio Di Fratta Edson Oliveira Joseph W. Jackson Andy Zou Muthu Chidambaram Timothy Manik Hector Haffenden Dashiell Stander Ali Dasouqi Alexander Shen Bita Golshani David Stap Egor Kretov Mikalai Uzhou Alina Borisovna Zhidkovskaya Nick Winter Miguel Orbegozo Rodriguez Robert Lauff Dustin Wehr Colin Tang Zaki Hossain Shaun Phillips Fortuna Samuele Fredrik Ekstr\u00f6m Angela Hammon Oam Patel Faraz Farhidi George Medley Forough Mohammadzadeh Madellene Pe\u00f1aflor Haile Kassahun Alena Friedrich Rayner Hernandez Perez Daniel Pyda Taom Sakal Omkar Dhamane Ali Khajegili Mirabadi Eric Hallman Kenchi Okutsu Mike Battaglia Mohammad Maghsoudimehrabani Alon Amit Dave Hulbert Roberto Pereira Simon Weber Handoko Anton Peristyy Stephen Malina Mustafa Mehkary Rami Aly Frank Reidegeld Anna-Katharina Dick Cary Friday Mukhwinder Singh Hassan Shapourian Wanyoung Kim Mariana Costa Hubeyb Gurdogan Harsh Kumar Chiara Ceconello Chao Zhuang Haon Park Micah Carroll Andrew R. Tawfeek Stefan Steinerberger Daattavya Aggarwal Michael Kirchhof Linjie Dai Evan Kim Johan Ferret Jainam Shah Yuzhou Wang Minghao Yan Krzysztof Burdzy Lixin Zhang Antonio Franca Diana T. Pham Kang Yong Loh Joshua Robinson Abram Jackson Paolo Giordano Philipp Petersen Adrian Cosma Jesus Colino Colin White Jacob Votava Vladimir Vinnikov Ethan Delaney Petr Spelda Vit Stritecky Syed M. Shahid Jean-Christophe Mourrat Lavr Vetoshkin Koen Sponselee Renas Bacho Zheng-Xin Yong Florencia de la Rosa Nathan Cho Xiuyu Li Guillaume Malod Orion Weller Guglielmo Albani Leon Lang Julien Laurendeau Dmitry Kazakov Fatimah Adesanya Julien Portier Lawrence Hollom Victor Souza Yuchen Anna Zhou Julien Degorre Yi\u011fit Yalin Gbenga Daniel Obikoya Rai Filippo Bigi M. C. Bosc\u00e1 Oleg Shumar Kaniuar Bacho Gabriel Recchia Mara Popescu Nikita Shulga Ngefor Mildred Tanwie Thomas C. H. Lux Ben Rank Colin Ni Matthew Brooks Alesia Yakimchyk Huanxu Liu Stefano Cavalleri Olle H\u00e4ggstr\u00f6m Emil Verkama Joshua Newbould Hans Gundlach Leonor Brito-Santana Brian Amaro Vivek Vajipey Rynaa Grover Ting Wang Yosi Kratish Wen-Ding Li Sivakanth Gopi Andrea Caciolai Christian Schroeder de Witt Pablo Hern\u00e1ndez-C\u00e1mara Emanuele Rodol\u00e0 Jules Robins Dominic Williamson Vincent Cheng Brad Raynor Hao Qi Ben Segev Jingxuan Fan Sarah Martinson Erik Y. Wang Kaylie Hausknecht Michael P. Brenner Mao Mao Christoph Demian Peyman Kassani Xinyu Zhang David Avagian Eshawn Jessica Scipio Alon Ragoler Justin Tan Blake Sims Rebeka Plecnik Aaron Kirtland Omer Faruk Bodur D. P. Shinde Yan Carlos Leyva Labrador Zahra Adoul Mohamed Zekry Ali Karakoc Tania C. B. Santos Samir Shamseldeen Loukmane Karim Anna Liakhovitskaia Nate Resman Nicholas Farina Juan Carlos Gonzalez Gabe Maayan Earth Anderson Rodrigo De Oliveira Pena Elizabeth Kelley Hodjat Mariji Rasoul Pouriamanesh Wentao Wu Ross Finocchio Ismail Alarab Joshua Cole Danyelle Ferreira Bryan Johnson Mohammad Safdari Liangti Dai Siriphan Arthornthurasuk Isaac C. McAlister Alejandro Jos\u00e9 Moyano Alexey Pronin Jing Fan Angel Ramirez-Trinidad Yana Malysheva Daphiny Pottmaier Omid Taheri Stanley Stepanic Samuel Perry Luke Askew Ra\u00fal Adri\u00e1n Huerta Rodr\u00edguez Ali M. R. Minissi Ricardo Lorena Krishnamurthy Iyer Arshad Anil Fasiludeen Ronald Clark Josh Ducey Matheus Piza Maja Somrak Eric Vergo Juehang Qin Benj\u00e1min Borb\u00e1s Eric Chu Jack Lindsey Antoine Jallon I. M. J. McInnis Evan Chen Avi Semler Luk Gloor Tej Shah Marc Carauleanu Pascal Lauer Tran \u0110uc Huy Hossein Shahrtash Emilien Duc Lukas Lewark Assaf Brown Samuel Albanie Brian Weber Warren S. Vaz Pierre Clavier Yiyang Fan Gabriel Poesia Reis e Silva Long Lian Marcus Abramovitch Xi Jiang Sandra Mendoza Murat Islam Juan Gonzalez Vasilios Mavroudis Justin Xu Pawan Kumar Laxman Prasad Goswami Daniel Bugas Nasser Heydari Ferenc Jeanplong Thorben Jansen Antonella Pinto Archimedes Apronti Abdallah Galal Ng Ze-An Ankit Singh Tong Jiang Joan of Arc Xavier Kanu Priya Agarwal Mohammed Berkani Gang Zhang Zhehang Du Benedito Alves de Oliveira Junior Dmitry Malishev Nicolas Remy Taylor D. Hartman Tim Tarver Stephen Mensah Gautier Abou Loume Wiktor Morak Farzad Habibi Sarah Hoback Will Cai Javier Gimenez Roselynn Grace Montecillo Jakub \u0141ucki Russell Campbell Asankhaya Sharma Khalida Meer Shreen Gul Daniel Espinosa Gonzalez Xavier Alapont Alex Hoover Gunjan Chhablani Freddie Vargus Arunim Agarwal Yibo Jiang Deepakkumar Patil David Outevsky Kevin Joseph Scaria Rajat Maheshwari Abdelkader Dendane Priti Shukla Ashley Cartwright Sergei Bogdanov Niels M\u00fcndler S\u00f6ren M\u00f6ller Luca Arnaboldi Kunvar Thaman Muhammad Rehan Siddiqi Prajvi Saxena Himanshu Gupta Tony Fruhauff Glen Sherman M\u00e1ty\u00e1s Vincze Siranut Usawasutsakorn Dylan Ler Anil Radhakrishnan Innocent Enyekwe Sk Md Salauddin Jiang Muzhen Aleksandr Maksapetyan Vivien Rossbach Chris Harjadi Mohsen Bahaloohoreh Claire Sparrow Jasdeep Sidhu Sam Ali Song Bian John Lai Eric Singer Justine Leon Uro Greg Bateman Mohamed Sayed Ahmed Menshawy Darling Duclosel Dario Bezzi Yashaswini Jain Ashley Aaron Murat Tiryakioglu Sheeshram Siddh Keith Krenek Imad Ali Shah Jun Jin Scott Creighton Denis Peskoff Zienab EL-Wasif Ragavendran P. V Michael Richmond Joseph McGowan Tejal Patwardhan Hao-Yu Sun Ting Sun Nikola Zubi\u0107 Samuele Sala Stephen Ebert Jean Kaddour Manuel Schottdorf Dianzhuo Wang Gerol Petruzella Alex Meiburg Tilen Medved Ali ElSheikh S. Ashwin Hebbar Lorenzo Vaquero Xianjun Yang Jason Poulos Vil\u00e9m Zouhar Sergey Bogdanik Mingfang Zhang Jorge Sanz-Ros David Anugraha Yinwei Dai Anh N. Nhu Xue Wang Ali Anil Demircali Zhibai Jia Yuyin Zhou Juncheng Wu Mike He Nitin Chandok Aarush Sinha Gaoxiang Luo Long Le Micka\u00ebl Noy\u00e9 Micha\u0142 Pere\u0142kiewicz Ioannis Pantidis Tianbo Qi Soham Sachin Purohit Letitia Parcalabescu Thai-Hoa Nguyen Genta Indra Winata Edoardo M. Ponti Hanchen Li Kaustubh Dhole Jongee Park Dario Abbondanza Yuanli Wang Anupam Nayak Diogo M. Caetano Antonio A. W. L. Wong Maria del Rio-Chanona D\u00e1niel Kondor Pieter Francois Ed Chalstrey Jakob Zsambok Dan Hoyer Jenny Reddish Jakob Hauser Francisco-Javier Rodrigo-Gin\u00e9s Suchandra Datta Maxwell Shepherd Thom Kamphuis Qizheng Zhang Hyunjun Kim Ruiji Sun Jianzhu Yao Franck Dernoncourt Satyapriya Krishna Sina Rismanchian Bonan Pu Francesco Pinto Yingheng Wang Kumar Shridhar Kalon J. Overholt Glib Briia Hieu Nguyen David Soler Bartomeu Tony CY Pang Adam Wecker Yifan Xiong Fanfei Li Lukas S. Huber Joshua Jaeger Romano De Maddalena Xing Han L\u00f9 Yuhui Zhang Claas Beger Patrick Tser Jern Kon Sean Li Vivek Sanker Ming Yin Yihao Liang Xinlu Zhang Ankit Agrawal Li S. Yifei Zechen Zhang Mu Cai Yasin Sonmez Costin Cozianu Changhao Li Alex Slen Shoubin Yu Hyun Kyu Park Gabriele Sarti Marcin Bria\u0148ski Alessandro Stolfo Truong An Nguyen Mike Zhang Yotam Perlitz Jose Hernandez-Orallo Runjia Li Amin Shabani Felix Juefei-Xu Shikhar Dhingra Orr Zohar My Chiffon Nguyen Alexander Pondaven Abdurrahim Yilmaz Xuandong Zhao Chuanyang Jin Muyan Jiang Stefan Todoran Xinyao Han Jules Kreuer Brian Rabern Anna Plassart Martino Maggetti Luther Yap Robert Geirhos Jonathon Kean Dingsu Wang Sina Mollaei Chenkai Sun Yifan Yin Shiqi Wang Rui Li Yaowen Chang Anjiang Wei Alice Bizeul Xiaohan Wang Alexandre Oliveira Arrais Kushin Mukherjee Jorge Chamorro-Padial Jiachen Liu Xingyu Qu Junyi Guan Adam Bouyamourn Shuyu Wu Martyna Plomecka Junda Chen Mengze Tang Jiaqi Deng Shreyas Subramanian Haocheng Xi Haoxuan Chen Weizhi Zhang Yinuo Ren Haoqin Tu Sejong Kim Yushun Chen Sara Vera Marjanovi\u0107 Junwoo Ha Grzegorz Luczyna Jeff J. Ma Zewen Shen Dawn Song Cedegao E. Zhang Zhun Wang Ga\u00ebl Gendron Yunze Xiao Leo Smucker Erica Weng Kwok Hao Lee Zhe Ye Stefano Ermon Ignacio D. Lopez-Miguel Theo Knights Anthony Gitter Namkyu Park Boyi Wei Hongzheng Chen Kunal Pai Ahmed Elkhanany Han Lin Philipp D. Siedler Jichao Fang Ritwik Mishra K\u00e1roly Zsolnai-Feh\u00e9r Xilin Jiang Shadab Khan Jun Yuan Rishab Kumar Jain Xi Lin Mike Peterson Zhe Wang Aditya Malusare Maosen Tang Isha Gupta Ivan Fosin Timothy Kang Barbara Dworakowska Kazuki Matsumoto Guangyao Zheng Gerben Sewuster Jorge Pretel Villanueva Ivan Rannev Igor Chernyavsky Jiale Chen Deepayan Banik Ben Racz Wenchao Dong Jianxin Wang Laila Bashmal Duarte V. Gon\u00e7alves Wei Hu Kaushik Bar Ondrej Bohdal Atharv Singh Patlan Shehzaad Dhuliawala Caroline Geirhos Julien Wist Yuval Kansal Bingsen Chen Kutay Tire Atak Talay Y\u00fccel Brandon Christof Veerupaksh Singla Zijian Song Sanxing Chen Jiaxin Ge Kaustubh Ponkshe Isaac Park Tianneng Shi Martin Q. Ma Joshua Mak Sherwin Lai Antoine Moulin Zhuo Cheng Zhanda Zhu Ziyi Zhang Vaidehi Patil Ketan Jha Qiutong Men Jiaxuan Wu Tianchi Zhang Bruno Hebling Vieira Alham Fikri Aji Jae-Won Chung Mohammed Mahfoud Ha Thi Hoang Marc Sperzel Wei Hao Kristof Meding Sihan Xu Vassilis Kostakos Davide Manini Yueying Liu Christopher Toukmaji Jay Paek Eunmi Yu Arif Engin Demircali Zhiyi Sun Ivan Dewerpe Hongsen Qin Roman Pflugfelder James Bailey Johnathan Morris Ville Heilala Sybille Rosset Zishun Yu Peter E. Chen Woongyeong Yeo Eeshaan Jain Ryan Yang Sreekar Chigurupati Julia Chernyavsky Sai Prajwal Reddy Subhashini Venugopalan Hunar Batra Core Francisco Park Hieu Tran Guilherme Maximiano Genghan Zhang Yizhuo Liang Hu Shiyu Rongwu Xu Rui Pan Siddharth Suresh Ziqi Liu Samaksh Gulati Songyang Zhang Peter Turchin Christopher W. Bartlett Christopher R. Scotese Phuong M. Cao Aakaash Nattanmai Gordon McKellips Anish Cheraku Asim Suhail Ethan Luo Marvin Deng Jason Luo Ashley Zhang Kavin Jindel Jay Paek Kasper Halevy Allen Baranov Michael Liu Advaith Avadhanam David Zhang Vincent Cheng Brad Ma Evan Fu Liam Do Joshua Lass Hubert Yang Surya Sunkari Vishruth Bharath Violet Ai James Leung Rishit Agrawal Alan Zhou Kevin Chen Tejas Kalpathi Ziqi Xu Gavin Wang Tyler Xiao Erik Maung Sam Lee Ryan Yang Roy Yue Ben Zhao Julia Yoon Sunny Sun Aryan Singh Ethan Luo Clark Peng Tyler Osbey Taozhi Wang Daryl Echeazu Hubert Yang Timothy Wu Spandan Patel Vidhi Kulkarni Vijaykaarti Sundarapandiyan Ashley Zhang Andrew Le Zafir Nasim Srikar Yalam Ritesh Kasamsetty Soham Samal Hubert Yang David Sun Nihar Shah Abhijeet Saha Alex Zhang Leon Nguyen Laasya Nagumalli Kaixin Wang Alan Zhou Aidan Wu Jason Luo Anwith Telluri Summer Yue Alexandr Wang and Dan Hendrycks. 2025. Humanity's Last Exam. arXiv:2501.14249 [cs] doi:10.48550\/arXiv.2501.14249","DOI":"10.48550\/arXiv.2501.14249"},{"key":"e_1_3_2_1_48_1","volume-title":"AI and the everything in the whole wide world benchmark. arXiv preprint arXiv:2111.15366","author":"Raji Inioluwa Deborah","year":"2021","unstructured":"Inioluwa Deborah Raji, Emily M Bender, Amandalynne Paullada, Emily Denton, and Alex Hanna. 2021. AI and the everything in the whole wide world benchmark. arXiv preprint arXiv:2111.15366 (2021)."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2311.12022"},{"key":"e_1_3_2_1_50_1","unstructured":"Kevin Roose. 2025. When A.I. Passes This Test Look Out. https:\/\/www.nytimes.com\/2025\/01\/23\/technology\/ai-test-humanitys-last-exam.html"},{"key":"e_1_3_2_1_51_1","volume-title":"None of the others: a general technique to distinguish reasoning from memorization in multiple-choice llm evaluation benchmarks. arXiv preprint arXiv:2502.12896","author":"Salido Eva S\u00e1nchez","year":"2025","unstructured":"Eva S\u00e1nchez Salido, Julio Gonzalo, and Guillermo Marco. 2025. None of the others: a general technique to distinguish reasoning from memorization in multiple-choice llm evaluation benchmarks. arXiv preprint arXiv:2502.12896 (2025)."},{"key":"e_1_3_2_1_52_1","unstructured":"David Sculley Jasper Snoek Alex Wiltschko and Ali Rahimi. 2018. Winner's curse? On pace progress and empirical rigor. (2018). https:\/\/openreview.net\/forum?id=rJWF0Fywf"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.70002"},{"key":"e_1_3_2_1_54_1","volume-title":"Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al.","author":"Srivastava Aarohi","year":"2023","unstructured":"Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al. 2023. Beyond the imitation game: Quantifying and extrapolating the capabilities of language models. Transactions on machine learning research (2023)."},{"key":"e_1_3_2_1_55_1","volume-title":"\u2018Improving ratings\u2019: audit in the British University system. European review 5, 3","author":"Strathern Marilyn","year":"1997","unstructured":"Marilyn Strathern. 1997. \u2018Improving ratings\u2019: audit in the British University system. European review 5, 3 (1997), 305\u2013321."},{"key":"e_1_3_2_1_56_1","volume-title":"Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society","volume":"7","author":"Thais Savannah","year":"2024","unstructured":"Savannah Thais. 2024. Misrepresented technological solutions in imagined futures: The origins and dangers of ai hype in the research community. In Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society, Vol. 7. 1455\u20131465."},{"key":"e_1_3_2_1_57_1","volume-title":"The 2025 Foundation Model Transparency Index. arXiv preprint arXiv:2512.10169","author":"Wan Alexander","year":"2025","unstructured":"Alexander Wan, Kevin Klyman, Sayash Kapoor, Nestor Maslej, Shayne Longpre, Betty Xiong, Percy Liang, and Rishi Bommasani. 2025. The 2025 Foundation Model Transparency Index. arXiv preprint arXiv:2512.10169 (2025)."},{"key":"e_1_3_2_1_58_1","volume-title":"Benchmark suites instead of leaderboards for evaluating AI fairness. Patterns 5, 11","author":"Wang Angelina","year":"2024","unstructured":"Angelina Wang, Aaron Hertzmann, and Olga Russakovsky. 2024. Benchmark suites instead of leaderboards for evaluating AI fairness. Patterns 5, 11 (2024)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","unstructured":"Yubo Wang Xueguang Ma Ge Zhang Yuansheng Ni Abhranil Chandra Shiguang Guo Weiming Ren Aaran Arulraj Xuan He Ziyan Jiang Tianle Li Max Ku Kai Wang Alex Zhuang Rongqi Fan Xiang Yue and Wenhu Chen. 2024. MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark. arXiv:2406.01574 [cs] doi:10.48550\/arXiv.2406.01574","DOI":"10.48550\/arXiv.2406.01574"},{"key":"e_1_3_2_1_60_1","volume-title":"Hanna Wallach, Margaret Mitchell, Angelina Wang, Olawale Salaudeen, Rishi Bommasani, Deep Ganguli, Sanmi Koyejo, and William Isaac.","author":"Weidinger Laura","year":"2025","unstructured":"Laura Weidinger, Inioluwa Deborah Raji, Hanna Wallach, Margaret Mitchell, Angelina Wang, Olawale Salaudeen, Rishi Bommasani, Deep Ganguli, Sanmi Koyejo, and William Isaac. 2025. Toward an evaluation science for generative AI systems. arXiv preprint arXiv:2503.05336 (2025)."},{"key":"e_1_3_2_1_61_1","unstructured":"Andrew White. 2025. About 30% of Humanity's Last Exam chemistry\/biology answers are likely wrong. https:\/\/www.futurehouse.org\/research-announcements\/hle-exam"},{"key":"e_1_3_2_1_62_1","volume-title":"Bo Li, Badih Ghazi, and Ravi Kumar.","author":"Xie Chulin","year":"2024","unstructured":"Chulin Xie, Yangsibo Huang, Chiyuan Zhang, Da Yu, Xinyun Chen, Bill Yuchen Lin, Bo Li, Badih Ghazi, and Ravi Kumar. 2024. On memorization of large language models in logical reasoning. arXiv preprint arXiv:2410.23123 (2024)."},{"key":"e_1_3_2_1_63_1","unstructured":"Cheng Xu Shuhao Guan Derek Greene M Kechadi et al. 2024. Benchmark data contamination of large language models: A survey. arXiv preprint arXiv:2406.04244 (2024)."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2311.16502"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1485"},{"key":"e_1_3_2_1_66_1","unstructured":"Hongli Zhou Hui Huang Ziqing Zhao Lvyuan Han Huicheng Wang Kehai Chen Muyun Yang Wei Bao Jian Dong Bing Xu et al. 2025. Lost in Benchmarks? Rethinking Large Language Model Benchmarking with Item Response Theory. arXiv preprint arXiv:2505.15055 (2025)."},{"key":"e_1_3_2_1_67_1","volume-title":"Xiang Zheng, Yaoyao Qian, Yunpeng Xiao, and Kai Shu.","author":"Zhou Kyrie Zhixuan","year":"2025","unstructured":"Kyrie Zhixuan Zhou, Justin Eric Chen, Xiang Zheng, Yaoyao Qian, Yunpeng Xiao, and Kai Shu. 2025. \u201cEveryone Else Does It\u201d: The Rise of Preprinting Culture in Computing Disciplines. arXiv preprint arXiv:2511.04081 (2025)."}],"event":{"name":"FAccT '26: The 2026 ACM Conference on Fairness, Accountability, and Transparency","location":"Montreal QC Canada","acronym":"FAccT '26","sponsor":["ACM\/SIG"]},"container-title":["Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3805689.3812240","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,29]],"date-time":"2026-06-29T18:34:39Z","timestamp":1782758079000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805689.3812240"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,25]]},"references-count":67,"alternative-id":["10.1145\/3805689.3812240","10.1145\/3805689"],"URL":"https:\/\/doi.org\/10.1145\/3805689.3812240","relation":{},"subject":[],"published":{"date-parts":[[2026,6,25]]},"assertion":[{"value":"2026-06-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}