{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T07:02:34Z","timestamp":1782802954822,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":78,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,5,26]],"date-time":"2026-05-26T00:00:00Z","timestamp":1779753600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"National Science Foundation","award":["2316768"],"award-info":[{"award-number":["2316768"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,5,26]]},"DOI":"10.1145\/3786335.3813140","type":"proceedings-article","created":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T03:16:22Z","timestamp":1779419782000},"page":"597-619","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Why Johnny Can\u2019t Use Agents: Industry Aspirations vs. User Realities with AI Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2768-9103","authenticated-orcid":false,"given":"Pradyumna","family":"Shome","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-0655-2754","authenticated-orcid":false,"given":"Sashreek","family":"Krishnan","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9073-8054","authenticated-orcid":false,"given":"Sauvik","family":"Das","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,5,26]]},"reference":[{"key":"e_1_3_3_2_2_2","volume-title":"Cursor - The AI Code Editor","year":"2025","unstructured":"Anysphere 2025. Cursor - The AI Code Editor. Anysphere. https:\/\/cursor.com\/ Accessed: 2025-09-01."},{"key":"e_1_3_3_2_3_2","unstructured":"2025. Expedia Trip Matching AI Agent. https:\/\/www.marketingdive.com\/news\/expedia-brings-generative-ai-trip-planning-tool-to-instagram\/748233\/. Accessed: 2025-09-12."},{"key":"e_1_3_3_2_4_2","unstructured":"2025. Gamma Presentation Builder. https:\/\/gamma.app\/. Accessed: 2025-09-12."},{"key":"e_1_3_3_2_5_2","unstructured":"2025. LiveX AI Agent. https:\/\/livex.ai\/. Accessed: 2025-09-12."},{"key":"e_1_3_3_2_6_2","volume-title":"Lovable","year":"2025","unstructured":"2025. Lovable. https:\/\/lovable.dev\/ Accessed: 2025-09-01."},{"key":"e_1_3_3_2_7_2","unstructured":"2025. Salesforce Agentforce. https:\/\/www.salesforce.com\/agentforce\/. Accessed: 2025-09-12."},{"key":"e_1_3_3_2_8_2","volume-title":"Introducing Perplexity Deep Research","author":"AI Perplexity","year":"2024","unstructured":"Perplexity AI. 2024. Introducing Perplexity Deep Research. https:\/\/www.perplexity.ai\/hub\/blog\/introducing-perplexity-deep-research Accessed: 2025-09-02."},{"key":"e_1_3_3_2_9_2","unstructured":"AI Agents Directory. 2025. AI Agent Marketplace & Directory | Find Top AI Agents & AI Agent Solutions. https:\/\/aiagentsdirectory.com\/ Accessed: 2025-09-01."},{"key":"e_1_3_3_2_10_2","unstructured":"Amazon. n. d.. Amazon Alexa. https:\/\/alexa.amazon.com\/about. Accessed: 2026-02-03."},{"key":"e_1_3_3_2_11_2","unstructured":"Apple. n. d.. Siri. https:\/\/www.apple.com\/siri\/. Accessed: 2026-02-03."},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","unstructured":"Gagan Bansal Besmira Nushi Ece Kamar Walter\u00a0S. Lasecki Daniel\u00a0S. Weld and Eric Horvitz. 2019. Beyond Accuracy: The Role of Mental Models in Human-AI Team Performance. Proceedings of the AAAI Conference on Human Computation and Crowdsourcing 7 1 (Oct. 2019) 2\u201311. 10.1609\/hcomp.v7i1.5285","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_3_2_13_2","unstructured":"Gagan Bansal Jennifer\u00a0Wortman Vaughan Saleema Amershi Eric Horvitz Adam Fourney Hussein Mozannar Victor Dibia and Daniel\u00a0S. Weld. 2024. Challenges in Human-Agent Communication. arxiv:https:\/\/arXiv.org\/abs\/2412.10380\u00a0[cs.HC] https:\/\/arxiv.org\/abs\/2412.10380"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","unstructured":"Virginia Braun and Victoria Clarke. 2006. Using thematic analysis in psychology. Qualitative Research in Psychology 3 2 (2006) 77\u2013101. 10.1191\/1478088706qp063oa","DOI":"10.1191\/1478088706qp063oa"},{"key":"e_1_3_3_2_15_2","unstructured":"John Brooke et\u00a0al. 1996. SUS-A quick and dirty usability scale. Usability evaluation in industry 189 194 (1996) 4\u20137."},{"key":"e_1_3_3_2_16_2","unstructured":"Tom\u00a0B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell Sandhini Agarwal Ariel Herbert-Voss Gretchen Krueger Tom Henighan Rewon Child Aditya Ramesh Daniel\u00a0M. Ziegler Jeffrey Wu Clemens Winter Christopher Hesse Mark Chen Eric Sigler Mateusz Litwin Scott Gray Benjamin Chess Jack Clark Christopher Berner Sam McCandlish Alec Radford Ilya Sutskever and Dario Amodei. 2020. Language Models are Few-Shot Learners. arxiv:https:\/\/arXiv.org\/abs\/2005.14165\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2005.14165"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3696410.3714842"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858498"},{"key":"e_1_3_3_2_19_2","volume-title":"Forty-first International Conference on Machine Learning","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios\u00a0Nikolas Angelopoulos, Tianle Li, Dacheng Li, Banghua Zhu, Hao Zhang, Michael Jordan, Joseph\u00a0E Gonzalez, et\u00a0al. 2024. Chatbot arena: An open platform for evaluating llms by human preference. In Forty-first International Conference on Machine Learning."},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025780"},{"key":"e_1_3_3_2_21_2","volume-title":"Watch what I do: programming by demonstration","author":"Cypher Allen","year":"1993","unstructured":"Allen Cypher, Daniel\u00a0C. Halbert, David Kurlander, Henry Lieberman, David Maulsby, Brad\u00a0A. Myers, and Alan Turransky (Eds.). 1993. Watch what I do: programming by demonstration. MIT Press, Cambridge, MA, USA."},{"key":"e_1_3_3_2_22_2","unstructured":"Alexandre Drouin Maxime Gasse Massimo Caccia Issam\u00a0H. Laradji Manuel\u00a0Del Verme Tom Marty L\u00e9o Boisvert Megh Thakkar Quentin Cappart David Vazquez Nicolas Chapados and Alexandre Lacoste. 2024. WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks? arxiv:https:\/\/arXiv.org\/abs\/2403.07718\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2403.07718"},{"key":"e_1_3_3_2_23_2","unstructured":"Dylan Freedman. 2025. The Day ChatGPT Went Cold. The New York Times (2025). https:\/\/www.nytimes.com\/2025\/08\/19\/business\/chatgpt-gpt-5-backlash-openai.html Accessed: 2025-09-04."},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"crossref","unstructured":"Greg Guest Arwen Bunce and Laura Johnson. 2006. How many interviews are enough? An experiment with data saturation and variability. Field methods 18 1 (2006) 59\u201382.","DOI":"10.1177\/1525822X05279903"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713218"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"crossref","unstructured":"Monique\u00a0M Hennink Bonnie\u00a0N Kaiser and Vincent\u00a0C Marconi. 2017. Code saturation versus meaning saturation: how many interviews are enough? Qualitative health research 27 4 (2017) 591\u2013608.","DOI":"10.1177\/1049732316665344"},{"key":"e_1_3_3_2_27_2","unstructured":"Ryan Hoover. 2013. Product Hunt. https:\/\/www.producthunt.com\/. Accessed: 2025-09-01."},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","unstructured":"Eric Horvitz. 1999. Principles of mixed-initiative user interfaces(CHI \u201999). Association for Computing Machinery New York NY USA 159\u2013166. 10.1145\/302979.303030302979.303030\"\/>","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_3_2_29_2","unstructured":"Xueyu Hu Tao Xiong Biao Yi Zishu Wei Ruixuan Xiao Yurun Chen Jiasheng Ye Meiling Tao Xiangxin Zhou Ziyu Zhao Yuhuai Li Shengze Xu Shenzhi Wang Xinchen Xu Shuofei Qiao Zhaokai Wang Kun Kuang Tieyong Zeng Liang Wang Jiwei Li Yuchen\u00a0Eleanor Jiang Wangchunshu Zhou Guoyin Wang Keting Yin Zhou Zhao Hongxia Yang Fan Wu Shengyu Zhang and Fei Wu. 2025. OS Agents: A Survey on MLLM-based Agents for General Computing Devices Use. arxiv:https:\/\/arXiv.org\/abs\/2508.04482\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2508.04482"},{"key":"e_1_3_3_2_30_2","unstructured":"Song Jiang Da Ju Andrew Cohen Sasha Mitts Aaron Foss Justine\u00a0T Kao Xian Li and Yuandong Tian. 2024. Towards Full Delegation: Designing Ideal Agentic Behaviors for Travel Planning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.13904 (2024)."},{"key":"e_1_3_3_2_31_2","unstructured":"Carlos\u00a0E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press and Karthik Narasimhan. 2024. SWE-bench: Can Language Models Resolve Real-World GitHub Issues? arxiv:https:\/\/arXiv.org\/abs\/2310.06770\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2310.06770"},{"key":"e_1_3_3_2_32_2","unstructured":"Sayash Kapoor Benedikt Stroebl Zachary\u00a0S. Siegel Nitya Nadgir and Arvind Narayanan. 2024. AI Agents That Matter. arxiv:https:\/\/arXiv.org\/abs\/2407.01502\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2407.01502"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300643"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","unstructured":"Pranav Khadpe Ranjay Krishna Li Fei-Fei Jeffrey\u00a0T. Hancock and Michael\u00a0S. Bernstein. 2020. Conceptual Metaphors Impact Perceptions of Human-AI Collaboration. Proc. ACM Hum.-Comput. Interact. 4 CSCW2 Article 163 (Oct. 2020) 26\u00a0pages. 10.1145\/34152343415234\"\/>","DOI":"10.1145\/3415234"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658941"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2004.47"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","unstructured":"Hui Li Jiasheng Zhang and Kun Huang. 2025. Meta-Analyzing the Trust-Performance Link in Collaboration: Moderating Effects of Conceptual and Contextual Factors. Public Performance & Management Review 48 1 (2025) 1\u201334. arXiv:10.1080\/15309576.2024.2405839","DOI":"10.1080\/15309576.2024.2405839"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/258549.258592"},{"key":"e_1_3_3_2_39_2","volume-title":"Proceedings of the ACM Conference on Human Factors in Computing Systems (CHI)","author":"Lieberman Henry","year":"1999","unstructured":"Henry Lieberman and Ted Selker. 1999. Agents for the User Interface. In Proceedings of the ACM Conference on Human Factors in Computing Systems (CHI). Media Laboratory, Massachusetts Institute of Technology. https:\/\/web.media.mit.edu\/\u00a0lieber\/Publications\/Agents_for_UI.pdf"},{"key":"e_1_3_3_2_40_2","unstructured":"Deloitte\u00a0Consulting LLP. 2025. AWS Marketplace: Care Finder Agent. https:\/\/aws.amazon.com\/marketplace\/pp\/prodview-mau5avpo5xog6. https:\/\/aws.amazon.com\/marketplace\/pp\/prodview-mau5avpo5xog6 Product listing for Care Finder Agent an AI-powered healthcare navigation solution by Deloitte.."},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"publisher","unstructured":"Pattie Maes. 1994. Agents that reduce work and information overload. Commun. ACM 37 7 (July 1994) 30\u201340. 10.1145\/176789.176792176789.176792\"\/>","DOI":"10.1145\/176789.176792"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"crossref","unstructured":"Manus AI. 2025. Manus: General AI agent that bridges mind and action. https:\/\/manus.im\/?index=1. Accessed: 2025-09-02.","DOI":"10.1016\/S1773-035X(25)76474-6"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/169059.169215"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3359256"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","unstructured":"Tim Miller. 2019. Explanation in artificial intelligence: Insights from the social sciences. Artificial Intelligence 267 (2019) 1\u201338. 10.1016\/j.artint.2018.07.007","DOI":"10.1016\/j.artint.2018.07.007"},{"key":"e_1_3_3_2_46_2","unstructured":"Hussein Mozannar Gagan Bansal Cheng Tan Adam Fourney Victor Dibia Jingya Chen Jack Gerrits Tyler Payne Matheus\u00a0Kunzler Maldaner Madeleine Grunde-McLaughlin Eric Zhu Griffin Bassman Jacob Alber Peter Chang Ricky Loynd Friederike Niedtner Ece Kamar Maya Murad Rafah Hosn and Saleema Amershi. 2025. Magentic-UI: Towards Human-in-the-loop Agentic Systems. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2507.22358 (2025)."},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","unstructured":"Robert\u00a0C. Nickerson Upkar Varshney and Jan Muntermann. 2013. A method for taxonomy development and its application in information systems. European Journal of Information Systems 22 3 (2013) 336\u2013359. 10.1057\/ejis.2012.26","DOI":"10.1057\/ejis.2012.26"},{"key":"e_1_3_3_2_48_2","first-page":"433","volume-title":"Proceedings of the 24th Australian Computer-Human Interaction Conference","author":"Ong Christopher","year":"2012","unstructured":"Christopher Ong, Kevin McGee, and Teong\u00a0Leong Chuah. 2012. Closing the human-AI team-mate gap: how changes to displayed information impact player behavior towards computer teammates. In Proceedings of the 24th Australian Computer-Human Interaction Conference. 433\u2013439."},{"key":"e_1_3_3_2_49_2","volume-title":"Introducing ChatGPT agent: bridging research and action","year":"2025","unstructured":"OpenAI. 2025. Introducing ChatGPT agent: bridging research and action. https:\/\/openai.com\/index\/introducing-chatgpt-agent\/ Accessed: 2025-09-02."},{"key":"e_1_3_3_2_50_2","unstructured":"OpenAI. 2025. Introducing Operator | OpenAI. https:\/\/openai.com\/index\/introducing-operator\/. Accessed: 2025-09-02."},{"key":"e_1_3_3_2_51_2","unstructured":"Yichen Pan Dehan Kong Sida Zhou Cheng Cui Yifei Leng Bing Jiang Hangyu Liu Yanyi Shang Shuyan Zhou Tongshuang Wu and Zhengyang Wu. 2024. WebCanvas: Benchmarking Web Agents in Online Environments. arxiv:https:\/\/arXiv.org\/abs\/2406.12373\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2406.12373"},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/3715336.3735753"},{"key":"e_1_3_3_2_53_2","unstructured":"Long Phan Alice Gatti Ziwen Han Nathaniel Li Josephina Hu Hugh Zhang Chen Bo\u00a0Calvin Zhang Mohamed Shaaban John Ling Sean Shi Michael Choi Anish Agrawal Arnav Chopra Adam Khoja Ryan Kim Richard Ren Jason Hausenloy Oliver Zhang Mantas Mazeika Dmitry Dodonov Tung Nguyen Jaeho Lee Daron Anderson Mikhail Doroshenko Alun\u00a0Cennyth Stokes Mobeen Mahmood Oleksandr Pokutnyi Oleg Iskra Jessica\u00a0P. Wang John-Clark Levin Mstyslav Kazakov Fiona Feng Steven\u00a0Y. Feng Haoran Zhao Michael Yu Varun Gangal Chelsea Zou Zihan Wang Serguei Popov Robert Gerbicz Geoff Galgon Johannes Schmitt Will Yeadon Yongki Lee Scott Sauers Alvaro Sanchez Fabian Giska Marc Roth S\u00f8ren Riis Saiteja Utpala Noah Burns Gashaw\u00a0M. Goshu Mohinder\u00a0Maheshbhai Naiya Chidozie Agu Zachary Giboney Antrell Cheatom Francesco Fournier-Facio Sarah-Jane Crowson Lennart Finke Zerui Cheng Jennifer Zampese Ryan\u00a0G. Hoerr Mark Nandor Hyunwoo Park Tim Gehrunger Jiaqi Cai Ben McCarty Alexis\u00a0C Garretson Edwin Taylor Damien Sileo Qiuyu Ren Usman Qazi Lianghui Li Jungbae Nam John\u00a0B. Wydallis Pavel Arkhipov Jack Wei\u00a0Lun Shi Aras Bacho Chris\u00a0G. Willcocks Hangrui Cao Sumeet Motwani Emily de Oliveira\u00a0Santos Johannes Veith Edward Vendrow Doru Cojoc Kengo Zenitani Joshua Robinson Longke Tang Yuqi Li Joshua Vendrow Natanael\u00a0Wildner Fraga Vladyslav Kuchkin Andrey\u00a0Pupasov Maksimov Pierre Marion Denis Efremov Jayson Lynch Kaiqu Liang Aleksandar Mikov Andrew Gritsevskiy Julien Guillod G\u00f6zdenur Demir Dakotah Martinez Ben Pageler Kevin Zhou Saeed Soori Ori Press Henry Tang Paolo Rissone Sean\u00a0R. Green Lina Br\u00fcssel Moon Twayana Aymeric Dieuleveut Joseph\u00a0Marvin Imperial Ameya Prabhu Jinzhou Yang Nick Crispino Arun Rao Dimitri Zvonkine Gabriel Loiseau Mikhail Kalinin Marco Lukas Ciprian Manolescu Nate Stambaugh Subrata Mishra Tad Hogg Carlo Bosio Brian\u00a0P Coppola Julian Salazar Jaehyeok Jin Rafael Sayous Stefan Ivanov Philippe Schwaller Shaipranesh Senthilkuma Andres\u00a0M Bran Andres Algaba Kelsey\u00a0Van den Houte Lynn Van\u00a0Der Sypt Brecht Verbeken David Noever Alexei Kopylov Benjamin Myklebust Bikun Li Lisa Schut Evgenii Zheltonozhskii Qiaochu Yuan Derek Lim Richard Stanley Tong Yang John Maar Julian Wykowski Mart\u00ed Oller Anmol Sahu Cesare\u00a0Giulio Ardito Yuzheng Hu Ariel Ghislain\u00a0Kemogne Kamdoum Alvin Jin Tobias\u00a0Garcia Vilchis Yuexuan Zu Martin Lackner James Koppel Gongbo Sun Daniil\u00a0S. Antonenko Steffi Chern Bingchen Zhao Pierrot Arsene Joseph\u00a0M Cavanagh Daofeng Li Jiawei Shen Donato Crisostomi Wenjin Zhang Ali Dehghan Sergey Ivanov David Perrella Nurdin Kaparov Allen Zang Ilia Sucholutsky Arina Kharlamova Daniil Orel Vladislav Poritski Shalev Ben-David Zachary Berger Parker Whitfill Michael Foster Daniel Munro Linh Ho Shankar Sivarajan Dan\u00a0Bar Hava Aleksey Kuchkin David Holmes Alexandra Rodriguez-Romero Frank Sommerhage Anji Zhang Richard Moat Keith Schneider Zakayo Kazibwe Don Clarke Dae\u00a0Hyun Kim Felipe\u00a0Meneguitti Dias Sara Fish Veit Elser Tobias Kreiman Victor Efren\u00a0Guadarrama Vilchis Immo Klose Ujjwala Anantheswaran Adam Zweiger Kaivalya Rawal Jeffery Li Jeremy Nguyen Nicolas Daans Haline Heidinger Maksim Radionov V\u00e1clav Rozho\u0148 Vincent Ginis Christian Stump Niv Cohen Rafa\u0142 Po\u015bwiata Josef Tkadlec Alan Goldfarb Chenguang Wang Piotr Padlewski Stanislaw Barzowski Kyle Montgomery Ryan Stendall Jamie Tucker-Foltz Jack Stade T.\u00a0Ryan Rogers Tom Goertzen Declan Grabb Abhishek Shukla Alan Givr\u00e9 John\u00a0Arnold Ambay Archan Sen Muhammad\u00a0Fayez Aziz Mark\u00a0H Inlow Hao He Ling Zhang Younesse Kaddar Ivar \u00c4ngquist Yanxu Chen Harrison\u00a0K Wang Kalyan Ramakrishnan Elliott Thornley Antonio Terpin Hailey Schoelkopf Eric Zheng Avishy Carmi Ethan D.\u00a0L. Brown Kelin Zhu Max Bartolo Richard Wheeler Martin Stehberger Peter Bradshaw JP Heimonen Kaustubh Sridhar Ido Akov Jennifer Sandlin Yury Makarychev Joanna Tam Hieu Hoang David\u00a0M. Cunningham Vladimir Goryachev Demosthenes Patramanis Michael Krause Andrew Redenti David Aldous Jesyin Lai Shannon Coleman Jiangnan Xu Sangwon Lee Ilias Magoulas Sandy Zhao Ning Tang Michael\u00a0K. Cohen Orr Paradise Jan\u00a0Hendrik Kirchner Maksym Ovchynnikov Jason\u00a0O. Matos Adithya Shenoy Michael Wang Yuzhou Nie Anna Sztyber-Betley Paolo Faraboschi Robin Riblet Jonathan Crozier Shiv Halasyamani Shreyas Verma Prashant Joshi Eli Meril Ziqiao Ma J\u00e9r\u00e9my Andr\u00e9oletti Raghav Singhal Jacob Platnick Volodymyr Nevirkovets Luke Basler Alexander Ivanov Seri Khoury Nils Gustafsson Marco Piccardo Hamid Mostaghimi Qijia Chen Virendra Singh Tran\u00a0Quoc Kh\u00e1nh Paul Rosu Hannah Szlyk Zachary Brown Himanshu Narayan Aline Menezes Jonathan Roberts William Alley Kunyang Sun Arkil Patel Max Lamparth Anka Reuel Linwei Xin Hanmeng Xu Jacob Loader Freddie Martin Zixuan Wang Andrea Achilleos Thomas Preu Tomek Korbak Ida Bosio Fereshteh Kazemi Ziye Chen Bir\u00f3 B\u00e1lint Eve J.\u00a0Y. Lo Jiaqi Wang Maria In\u00eas\u00a0S. Nunes Jeremiah Milbauer M\u00a0Saiful Bari Zihao Wang Behzad Ansarinejad Yewen Sun Stephane Durand Hossam Elgnainy Guillaume Douville Daniel Tordera George Balabanian Hew Wolff Lynna Kvistad Hsiaoyun Milliron Ahmad Sakor Murat Eron Andrew Favre\u00a0D. O. Shailesh Shah Xiaoxiang Zhou Firuz Kamalov Sherwin Abdoli Tim Santens Shaul Barkan Allison Tee Robin Zhang Alessandro Tomasiello G.\u00a0Bruno\u00a0De Luca Shi-Zhuo Looi Vinh-Kha Le Noam Kolt Jiayi Pan Emma Rodman Jacob Drori Carl\u00a0J Fossum Niklas Muennighoff Milind Jagota Ronak Pradeep Honglu Fan Jonathan Eicher Michael Chen Kushal Thaman William Merrill Moritz Firsching Carter Harris Stefan Ciob\u00e2c\u0103 Jason Gross Rohan Pandey Ilya Gusev Adam Jones Shashank Agnihotri Pavel Zhelnov Mohammadreza Mofayezi Alexander Piperski David\u00a0K. Zhang Kostiantyn Dobarskyi Roman Leventov Ignat Soroko Joshua Duersch Vage Taamazyan Andrew Ho Wenjie Ma William Held Ruicheng Xian Armel\u00a0Randy Zebaze Mohanad Mohamed Julian\u00a0Noah Leser Michelle\u00a0X Yuan Laila Yacar Johannes Lengler Katarzyna Olszewska Claudio\u00a0Di Fratta Edson Oliveira Joseph\u00a0W. Jackson Andy Zou Muthu Chidambaram Timothy Manik Hector Haffenden Dashiell Stander Ali Dasouqi Alexander Shen Bita Golshani David Stap Egor Kretov Mikalai Uzhou Alina\u00a0Borisovna Zhidkovskaya Nick Winter Miguel\u00a0Orbegozo Rodriguez Robert Lauff Dustin Wehr Colin Tang Zaki Hossain Shaun Phillips Fortuna Samuele Fredrik Ekstr\u00f6m Angela Hammon Oam Patel Faraz Farhidi George Medley Forough Mohammadzadeh Madellene Pe\u00f1aflor Haile Kassahun Alena Friedrich Rayner\u00a0Hernandez Perez Daniel Pyda Taom Sakal Omkar Dhamane Ali\u00a0Khajegili Mirabadi Eric Hallman Kenchi Okutsu Mike Battaglia Mohammad Maghsoudimehrabani Alon Amit Dave Hulbert Roberto Pereira Simon Weber Handoko Anton Peristyy Stephen Malina Mustafa Mehkary Rami Aly Frank Reidegeld Anna-Katharina Dick Cary Friday Mukhwinder Singh Hassan Shapourian Wanyoung Kim Mariana Costa Hubeyb Gurdogan Harsh Kumar Chiara Ceconello Chao Zhuang Haon Park Micah Carroll Andrew\u00a0R. Tawfeek Stefan Steinerberger Daattavya Aggarwal Michael Kirchhof Linjie Dai Evan Kim Johan Ferret Jainam Shah Yuzhou Wang Minghao Yan Krzysztof Burdzy Lixin Zhang Antonio Franca Diana\u00a0T. Pham Kang\u00a0Yong Loh Joshua Robinson Abram Jackson Paolo Giordano Philipp Petersen Adrian Cosma Jesus Colino Colin White Jacob Votava Vladimir Vinnikov Ethan Delaney Petr Spelda Vit Stritecky Syed\u00a0M. Shahid Jean-Christophe Mourrat Lavr Vetoshkin Koen Sponselee Renas Bacho Zheng-Xin Yong Florencia de\u00a0la Rosa Nathan Cho Xiuyu Li Guillaume Malod Orion Weller Guglielmo Albani Leon Lang Julien Laurendeau Dmitry Kazakov Fatimah Adesanya Julien Portier Lawrence Hollom Victor Souza Yuchen\u00a0Anna Zhou Julien Degorre Yi\u011fit Yal\u0131n Gbenga\u00a0Daniel Obikoya Rai Filippo Bigi M.\u00a0C. Bosc\u00e1 Oleg Shumar Kaniuar Bacho Gabriel Recchia Mara Popescu Nikita Shulga Ngefor\u00a0Mildred Tanwie Thomas C.\u00a0H. Lux Ben Rank Colin Ni Matthew Brooks Alesia Yakimchyk Huanxu Liu Stefano Cavalleri Olle H\u00e4ggstr\u00f6m Emil Verkama Joshua Newbould Hans Gundlach Leonor Brito-Santana Brian Amaro Vivek Vajipey Rynaa Grover Ting Wang Yosi Kratish Wen-Ding Li Sivakanth Gopi Andrea Caciolai Christian\u00a0Schroeder de Witt Pablo Hern\u00e1ndez-C\u00e1mara Emanuele Rodol\u00e0 Jules Robins Dominic Williamson Vincent Cheng Brad Raynor Hao Qi Ben Segev Jingxuan Fan Sarah Martinson Erik\u00a0Y. Wang Kaylie Hausknecht Michael\u00a0P. Brenner Mao Mao Christoph Demian Peyman Kassani Xinyu Zhang David Avagian Eshawn\u00a0Jessica Scipio Alon Ragoler Justin Tan Blake Sims Rebeka Plecnik Aaron Kirtland Omer\u00a0Faruk Bodur D.\u00a0P. Shinde Yan Carlos\u00a0Leyva Labrador Zahra Adoul Mohamed Zekry Ali Karakoc Tania C.\u00a0B. Santos Samir Shamseldeen Loukmane Karim Anna Liakhovitskaia Nate Resman Nicholas Farina Juan\u00a0Carlos Gonzalez Gabe Maayan Earth Anderson Rodrigo De\u00a0Oliveira Pena Elizabeth Kelley Hodjat Mariji Rasoul Pouriamanesh Wentao Wu Ross Finocchio Ismail Alarab Joshua Cole Danyelle Ferreira Bryan Johnson Mohammad Safdari Liangti Dai Siriphan Arthornthurasuk Isaac\u00a0C. McAlister Alejandro\u00a0Jos\u00e9 Moyano Alexey Pronin Jing Fan Angel Ramirez-Trinidad Yana Malysheva Daphiny Pottmaier Omid Taheri Stanley Stepanic Samuel Perry Luke Askew Ra\u00fal Adri\u00e1n\u00a0Huerta Rodr\u00edguez Ali M.\u00a0R. Minissi Ricardo Lorena Krishnamurthy Iyer Arshad\u00a0Anil Fasiludeen Ronald Clark Josh Ducey Matheus Piza Maja Somrak Eric Vergo Juehang Qin Benj\u00e1min Borb\u00e1s Eric Chu Jack Lindsey Antoine Jallon I.\u00a0M.\u00a0J. McInnis Evan Chen Avi Semler Luk Gloor Tej Shah Marc Carauleanu Pascal Lauer Tran \u0110uc Huy Hossein Shahrtash Emilien Duc Lukas Lewark Assaf Brown Samuel Albanie Brian Weber Warren\u00a0S. Vaz Pierre Clavier Yiyang Fan Gabriel Poesia\u00a0Reis e Silva Long Lian Marcus Abramovitch Xi Jiang Sandra Mendoza Murat Islam Juan Gonzalez Vasilios Mavroudis Justin Xu Pawan Kumar Laxman\u00a0Prasad Goswami Daniel Bugas Nasser Heydari Ferenc Jeanplong Thorben Jansen Antonella Pinto Archimedes Apronti Abdallah Galal Ng Ze-An Ankit Singh Tong Jiang Joan of Arc\u00a0Xavier Kanu\u00a0Priya Agarwal Mohammed Berkani Gang Zhang Zhehang Du Benedito\u00a0Alves de Oliveira\u00a0Junior Dmitry Malishev Nicolas Remy Taylor\u00a0D. Hartman Tim Tarver Stephen Mensah Gautier\u00a0Abou Loume Wiktor Morak Farzad Habibi Sarah Hoback Will Cai Javier Gimenez Roselynn\u00a0Grace Montecillo Jakub \u0141ucki Russell Campbell Asankhaya Sharma Khalida Meer Shreen Gul Daniel\u00a0Espinosa Gonzalez Xavier Alapont Alex Hoover Gunjan Chhablani Freddie Vargus Arunim Agarwal Yibo Jiang Deepakkumar Patil David Outevsky Kevin\u00a0Joseph Scaria Rajat Maheshwari Abdelkader Dendane Priti Shukla Ashley Cartwright Sergei Bogdanov Niels M\u00fcndler S\u00f6ren M\u00f6ller Luca Arnaboldi Kunvar Thaman Muhammad\u00a0Rehan Siddiqi Prajvi Saxena Himanshu Gupta Tony Fruhauff Glen Sherman M\u00e1ty\u00e1s Vincze Siranut Usawasutsakorn Dylan Ler Anil Radhakrishnan Innocent Enyekwe Sk\u00a0Md Salauddin Jiang Muzhen Aleksandr Maksapetyan Vivien Rossbach Chris Harjadi Mohsen Bahaloohoreh Claire Sparrow Jasdeep Sidhu Sam Ali Song Bian John Lai Eric Singer Justine\u00a0Leon Uro Greg Bateman Mohamed Sayed Ahmed Menshawy Darling Duclosel Dario Bezzi Yashaswini Jain Ashley Aaron Murat Tiryakioglu Sheeshram Siddh Keith Krenek Imad\u00a0Ali Shah Jun Jin Scott Creighton Denis Peskoff Zienab EL-Wasif Ragavendran\u00a0P V Michael Richmond Joseph McGowan Tejal Patwardhan Hao-Yu Sun Ting Sun Nikola Zubi\u0107 Samuele Sala Stephen Ebert Jean Kaddour Manuel Schottdorf Dianzhuo Wang Gerol Petruzella Alex Meiburg Tilen Medved Ali ElSheikh S\u00a0Ashwin Hebbar Lorenzo Vaquero Xianjun Yang Jason Poulos Vil\u00e9m Zouhar Sergey Bogdanik Mingfang Zhang Jorge Sanz-Ros David Anugraha Yinwei Dai Anh\u00a0N. Nhu Xue Wang Ali\u00a0Anil Demircali Zhibai Jia Yuyin Zhou Juncheng Wu Mike He Nitin Chandok Aarush Sinha Gaoxiang Luo Long Le Micka\u00ebl Noy\u00e9 Micha\u0142 Pere\u0142kiewicz Ioannis Pantidis Tianbo Qi Soham\u00a0Sachin Purohit Letitia Parcalabescu Thai-Hoa Nguyen Genta\u00a0Indra Winata Edoardo\u00a0M. Ponti Hanchen Li Kaustubh Dhole Jongee Park Dario Abbondanza Yuanli Wang Anupam Nayak Diogo\u00a0M. Caetano Antonio A. W.\u00a0L. Wong Maria del Rio-Chanona D\u00e1niel Kondor Pieter Francois Ed Chalstrey Jakob Zsambok Dan Hoyer Jenny Reddish Jakob Hauser Francisco-Javier Rodrigo-Gin\u00e9s Suchandra Datta Maxwell Shepherd Thom Kamphuis Qizheng Zhang Hyunjun Kim Ruiji Sun Jianzhu Yao Franck Dernoncourt Satyapriya Krishna Sina Rismanchian Bonan Pu Francesco Pinto Yingheng Wang Kumar Shridhar Kalon\u00a0J. Overholt Glib Briia Hieu Nguyen David Soler Bartomeu Tony\u00a0CY Pang Adam Wecker Yifan Xiong Fanfei Li Lukas\u00a0S. Huber Joshua Jaeger Romano\u00a0De Maddalena Xing\u00a0Han L\u00f9 Yuhui Zhang Claas Beger Patrick Tser\u00a0Jern Kon Sean Li Vivek Sanker Ming Yin Yihao Liang Xinlu Zhang Ankit Agrawal Li\u00a0S. Yifei Zechen Zhang Mu Cai Yasin Sonmez Costin Cozianu Changhao Li Alex Slen Shoubin Yu Hyun\u00a0Kyu Park Gabriele Sarti Marcin Bria\u0144ski Alessandro Stolfo Truong\u00a0An Nguyen Mike Zhang Yotam Perlitz Jose Hernandez-Orallo Runjia Li Amin Shabani Felix Juefei-Xu Shikhar Dhingra Orr Zohar My\u00a0Chiffon Nguyen Alexander Pondaven Abdurrahim Yilmaz Xuandong Zhao Chuanyang Jin Muyan Jiang Stefan Todoran Xinyao Han Jules Kreuer Brian Rabern Anna Plassart Martino Maggetti Luther Yap Robert Geirhos Jonathon Kean Dingsu Wang Sina Mollaei Chenkai Sun Yifan Yin Shiqi Wang Rui Li Yaowen Chang Anjiang Wei Alice Bizeul Xiaohan Wang Alexandre\u00a0Oliveira Arrais Kushin Mukherjee Jorge Chamorro-Padial Jiachen Liu Xingyu Qu Junyi Guan Adam Bouyamourn Shuyu Wu Martyna Plomecka Junda Chen Mengze Tang Jiaqi Deng Shreyas Subramanian Haocheng Xi Haoxuan Chen Weizhi Zhang Yinuo Ren Haoqin Tu Sejong Kim Yushun Chen Sara\u00a0Vera Marjanovi\u0107 Junwoo Ha Grzegorz Luczyna Jeff\u00a0J. Ma Zewen Shen Dawn Song Cedegao\u00a0E. Zhang Zhun Wang Ga\u00ebl Gendron Yunze Xiao Leo Smucker Erica Weng Kwok\u00a0Hao Lee Zhe Ye Stefano Ermon Ignacio\u00a0D. Lopez-Miguel Theo Knights Anthony Gitter Namkyu Park Boyi Wei Hongzheng Chen Kunal Pai Ahmed Elkhanany Han Lin Philipp\u00a0D. Siedler Jichao Fang Ritwik Mishra K\u00e1roly Zsolnai-Feh\u00e9r Xilin Jiang Shadab Khan Jun Yuan Rishab\u00a0Kumar Jain Xi Lin Mike Peterson Zhe Wang Aditya Malusare Maosen Tang Isha Gupta Ivan Fosin Timothy Kang Barbara Dworakowska Kazuki Matsumoto Guangyao Zheng Gerben Sewuster Jorge\u00a0Pretel Villanueva Ivan Rannev Igor Chernyavsky Jiale Chen Deepayan Banik Ben Racz Wenchao Dong Jianxin Wang Laila Bashmal Duarte\u00a0V. Gon\u00e7alves Wei Hu Kaushik Bar Ondrej Bohdal Atharv\u00a0Singh Patlan Shehzaad Dhuliawala Caroline Geirhos Julien Wist Yuval Kansal Bingsen Chen Kutay Tire Atak\u00a0Talay Y\u00fccel Brandon Christof Veerupaksh Singla Zijian Song Sanxing Chen Jiaxin Ge Kaustubh Ponkshe Isaac Park Tianneng Shi Martin\u00a0Q. Ma Joshua Mak Sherwin Lai Antoine Moulin Zhuo Cheng Zhanda Zhu Ziyi Zhang Vaidehi Patil Ketan Jha Qiutong Men Jiaxuan Wu Tianchi Zhang Bruno\u00a0Hebling Vieira Alham\u00a0Fikri Aji Jae-Won Chung Mohammed Mahfoud Ha\u00a0Thi Hoang Marc Sperzel Wei Hao Kristof Meding Sihan Xu Vassilis Kostakos Davide Manini Yueying Liu Christopher Toukmaji Jay Paek Eunmi Yu Arif\u00a0Engin Demircali Zhiyi Sun Ivan Dewerpe Hongsen Qin Roman Pflugfelder James Bailey Johnathan Morris Ville Heilala Sybille Rosset Zishun Yu Peter\u00a0E. Chen Woongyeong Yeo Eeshaan Jain Ryan Yang Sreekar Chigurupati Julia Chernyavsky Sai\u00a0Prajwal Reddy Subhashini Venugopalan Hunar Batra Core\u00a0Francisco Park Hieu Tran Guilherme Maximiano Genghan Zhang Yizhuo Liang Hu Shiyu Rongwu Xu Rui Pan Siddharth Suresh Ziqi Liu Samaksh Gulati Songyang Zhang Peter Turchin Christopher\u00a0W. Bartlett Christopher\u00a0R. Scotese Phuong\u00a0M. Cao Aakaash Nattanmai Gordon McKellips Anish Cheraku Asim Suhail Ethan Luo Marvin Deng Jason Luo Ashley Zhang Kavin Jindel Jay Paek Kasper Halevy Allen Baranov Michael Liu Advaith Avadhanam David Zhang Vincent Cheng Brad Ma Evan Fu Liam Do Joshua Lass Hubert Yang Surya Sunkari Vishruth Bharath Violet Ai James Leung Rishit Agrawal Alan Zhou Kevin Chen Tejas Kalpathi Ziqi Xu Gavin Wang Tyler Xiao Erik Maung Sam Lee Ryan Yang Roy Yue Ben Zhao Julia Yoon Sunny Sun Aryan Singh Ethan Luo Clark Peng Tyler Osbey Taozhi Wang Daryl Echeazu Hubert Yang Timothy Wu Spandan Patel Vidhi Kulkarni Vijaykaarti Sundarapandiyan Ashley Zhang Andrew Le Zafir Nasim Srikar Yalam Ritesh Kasamsetty Soham Samal Hubert Yang David Sun Nihar Shah Abhijeet Saha Alex Zhang Leon Nguyen Laasya Nagumalli Kaixin Wang Alan Zhou Aidan Wu Jason Luo Anwith Telluri Summer Yue Alexandr Wang and Dan Hendrycks. 2025. Humanity\u2019s Last Exam. arxiv:https:\/\/arXiv.org\/abs\/2501.14249\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2501.14249"},{"key":"e_1_3_3_2_54_2","unstructured":"Christopher Rawles Sarah Clinckemaillie Yifan Chang Jonathan Waltz Gabrielle Lau Marybeth Fair Alice Li William Bishop Wei Li Folawiyo Campbell-Ajala et\u00a0al. 2024. Androidworld: A dynamic benchmarking environment for autonomous agents. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.14573 (2024)."},{"key":"e_1_3_3_2_55_2","volume-title":"601 real-world gen AI use cases from the world\u2019s leading organizations","author":"Renner Matt","year":"2025","unstructured":"Matt Renner and Matt\u00a0A.V. Chaban. 2025. 601 real-world gen AI use cases from the world\u2019s leading organizations. https:\/\/cloud.google.com\/transform\/101-real-world-generative-ai-use-cases-from-industry-leaders Published April 12, 2024; last updated April 9, 2025. Google Cloud.."},{"key":"e_1_3_3_2_56_2","unstructured":"Olawale Salaudeen Anka Reuel Ahmed Ahmed Suhana Bedi Zachary Robertson Sudharsan Sundar Ben Domingue Angelina Wang and Sanmi Koyejo. 2025. Measurement to Meaning: A Validity-Centered Framework for AI Evaluation. arxiv:https:\/\/arXiv.org\/abs\/2505.10573\u00a0[cs.CY] https:\/\/arxiv.org\/abs\/2505.10573"},{"key":"e_1_3_3_2_57_2","volume-title":"The Coding Manual for Qualitative Researchers (4 ed.)","author":"Salda\u00f1a Johnny","year":"2021","unstructured":"Johnny Salda\u00f1a. 2021. The Coding Manual for Qualitative Researchers (4 ed.). Sage Publications, London."},{"key":"e_1_3_3_2_58_2","volume-title":"Advances in Neural Information Processing Systems (NeurIPS)","author":"Schick Timo","year":"2023","unstructured":"Timo Schick, Jane Dwivedi-Yu, Roberto Dess\u00ec, Roberta Raileanu, Maria Lomeli, Luke Zettlemoyer, Nicola Cancedda, and Thomas Scialom. 2023. Toolformer: Language Models Can Teach Themselves to Use Tools. In Advances in Neural Information Processing Systems (NeurIPS), Vol.\u00a036. arxiv:https:\/\/arXiv.org\/abs\/2302.04761\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2302.04761"},{"key":"e_1_3_3_2_59_2","unstructured":"Yijia Shao Humishka Zope Yucheng Jiang Jiaxin Pei David Nguyen Erik Brynjolfsson and Diyi Yang. 2025. Future of Work with AI Agents: Auditing Automation and Augmentation Potential across the U.S. Workforce. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2506.06576 (2025)."},{"key":"e_1_3_3_2_60_2","doi-asserted-by":"publisher","DOI":"10.5555\/134291"},{"key":"e_1_3_3_2_61_2","doi-asserted-by":"crossref","unstructured":"Ben Shneiderman. 1983. Direct manipulation: A step beyond programming languages. Computer 16 08 (1983) 57\u201369.","DOI":"10.1109\/MC.1983.1654471"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"publisher","unstructured":"Ben Shneiderman and Pattie Maes. 1997. Direct manipulation vs. interface agents. Interactions 4 6 (Nov. 1997) 42\u201361. 10.1145\/267505.267514267505.267514\"\/>","DOI":"10.1145\/267505.267514"},{"key":"e_1_3_3_2_63_2","unstructured":"Leon Staufer Mick Yang Anka Reuel and Stephen Casper. 2025. Audit Cards: Contextualizing AI Evaluations. arxiv:https:\/\/arXiv.org\/abs\/2504.13839\u00a0[cs.CY] https:\/\/arxiv.org\/abs\/2504.13839"},{"key":"e_1_3_3_2_64_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713166"},{"key":"e_1_3_3_2_65_2","unstructured":"Hanna Wallach Meera Desai A\u00a0Feder Cooper Angelina Wang Chad Atalla Solon Barocas Su\u00a0Lin Blodgett Alexandra Chouldechova Emily Corvi P\u00a0Alex Dow et\u00a0al. 2025. Position: Evaluating generative ai systems is a social science measurement challenge. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2502.00561 (2025)."},{"key":"e_1_3_3_2_66_2","unstructured":"Zora\u00a0Zhiruo Wang Yijia Shao Omar Shaikh Daniel Fried Graham Neubig and Diyi Yang. 2025. How Do AI Agents Do Human Work? Comparing AI and Human Workflows Across Diverse Occupations. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2510.22780 (2025)."},{"key":"e_1_3_3_2_67_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642466"},{"key":"e_1_3_3_2_68_2","first-page":"169","volume-title":"USENIX Security Symposium","author":"Whitten Alma","year":"1999","unstructured":"Alma Whitten and J\u00a0Doug Tygar. 1999. Why Johnny Can\u2019t Encrypt: A Usability Evaluation of PGP 5.0.. In USENIX Security Symposium, Vol.\u00a0348. 169\u2013184."},{"key":"e_1_3_3_2_69_2","unstructured":"Wikipedia contributors. 2025. Office Assistant. https:\/\/en.wikipedia.org\/wiki\/Office_Assistant. Accessed: 2025-09-04."},{"key":"e_1_3_3_2_70_2","series-title":"(NIPS \u201924)","volume-title":"Proceedings of the 38th International Conference on Neural Information Processing Systems","author":"Xie Tianbao","year":"2025","unstructured":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh\u00a0Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, and Tao Yu. 2025. OSWORLD: benchmarking multimodal agents for open-ended tasks in real computer environments. In Proceedings of the 38th International Conference on Neural Information Processing Systems (Vancouver, BC, Canada) (NIPS \u201924). Curran Associates Inc., Red Hook, NY, USA, Article 1650, 55\u00a0pages."},{"key":"e_1_3_3_2_71_2","unstructured":"Deshraj Yadav Rishabh Jain Harsh Agrawal Prithvijit Chattopadhyay Taranjeet Singh Akash Jain Shiv\u00a0Baran Singh Stefan Lee and Dhruv Batra. 2019. EvalAI: Towards Better Evaluation Systems for AI Agents. arxiv:https:\/\/arXiv.org\/abs\/1902.03570\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/1902.03570"},{"key":"e_1_3_3_2_72_2","volume-title":"International Conference on Learning Representations (ICLR)","author":"Yao Shunyu","year":"2023","unstructured":"Shunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik Narasimhan, and Yuan Cao. 2023. ReAct: Synergizing Reasoning and Acting in Language Models. In International Conference on Learning Representations (ICLR). arxiv:https:\/\/arXiv.org\/abs\/2210.03629\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2210.03629"},{"key":"e_1_3_3_2_73_2","volume-title":"CHI 2019","author":"Yin Ming","year":"2019","unstructured":"Ming Yin, Jennifer\u00a0Wortman Vaughan, and Hanna Wallach. 2019. Understanding the Effect of Accuracy on Trust in Machine Learning Models. In CHI 2019. ACM. https:\/\/www.microsoft.com\/en-us\/research\/publication\/understanding-the-effect-of-accuracy-on-trust-in-machine-learning-models\/"},{"key":"e_1_3_3_2_74_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_2_75_2","unstructured":"Chaoyun Zhang Shilin He Jiaxu Qian Bowen Li Liqun Li Si Qin Yu Kang Minghua Ma Guyue Liu Qingwei Lin Saravan Rajmohan Dongmei Zhang and Qi Zhang. 2025. Large Language Model-Brained GUI Agents: A Survey. arxiv:https:\/\/arXiv.org\/abs\/2411.18279\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2411.18279"},{"key":"e_1_3_3_2_76_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517791"},{"key":"e_1_3_3_2_77_2","doi-asserted-by":"publisher","unstructured":"Rui Zhang Nathan\u00a0J. McNeese Guo Freeman and Geoff Musick. 2020. \"An Ideal Human\": Expectations of AI Teammates in Human-AI Teaming. Proceedings of the ACM on Human-Computer Interaction 4 CSCW3 (December 2020) 246:1\u2013246:25. 10.1145\/34329453432945\"\/>","DOI":"10.1145\/3432945"},{"key":"e_1_3_3_2_78_2","unstructured":"Yuxuan Zhu Tengjun Jin Yada Pruksachatkun Andy Zhang Shu Liu Sasha Cui Sayash Kapoor Shayne Longpre Kevin Meng Rebecca Weiss et\u00a0al. 2025. Establishing best practices for building rigorous agentic benchmarks. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2507.02825 (2025)."},{"key":"e_1_3_3_2_79_2","unstructured":"Mingchen Zhuge Changsheng Zhao Dylan Ashley Wenyi Wang Dmitrii Khizbullin Yunyang Xiong Zechun Liu Ernie Chang Raghuraman Krishnamoorthi Yuandong Tian Yangyang Shi Vikas Chandra and J\u00fcrgen Schmidhuber. 2024. Agent-as-a-Judge: Evaluate Agents with Agents. arxiv:https:\/\/arXiv.org\/abs\/2410.10934\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2410.10934"}],"event":{"name":"CAIS '26: ACM Conference on AI and Agentic Systems","location":"San Jose CA USA","acronym":"CAIS '26"},"container-title":["Proceedings of the ACM Conference on AI and Agentic Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3786335.3813140","content-type":"text\/html","content-version":"vor","intended-application":"syndication"}],"deposited":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T03:32:05Z","timestamp":1779420725000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786335.3813140"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,26]]},"references-count":78,"alternative-id":["10.1145\/3786335.3813140","10.1145\/3786335"],"URL":"https:\/\/doi.org\/10.1145\/3786335.3813140","relation":{},"subject":[],"published":{"date-parts":[[2026,5,26]]},"assertion":[{"value":"2026-05-26","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}