{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T12:34:18Z","timestamp":1776083658557,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706599.3706674","type":"proceedings-article","created":{"date-parts":[[2025,4,23]],"date-time":"2025-04-23T20:48:52Z","timestamp":1745441332000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Creating benchmarkable components to measure the quality of AI-enhanced developer tools"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9103-4721","authenticated-orcid":false,"given":"Elise","family":"Paradis","sequence":"first","affiliation":[{"name":"Google, Mountain View, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-5911-7673","authenticated-orcid":false,"given":"Ambar","family":"Murillo","sequence":"additional","affiliation":[{"name":"Google, Munich, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0543-3088","authenticated-orcid":false,"given":"Maulishree","family":"Pandey","sequence":"additional","affiliation":[{"name":"Google, Mountain View, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9104-8365","authenticated-orcid":false,"given":"Sarah","family":"D'Angelo","sequence":"additional","affiliation":[{"name":"Google, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8215-8513","authenticated-orcid":false,"given":"Andrew","family":"Macvean","sequence":"additional","affiliation":[{"name":"Google, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-4928-970X","authenticated-orcid":false,"given":"Ben","family":"Ferrari-Church","sequence":"additional","affiliation":[{"name":"Google, San Francisco, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2642-2223","authenticated-orcid":false,"given":"Matthew","family":"Hughes","sequence":"additional","affiliation":[{"name":"ServiceNow, Santa Clara, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","unstructured":"Paul\u00a0C. Beatty and Gordon\u00a0B. Willis. 2007. Research Synthesis: The Practice of Cognitive Interviewing. Public Opinion Quarterly 71 2 (05 2007) 287\u2013311. 10.1093\/poq\/nfm006 arXiv:https:\/\/academic.oup.com\/poq\/article-pdf\/71\/2\/287\/5436696\/nfm006.pdf","DOI":"10.1093\/poq\/nfm006"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","unstructured":"Jenna Bergdahl Rita Latikka Magdalena Celuch Iina Savolainen Eerik Soares Mantere Nina Savela and Atte Oksanen. 2023. Self-determination and attitudes toward artificial intelligence: Cross-national and longitudinal perspectives. Telematics and Informatics 82 (2023) 102013. 10.1016\/j.tele.2023.102013","DOI":"10.1016\/j.tele.2023.102013"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"Satish Chandra and Maxim Tabachnyk. 2024. AI in software engineering at Google: Progress and the path ahead. https:\/\/research.google\/blog\/ai-in-software-engineering-at-google-progress-and-the-path-ahead\/. Accessed: 2024-07-10.","DOI":"10.1145\/3664646.3676277"},{"key":"e_1_3_3_2_5_2","unstructured":"CoderPad. 2024. CoderPad State of Tech Hiring. https:\/\/coderpad.io\/survey-reports\/coderpad-and-codingame-state-of-tech-hiring-2024\/. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_6_2","unstructured":"Zheyuan\u00a0Kevin Cui Mert Demirer Sonia Jaffe Leon Musolff Sida Peng and Tobias Salz. 2024. The Effects of Generative AI on High Skilled Work: Evidence from Three Field Experiments with Software Developers. https:\/\/papers.ssrn.com\/sol3\/papers.cfm?abstract_id=4945566. 22\u00a0pages."},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","unstructured":"Angus Deaton and Nancy Cartwright. 2018. Understanding and misunderstanding randomized controlled trials. Social Science & Medicine 210 (2018) 2\u201321. 10.1016\/j.socscimed.2017.12.005Randomized Controlled Trials and Evidence-based Policy: A Multidisciplinary Dialogue.","DOI":"10.1016\/j.socscimed.2017.12.005"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","unstructured":"Sarah D\u2019Angelo Jessica Lin Jill Dicker Carolyn Egelman Maggie Hodges Collin Green and Ciera Jaspan. 2024. Measuring Developer Experience With a Longitudinal Survey. IEEE Software 41 4 (2024) 19\u201324. 10.1109\/MS.2024.3386027","DOI":"10.1109\/MS.2024.3386027"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","unstructured":"Steve\u00a0W Edison and Gary\u00a0L Geissler. 2003. Measuring attitudes towards general technology: Antecedents hypotheses and scale development. Journal of Targeting Measurement and Analysis for Marketing 12 (2003) 137\u2013156. 10.1057\/palgrave.jt.5740104","DOI":"10.1057\/palgrave.jt.5740104"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"crossref","unstructured":"Nicole Forsgren Margaret-Anne Storey Chandra Maddila Thomas Zimmermann Brian Houck and Jenna Butler. 2021. The SPACE of Developer Productivity: There\u2019s more to it than you think. Queue 19 1 (2021) 20\u201348.","DOI":"10.1145\/3454122.3454124"},{"key":"e_1_3_3_2_11_2","unstructured":"Ya Gao and Github\u00a0Customer Research. 2024. Research: Quantifying GitHub Copilot\u2019s impact in the enterprise with Accenture. https:\/\/github.blog\/2024-05-13-research-quantifying-github-copilots-impact-in-the-enterprise-with-accenture\/. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_12_2","unstructured":"William Harding and Matthew Kloster. 2024. Coding on Copilot: 2023 Data Suggests Downward Pressure on Code Quality. https:\/\/www.gitclear.com\/coding_on_copilot_data_shows_ais_downward_pressure_on_code_quality\/. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3522684"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Sage Kelly Sherrie-Anne Kaye and Oscar Oviedo-Trespalacios. 2023. What factors contribute to the acceptance of artificial intelligence? A systematic review. Telematics and Informatics 77 (2023) 101925.","DOI":"10.1016\/j.tele.2022.101925"},{"key":"e_1_3_3_2_15_2","unstructured":"JetBrains Tech\u00a0Insights Lab. 2023. Jetbrains The State of Developer Ecosystem 2023. https:\/\/www.jetbrains.com\/lp\/devecosystem-2023\/. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_16_2","unstructured":"LeetCode. 2015. LeetCode - The World\u2019s Leading Online Programming Learning Platform. https:\/\/leetcode.com\/. Accessed: 2024-06-20."},{"key":"e_1_3_3_2_17_2","unstructured":"Timothy\u00a0R. McIntosh Teo Susnjak Nalin Arachchilage Tong Liu Paul Watters and Malka\u00a0N. Halgamuge. 2024. Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence. arxiv:https:\/\/arXiv.org\/abs\/2402.09880\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2402.09880"},{"key":"e_1_3_3_2_18_2","unstructured":"Lisa-Maria Neudert Aleksi Knuutila and Philip\u00a0N. Howard. 2023. Global attitudes towards AI Machine Learning & Automated Decision Making\u2013Implications for involving artificial intelligence in public service and good governance. https:\/\/perma.cc\/6PB6-X56B. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_19_2","unstructured":"Stack Overflow. 2023. Stack Overflow 2023 Developer Survey AI sentiment and usage. https:\/\/survey.stackoverflow.co\/2023\/#ai-sentiment-and-usage. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_20_2","unstructured":"Oxford University Press. 2024. benchmark n. https:\/\/languages.oup.com\/google-dictionary-en\/. Accessed: 2024-07-10."},{"key":"e_1_3_3_2_21_2","unstructured":"Elise Paradis Kate Grey Quinn Madison Daye Nam Andrew Macvean Vahid Meimand Nan Zhang Ben Ferrari-Church and Satish Chandra. 2024. How much does AI impact development speed? An enterprise-based randomized controlled trial. https:\/\/arxiv.org\/pdf\/2410.12944. 12\u00a0pages."},{"key":"e_1_3_3_2_22_2","unstructured":"Sida Peng Eirini Kalliamvakou Peter Cihon and Mert Demirer. 2023. The Impact of AI on Developer Productivity: Evidence from GitHub Copilot. arxiv:https:\/\/arXiv.org\/abs\/2302.06590\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2302.06590"},{"key":"e_1_3_3_2_23_2","unstructured":"and Phoronix\u00a0Media. 2010. PyBench Benchmark. https:\/\/openbenchmarking.org\/test\/pts\/pybench. Accessed: 2024-06-20."},{"key":"e_1_3_3_2_24_2","unstructured":"Meta\u00a0AI Research. 2018. Papers with Code. https:\/\/paperswithcode.com\/. Accessed: 2024-06-24."},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","unstructured":"Astrid Schepman and Paul Rodway. 2020. Initial validation of the general attitudes towards Artificial Intelligence Scale. Computers in Human Behavior Reports 1 (2020) 100014. 10.1016\/j.chbr.2020.100014","DOI":"10.1016\/j.chbr.2020.100014"},{"key":"e_1_3_3_2_26_2","unstructured":"Aarohi Srivastava Abhinav Rastogi Abhishek Rao Abu Awal\u00a0Md Shoeb and et al.2023. Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models. arxiv:https:\/\/arXiv.org\/abs\/2206.04615\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2206.04615"},{"key":"e_1_3_3_2_27_2","unstructured":"Catherine Thorbecke. 2023. A year after ChatGPT\u2019s release the AI revolution is just beginning. https:\/\/www.cnn.com\/2023\/11\/30\/tech\/chatgpt-openai-revolution-one-year\/index.html. Accessed: 2024-06-24."},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_3_2_29_2","unstructured":"Yanlin Wang Wanjun Zhong Yanxian Huang Ensheng Shi Min Yang Jiachi Chen Hui Li Yuchi Ma Qianxiang Wang and Zibin Zheng. 2024. Agents in Software Engineering: Survey Landscape and Vision. arxiv:https:\/\/arXiv.org\/abs\/2409.09030\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2409.09030"},{"key":"e_1_3_3_2_30_2","first-page":"11","volume-title":"Advances in Neural Information Processing Systems","author":"Waswani Ashish","year":"2017","unstructured":"Ashish Waswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan\u00a0N. Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in Neural Information Processing Systems. NeurIPS Proceedings, Online, 11. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2017\/file\/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.18293\/seke2023-077"}],"event":{"name":"CHI EA '25: Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI EA '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the Extended Abstracts of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3706674","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706599.3706674","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:57:12Z","timestamp":1750298232000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3706674"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":30,"alternative-id":["10.1145\/3706599.3706674","10.1145\/3706599"],"URL":"https:\/\/doi.org\/10.1145\/3706599.3706674","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}