{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T21:11:14Z","timestamp":1776114674642,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":67,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706598.3713271","type":"proceedings-article","created":{"date-parts":[[2025,4,24]],"date-time":"2025-04-24T04:45:58Z","timestamp":1745469958000},"page":"1-34","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["How Humans Communicate Programming Tasks in Natural Language and Implications For End-User Programming with LLMs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-4000-2951","authenticated-orcid":false,"given":"Madison","family":"Pickering","sequence":"first","affiliation":[{"name":"University of Chicago, Chicago, Illinois, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-6612-6031","authenticated-orcid":false,"given":"Helena","family":"Williams","sequence":"additional","affiliation":[{"name":"University of Chicago, Chicago, Illinois, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-2109-4987","authenticated-orcid":false,"given":"Alison","family":"Gan","sequence":"additional","affiliation":[{"name":"University of Chicago, Chicago, Illinois, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-1189-7063","authenticated-orcid":false,"given":"Weijia","family":"He","sequence":"additional","affiliation":[{"name":"University of Southampton, Southampton, United Kingdom and University of Chicago, Chicago, Illinois, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8686-5600","authenticated-orcid":false,"given":"Hyojae","family":"Park","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1222-0783","authenticated-orcid":false,"given":"Francisco","family":"Piedrahita Velez","sequence":"additional","affiliation":[{"name":"Brown University, Providence, Rhode Island, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5596-1840","authenticated-orcid":false,"given":"Michael L.","family":"Littman","sequence":"additional","affiliation":[{"name":"Brown University, Providence, Rhode Island, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9365-3155","authenticated-orcid":false,"given":"Blase","family":"Ur","sequence":"additional","affiliation":[{"name":"University of Chicago, Chicago, Illinois, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le and Charles Sutton. 2021. Program Synthesis with Large Language Models. arXiv:2108.07732."},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.5555\/3721488.3721583"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","DOI":"10.1109\/VL\/HCC60511.2024.00011"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"crossref","unstructured":"Barbara\u00a0Rita Barricelli Fabio Cassano Daniela Fogli and Antonio Piccinno. 2019. End-User Development End-User Programming and End-User Software Engineering: A Systematic Mapping Study. Journal of Systems and Software 149 (2019) 101\u2013137.","DOI":"10.1016\/j.jss.2018.11.041"},{"key":"e_1_3_3_2_6_2","unstructured":"Tom\u00a0B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell Sandhini Agarwal Ariel Herbert-Voss Gretchen Krueger Tom Henighan Rewon Child Aditya Ramesh Daniel\u00a0M. Ziegler Jeffrey Wu Clemens Winter Christopher Hesse Mark Chen Eric Sigler Mateusz Litwin Scott Gray Benjamin Chess Jack Clark Christopher Berner Sam McCandlish Alec Radford Ilya Sutskever and Dario Amodei. 2020. Language Models are Few-Shot Learners. arXiv:2005.14165."},{"key":"e_1_3_3_2_7_2","unstructured":"Sarina Canelake. 2011. A Gentle Introduction To Programming Using Python. https:\/\/ocw.mit.edu\/courses\/6-189-a-gentle-introduction-to-programming-using-python-january-iap-2011\/"},{"key":"e_1_3_3_2_8_2","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de\u00a0Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe\u00a0Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William\u00a0Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew\u00a0N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. arXiv:2107.03374."},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00057"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","DOI":"10.1145\/3663384.3663389"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"crossref","unstructured":"Jean-Baptiste D\u00f6derlein Mathieu Acher Djamel\u00a0Eddine Khelladi and Benoit Combemale. 2023. Piloting Copilot and Codex: Hot Temperature Cold Prompts or Black Magic? arXiv:2210.14699.","DOI":"10.2139\/ssrn.4496380"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1109\/VL-HCC57772.2023.00017"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","DOI":"10.1145\/3576123.3576134"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","DOI":"10.1109\/BLOCKS.2015.7369000"},{"key":"e_1_3_3_2_16_2","unstructured":"Gemini Team Google. 2024. Gemini: A Family of Highly Capable Multimodal Models. arXiv:2312.11805."},{"key":"e_1_3_3_2_17_2","unstructured":"Dan Hendrycks Steven Basart Saurav Kadavath Mantas Mazeika Akul Arora Ethan Guo Collin Burns Samir Puranik Horace He Dawn Song and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. arXiv:2105.09938."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/2750858.2805830"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3526113.3545659"},{"key":"e_1_3_3_2_20_2","unstructured":"Eirini Kalliamvakou. 2022. Research: Quantifying GitHub Copilot\u2019s Impact on Developer Productivity and Happiness. https:\/\/github.blog\/2022-09-07-research-quantifying-github-copilots-impact-on-developer-productivity-and-happiness\/"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634969"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"crossref","unstructured":"Mohammad\u00a0Amin Kuhail Shahbano Farooq Rawad Hammad and Mohammed Bahja. 2021. Characterizing Visual Programming Approaches For End-User Developers: A Systematic Review. IEEE Access 9 (2021) 14181\u201314202.","DOI":"10.1109\/ACCESS.2021.3051043"},{"key":"e_1_3_3_2_23_2","unstructured":"Mina Lee Megha Srivastava Amelia Hardy John Thickstun Esin Durmus Ashwin Paranjape Ines Gerard-Ursin Xiang\u00a0Lisa Li Faisal Ladhak Frieda Rong Rose\u00a0E. Wang Minae Kwon Joon\u00a0Sung Park Hancheng Cao Tony Lee Rishi Bommasani Michael Bernstein and Percy Liang. 2024. Evaluating Human-Language Model Interaction. arXiv:2212.09746."},{"key":"e_1_3_3_2_24_2","unstructured":"LeetCode. 2023. LeetCode: A New Way to Learn. https:\/\/leetcode.com\/"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300675"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1007\/1-4020-5386-X_1"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240767"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580817"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580940"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Andr\u00e9\u00a0N. Meyer Earl\u00a0T. Barr Christian Bird and Thomas Zimmermann. 2021. Today Was a Good Day: The Daily Life of Software Developers. IEEE Transactions on Software Engineering 47 5 (2021) 863\u2013880.","DOI":"10.1109\/TSE.2019.2904957"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.1145\/3131365.3131369"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/1449715.1449737"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"crossref","unstructured":"Arghavan Moradi Dakhel Vahid Majdinasab Amin Nikanjam Foutse Khomh Michel\u00a0C. Desmarais and Zhen Ming\u00a0(Jack) Jiang. 2023. GitHub Copilot AI Pair Programmer: Asset or Liability? Journal of Systems and Software 203 (2023).","DOI":"10.1016\/j.jss.2023.111734"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/1125451.1125472"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/1020.001.0001"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3528470"},{"key":"e_1_3_3_2_37_2","unstructured":"OpenAI. 2022. Introducing ChatGPT. https:\/\/openai.com\/blog\/chatgpt"},{"key":"e_1_3_3_2_38_2","unstructured":"OpenAI. 2024. GPT-4 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2303.08774"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"crossref","unstructured":"Hammond Pearce Baleegh Ahmad Benjamin Tan Brendan Dolan-Gavitt and Ramesh Karri. 2021. Asleep at the Keyboard? Assessing the Security of GitHub Copilot\u2019s Code Contributions. arxiv:https:\/\/arXiv.org\/abs\/2108.09293","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3576915.3623157"},{"key":"e_1_3_3_2_41_2","unstructured":"Madison Pickering Helena Williams Alison Gan Weijia He Hyojae Park Francisco\u00a0Piedrahita Velez Michael\u00a0L. Littman and Blase Ur. 2025. Data Release. https:\/\/github.com\/UChicagoSUPERgroup\/chi25"},{"key":"e_1_3_3_2_42_2","unstructured":"Prolific. 2025. Prolific: A Higher Standard of Online Research. https:\/\/www.prolific.co\/"},{"key":"e_1_3_3_2_43_2","unstructured":"Alec Radford Jeff Wu Rewon Child David Luan Dario Amodei and Ilya Sutskever. 2019. Language Models are Unsupervised Multitask Learners. OpenAI Blog."},{"key":"e_1_3_3_2_44_2","unstructured":"Amir Rahmati Earlence Fernandes Jaeyeon Jung and Atul Prakash. 2017. IFTTT vs. Zapier: A Comparative Study of Trigger-Action Programming Frameworks. arXiv:1709.02788."},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"crossref","unstructured":"Mitchel Resnick John Maloney Andr\u00e9s Monroy-Hern\u00e1ndez Natalie Rusk Evelyn Eastmond Karen Brennan Amon Millner Eric Rosenbaum Jay Silver Brian Silverman and Yasmin Kafai. 2009. Scratch: Programming For All. Commun. ACM 52 11 (2009) 60\u201367.","DOI":"10.1145\/1592761.1592779"},{"key":"e_1_3_3_2_46_2","unstructured":"Kevin Roose. 2023. How Schools Can Survive (and Maybe Even Thrive) With A.I. This Fall. New York Times. https:\/\/www.nytimes.com\/2023\/08\/24\/technology\/how-schools-can-survive-and-maybe-even-thrive-with-ai-this-fall.html"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584037"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"publisher","DOI":"10.5555\/3495724.3497454"},{"key":"e_1_3_3_2_49_2","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing\u00a0Ellen Tan Yossi Adi Jingyu Liu Romain Sauvestre Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian\u00a0Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2024. Code Llama: Open Foundation Models for Code. arxiv:https:\/\/arXiv.org\/abs\/2308.12950"},{"key":"e_1_3_3_2_50_2","volume-title":"Proceedings of the 32nd USENIX Security Symposium","author":"Sandoval Gustavo","year":"2023","unstructured":"Gustavo Sandoval, Hammond Pearce, Teo Nys, Ramesh Karri, Siddharth Garg, and Brendan Dolan-Gavitt. 2023. Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants. In Proceedings of the 32nd USENIX Security Symposium."},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3622758.3622882"},{"key":"e_1_3_3_2_52_2","unstructured":"Advait Sarkar. 2024. Intention Is All You Need. arxiv:https:\/\/arXiv.org\/abs\/2410.18851"},{"key":"e_1_3_3_2_53_2","unstructured":"Advait Sarkar Andrew\u00a0D. Gordon Carina Negreanu Christian Poelitz Sruti\u00a0Srinivasa Ragavan and Ben Zorn. 2022. What Is It Like to Program With Artificial Intelligence? arxiv:https:\/\/arXiv.org\/abs\/2208.06213"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_3_2_55_2","unstructured":"Aarohi Srivastava et\u00a0al. 2022. Beyond the Imitation Game: Quantifying and Extrapolating the Capabilities of Language Models. arXiv:2206.04615."},{"key":"e_1_3_3_2_56_2","doi-asserted-by":"crossref","unstructured":"Florian Tambon Arghavan\u00a0Moradi Dakhel Amin Nikanjam Foutse Khomh Michel\u00a0C. Desmarais and Giuliano Antoniol. 2024. Bugs in Large Language Models Generated Code: An Empirical Study. arxiv:https:\/\/arXiv.org\/abs\/2403.08937","DOI":"10.1007\/s10664-025-10614-4"},{"key":"e_1_3_3_2_57_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642902"},{"key":"e_1_3_3_2_58_2","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar Aurelien Rodriguez Armand Joulin Edouard Grave and Guillaume Lample. 2023. LLaMA: Open and Efficient Foundation Language Models. arxiv:https:\/\/arXiv.org\/abs\/2302.13971"},{"key":"e_1_3_3_2_59_2","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian\u00a0Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit\u00a0Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric\u00a0Michael Smith Ranjan Subramanian Xiaoqing\u00a0Ellen Tan Binh Tang Ross Taylor Adina Williams Jian\u00a0Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arXiv:2307.09288."},{"key":"e_1_3_3_2_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557420"},{"key":"e_1_3_3_2_61_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858556"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_3_2_63_2","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan\u00a0N. Gomez Lukasz Kaiser and Illia Polosukhin. 2017. Attention Is All You Need. arXiv:1706.03762."},{"key":"e_1_3_3_2_64_2","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240842"},{"key":"e_1_3_3_2_65_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_2_66_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00043"},{"key":"e_1_3_3_2_67_2","doi-asserted-by":"crossref","unstructured":"Lefan Zhang Weijia He Olivia Morkved Valerie Zhao Michael\u00a0L. Littman Shan Lu and Blase Ur. 2020. Trace2tap: Synthesizing Trigger-Action Programs From Traces of Behavior. Proceedings of the ACM on Interactive Mobile Wearable and Ubiquitous Technologies 4 3 (2020).","DOI":"10.1145\/3411838"},{"key":"e_1_3_3_2_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445567"}],"event":{"name":"CHI 2025: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3713271","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706598.3713271","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T05:52:19Z","timestamp":1751608339000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3713271"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":67,"alternative-id":["10.1145\/3706598.3713271","10.1145\/3706598"],"URL":"https:\/\/doi.org\/10.1145\/3706598.3713271","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}