{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,20]],"date-time":"2026-06-20T00:01:20Z","timestamp":1781913680834,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706598.3714154","type":"proceedings-article","created":{"date-parts":[[2025,4,28]],"date-time":"2025-04-28T14:16:17Z","timestamp":1745849777000},"page":"1-17","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":32,"title":["Beyond Code Generation: LLM-supported Exploration of the Program Design Space"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5310-6728","authenticated-orcid":false,"given":"J.D.","family":"Zamfirescu-Pereira","sequence":"first","affiliation":[{"name":"EECS, UC Berkeley, Berkeley, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4050-4284","authenticated-orcid":false,"given":"Eunice","family":"Jun","sequence":"additional","affiliation":[{"name":"Computer Science, UCLA, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1941-939X","authenticated-orcid":false,"given":"Michael","family":"Terry","sequence":"additional","affiliation":[{"name":"Google DeepMind, Cambridge, Massachusetts, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3548-2535","authenticated-orcid":false,"given":"Qian","family":"Yang","sequence":"additional","affiliation":[{"name":"Computing and Information Science, Cornell University, Ithaca, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0693-0829","authenticated-orcid":false,"given":"Bjoern","family":"Hartmann","sequence":"additional","affiliation":[{"name":"EECS, UC Berkeley, Berkeley, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"2024. Artifacts are now generally available. https:\/\/www.anthropic.com\/news\/artifacts"},{"key":"e_1_3_3_3_3_2","unstructured":"2024. What is User Centered Design (UCD)?https:\/\/www.interaction-design.org\/literature\/topics\/user-centered-design"},{"key":"e_1_3_3_3_4_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642858"},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/3635636.3656204"},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606719"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"crossref","unstructured":"Ian Arawjo Priyan Vaithilingam Martin Wattenberg and Elena Glassman. 2023. ChainForge: An open-source visual programming environment for prompt engineering. 1\u20133.","DOI":"10.1145\/3586182.3616660"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"crossref","unstructured":"Shraddha Barke Michael\u00a0B James and Nadia Polikarpova. 2023. Grounded copilot: How programmers interact with code-generating models. Proceedings of the ACM on Programming Languages 7 OOPSLA1 (2023) 85\u2013111. Publisher: ACM New York NY USA.","DOI":"10.1145\/3586030"},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"crossref","unstructured":"Shaon Barman Sarah Chasins Rastislav Bodik and Sumit Gulwani. 2016. Ringer: web automation by demonstration. 748\u2013764.","DOI":"10.1145\/2983990.2984020"},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2017.8103446"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"crossref","unstructured":"Mary\u00a0Jo Bitner Amy\u00a0L Ostrom and Felicia\u00a0N Morgan. 2008. Service blueprinting: a practical technique for service innovation. California management review 50 3 (2008) 66\u201394. Publisher: SAGE Publications Sage CA: Los Angeles CA.","DOI":"10.2307\/41166446"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606725"},{"key":"e_1_3_3_3_13_2","doi-asserted-by":"publisher","unstructured":"J. Brandt P.J. Guo J. Lewenstein S.R. Klemmer and M. Dontcheva. 2009. Writing Code to Prototype Ideate and Discover. IEEE Software 26 5 (Sept. 2009) 18\u201324. 10.1109\/MS.2009.147","DOI":"10.1109\/MS.2009.147"},{"key":"e_1_3_3_3_14_2","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1518944"},{"key":"e_1_3_3_3_15_2","volume-title":"Sketching user experiences: getting the design right and the right design","author":"Buxton Bill","year":"2010","unstructured":"Bill Buxton. 2010. Sketching user experiences: getting the design right and the right design. Morgan kaufmann."},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"publisher","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de\u00a0Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe\u00a0Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William\u00a0Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew\u00a0N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. 10.48550\/arXiv.2107.03374arXiv:https:\/\/arXiv.org\/abs\/2107.03374 [cs] version: 2.","DOI":"10.48550\/arXiv.2107.03374"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858512"},{"key":"e_1_3_3_3_18_2","unstructured":"Design Council. 2004. Framework for Innovation. https:\/\/www.designcouncil.org.uk\/our-resources\/framework-for-innovation\/"},{"key":"e_1_3_3_3_19_2","volume-title":"Flow: The psychology of optimal experience","author":"Czikszentmihalyi Mihaly","year":"1990","unstructured":"Mihaly Czikszentmihalyi. 1990. Flow: The psychology of optimal experience. New York: Harper & Row."},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"crossref","unstructured":"Paul Denny Viraj Kumar and Nasser Giacaman. 2023. Conversing with copilot: Exploring prompt engineering for solving cs1 problems using natural language. 1136\u20131142.","DOI":"10.1145\/3545945.3569823"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3527927.3535197"},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"publisher","unstructured":"Steven\u00a0P. Dow Alana Glassco Jonathan Kass Melissa Schwarz Daniel\u00a0L. Schwartz and Scott\u00a0R. Klemmer. 2010. Parallel prototyping leads to better design results more divergence and increased self-efficacy. ACM Transactions on Computer-Human Interaction 17 4 (Dec. 2010) 1\u201324. 10.1145\/1879831.1879836","DOI":"10.1145\/1879831.1879836"},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"crossref","unstructured":"Laura Faulkner. 2003. Beyond the five-user assumption: Benefits of increased sample sizes in usability testing. Behavior Research Methods Instruments & Computers 35 (2003) 379\u2013383. Publisher: Springer.","DOI":"10.3758\/BF03195514"},{"key":"e_1_3_3_3_24_2","unstructured":"GitHub. 2021. GitHub Copilot \u2022 Your AI Pair Programmer. http:\/\/github.com\/features\/copilot"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641891"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"crossref","unstructured":"Sumit Gulwani. 2011. Automating string processing in spreadsheets using input-output examples. ACM Sigplan Notices 46 1 (2011) 317\u2013330. Publisher: ACM New York NY USA.","DOI":"10.1145\/1925844.1926423"},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/1449715.1449732"},{"key":"e_1_3_3_3_28_2","unstructured":"Sirui Hong Mingchen Zhuge Jonathan Chen Xiawu Zheng Yuheng Cheng Jinlin Wang Ceyao Zhang Zili Wang Steven Ka\u00a0Shing Yau Zijuan Lin Liyang Zhou Chenyu Ran Lingfeng Xiao Chenglin Wu and J\u00fcrgen Schmidhuber. 2023. MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework. https:\/\/openreview.net\/forum?id=VtmBAGCN7o"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"publisher","DOI":"10.1016\/B978-044481862-1.50082-0"},{"key":"e_1_3_3_3_31_2","doi-asserted-by":"crossref","unstructured":"Dhanya Jayagopal Justin Lubin and Sarah\u00a0E Chasins. 2022. Exploring the learnability of program synthesizers by novice programmers. 1\u201315.","DOI":"10.1145\/3526113.3545659"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"publisher","unstructured":"Majeed Kazemitabaar Jack Williams Ian Drosos Tovi Grossman Austin Henley Carina Negreanu and Advait Sarkar. 2024. Improving Steering and Verification in AI-Assisted Data Analysis with Interactive Task Decomposition. 10.1145\/3654777.3676345arXiv:https:\/\/arXiv.org\/abs\/2407.02651 [cs].","DOI":"10.1145\/3654777.3676345"},{"key":"e_1_3_3_3_33_2","doi-asserted-by":"crossref","unstructured":"Mary\u00a0Beth Kery. 2017. Designing Effective History Support for Exploratory Programming Data Work. (2017).","DOI":"10.1109\/VLHCC.2017.8103490"},{"key":"e_1_3_3_3_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025626"},{"key":"e_1_3_3_3_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300322"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"crossref","unstructured":"Pranav Khadpe Ranjay Krishna Li Fei-Fei Jeffrey\u00a0T Hancock and Michael\u00a0S Bernstein. 2020. Conceptual metaphors impact perceptions of human-ai collaboration. Proceedings of the ACM on Human-Computer Interaction 4 CSCW2 (2020) 1\u201326. Publisher: ACM New York NY USA.","DOI":"10.1145\/3415234"},{"key":"e_1_3_3_3_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642830"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/223904.223910"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/3563657.3595977"},{"key":"e_1_3_3_3_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606831"},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025483"},{"key":"e_1_3_3_3_42_2","unstructured":"Geoffrey Litt. 2024. Patchwork lab notebook: Version control for everything. https:\/\/www.inkandswitch.com\/patchwork\/notebook\/"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"crossref","unstructured":"Michael\u00a0Xieyang Liu Advait Sarkar Carina Negreanu Benjamin Zorn Jack Williams Neil Toronto and Andrew\u00a0D Gordon. 2023. \u201cWhat it wants me to say\u201d: Bridging the abstraction gap between end-user programmers and code-generating large language models. 1\u201331.","DOI":"10.1145\/3544548.3580817"},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501825"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"crossref","unstructured":"Justin Lubin Nick Collins Cyrus Omar and Ravi Chugh. 2019. Program Synthesis with Live Bidirectional Evaluation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1911.00583 (2019).","DOI":"10.1145\/zenodo.3926830"},{"key":"e_1_3_3_3_46_2","doi-asserted-by":"publisher","unstructured":"Aran Lunzer and Kasper Hornb\u00e6k. 2008. Subjunctive Interfaces: Extending Applications to Support Parallel Setup Viewing and Control of Alternative Scenarios. ACM Trans. Comput.-Hum. Interact. 14 4 (Jan. 2008). 10.1145\/1314683.1314685Place: New York NY USA Publisher: Association for Computing Machinery.","DOI":"10.1145\/1314683.1314685"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/258734.258887"},{"key":"e_1_3_3_3_48_2","unstructured":"Andrew\u00a0M. McNutt Chenglong Wang Robert\u00a0A. DeLine and Steven\u00a0M. Drucker. 2023. On the Design of AI-powered Code Assistants for Notebooks. http:\/\/arxiv.org\/abs\/2301.11178 arXiv:https:\/\/arXiv.org\/abs\/2301.11178 [cs]."},{"key":"e_1_3_3_3_49_2","doi-asserted-by":"publisher","DOI":"10.5555\/1196347"},{"key":"e_1_3_3_3_50_2","unstructured":"Jakob Nielsen. [n. d.]. 10 Usability Heuristics for User Interface Design. https:\/\/www.nngroup.com\/articles\/ten-usability-heuristics\/"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"publisher","unstructured":"Amy Orben Andrew\u00a0K. Przybylski Sarah-Jayne Blakemore and Rogier\u00a0A. Kievit. 2022. Windows of developmental sensitivity to social media. Nature Communications 13 1 (March 2022) 1649. 10.1038\/s41467-022-29296-3Publisher: Nature Publishing Group.","DOI":"10.1038\/s41467-022-29296-3"},{"key":"e_1_3_3_3_52_2","doi-asserted-by":"crossref","unstructured":"James Prather Brent\u00a0N Reeves Paul Denny Brett\u00a0A Becker Juho Leinonen Andrew Luxton-Reilly Garrett Powell James Finnie-Ansley and Eddie\u00a0Antonio Santos. 2023. \u201cIt\u2019s Weird That it Knows What I Want\u201d: Usability and Interactions with Copilot for Novice Programmers. ACM Transactions on Computer-Human Interaction 31 1 (2023) 1\u201331. Publisher: ACM New York NY.","DOI":"10.1145\/3617367"},{"key":"e_1_3_3_3_53_2","volume-title":"Proceedings of the SIGCHI conference on human factors in computing systems","author":"Rawn Eric","year":"2023","unstructured":"Eric Rawn, Jingyi Li, Eric Paulos, and Sarah Chasins. 2023. Understanding Version Control as Material Interaction with Quickpose. In Proceedings of the SIGCHI conference on human factors in computing systems."},{"key":"e_1_3_3_3_54_2","volume-title":"Quantifying the user experience: Practical statistics for user research","author":"Sauro Jeff","year":"2016","unstructured":"Jeff Sauro and James\u00a0R Lewis. 2016. Quantifying the user experience: Practical statistics for user research. Morgan Kaufmann."},{"key":"e_1_3_3_3_55_2","volume-title":"The Reflective Practitioner: How Professionals Think In Action","author":"Schon Donald\u00a0A","year":"1984","unstructured":"Donald\u00a0A Schon. 1984. The Reflective Practitioner: How Professionals Think In Action. Vol.\u00a05126. Basic Books."},{"key":"e_1_3_3_3_56_2","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676450"},{"key":"e_1_3_3_3_57_2","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-934613-12-5.50048-3"},{"key":"e_1_3_3_3_58_2","unstructured":"Chenglei Si Yanzhe Zhang Zhengyuan Yang Ruibo Liu and Diyi Yang. 2024. Design2Code: How Far Are We From Automating Front-End Engineering?http:\/\/arxiv.org\/abs\/2403.03163 arXiv:https:\/\/arXiv.org\/abs\/2403.03163 [cs]."},{"key":"e_1_3_3_3_59_2","doi-asserted-by":"crossref","unstructured":"Hari Subramonyam Roy Pea Christopher Pondoc Maneesh Agrawala and Colleen Seifert. 2024. Bridging the Gulf of Envisioning: Cognitive Challenges in Prompt Based Interactions with LLMs. 1\u201319.","DOI":"10.1145\/3613904.3642754"},{"key":"e_1_3_3_3_60_2","unstructured":"Sangho Suh Bryan Min Srishti Palani and Haijun Xia. 2023. Sensecape: Enabling Multilevel Exploration and Sensemaking with Large Language Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2305.11483 (2023)."},{"key":"e_1_3_3_3_61_2","doi-asserted-by":"publisher","unstructured":"Theodore\u00a0R. Sumers Shunyu Yao Karthik Narasimhan and Thomas\u00a0L. Griffiths. 2024. Cognitive Architectures for Language Agents. 10.48550\/arXiv.2309.02427arXiv:https:\/\/arXiv.org\/abs\/2309.02427 [cs].","DOI":"10.48550\/arXiv.2309.02427"},{"key":"e_1_3_3_3_62_2","volume-title":"How You Can Write Plays: A Practical Guide-book","author":"Swan M.","year":"1927","unstructured":"M. Swan. 1927. How You Can Write Plays: A Practical Guide-book. S. French. https:\/\/books.google.com\/books?id=UyYrAAAAIAAJ"},{"key":"e_1_3_3_3_63_2","unstructured":"Michael Terry Chinmay Kulkarni Martin Wattenberg Lucas Dixon and Meredith\u00a0Ringel Morris. 2023. AI Alignment in the Design of Interactive AI: Specification Alignment Process Alignment and Evaluation Support. http:\/\/arxiv.org\/abs\/2311.00710 arXiv:https:\/\/arXiv.org\/abs\/2311.00710 [cs]."},{"key":"e_1_3_3_3_64_2","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985782"},{"key":"e_1_3_3_3_65_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643834.3661525"},{"key":"e_1_3_3_3_66_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_3_67_2","doi-asserted-by":"publisher","DOI":"10.1145\/3563657.3596138"},{"key":"e_1_3_3_3_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_3_69_2","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240704"}],"event":{"name":"CHI 2025: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3714154","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706598.3714154","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T05:33:16Z","timestamp":1751607196000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3714154"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":68,"alternative-id":["10.1145\/3706598.3714154","10.1145\/3706598"],"URL":"https:\/\/doi.org\/10.1145\/3706598.3714154","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}