{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T18:03:34Z","timestamp":1777399414962,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":80,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706598.3714259","type":"proceedings-article","created":{"date-parts":[[2025,4,24]],"date-time":"2025-04-24T04:29:11Z","timestamp":1745468951000},"page":"1-18","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":22,"title":["AI-Instruments: Embodying Prompts as Instruments to Abstract & Reflect Graphical Interface Commands as General-Purpose Tools"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1759-7512","authenticated-orcid":false,"given":"Nathalie","family":"Riche","sequence":"first","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9560-201X","authenticated-orcid":false,"given":"Anna","family":"Offenwanger","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA and CNRS, Inria, LISN, Universit\u00e9 Paris-Saclay, Orsay, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4106-1881","authenticated-orcid":false,"given":"Frederic","family":"Gmeiner","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA and Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6013-2713","authenticated-orcid":false,"given":"David","family":"Brown","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9194-8889","authenticated-orcid":false,"given":"Hugo","family":"Romat","sequence":"additional","affiliation":[{"name":"Microsoft, Seattle, Washington, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3924-1189","authenticated-orcid":false,"given":"Michel","family":"Pahud","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5473-2448","authenticated-orcid":false,"given":"Nicolai","family":"Marquardt","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5661-4659","authenticated-orcid":false,"given":"Kori","family":"Inkpen","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4733-4927","authenticated-orcid":false,"given":"Ken","family":"Hinckley","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, Washington, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Adobe. 2024. Firefly. https:\/\/www.adobe.com\/products\/firefly"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642858"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","DOI":"10.1007\/1-84628-062-1_18"},{"key":"e_1_3_3_2_5_2","unstructured":"Yavar Bathaee. 2017. The artificial intelligence black box and the failure of intent and causation. Harv. JL & Tech. 31 (2017) 889."},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/332040.332473"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"crossref","unstructured":"Michel Beaudouin-Lafon Susanne B\u00f8dker and Wendy\u00a0E Mackay. 2021. Generative theories of interaction. ACM Transactions on Computer-Human Interaction (TOCHI) 28 6 (2021) 1\u201354.","DOI":"10.1145\/3468505"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/345513.345267"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/259963.260447"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","unstructured":"Aras Bozkurt. 2024. Tell Me Your Prompts and I Will Make Them True: The Alchemy of Prompt Engineering and Generative AI. Open Praxis 16 2 (April 2024) 111\u2013118. 10.55982\/openpraxis.16.2.661","DOI":"10.55982\/openpraxis.16.2.661"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606725"},{"key":"e_1_3_3_2_12_2","unstructured":"S\u00e9bastien Bubeck Varun Chandrasekaran Ronen Eldan Johannes Gehrke Eric Horvitz Ece Kamar Peter Lee Yin\u00a0Tat Lee Yuanzhi Li Scott Lundberg et\u00a0al. 2023. Sparks of artificial general intelligence: Early experiments with gpt-4. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.12712 (2023)."},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-12-374037-3.X5043-3"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","unstructured":"William Buxton. 1983. Lexical and pragmatic considerations of input structures. SIGGRAPH Comput. Graph. 17 1 (jan 1983) 31\u201337. 10.1145\/988584.988586","DOI":"10.1145\/988584.988586"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","DOI":"10.5555\/212925.212970"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"crossref","unstructured":"Tracy\u00a0Diane Cassidy. 2008. Mood boards: Current practice in learning and teaching strategies and students\u2019 understanding of the process. International journal of fashion design 1 1 (2008) 43\u201354.","DOI":"10.1080\/17543260802015154"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642794"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606777"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501819"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586182.3615821"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"crossref","unstructured":"Nicholas Davis Chih-Pin Hsiao Yanna Popova and Brian Magerko. 2015. An enactive model of creativity for computational collaboration and co-creation. Creativity in the digital age (2015) 109\u2013133.","DOI":"10.1007\/978-1-4471-6681-8_7"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1145\/1099203.1099242"},{"key":"e_1_3_3_2_23_2","unstructured":"OpenJS Foundation. 2024. Node.js. https:\/\/nodejs.org\/en"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"crossref","unstructured":"Charles Freeman Sara Marcketti and Elena Karpova. 2017. Creativity of images: using digital consensual assessment to evaluate mood boards. Fashion and Textiles 4 (2017) 1\u201315.","DOI":"10.1186\/s40691-017-0102-4"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642139"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"crossref","unstructured":"James\u00a0J Gibson. 1977. The theory of affordances. Hilldale USA 1 2 (1977) 67\u201382.","DOI":"10.2307\/3171580"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/1254960.1254987"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","unstructured":"Christine\u00a0A. Halverson. 2002. Activity Theory and Distributed Cognition: Or What Does CSCW Need to DO with Theories? Computer Supported Cooperative Work (CSCW) 11 1 (March 2002) 243\u2013267. 10.1023\/A:1015298005381","DOI":"10.1023\/A:1015298005381"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/1166253.1166300"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858079"},{"key":"e_1_3_3_2_31_2","first-page":"197","volume-title":"Proceedings of the human factors and ergonomics society annual meeting","volume":"62","author":"Hoffman Robert\u00a0R","year":"2018","unstructured":"Robert\u00a0R Hoffman, Gary Klein, and Shane\u00a0T Mueller. 2018. Explaining explanation for \u201cexplainable AI\u201d. In Proceedings of the human factors and ergonomics society annual meeting , Vol.\u00a062. SAGE Publications Sage CA: Los Angeles, CA, SAGE Publications, Los Angeles, CA, USA, 197\u2013201."},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025765"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"crossref","unstructured":"Edwin\u00a0L Hutchins James\u00a0D Hollan and Donald\u00a0A Norman. 1985. Direct manipulation interfaces. Human\u2013computer interaction 1 4 (1985) 311\u2013338.","DOI":"10.1207\/s15327051hci0104_2"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/642611.642616"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/1281500.1281531"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357089"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606737"},{"key":"e_1_3_3_2_39_2","unstructured":"Juriy Zaytsev Stefan Kienzle and Andrea Bogazzi. 2024. Fabric.js. https:\/\/github.com\/fabricjs\/fabric.js"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606833"},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"crossref","unstructured":"Alexander Kirillov Eric Mintun Nikhila Ravi Hanzi Mao Chloe Rolland Laura Gustafson Tete Xiao Spencer Whitehead Alexander\u00a0C. Berg Wan-Yen Lo Piotr Doll\u00e1r and Ross Girshick. 2023. Segment Anything. https:\/\/arxiv.org\/abs\/2304.02643v1","DOI":"10.1109\/ICCV51070.2023.00371"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","unstructured":"David Kirsh. 1995. The intelligent use of space. Artif. Intell. 73 1\u20132 (feb 1995) 31\u201368. 10.1016\/0004-3702(94)00017-U","DOI":"10.1016\/0004-3702(94)00017-U"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2004.47"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606831"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3577043"},{"key":"e_1_3_3_2_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501825"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/1556262.1556294"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"publisher","DOI":"10.1145\/263552.263612"},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642861"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/258734.258887"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642462"},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"crossref","unstructured":"Donald\u00a0A Norman. 1986. Cognitive engineering. User centered system design 31 61 (1986) 2.","DOI":"10.1201\/b15703-3"},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174223"},{"key":"e_1_3_3_2_54_2","unstructured":"OpenAI. 2024. GPT-4o System Card. https:\/\/openai.com\/index\/gpt-4o-system-card\/"},{"key":"e_1_3_3_2_55_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643834.3661588"},{"key":"e_1_3_3_2_56_2","unstructured":"Ben Poole Ajay Jain Jonathan\u00a0T Barron and Ben Mildenhall. 2022. Dreamfusion: Text-to-3d using 2d diffusion. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2209.14988 (2022)."},{"key":"e_1_3_3_2_57_2","doi-asserted-by":"crossref","unstructured":"Yvonne Rogers. 2004. New Theoretical Approaches for Human-Computer Interaction. Annual Review of Information Science and Technology (ARIST) 38 (2004) 87\u2013143. ERIC Number: EJ678114.","DOI":"10.1002\/aris.1440380103"},{"key":"e_1_3_3_2_58_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02197-8"},{"key":"e_1_3_3_2_59_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501988"},{"key":"e_1_3_3_2_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3651089"},{"key":"e_1_3_3_2_61_2","unstructured":"Runway. 2024. Motion Brushes. https:\/\/youtu.be\/zQ3fQt8swEI"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"publisher","unstructured":"D.\u00a0A. Sch\u00f6n. 1992. Designing as reflective conversation with the materials of a design situation. Know.-Based Syst. 5 1 (mar 1992) 3\u201314. 10.1016\/0950-7051(92)90020-G","DOI":"10.1016\/0950-7051(92)90020-G"},{"key":"e_1_3_3_2_63_2","doi-asserted-by":"publisher","unstructured":"Ben Shneiderman. 2007. Creativity support tools: accelerating discovery and innovation. Commun. ACM 50 12 (Dec. 2007) 20\u201332. 10.1145\/1323688.1323689","DOI":"10.1145\/1323688.1323689"},{"key":"e_1_3_3_2_64_2","unstructured":"StabilityAI. 2024. CompVis\/stable-diffusion \u00b7 Hugging Face. https:\/\/huggingface.co\/CompVis\/stable-diffusion"},{"key":"e_1_3_3_2_65_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642754"},{"key":"e_1_3_3_2_66_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642400"},{"key":"e_1_3_3_2_67_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606756"},{"key":"e_1_3_3_2_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/581710.581718"},{"key":"e_1_3_3_2_69_2","doi-asserted-by":"publisher","DOI":"10.1145\/571985.571996"},{"key":"e_1_3_3_2_70_2","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124960"},{"key":"e_1_3_3_2_71_2","volume-title":"International Conference on Learning Representations","author":"Villegas Ruben","year":"2022","unstructured":"Ruben Villegas, Mohammad Babaeizadeh, Pieter-Jan Kindermans, Hernan Moraldo, Han Zhang, Mohammad\u00a0Taghi Saffar, Santiago Castro, Julius Kunze, and Dumitru Erhan. 2022. Phenaki: Variable length video generation from open domain textual descriptions. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_72_2","doi-asserted-by":"publisher","unstructured":"Yael Vinker Andrey Voynov Daniel Cohen-Or and Ariel Shamir. 2023. Concept Decomposition for Visual Exploration and Inspiration. ACM Trans. Graph. 42 6 Article 241 (Dec. 2023) 13\u00a0pages. 10.1145\/3618315","DOI":"10.1145\/3618315"},{"key":"e_1_3_3_2_73_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642803"},{"key":"e_1_3_3_2_74_2","doi-asserted-by":"crossref","unstructured":"Xingjiao Wu Luwei Xiao Yixuan Sun Junhang Zhang Tianlong Ma and Liang He. 2022. A survey of human-in-the-loop for machine learning. Future Generation Computer Systems 135 (2022) 364\u2013381.","DOI":"10.1016\/j.future.2022.05.014"},{"key":"e_1_3_3_2_75_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858075"},{"key":"e_1_3_3_2_76_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025664"},{"key":"e_1_3_3_2_77_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606826"},{"key":"e_1_3_3_2_78_2","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676388"},{"key":"e_1_3_3_2_79_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_2_80_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501914"},{"key":"e_1_3_3_2_81_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00355"}],"event":{"name":"CHI 2025: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3714259","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706598.3714259","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T05:29:34Z","timestamp":1751606974000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3714259"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":80,"alternative-id":["10.1145\/3706598.3714259","10.1145\/3706598"],"URL":"https:\/\/doi.org\/10.1145\/3706598.3714259","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}