{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,21]],"date-time":"2026-07-21T23:20:47Z","timestamp":1784676047722,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T00:00:00Z","timestamp":1715385600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,11]]},"DOI":"10.1145\/3613905.3650756","type":"proceedings-article","created":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T08:15:21Z","timestamp":1715415321000},"page":"1-9","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":67,"title":["\"We Need Structured Output\": Towards User-centered Constraints on Large Language Model Output"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8246-8736","authenticated-orcid":false,"given":"Michael Xieyang","family":"Liu","sequence":"first","affiliation":[{"name":"Google Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7468-8288","authenticated-orcid":false,"given":"Frederick","family":"Liu","sequence":"additional","affiliation":[{"name":"Google Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8981-1114","authenticated-orcid":false,"given":"Alexander J.","family":"Fiannaca","sequence":"additional","affiliation":[{"name":"Google Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8851-4654","authenticated-orcid":false,"given":"Terry","family":"Koo","sequence":"additional","affiliation":[{"name":"Google, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1094-1675","authenticated-orcid":false,"given":"Lucas","family":"Dixon","sequence":"additional","affiliation":[{"name":"Google Research, France"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1941-939X","authenticated-orcid":false,"given":"Michael","family":"Terry","sequence":"additional","affiliation":[{"name":"Google Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9421-7128","authenticated-orcid":false,"given":"Carrie J.","family":"Cai","sequence":"additional","affiliation":[{"name":"Google Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,5,11]]},"reference":[{"key":"e_1_3_3_2_1_1","unstructured":"2023. guidance-ai\/guidance. https:\/\/github.com\/guidance-ai\/guidance original-date: 2022-11-10T18:21:45Z."},{"key":"e_1_3_3_2_2_1","doi-asserted-by":"publisher","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan Nicholas Joseph Saurav Kadavath Jackson Kernion Tom Conerly Sheer El-Showk Nelson Elhage Zac Hatfield-Dodds Danny Hernandez Tristan Hume Scott Johnston Shauna Kravec Liane Lovitt Neel Nanda Catherine Olsson Dario Amodei Tom Brown Jack Clark Sam McCandlish Chris Olah Ben Mann and Jared Kaplan. 2022. Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback. https:\/\/doi.org\/10.48550\/arXiv.2204.05862 arXiv:2204.05862 [cs].","DOI":"10.48550\/arXiv.2204.05862"},{"key":"e_1_3_3_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1963405.1963424"},{"key":"e_1_3_3_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3591300"},{"key":"e_1_3_3_2_5_1","doi-asserted-by":"publisher","unstructured":"Tom\u00a0B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell Sandhini Agarwal Ariel Herbert-Voss Gretchen Krueger Tom Henighan Rewon Child Aditya Ramesh Daniel\u00a0M. Ziegler Jeffrey Wu Clemens Winter Christopher Hesse Mark Chen Eric Sigler Mateusz Litwin Scott Gray Benjamin Chess Jack Clark Christopher Berner Sam McCandlish Alec Radford Ilya Sutskever and Dario Amodei. 2020. Language Models are Few-Shot Learners. https:\/\/doi.org\/10.48550\/arXiv.2005.14165 arXiv:2005.14165 [cs].","DOI":"10.48550\/arXiv.2005.14165"},{"key":"e_1_3_3_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2984511.2984538"},{"key":"e_1_3_3_2_7_1","doi-asserted-by":"publisher","unstructured":"Aakanksha Chowdhery Sharan Narang Jacob Devlin Maarten Bosma Gaurav Mishra Adam Roberts Paul Barham Hyung\u00a0Won Chung Charles Sutton Sebastian Gehrmann Parker Schuh Kensen Shi Sasha Tsvyashchenko Joshua Maynez Abhishek Rao Parker Barnes Yi Tay Noam Shazeer Vinodkumar Prabhakaran Emily Reif Nan Du Ben Hutchinson Reiner Pope James Bradbury Jacob Austin Michael Isard Guy Gur-Ari Pengcheng Yin Toju Duke Anselm Levskaya Sanjay Ghemawat Sunipa Dev Henryk Michalewski Xavier Garcia Vedant Misra Kevin Robinson Liam Fedus Denny Zhou Daphne Ippolito David Luan Hyeontaek Lim Barret Zoph Alexander Spiridonov Ryan Sepassi David Dohan Shivani Agrawal Mark Omernick Andrew\u00a0M. Dai Thanumalayan\u00a0Sankaranarayana Pillai Marie Pellat Aitor Lewkowycz Erica Moreira Rewon Child Oleksandr Polozov Katherine Lee Zongwei Zhou Xuezhi Wang Brennan Saeta Mark Diaz Orhan Firat Michele Catasta Jason Wei Kathy Meier-Hellstern Douglas Eck Jeff Dean Slav Petrov and Noah Fiedel. 2022. PaLM: Scaling Language Modeling with Pathways. https:\/\/doi.org\/10.48550\/arXiv.2204.02311 arXiv:2204.02311 [cs].","DOI":"10.48550\/arXiv.2204.02311"},{"key":"e_1_3_3_2_8_1","unstructured":"Google Cloud. 2023. Function calling | Vertex AI. https:\/\/cloud.google.com\/vertex-ai\/docs\/generative-ai\/multimodal\/function-calling"},{"key":"e_1_3_3_2_9_1","doi-asserted-by":"publisher","unstructured":"Ronen Eldan and Yuanzhi Li. 2023. TinyStories: How Small Can Language Models Be and Still Speak Coherent English?https:\/\/doi.org\/10.48550\/arXiv.2305.07759 arXiv:2305.07759 [cs].","DOI":"10.48550\/arXiv.2305.07759"},{"key":"e_1_3_3_2_10_1","doi-asserted-by":"publisher","unstructured":"Angela Fan Beliz Gokkaya Mark Harman Mitya Lyubarskiy Shubho Sengupta Shin Yoo and Jie\u00a0M. Zhang. 2023. Large Language Models for Software Engineering: Survey and Open Problems. https:\/\/doi.org\/10.48550\/arXiv.2310.03533 arXiv:2310.03533 [cs].","DOI":"10.48550\/arXiv.2310.03533"},{"key":"e_1_3_3_2_11_1","unstructured":"Google. 2023. Google AI Studio quickstart. https:\/\/ai.google.dev\/tutorials\/ai-studio_quickstart"},{"key":"e_1_3_3_2_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1141"},{"key":"e_1_3_3_2_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1090"},{"key":"e_1_3_3_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3503564"},{"key":"e_1_3_3_2_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.96"},{"key":"e_1_3_3_2_16_1","doi-asserted-by":"publisher","unstructured":"Minsuk Kahng Ian Tenney Mahima Pushkarna Michael\u00a0Xieyang Liu James Wexler Emily Reif Krystal Kallarackal Minsuk Chang Michael Terry and Lucas Dixon. 2024. LLM Comparator: Visual Analytics for Side-by-Side Evaluation of Large Language Models. https:\/\/doi.org\/10.48550\/arXiv.2402.10524 arXiv:2402.10524 [cs].","DOI":"10.48550\/arXiv.2402.10524"},{"key":"e_1_3_3_2_17_1","doi-asserted-by":"publisher","unstructured":"Percy Liang Rishi Bommasani Tony Lee Dimitris Tsipras Dilara Soylu Michihiro Yasunaga Yian Zhang Deepak Narayanan Yuhuai Wu Ananya Kumar Benjamin Newman Binhang Yuan Bobby Yan Ce Zhang Christian Cosgrove Christopher\u00a0D. Manning Christopher R\u00e9 Diana Acosta-Navas Drew\u00a0A. Hudson Eric Zelikman Esin Durmus Faisal Ladhak Frieda Rong Hongyu Ren Huaxiu Yao Jue Wang Keshav Santhanam Laurel Orr Lucia Zheng Mert Yuksekgonul Mirac Suzgun Nathan Kim Neel Guha Niladri Chatterji Omar Khattab Peter Henderson Qian Huang Ryan Chi Sang\u00a0Michael Xie Shibani Santurkar Surya Ganguli Tatsunori Hashimoto Thomas Icard Tianyi Zhang Vishrav Chaudhary William Wang Xuechen Li Yifan Mai Yuhui Zhang and Yuta Koreeda. 2023. Holistic Evaluation of Language Models. https:\/\/doi.org\/10.48550\/arXiv.2211.09110 arXiv:2211.09110 [cs].","DOI":"10.48550\/arXiv.2211.09110"},{"key":"e_1_3_3_2_18_1","volume-title":"Tool Support for Knowledge Foraging, Structuring, and Transfer during Online Sensemaking. Ph.\u00a0D. Dissertation","author":"Liu Michael\u00a0Xieyang","unstructured":"Michael\u00a0Xieyang Liu. 2023. Tool Support for Knowledge Foraging, Structuring, and Transfer during Online Sensemaking. Ph.\u00a0D. Dissertation. Carnegie Mellon University. http:\/\/reports-archive.adm.cs.cmu.edu\/anon\/anon\/usr0\/ftp\/usr\/ftp\/hcii\/abstracts\/23-105.html"},{"key":"e_1_3_3_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501968"},{"key":"e_1_3_3_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3526113.3545661"},{"key":"e_1_3_3_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580817"},{"key":"e_1_3_3_2_22_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2310.02161"},{"key":"e_1_3_3_2_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_24_1","doi-asserted-by":"publisher","unstructured":"Sidharth Mudgal Jong Lee Harish Ganapathy YaGuang Li Tao Wang Yanping Huang Zhifeng Chen Heng-Tze Cheng Michael Collins Trevor Strohman Jilin Chen Alex Beutel and Ahmad Beirami. 2023. Controlled Decoding from Language Models. https:\/\/doi.org\/10.48550\/arXiv.2310.17022 arXiv:2310.17022 [cs].","DOI":"10.48550\/arXiv.2310.17022"},{"key":"e_1_3_3_2_25_1","unstructured":"OpenAI. 2023. ChatGPT. https:\/\/chat.openai.com"},{"key":"e_1_3_3_2_26_1","unstructured":"OpenAI. 2023. Function calling | OpenAI Platform. https:\/\/platform.openai.com\/docs\/guides\/function-calling"},{"key":"e_1_3_3_2_27_1","unstructured":"OpenAI. 2023. JSON mode - Text generation. https:\/\/platform.openai.com\/docs\/guides\/text-generation\/json-mode"},{"key":"e_1_3_3_2_28_1","unstructured":"OpenAI. 2023. Playground - OpenAI API. https:\/\/platform.openai.com\/playground"},{"key":"e_1_3_3_2_29_1","doi-asserted-by":"publisher","unstructured":"Long Ouyang Jeff Wu Xu Jiang Diogo Almeida Carroll\u00a0L. Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray John Schulman Jacob Hilton Fraser Kelton Luke Miller Maddie Simens Amanda Askell Peter Welinder Paul Christiano Jan Leike and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. https:\/\/doi.org\/10.48550\/arXiv.2203.02155 arXiv:2203.02155 [cs].","DOI":"10.48550\/arXiv.2203.02155"},{"key":"e_1_3_3_2_30_1","doi-asserted-by":"publisher","unstructured":"Chris Parnin Gustavo Soares Rahul Pandita Sumit Gulwani Jessica Rich and Austin\u00a0Z. Henley. 2023. Building Your Own Product Copilot: Challenges Opportunities and Needs. https:\/\/doi.org\/10.48550\/arXiv.2312.14231 arXiv:2312.14231 [cs].","DOI":"10.48550\/arXiv.2312.14231"},{"key":"e_1_3_3_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3585628"},{"key":"e_1_3_3_2_32_1","doi-asserted-by":"publisher","unstructured":"Savvas Petridis Ben Wedin James Wexler Aaron Donsbach Mahima Pushkarna Nitesh Goyal Carrie\u00a0J. Cai and Michael Terry. 2023. ConstitutionMaker: Interactively Critiquing Large Language Models by Converting Feedback into Principles. https:\/\/doi.org\/10.48550\/arXiv.2310.15428 arXiv:2310.15428 [cs].","DOI":"10.48550\/arXiv.2310.15428"},{"key":"e_1_3_3_2_33_1","volume-title":"Advances in Neural Information Processing Systems, Vol.\u00a033. Curran Associates","author":"Stiennon Nisan","year":"2020","unstructured":"Nisan Stiennon, Long Ouyang, Jeffrey Wu, Daniel Ziegler, Ryan Lowe, Chelsea Voss, Alec Radford, Dario Amodei, and Paul\u00a0F Christiano. 2020. Learning to summarize with human feedback. In Advances in Neural Information Processing Systems, Vol.\u00a033. Curran Associates, Inc., 3008\u20133021. https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1f89885d556929e98d3ef9b86448f951-Abstract.html"},{"key":"e_1_3_3_2_34_1","volume-title":"Basics of qualitative research","author":"Strauss Anselm","unstructured":"Anselm Strauss and Juliet Corbin. 1990. Basics of qualitative research. Sage publications."},{"key":"e_1_3_3_2_35_1","doi-asserted-by":"publisher","unstructured":"Jiao Sun Xuezhe Ma and Nanyun Peng. 2021. AESOP: Paraphrase Generation with Adaptive Syntactic Control. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing Marie-Francine Moens Xuanjing Huang Lucia Specia and Scott Wen-tau Yih (Eds.). Association for Computational Linguistics Online and Punta Cana Dominican Republic 5176\u20135189. https:\/\/doi.org\/10.18653\/v1\/2021.emnlp-main.420","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.190"},{"key":"e_1_3_3_2_37_1","doi-asserted-by":"publisher","unstructured":"Jialiang Tan Yu Chen and Shuyin Jiao. 2023. Visual Studio Code in Introductory Computer Science Course: An Experience Report. https:\/\/doi.org\/10.48550\/arXiv.2303.10174 arXiv:2303.10174 [cs].","DOI":"10.48550\/arXiv.2303.10174"},{"key":"e_1_3_3_2_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_39_1","volume-title":"Willard and R\u00e9mi Louf","author":"T.","year":"2023","unstructured":"Brandon\u00a0T. Willard and R\u00e9mi Louf. 2023. Efficient Guided Generation for Large Language Models. https:\/\/arxiv.org\/abs\/2307.09702v4"},{"key":"e_1_3_3_2_40_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.66"},{"key":"e_1_3_3_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_2_42_1","doi-asserted-by":"publisher","unstructured":"Zhiyuan Zeng Jiatong Yu Tianyu Gao Yu Meng Tanya Goyal and Danqi Chen. 2023. Evaluating Large Language Models at Evaluating Instruction Following. https:\/\/doi.org\/10.48550\/arXiv.2310.07641 arXiv:2310.07641 [cs].","DOI":"10.48550\/arXiv.2310.07641"},{"key":"e_1_3_3_2_43_1","doi-asserted-by":"publisher","unstructured":"Jeffrey Zhou Tianjian Lu Swaroop Mishra Siddhartha Brahma Sujoy Basu Yi Luan Denny Zhou and Le Hou. 2023. Instruction-Following Evaluation for Large Language Models. https:\/\/doi.org\/10.48550\/arXiv.2311.07911 arXiv:2311.07911 [cs].","DOI":"10.48550\/arXiv.2311.07911"}],"event":{"name":"CHI '24: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGACCESS ACM Special Interest Group on Accessible Computing"]},"container-title":["Extended Abstracts of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613905.3650756","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3613905.3650756","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:16Z","timestamp":1750290256000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613905.3650756"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,11]]},"references-count":43,"alternative-id":["10.1145\/3613905.3650756","10.1145\/3613905"],"URL":"https:\/\/doi.org\/10.1145\/3613905.3650756","relation":{},"subject":[],"published":{"date-parts":[[2024,5,11]]},"assertion":[{"value":"2024-05-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}