{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T07:57:36Z","timestamp":1776931056287,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,3,23]]},"DOI":"10.1145\/3742413.3789088","type":"proceedings-article","created":{"date-parts":[[2026,3,3]],"date-time":"2026-03-03T11:32:24Z","timestamp":1772537544000},"page":"325-348","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["DSCode Comparator: An Interactive Interface for Comparing Models and Evaluating Code for Data Science Tasks"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-4666-7595","authenticated-orcid":false,"given":"Xinxin","family":"Yu","sequence":"first","affiliation":[{"name":"Cheriton School of Computer Science, University of Waterloo, Waterloo, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-0500-6802","authenticated-orcid":false,"given":"Victor","family":"Zhong","sequence":"additional","affiliation":[{"name":"Cheriton School of Computer Science, University of Waterloo, Waterloo, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3445-3414","authenticated-orcid":false,"given":"Anamaria","family":"Crisan","sequence":"additional","affiliation":[{"name":"Cheriton School of Computer Science, University of Waterloo, Waterloo, Ontario, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,3,22]]},"reference":[{"key":"e_1_3_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642016"},{"key":"e_1_3_3_3_3_2","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le and Charles Sutton. 2021. Program Synthesis with Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2108.07732\u00a0[cs.PL] https:\/\/arxiv.org\/abs\/2108.07732"},{"key":"e_1_3_3_3_4_2","unstructured":"Aaron Bangor Phil Kortum and James Miller. 2009. Determining What Individual SUS Scores Mean: Adding an Adjective Rating Scale. J. Usability Stud. 4 (04 2009) 114\u2013123."},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"crossref","unstructured":"Barry Boehm and Victor\u00a0R. Basili. 2001. Software Defect Reduction Top 10 List. Computer 34 1 (2001) 135\u2013137. https:\/\/www.cs.umd.edu\/projects\/SoftEng\/ESEG\/papers\/82.78.pdf","DOI":"10.1109\/2.962984"},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"publisher","unstructured":"Virginia Braun and Victoria Clarke. 2019. Reflecting on reflexive thematic analysis. Qualitative Research in Sport Exercise and Health 11 4 (2019) 589\u2013597. 10.1080\/2159676X.2019.1628806 arXiv:https:\/\/doi.org\/10.1080\/2159676X.2019.1628806","DOI":"10.1080\/2159676X.2019.1628806"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"publisher","DOI":"10.1145\/1390630.1390647"},{"key":"e_1_3_3_3_8_2","unstructured":"Federico Cassano John Gouwar Daniel Nguyen Sydney Nguyen Luna Phipps-Costin Donald Pinckney Ming-Ho Yee Yangtian Zi Carolyn\u00a0Jane Anderson Molly\u00a0Q Feldman Arjun Guha Michael Greenberg and Abhinav Jangda. 2022. MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation. arxiv:https:\/\/arXiv.org\/abs\/2208.08227\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2208.08227"},{"key":"e_1_3_3_3_9_2","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique\u00a0Ponde de Oliveira\u00a0Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe\u00a0Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William\u00a0Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew\u00a0N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. arxiv:https:\/\/arXiv.org\/abs\/2107.03374\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_3_3_10_2","unstructured":"Xinyun Chen Maxwell Lin Nathanael Sch\u00e4rli and Denny Zhou. 2023. Teaching Large Language Models to Self-Debug. arxiv:https:\/\/arXiv.org\/abs\/2304.05128\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2304.05128"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"publisher","unstructured":"Anamaria Crisan Brittany Fiore-Gartland and Melanie Tory. 2021. Passing the Data Baton : A Retrospective Analysis on Data Science Work and Workers. IEEE Transactions on Visualization and Computer Graphics 27 2 (2021) 1860\u20131870. 10.1109\/TVCG.2020.3030340","DOI":"10.1109\/TVCG.2020.3030340"},{"key":"e_1_3_3_3_12_2","unstructured":"Igor\u00a0Regis da Silva\u00a0Sim\u00f5es and Elaine Venson. 2024. Evaluating Source Code Quality with Large Language Models: a comparative study. arxiv:https:\/\/arXiv.org\/abs\/2408.07082\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2408.07082"},{"key":"e_1_3_3_3_13_2","doi-asserted-by":"publisher","unstructured":"Cristina David and Daniel Kroening. 2017. Program synthesis: challenges and opportunities. Philosophical Transactions of the Royal Society A: Mathematical Physical and Engineering Sciences 375 2104 (2017) 20150403. 10.1098\/rsta.2015.0403 arXiv:https:\/\/royalsocietypublishing.org\/doi\/pdf\/10.1098\/rsta.2015.0403","DOI":"10.1098\/rsta.2015.0403"},{"key":"e_1_3_3_3_14_2","doi-asserted-by":"crossref","unstructured":"Victor Dibia Adam Fourney Gagan Bansal Forough Poursabzi-Sangdeh Han Liu and Saleema Amershi. 2023. Aligning Offline Metrics and Human Judgments of Value for Code Generation Models. arxiv:https:\/\/arXiv.org\/abs\/2210.16494\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2210.16494","DOI":"10.18653\/v1\/2023.findings-acl.540"},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.540"},{"key":"e_1_3_3_3_16_2","unstructured":"Xueying Du Mingwei Liu Kaixin Wang Hanlin Wang Junwei Liu Yixuan Chen Jiayi Feng Chaofeng Sha Xin Peng and Yiling Lou. 2023. ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation. arxiv:https:\/\/arXiv.org\/abs\/2308.01861\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2308.01861"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643916.3644409"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"publisher","unstructured":"Sarah Fakhoury Aaditya Naik Georgios Sakkas Saikat Chakraborty and Shuvendu\u00a0K. Lahiri. 2024. LLM-Based Test-Driven Interactive Code Generation: User Study and Empirical Evaluation. IEEE Transactions on Software Engineering 50 9 (2024) 2254\u20132268. 10.1109\/TSE.2024.3428972","DOI":"10.1109\/TSE.2024.3428972"},{"key":"e_1_3_3_3_19_2","unstructured":"Daniel Fried Armen Aghajanyan Jessy Lin Sida Wang Eric Wallace Freda Shi Ruiqi Zhong Wen tau Yih Luke Zettlemoyer and Mike Lewis. 2023. InCoder: A Generative Model for Code Infilling and Synthesis. arxiv:https:\/\/arXiv.org\/abs\/2204.05999\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2204.05999"},{"key":"e_1_3_3_3_20_2","unstructured":"Daya Guo Qihao Zhu Dejian Yang Zhenda Xie Kai Dong Wentao Zhang Guanting Chen Xiao Bi Y. Wu Y.\u00a0K. Li Fuli Luo Yingfei Xiong and Wenfeng Liang. 2024. DeepSeek-Coder: When the Large Language Model Meets Programming \u2013 The Rise of Code Intelligence. arxiv:https:\/\/arXiv.org\/abs\/2401.14196\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2401.14196"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","unstructured":"Robert Haase Christian Tischer Jean-Karim H\u00e9rich\u00e9 and Nico Scherf. 2024. Benchmarking Large Language Models for Bio-Image Analysis Code Generation. bioRxiv (2024). 10.1101\/2024.04.19.590278 arXiv:https:\/\/www.biorxiv.org\/content\/early\/2024\/04\/25\/2024.04.19.590278.full.pdf","DOI":"10.1101\/2024.04.19.590278"},{"key":"e_1_3_3_3_22_2","unstructured":"William Harding and Matthew Kloster. 2023. Coding on Copilot: 2023 Data Suggests Downward Pressure on Code Quality. https:\/\/www.gitclear.com\/coding_on_copilot_data_shows_ais_downward_pressure_on_code_quality. Accessed: 2024-05-20."},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"publisher","DOI":"10.1016\/S0166-4115(08)62386-9"},{"key":"e_1_3_3_3_24_2","unstructured":"Dan Hendrycks Steven Basart Saurav Kadavath Mantas Mazeika Akul Arora Ethan Guo Collin Burns Samir Puranik Horace He Dawn Song and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. CoRR abs\/2105.09938 (2021). arXiv:https:\/\/arXiv.org\/abs\/2105.09938https:\/\/arxiv.org\/abs\/2105.09938"},{"key":"e_1_3_3_3_25_2","unstructured":"Yiming Huang Jianwen Luo Yan Yu Yitong Zhang Fangyu Lei Yifan Wei Shizhu He Lifu Huang Xiao Liu Jun Zhao and Kang Liu. 2024. DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2410.07331\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2410.07331"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"crossref","unstructured":"Minsuk Kahng Ian Tenney Mahima Pushkarna Michael\u00a0Xieyang Liu James Wexler Emily Reif Krystal Kallarackal Minsuk Chang Michael Terry and Lucas Dixon. 2024. LLM Comparator: Visual Analytics for Side-by-Side Evaluation of Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2402.10524\u00a0[cs.HC] https:\/\/arxiv.org\/abs\/2402.10524","DOI":"10.1145\/3613905.3650755"},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642216"},{"key":"e_1_3_3_3_28_2","unstructured":"Philippe Laban Hiroaki Hayashi Yingbo Zhou and Jennifer Neville. 2025. LLMs Get Lost In Multi-Turn Conversation. https:\/\/www.microsoft.com\/en-us\/research\/publication\/llms-get-lost-in-multi-turn-conversation\/"},{"key":"e_1_3_3_3_29_2","unstructured":"Yuhang Lai Chengxi Li Yiming Wang Tianyi Zhang Ruiqi Zhong Luke Zettlemoyer Scott\u00a0Wen tau Yih Daniel Fried Sida Wang and Tao Yu. 2022. DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation. arxiv:https:\/\/arXiv.org\/abs\/2211.11501\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2211.11501"},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"publisher","unstructured":"Jia Li Ge Li Zhuo Li Zhi Jin Xing Hu Kechi Zhang and Zhiyi Fu. 2023. CodeEditor: Learning to Edit Source Code with Pre-trained Models. ACM Transactions on Software Engineering and Methodology 32 6 (Sept. 2023) 1\u201322. 10.1145\/3597207","DOI":"10.1145\/3597207"},{"key":"e_1_3_3_3_31_2","unstructured":"Jiawei Liu Chunqiu\u00a0Steven Xia Yuyao Wang and Lingming Zhang. 2023. Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation. arxiv:https:\/\/arXiv.org\/abs\/2305.01210\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2305.01210"},{"key":"e_1_3_3_3_32_2","unstructured":"Aman Madaan Niket Tandon Prakhar Gupta Skyler Hallinan Luyu Gao Sarah Wiegreffe Uri Alon Nouha Dziri Shrimai Prabhumoye Yiming Yang Shashank Gupta Bodhisattwa\u00a0Prasad Majumder Katherine Hermann Sean Welleck Amir Yazdanbakhsh and Peter Clark. 2023. Self-Refine: Iterative Refinement with Self-Feedback. arxiv:https:\/\/arXiv.org\/abs\/2303.17651\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2303.17651"},{"key":"e_1_3_3_3_33_2","unstructured":"Erik Nijkamp Bo Pang Hiroaki Hayashi Lifu Tu Huan Wang Yingbo Zhou Silvio Savarese and Caiming Xiong. 2023. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. arxiv:https:\/\/arXiv.org\/abs\/2203.13474\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2203.13474"},{"key":"e_1_3_3_3_34_2","unstructured":"Baolin Peng Michel Galley Pengcheng He Hao Cheng Yujia Xie Yu Hu Qiuyuan Huang Lars Liden Zhou Yu Weizhu Chen and Jianfeng Gao. 2023. Check Your Facts and Try Again: Improving Large Language Models with External Knowledge and Automated Feedback. arxiv:https:\/\/arXiv.org\/abs\/2302.12813\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2302.12813"},{"key":"e_1_3_3_3_35_2","unstructured":"Yun Peng Akhilesh\u00a0Deepak Gotmare Michael Lyu Caiming Xiong Silvio Savarese and Doyen Sahoo. 2024. PerfCodeGen: Improving Performance of LLM Generated Code with Execution Feedback. arxiv:https:\/\/arXiv.org\/abs\/2412.03578\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2412.03578"},{"key":"e_1_3_3_3_36_2","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing\u00a0Ellen Tan Yossi Adi Jingyu Liu Romain Sauvestre Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian\u00a0Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2024. Code Llama: Open Foundation Models for Code. arxiv:https:\/\/arXiv.org\/abs\/2308.12950\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2308.12950"},{"key":"e_1_3_3_3_37_2","unstructured":"John Schulman Barret Zoph Christina Kim Jacob Hilton Jacob Menick Jiayi Weng Juan\u00a0Felipe Ceron\u00a0Uribe Liam Fedus Luke Metz Michael Pokorny et\u00a0al. 2022. ChatGPT: Optimizing Language Models for Dialogue. OpenAI Blog (2022). https:\/\/openai.com\/blog\/chatgpt."},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","unstructured":"Agnia Sergeyuk Yaroslav Golubev Timofey Bryksin and Iftekhar Ahmed. 2025. Using AI-based coding assistants in practice: State of affairs perceptions and ways forward. Information and Software Technology 178 (2025) 107610. 10.1016\/j.infsof.2024.107610","DOI":"10.1016\/j.infsof.2024.107610"},{"key":"e_1_3_3_3_39_2","unstructured":"Noah Shinn Federico Cassano Edward Berman Ashwin Gopinath Karthik Narasimhan and Shunyu Yao. 2023. Reflexion: Language Agents with Verbal Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2303.11366\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2303.11366"},{"key":"e_1_3_3_3_40_2","unstructured":"Manav Singhal Tushar Aggarwal Abhijeet Awasthi Nagarajan Natarajan and Aditya Kanade. 2024. NoFunEval: Funny How Code LMs Falter on Requirements Beyond Functional Correctness. arxiv:https:\/\/arXiv.org\/abs\/2401.15963\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2401.15963"},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","unstructured":"Matthew\u00a0A. Waller and Stanley\u00a0E. Fawcett. 2013. Data Science Predictive Analytics and Big Data: A Revolution That Will Transform Supply Chain Design and Management. 34 2 (2013) 77\u201384. 10.1111\/jbl.12010_eprint: https:\/\/onlinelibrary.wiley.com\/doi\/pdf\/10.1111\/jbl.12010.","DOI":"10.1111\/jbl.12010"},{"key":"e_1_3_3_3_43_2","unstructured":"Jason Wei Xuezhi Wang Dale Schuurmans Maarten Bosma Brian Ichter Fei Xia Ed Chi Quoc Le and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2201.11903\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2201.11903"},{"key":"e_1_3_3_3_44_2","unstructured":"Sean Welleck Ximing Lu Peter West Faeze Brahman Tianxiao Shen Daniel Khashabi and Yejin Choi. 2022. Generating Sequences by Learning to Self-Correct. arxiv:https:\/\/arXiv.org\/abs\/2211.00053\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2211.00053"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_3_46_2","unstructured":"Frank\u00a0F. Xu Uri Alon Graham Neubig and Vincent\u00a0J. Hellendoorn. 2022. A Systematic Evaluation of Large Language Models of Code. arxiv:https:\/\/arXiv.org\/abs\/2202.13169\u00a0[cs.PL] https:\/\/arxiv.org\/abs\/2202.13169"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.996"},{"key":"e_1_3_3_3_48_2","unstructured":"John Yang Akshara Prabhakar Karthik Narasimhan and Shunyu Yao. 2023. InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback. arxiv:https:\/\/arXiv.org\/abs\/2306.14898\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2306.14898"},{"key":"e_1_3_3_3_49_2","unstructured":"Ryan Yen Jiawen Zhu Sangho Suh Haijun Xia and Jian Zhao. 2023. CoLadder: Supporting Programmers with Hierarchical Code Generation in Multi-Level Abstraction. arxiv:https:\/\/arXiv.org\/abs\/2310.08699\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2310.08699"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623316"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"publisher","unstructured":"Jingyue Zhang and Ian Arawjo. 2025. ChainBuddy: An AI Agent System for Generating LLM Pipelines. 10.1145\/3706598.3714085 arxiv:https:\/\/arXiv.org\/abs\/2409.13588\u00a0[cs.HC]","DOI":"10.1145\/3706598.3714085"}],"event":{"name":"IUI '26: 31st International Conference on Intelligent User Interfaces","location":"Paphos Cyprus","acronym":"IUI '26","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGAI ACM Special Interest Group on Artificial Intelligence"]},"container-title":["Proceedings of the 31st International Conference on Intelligent User Interfaces"],"original-title":[],"deposited":{"date-parts":[[2026,3,14]],"date-time":"2026-03-14T12:59:48Z","timestamp":1773493188000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3742413.3789088"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,3,22]]},"references-count":50,"alternative-id":["10.1145\/3742413.3789088","10.1145\/3742413"],"URL":"https:\/\/doi.org\/10.1145\/3742413.3789088","relation":{},"subject":[],"published":{"date-parts":[[2026,3,22]]},"assertion":[{"value":"2026-03-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}