{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,30]],"date-time":"2026-01-30T04:15:01Z","timestamp":1769746501730,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,17]]},"DOI":"10.1145\/3756681.3757003","type":"proceedings-article","created":{"date-parts":[[2025,12,24]],"date-time":"2025-12-24T08:30:04Z","timestamp":1766565004000},"page":"1086-1096","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["LLM Code Customization with Visual Results: A Benchmark on TikZ"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-8708-8362","authenticated-orcid":false,"given":"Charly","family":"Reux","sequence":"first","affiliation":[{"name":"Univ Rennes, Rennes, France; DiverSE, Inria, Rennes, France; IRISA, Rennes, France and INSA, Rennes, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1483-3858","authenticated-orcid":false,"given":"Mathieu","family":"Acher","sequence":"additional","affiliation":[{"name":"Univ Rennes, Rennes, France; Inria, Rennes, France; CNRS, Rennes, France and INSA, Rennes, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2218-650X","authenticated-orcid":false,"given":"Djamel Eddine","family":"Khelladi","sequence":"additional","affiliation":[{"name":"Univ Rennes, Rennes, France; Inria, Rennes, France; CNRS, Rennes, France and IRISA, Rennes, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3203-6107","authenticated-orcid":false,"given":"Cl\u00e9ment","family":"Quinton","sequence":"additional","affiliation":[{"name":"Univ. Lille, Lille, France; CNRS, Lille, France and Inria, Lille, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4551-8562","authenticated-orcid":false,"given":"Olivier","family":"Barais","sequence":"additional","affiliation":[{"name":"Univ. Rennes, Rennes, France; IRISA, Rennes, France and Inria, Rennes, France"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,12,24]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","unstructured":"Mathieu Acher Jos\u00e9\u00a0Galindo Duarte and Jean-Marc J\u00e9z\u00e9quel. 2023. On Programming Variability with Large Language Model-based Assistant(SPLC \u201923 Vol.\u00a0A). 8\u201314. 10.1145\/3579027.3608972","DOI":"10.1145\/3579027.3608972"},{"key":"e_1_3_3_2_3_2","unstructured":"Reem Aleithan Haoran Xue Mohammad\u00a0Mahdi Mohajer Elijah Nnorom Gias Uddin and Song Wang. 2024. SWE-Bench+: Enhanced Coding Benchmark for LLMs. arXiv:https:\/\/arXiv.org\/abs\/2410.06992 version: 1."},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"Tyler Angert Miroslav Suzara Jenny Han Christopher Pondoc and Hariharan Subramonyam. 2023. Spellburst: A Node-based Interface for Exploratory Creative Coding with Natural Language Prompts(UIST \u201923). New York NY USA 1\u201322.","DOI":"10.1145\/3586183.3606719"},{"key":"e_1_3_3_2_5_2","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le and Charles Sutton. 2021. Program Synthesis with Large Language Models. arXiv:https:\/\/arXiv.org\/abs\/2108.07732."},{"key":"e_1_3_3_2_6_2","unstructured":"Eslam\u00a0Mohamed Bakr Pengzhan Sun Xiaoqian Shen Faizan\u00a0Farooq Khan Li\u00a0Erran Li and Mohamed Elhoseiny. 2023. HRS-Bench: Holistic Reliable and Scalable Benchmark for Text-to-Image Models. arXiv:https:\/\/arXiv.org\/abs\/2304.05390."},{"key":"e_1_3_3_2_7_2","unstructured":"Ayan Banerjee Nityanand Mathur Josep Llad\u00f3s Umapada Pal and Anjan Dutta. 2024. SVGCraft: Beyond Single Object Text-to-SVG Synthesis with Comprehensive Canvas Layout. arXiv:https:\/\/arXiv.org\/abs\/2404.00412."},{"key":"e_1_3_3_2_8_2","unstructured":"Jonas Belouadi Anne Lauscher and Steffen Eger. 2024. AutomaTikZ: Text-Guided Synthesis of Scientific Vector Graphics with TikZ. arXiv:https:\/\/arXiv.org\/abs\/2310.00367 [cs]."},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"crossref","unstructured":"Jonas Belouadi Simone\u00a0Paolo Ponzetto and Steffen Eger. 2024. DeTikZify: Synthesizing Graphics Programs for Scientific Figures and Sketches with TikZ. arXiv:https:\/\/arXiv.org\/abs\/2405.15306 [cs].","DOI":"10.52202\/079017-2701"},{"key":"e_1_3_3_2_10_2","unstructured":"S\u00e9bastien Bubeck Varun Chandrasekaran Ronen Eldan Johannes Gehrke Eric Horvitz Ece Kamar Peter Lee Yin\u00a0Tat Lee Yuanzhi Li Scott Lundberg Harsha Nori Hamid Palangi Marco\u00a0Tulio Ribeiro and Yi Zhang. 2023. Sparks of Artificial General Intelligence: Early experiments with GPT-4. arXiv:https:\/\/arXiv.org\/abs\/2303.12712."},{"key":"e_1_3_3_2_11_2","unstructured":"Mark Chen and al.2021. Evaluating Large Language Models Trained on Code. arXiv:https:\/\/arXiv.org\/abs\/2107.03374."},{"key":"e_1_3_3_2_12_2","unstructured":"Jaemin Cho Abhay Zala and Mohit Bansal. 2023. Visual Programming for Text-to-Image Generation and Evaluation. arXiv:https:\/\/arXiv.org\/abs\/2305.15328."},{"key":"e_1_3_3_2_13_2","unstructured":"Yi Cui. 2024. WebApp1K: A Practical Code-Generation Benchmark for Web App Development. arXiv:https:\/\/arXiv.org\/abs\/2408.00019 version: 1."},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Aryaz Eghbali and Michael Pradel. 2023. CrystalBLEU: Precisely and Efficiently Measuring the Similarity of Code(ASE \u201922). New York NY USA 1\u201312.","DOI":"10.1145\/3551349.3556903"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"crossref","unstructured":"Kanika Goswami Puneet Mathur Ryan Rossi and Franck Dernoncourt. 2025. PlotGen: Multi-Agent LLM-based Scientific Data Visualization via Multimodal Feedback. arXiv:https:\/\/arXiv.org\/abs\/2502.00988 version: 1.","DOI":"10.1145\/3701716.3716888"},{"key":"e_1_3_3_2_16_2","unstructured":"Patrick Haller Jonas Golde and Alan Akbik. 2024. PECC: Problem Extraction and Coding Challenges. arXiv:https:\/\/arXiv.org\/abs\/2404.18766."},{"key":"e_1_3_3_2_17_2","unstructured":"Shuhao Han Haotian Fan Jiachen Fu Liang Li Tao Li Junhui Cui Yunqiu Wang Yang Tai Jingwei Sun Chunle Guo and Chongyi Li. 2024. EvalMuse-40K: A Reliable and Fine-Grained Benchmark with Comprehensive Human Annotations for Text-to-Image Generation Model Evaluation. arXiv:https:\/\/arXiv.org\/abs\/2412.18150."},{"key":"e_1_3_3_2_18_2","unstructured":"Dan Hendrycks Steven Basart Saurav Kadavath Mantas Mazeika Akul Arora Ethan Guo Collin Burns Samir Puranik Horace He Dawn Song and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. arXiv:https:\/\/arXiv.org\/abs\/2105.09938."},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","unstructured":"Xinyi Hou Yanjie Zhao Yue Liu Zhou Yang Kailong Wang Li Li Xiapu Luo David Lo John Grundy and Haoyu Wang. 2024. Large Language Models for Software Engineering: A Systematic Literature Review. ACM Trans. Softw. Eng. Methodol. 33 8 (Dec. 2024) 220:1\u2013220:79. 10.1145\/3695988","DOI":"10.1145\/3695988"},{"key":"e_1_3_3_2_20_2","unstructured":"Kaiyi Huang Kaiyue Sun Enze Xie Zhenguo Li and Xihui Liu. 2023. T2I-CompBench: A Comprehensive Benchmark for Open-world Compositional Text-to-image Generation. arXiv:https:\/\/arXiv.org\/abs\/2307.06350."},{"key":"e_1_3_3_2_21_2","unstructured":"Naman Jain King Han Alex Gu Wen-Ding Li Fanjia Yan Tianjun Zhang Sida Wang Armando Solar-Lezama Koushik Sen and Ion Stoica. 2024. LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code. arXiv:https:\/\/arXiv.org\/abs\/2403.07974."},{"key":"e_1_3_3_2_22_2","unstructured":"Carlos\u00a0E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press and Karthik Narasimhan. 2024. SWE-bench: Can Language Models Resolve Real-World GitHub Issues?arXiv:https:\/\/arXiv.org\/abs\/2310.06770."},{"key":"e_1_3_3_2_23_2","unstructured":"Sayash Kapoor Benedikt Stroebl Zachary\u00a0S. Siegel Nitya Nadgir and Arvind Narayanan. 2024. AI Agents That Matter. arXiv:https:\/\/arXiv.org\/abs\/2407.01502 [cs]."},{"key":"e_1_3_3_2_24_2","unstructured":"Beck LaBash August Rosedale Alex Reents Lucas Negritto and Colin Wiel. 2024. RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale. arXiv:https:\/\/arXiv.org\/abs\/2406.16801."},{"key":"e_1_3_3_2_25_2","unstructured":"Shanchao Liang Yiran Hu Nan Jiang and Lin Tan. 2024. Can Language Models Replace Programmers? REPOCOD Says \u2019Not Yet\u2019. arXiv:https:\/\/arXiv.org\/abs\/2410.21647."},{"key":"e_1_3_3_2_26_2","unstructured":"Junwei Liu Kaixin Wang Yixuan Chen Xin Peng Zhenpeng Chen Lingming Zhang and Yiling Lou. 2024. Large Language Model-Based Agents for Software Engineering: A Survey. arXiv:https:\/\/arXiv.org\/abs\/2409.02977."},{"key":"e_1_3_3_2_27_2","unstructured":"Vivian Liu Rubaiat\u00a0Habib Kazi Li-Yi Wei Matthew Fisher Timothy Langlois Seth Walker and Lydia Chilton. 2024. LogoMotion: Visually Grounded Code Generation for Content-Aware Animation. arXiv:https:\/\/arXiv.org\/abs\/2405.07065."},{"key":"e_1_3_3_2_28_2","unstructured":"Yongkun Liu Jiachi Chen Tingting Bi John Grundy Yanlin Wang Jianxing Yu Ting Chen Yutian Tang and Zibin Zheng. 2024. An Empirical Study on Low Code Programming using Traditional vs Large Language Model Support. arXiv:https:\/\/arXiv.org\/abs\/2402.01156."},{"key":"e_1_3_3_2_29_2","unstructured":"Yadong Lu Jianwei Yang Yelong Shen and Ahmed Awadallah. 2024. OmniParser for Pure Vision Based GUI Agent. arxiv:https:\/\/arXiv.org\/abs\/2408.00203\u00a0[cs.CV] arXiv:https:\/\/arXiv.org\/abs\/2408.00203."},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Juan\u00a0A. Rodriguez Abhay Puri Shubham Agarwal Issam\u00a0H. Laradji Pau Rodriguez Sai Rajeswar David Vazquez Christopher Pal and Marco Pedersoli. 2024. StarVector: Generating Scalable Vector Graphics Code from Images and Text. arXiv:https:\/\/arXiv.org\/abs\/2312.11556.","DOI":"10.1109\/CVPR52734.2025.01508"},{"key":"e_1_3_3_2_31_2","unstructured":"Chenglei Si Yanzhe Zhang Zhengyuan Yang Ruibo Liu and Diyi Yang. 2024. Design2Code: How Far Are We From Automating Front-End Engineering?arXiv:https:\/\/arXiv.org\/abs\/2403.03163."},{"key":"e_1_3_3_2_32_2","unstructured":"George Thomas Alex\u00a0J. Chan Jikun Kang Wenqi Wu Filippos Christianos Fraser Greenlee Andy Toulis and Marvin Purtorab. 2025. WebGames: Challenging General-Purpose Web-Browsing AI Agents. arXiv:https:\/\/arXiv.org\/abs\/2502.18356 [cs]."},{"key":"e_1_3_3_2_33_2","unstructured":"Tiffany Tseng Ruijia Cheng and Jeffrey Nichols. 2024. Keyframer: Empowering Animation Design using Large Language Models. arXiv:https:\/\/arXiv.org\/abs\/2402.06071."},{"key":"e_1_3_3_2_34_2","unstructured":"Xingyao Wang Boxuan Li Yufan Song Frank\u00a0F. Xu Xiangru Tang Mingchen Zhuge Jiayi Pan Yueqi Song Bowen Li Jaskirat Singh Hoang\u00a0H. Tran Fuqiang Li Ren Ma Mingzhang Zheng Bill Qian Yanjun Shao Niklas Muennighoff Yizhe Zhang Binyuan Hui Junyang Lin Robert Brennan Hao Peng Heng Ji and Graham Neubig. 2025. OpenHands: An Open Platform for AI Software Developers as Generalist Agents. arXiv:https:\/\/arXiv.org\/abs\/2407.16741 [cs]."},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"crossref","unstructured":"Zhiruo Wang Grace Cuenca Shuyan Zhou Frank\u00a0F. Xu and Graham Neubig. 2023. MCoNaLa: A Benchmark for Code Generation from Multiple Natural Languages. arXiv:https:\/\/arXiv.org\/abs\/2203.08388.","DOI":"10.18653\/v1\/2023.findings-eacl.20"},{"key":"e_1_3_3_2_36_2","unstructured":"Jingxuan Wei Cheng Tan Qi Chen Gaowei Wu Siyuan Li Zhangyang Gao Linzhuang Sun Bihui Yu and Ruifeng Guo. 2024. From Words to Structured Visuals: A Benchmark and Framework for Text-to-Diagram Generation and Editing. arXiv:https:\/\/arXiv.org\/abs\/2411.11916."},{"key":"e_1_3_3_2_37_2","unstructured":"Simon Willison. 2025. Notes on Google\u2019s Gemma 3. https:\/\/simonwillison.net\/2025\/Mar\/12\/gemma-3\/"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"crossref","unstructured":"Ronghuan Wu Wanchao Su Kede Ma and Jing Liao. 2023. IconShop: Text-Guided Vector Icon Synthesis with Autoregressive Transformers. ACM Trans. Graph. 42 6 (Dec. 2023) 230:1\u2013230:14.","DOI":"10.1145\/3618364"},{"key":"e_1_3_3_2_39_2","unstructured":"Ximing Xing Juncheng Hu Jing Zhang Dong Xu and Qian Yu. 2024. SVGFusion: Scalable Text-to-SVG Generation via Vector Space Diffusion. arXiv:https:\/\/arXiv.org\/abs\/2412.10437."},{"key":"e_1_3_3_2_40_2","unstructured":"Ximing Xing Haitao Zhou Chuang Wang Jing Zhang Dong Xu and Qian Yu. 2024. SVGDreamer: Text Guided SVG Generation with Diffusion Model. arXiv:https:\/\/arXiv.org\/abs\/2312.16476."},{"key":"e_1_3_3_2_41_2","unstructured":"Abhay Zala Han Lin Jaemin Cho and Mohit Bansal. 2024. DiagrammerGPT: Generating Open-Domain Open-Platform Diagrams via LLM Planning. arXiv:https:\/\/arXiv.org\/abs\/2310.12128."},{"key":"e_1_3_3_2_42_2","unstructured":"Fengji Zhang Linquan Wu Huiyu Bai Guancheng Lin Xiao Li Xiao Yu Yue Wang Bei Chen and Jacky Keung. 2024. HumanEval-V: Evaluating Visual Understanding and Reasoning Abilities of Large Multimodal Models Through Coding Tasks. arXiv:https:\/\/arXiv.org\/abs\/2410.12381."},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"crossref","unstructured":"Kaizhong Zhang and Dennis Shasha. 1989. Simple Fast Algorithms for the Editing Distance between Trees and Related Problems. SIAM J. Comput. 18 6 (Dec. 1989) 1245\u20131262. Publisher: Society for Industrial and Applied Mathematics.","DOI":"10.1137\/0218082"},{"key":"e_1_3_3_2_44_2","unstructured":"Dewu Zheng Yanlin Wang Ensheng Shi Hongyu Zhang and Zibin Zheng. 2024. How Well Do LLMs Generate Code for Different Application Domains? Benchmark and Evaluation. arXiv:https:\/\/arXiv.org\/abs\/2412.18573 version: 1."},{"key":"e_1_3_3_2_45_2","unstructured":"Shuyan Zhou Frank\u00a0F Xu Hao Zhu Xuhui Zhou Robert Lo Abishek Sridhar Xianyi Cheng Yonatan Bisk Daniel Fried Uri Alon et\u00a0al. 2023. WebArena: A Realistic Web Environment for Building Autonomous Agents. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.13854 (2023). https:\/\/webarena.dev"},{"key":"e_1_3_3_2_46_2","unstructured":"Terry\u00a0Yue Zhuo Minh\u00a0Chien Vu Jenny Chim Han Hu Wenhao Yu Ratnadira Widyasari Imam Nur\u00a0Bani Yusuf Haolan Zhan Junda He Indraneil Paul Simon Brunner Chen Gong Thong Hoang Armel\u00a0Randy Zebaze Xiaoheng Hong Wen-Ding Li Jean Kaddour Ming Xu Zhihan Zhang Prateek Yadav Naman Jain Alex Gu Zhoujun Cheng Jiawei Liu Qian Liu Zijian Wang David Lo Binyuan Hui Niklas Muennighoff Daniel Fried Xiaoning Du Harm\u00a0de Vries and Leandro\u00a0Von Werra. 2024. BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions. arXiv:https:\/\/arXiv.org\/abs\/2406.15877."}],"event":{"name":"EASE '25: Evaluation and Assessment in Software Engineering","location":"Istanbul Turkiye","acronym":"EASE '25"},"container-title":["Proceedings of the 29th International Conference on Evaluation and Assessment in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3756681.3757003","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,24]],"date-time":"2025-12-24T08:43:06Z","timestamp":1766565786000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3756681.3757003"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,17]]},"references-count":45,"alternative-id":["10.1145\/3756681.3757003","10.1145\/3756681"],"URL":"https:\/\/doi.org\/10.1145\/3756681.3757003","relation":{},"subject":[],"published":{"date-parts":[[2025,6,17]]},"assertion":[{"value":"2025-12-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}