{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T07:17:06Z","timestamp":1783408626205,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T00:00:00Z","timestamp":1775952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,12]]},"DOI":"10.1145\/3786181.3788710","type":"proceedings-article","created":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T06:28:24Z","timestamp":1783405704000},"page":"113-120","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["RuberDuckBench: A Benchmark for AI Coding Assistants"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-5474-2322","authenticated-orcid":false,"given":"Ferida","family":"Mohammed","sequence":"first","affiliation":[{"name":"Bryn Mawr College, Bryn Mawr, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-2803-176X","authenticated-orcid":false,"given":"Fatma","family":"Ayad","sequence":"additional","affiliation":[{"name":"Bryn Mawr College, Bryn Mawr, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3777-5291","authenticated-orcid":false,"given":"Petros","family":"Maniatis","sequence":"additional","affiliation":[{"name":"Google DeepMind, Mountain View, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2546-9000","authenticated-orcid":false,"given":"Satish","family":"Chandra","sequence":"additional","affiliation":[{"name":"Meta Platforms, Mountain View, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0738-8813","authenticated-orcid":false,"given":"Elizabeth","family":"Dinella","sequence":"additional","affiliation":[{"name":"Bryn Mawr College, Bryn Mawr, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,6]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"2025. AlphaCode. https:\/\/alphacode.deepmind.com\/. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_3_2","unstructured":"2025. Claude 4. https:\/\/www.anthropic.com\/news\/claude-4."},{"key":"e_1_3_3_2_4_2","unstructured":"2025. Claude Opus 4.1. https:\/\/www.anthropic.com\/news\/claude-opus-4-1."},{"key":"e_1_3_3_2_5_2","unstructured":"2025. Claude Sonnet. https:\/\/www.anthropic.com\/news\/claude-3-7-sonnet."},{"key":"e_1_3_3_2_6_2","unstructured":"2025. Cursor. https:\/\/cursor.com\/. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_7_2","unstructured":"2025. Gemini 2.5 Flash. https:\/\/deepmind.google\/models\/gemini\/flash\/."},{"key":"e_1_3_3_2_8_2","unstructured":"2025. Gemini 2.5 Pro. https:\/\/deepmind.google\/models\/gemini\/pro\/."},{"key":"e_1_3_3_2_9_2","unstructured":"2025. GitHub Copilot. https:\/\/github.com\/features\/copilot. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_10_2","unstructured":"2025. gpt-4.1. https:\/\/platform.openai.com\/docs\/models\/gpt-4.1."},{"key":"e_1_3_3_2_11_2","unstructured":"2025. gpt-5. https:\/\/openai.com\/index\/introducing-gpt-5. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_12_2","unstructured":"2025. gpt-oss-120b. https:\/\/platform.openai.com\/docs\/models\/gpt-oss-120b."},{"key":"e_1_3_3_2_13_2","unstructured":"2025. gpt-oss-20b. https:\/\/platform.openai.com\/docs\/models\/gpt-oss-20b."},{"key":"e_1_3_3_2_14_2","unstructured":"2025. Grok 3. https:\/\/x.ai\/news\/grok-3. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_15_2","unstructured":"2025. Grok 4. https:\/\/x.ai\/news\/grok-4. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_16_2","unstructured":"2025. Kiro. https:\/\/kiro.dev\/. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_17_2","unstructured":"2025. llama-3.3. https:\/\/www.llama.com\/models\/llama-3\/. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_18_2","unstructured":"2025. llama-4. https:\/\/www.llama.com\/models\/llama-4\/. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_19_2","unstructured":"2025. Mistral. https:\/\/mistral.ai\/news\/mistral-large. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_20_2","unstructured":"2025. o3. https:\/\/platform.openai.com\/docs\/models\/o3. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_21_2","unstructured":"2025. Tabnine. https:\/\/www.tabnine.com\/. Accessed: 2025-10-10."},{"key":"e_1_3_3_2_22_2","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le et\u00a0al. 2021. Program Synthesis with Large Language Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2108.07732 (2021)."},{"key":"e_1_3_3_2_23_2","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique\u00a0Ponde de Oliveira\u00a0Pinto Jared Kaplan Harrison Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe\u00a0Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William\u00a0Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew\u00a0N. Carr Jan Leike Joshua Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107.03374 (2021). arXiv:https:\/\/arXiv.org\/abs\/2107.03374https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_3_2_24_2","unstructured":"DeepSeek-AI Daya Guo Dejian Yang Haowei Zhang Junxiao Song Ruoyu Zhang Runxin Xu Qihao Zhu Shirong Ma Peiyi Wang Xiao Bi Xiaokang Zhang Xingkai Yu Yu Wu Z.\u00a0F. Wu Zhibin Gou Zhihong Shao Zhuoshu Li Ziyi Gao Aixin Liu Bing Xue Bingxuan Wang Bochao Wu Bei Feng Chengda Lu Chenggang Zhao Chengqi Deng Chenyu Zhang Chong Ruan Damai Dai Deli Chen Dongjie Ji Erhang Li Fangyun Lin Fucong Dai Fuli Luo Guangbo Hao Guanting Chen Guowei Li H. Zhang Han Bao Hanwei Xu Haocheng Wang Honghui Ding Huajian Xin Huazuo Gao Hui Qu Hui Li Jianzhong Guo Jiashi Li Jiawei Wang Jingchang Chen Jingyang Yuan Junjie Qiu Junlong Li J.\u00a0L. Cai Jiaqi Ni Jian Liang Jin Chen Kai Dong Kai Hu Kaige Gao Kang Guan Kexin Huang Kuai Yu Lean Wang Lecong Zhang Liang Zhao Litong Wang Liyue Zhang Lei Xu Leyi Xia Mingchuan Zhang Minghua Zhang Minghui Tang Meng Li Miaojun Wang Mingming Li Ning Tian Panpan Huang Peng Zhang Qiancheng Wang Qinyu Chen Qiushi Du Ruiqi Ge Ruisong Zhang Ruizhe Pan Runji Wang R.\u00a0J. Chen R.\u00a0L. Jin Ruyi Chen Shanghao Lu Shangyan Zhou Shanhuang Chen Shengfeng Ye Shiyu Wang Shuiping Yu Shunfeng Zhou Shuting Pan S.\u00a0S. Li Shuang Zhou Shaoqing Wu Shengfeng Ye Tao Yun Tian Pei Tianyu Sun T. Wang Wangding Zeng Wanjia Zhao Wen Liu Wenfeng Liang Wenjun Gao Wenqin Yu Wentao Zhang W.\u00a0L. Xiao Wei An Xiaodong Liu Xiaohan Wang Xiaokang Chen Xiaotao Nie Xin Cheng Xin Liu Xin Xie Xingchao Liu Xinyu Yang Xinyuan Li Xuecheng Su Xuheng Lin X.\u00a0Q. Li Xiangyue Jin Xiaojin Shen Xiaosha Chen Xiaowen Sun Xiaoxiang Wang Xinnan Song Xinyi Zhou Xianzu Wang Xinxia Shan Y.\u00a0K. Li Y.\u00a0Q. Wang Y.\u00a0X. Wei Yang Zhang Yanhong Xu Yao Li Yao Zhao Yaofeng Sun Yaohui Wang Yi Yu Yichao Zhang Yifan Shi Yiliang Xiong Ying He Yishi Piao Yisong Wang Yixuan Tan Yiyang Ma Yiyuan Liu Yongqiang Guo Yuan Ou Yuduan Wang Yue Gong Yuheng Zou Yujia He Yunfan Xiong Yuxiang Luo Yuxiang You Yuxuan Liu Yuyang Zhou Y.\u00a0X. Zhu Yanhong Xu Yanping Huang Yaohui Li Yi Zheng Yuchen Zhu Yunxian Ma Ying Tang Yukun Zha Yuting Yan Z.\u00a0Z. Ren Zehui Ren Zhangli Sha Zhe Fu Zhean Xu Zhenda Xie Zhengyan Zhang Zhewen Hao Zhicheng Ma Zhigang Yan Zhiyu Wu Zihui Gu Zijia Zhu Zijun Liu Zilin Li Ziwei Xie Ziyang Song Zizheng Pan Zhen Huang Zhipeng Xu Zhongyu Zhang and Zhen Zhang. 2025. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2501.12948\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2501.12948"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287549"},{"key":"e_1_3_3_2_26_2","unstructured":"Carlos\u00a0E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press and Karthik Narasimhan. 2023. SWE-bench: Can Language Models Resolve Real-World GitHub Issues? arxiv:https:\/\/arXiv.org\/abs\/2310.06770\u00a0[cs.CL]"},{"key":"e_1_3_3_2_27_2","unstructured":"Avishree Khare Saikat Dutta Ziyang Li Alaia Solko-Breslin Rajeev Alur and Mayur Naik. 2024. Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities. arxiv:https:\/\/arXiv.org\/abs\/2311.16169\u00a0[cs.CR] https:\/\/arxiv.org\/abs\/2311.16169"},{"key":"e_1_3_3_2_28_2","unstructured":"Zhiyu Li Shuai Lu Daya Guo Nan Duan Shailesh Jannu Grant Jenks Deep Majumder Jared Green Alexey Svyatkovskiy Shengyu Fu and Neel Sundaresan. 2022. Automating Code Review Activities by Large-Scale Pre-training. arxiv:https:\/\/arXiv.org\/abs\/2203.09095\u00a0[cs.SE]"},{"key":"e_1_3_3_2_29_2","unstructured":"Shuai Lu Daya Guo Shuo Ren Junjie Huang Alexey Svyatkovskiy Ambrosio Blanco Colin Clement Dawn Drain Daxin Jiang Duyu Tang Ge Li Lidong Zhou Linjun Shou Long Zhou Michele Tufano Ming Gong Ming Zhou Nan Duan Neel Sundaresan Shao\u00a0Kun Deng Shengyu Fu and Shujie Liu. 2021. CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. arxiv:https:\/\/arXiv.org\/abs\/2102.04664\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2102.04664"},{"key":"e_1_3_3_2_30_2","unstructured":"Elise Paradis Kate Grey Quinn Madison Daye Nam Andrew Macvean Vahid Meimand Nan Zhang Ben Ferrari-Church and Satish Chandra. 2024. How much does AI impact development speed? An enterprise-based randomized controlled trial. arxiv:https:\/\/arXiv.org\/abs\/2410.12944\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2410.12944"},{"key":"e_1_3_3_2_31_2","unstructured":"Pranab Sahoo Ayush\u00a0Kumar Singh Sriparna Saha Vinija Jain Samrat Mondal and Aman Chadha. 2025. A Systematic Survey of Prompt Engineering in Large Language Models: Techniques and Applications. arxiv:https:\/\/arXiv.org\/abs\/2402.07927\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2402.07927"},{"key":"e_1_3_3_2_32_2","unstructured":"Nidhish Shah Zulkuf Genc and Dogu Araci. 2024. StackEval: Benchmarking LLMs in Coding Assistance. arxiv:https:\/\/arXiv.org\/abs\/2412.05288\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2412.05288"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"crossref","unstructured":"Patrick\u00a0E Shrout and Joseph\u00a0L Fleiss. 1979. Intraclass correlations: uses in assessing rater reliability. Psychological Bulletin 86 2 (1979) 420\u2013428.","DOI":"10.1037\/\/0033-2909.86.2.420"},{"key":"e_1_3_3_2_34_2","unstructured":"Andr\u00e9 Silva and Martin Monperrus. 2024. RepairBench: Leaderboard of Frontier Models for Program Repair. arxiv:https:\/\/arXiv.org\/abs\/2409.18952\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2409.18952"},{"key":"e_1_3_3_2_35_2","unstructured":"Stack Overflow. 2025. Stack Overflow Developer Survey 2025. https:\/\/survey.stackoverflow.co\/2025\/. Accessed: 10-06-2025."},{"key":"e_1_3_3_2_36_2","unstructured":"Mark Vero Niels M\u00fcndler Victor Chibotaru Veselin Raychev Maximilian Baader Nikola Jovanovi\u0107 Jingxuan He and Martin Vechev. 2025. BaxBench: Can LLMs Generate Correct and Secure Backends? arxiv:https:\/\/arXiv.org\/abs\/2502.11844\u00a0[cs.CR] https:\/\/arxiv.org\/abs\/2502.11844"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.52202\/068431-1800"},{"key":"e_1_3_3_2_38_2","unstructured":"Shihao Xia Mengting He Linhai Song and Yiying Zhang. 2025. SC-Bench: A Large-Scale Dataset for Smart Contract Auditing. arxiv:https:\/\/arXiv.org\/abs\/2410.06176\u00a0[cs.CR] https:\/\/arxiv.org\/abs\/2410.06176"},{"key":"e_1_3_3_2_39_2","unstructured":"Kai Xu YiWei Mao XinYi Guan and ZiLong Feng. 2025. Web-Bench: A LLM Code Benchmark Based on Web Standards and Frameworks. arxiv:https:\/\/arXiv.org\/abs\/2505.07473\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2505.07473"},{"key":"e_1_3_3_2_40_2","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chang Gao Chengen Huang Chenxu Lv Chujie Zheng Dayiheng Liu Fan Zhou Fei Huang Feng Hu Hao Ge Haoran Wei Huan Lin Jialong Tang Jian Yang Jianhong Tu Jianwei Zhang Jianxin Yang Jiaxi Yang Jing Zhou Jingren Zhou Junyang Lin Kai Dang Keqin Bao Kexin Yang Le Yu Lianghao Deng Mei Li Mingfeng Xue Mingze Li Pei Zhang Peng Wang Qin Zhu Rui Men Ruize Gao Shixuan Liu Shuang Luo Tianhao Li Tianyi Tang Wenbiao Yin Xingzhang Ren Xinyu Wang Xinyu Zhang Xuancheng Ren Yang Fan Yang Su Yichang Zhang Yinger Zhang Yu Wan Yuqiong Liu Zekun Wang Zeyu Cui Zhenru Zhang Zhipeng Zhou and Zihan Qiu. 2025. Qwen3 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2505.09388\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2505.09388"},{"key":"e_1_3_3_2_41_2","unstructured":"Quanjun Zhang Chunrong Fang Yang Xie Yaxin Zhang Yun Yang Weisong Sun Shengcheng Yu and Zhenyu Chen. 2024. A Survey on Large Language Models for Software Engineering. arxiv:https:\/\/arXiv.org\/abs\/2312.15223\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2312.15223"},{"key":"e_1_3_3_2_42_2","unstructured":"Li Zhong and Zilong Wang. 2023. Can ChatGPT replace StackOverflow? A Study on Robustness and Reliability of Large Language Model Code Generation. arxiv:https:\/\/arXiv.org\/abs\/2308.10335\u00a0[cs.CL]"},{"key":"e_1_3_3_2_43_2","unstructured":"Albert Ziegler Eirini Kalliamvakou Shawn Simister Ganesh Sittampalam Alice Li Andrew Rice Devon Rifkin and Edward Aftandilian. 2022. Productivity Assessment of Neural Code Completion. arxiv:https:\/\/arXiv.org\/abs\/2205.06537\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2205.06537"}],"event":{"name":"LLM4Code '26: 3rd International Workshop on Large Language Models For Code","location":"Rio de Janeiro Brazil","acronym":"LLM4Code '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 3rd International Workshop on Large Language Models For Code"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786181.3788710","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T06:29:39Z","timestamp":1783405779000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786181.3788710"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,12]]},"references-count":42,"alternative-id":["10.1145\/3786181.3788710","10.1145\/3786181"],"URL":"https:\/\/doi.org\/10.1145\/3786181.3788710","relation":{},"subject":[],"published":{"date-parts":[[2026,4,12]]},"assertion":[{"value":"2026-07-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}