{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,22]],"date-time":"2026-04-22T03:19:21Z","timestamp":1776827961648,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,14]],"date-time":"2024-04-14T00:00:00Z","timestamp":1713052800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Natural Sciences and Engineering Research Council of Canada (NSERC)","award":["RGPIN-2019-05071"],"award-info":[{"award-number":["RGPIN-2019-05071"]}]},{"DOI":"10.13039\/501100019117","name":"Vector Institute","doi-asserted-by":"publisher","award":["Vector Scholarship in AI"],"award-info":[{"award-number":["Vector Scholarship in AI"]}],"id":[{"id":"10.13039\/501100019117","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,14]]},"DOI":"10.1145\/3650105.3652301","type":"proceedings-article","created":{"date-parts":[[2024,6,12]],"date-time":"2024-06-12T16:01:35Z","timestamp":1718208095000},"page":"57-68","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":21,"title":["Exploring the Impact of the Output Format on the Evaluation of Large Language Models for Code Translation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-1008-1073","authenticated-orcid":false,"given":"Marcos","family":"Macedo","sequence":"first","affiliation":[{"name":"Queen's University, Kingston, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2208-3893","authenticated-orcid":false,"given":"Yuan","family":"Tian","sequence":"additional","affiliation":[{"name":"Queen's University, Kingston, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5494-685X","authenticated-orcid":false,"given":"Filipe","family":"Cogo","sequence":"additional","affiliation":[{"name":"Centre for Software Excellence - Huawei Canada, Kingston, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7213-4006","authenticated-orcid":false,"given":"Bram","family":"Adams","sequence":"additional","affiliation":[{"name":"Queen's University, Kingston, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,6,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2024. Airoboros 13B HF fp16. hhttps:\/\/huggingface.co\/TheBloke\/airoboros-13B-HF Accessed: date-of-access."},{"key":"e_1_3_2_1_2_1","unstructured":"2024. Aizu online judge. https:\/\/onlinejudge.u-aizu.ac.jp\/ Accessed: date-of-access."},{"key":"e_1_3_2_1_3_1","unstructured":"2024. Atcoder. https:\/\/atcoder.jp\/ Accessed: date-of-access."},{"key":"e_1_3_2_1_4_1","unstructured":"2024. C to Go translator. https:\/\/github.com\/gotranspile\/cxgo Accessed: date-of-access."},{"key":"e_1_3_2_1_5_1","unstructured":"2024. C2Rust. https:\/\/github.com\/immunant\/c2rust Accessed: date-of-access."},{"key":"e_1_3_2_1_6_1","unstructured":"2024. GeeksForGeeks. https:\/\/www.geeksforgeeks.org\/ Accessed: date-of-access."},{"key":"e_1_3_2_1_7_1","unstructured":"2024. Java 2 CSharp Translator for Eclipse. https:\/\/sourceforge.net\/projects\/j2cstranslator\/ Accessed: date-of-access."},{"key":"e_1_3_2_1_8_1","unstructured":"2024. LLama 2. https:\/\/ai.meta.com\/research\/publications\/llama-2-open-foundation-and-fine-tuned-chat-models\/ Accessed: date-of-access."},{"key":"e_1_3_2_1_9_1","unstructured":"2024. Sharpen - Automated Java->C coversion. https:\/\/github.com\/mono\/sharpen Accessed: date-of-access."},{"key":"e_1_3_2_1_10_1","unstructured":"2024. Wizard-Vicuna-13B-Uncensored float16 HF. https:\/\/huggingface.co\/TheBloke\/Wizard-Vicuna-13B-Uncensored-HF Accessed: date-of-access."},{"key":"e_1_3_2_1_13_1","volume-title":"Tree-to-tree neural networks for program translation. Advances in neural information processing systems 31","author":"Chen Xinyun","year":"2018","unstructured":"Xinyun Chen, Chang Liu, and Dawn Song. 2018. Tree-to-tree neural networks for program translation. Advances in neural information processing systems 31 (2018)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3528648"},{"key":"e_1_3_2_1_15_1","volume-title":"Large language models for software engineering: Survey and open problems. arXiv preprint arXiv:2310.03533","author":"Fan Angela","year":"2023","unstructured":"Angela Fan, Beliz Gokkaya, Mark Harman, Mitya Lyubarskiy, Shubho Sengupta, Shin Yoo, and Jie M Zhang. 2023. Large language models for software engineering: Survey and open problems. arXiv preprint arXiv:2310.03533 (2023)."},{"key":"e_1_3_2_1_16_1","volume-title":"Prompting Is All Your Need: Automated Android Bug Replay with Large Language Models. arXiv preprint arXiv:2306.01987","author":"Feng Sidong","year":"2023","unstructured":"Sidong Feng and Chunyang Chen. 2023. Prompting Is All Your Need: Automated Android Bug Replay with Large Language Models. arXiv preprint arXiv:2306.01987 (2023)."},{"key":"e_1_3_2_1_17_1","volume-title":"Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155","author":"Feng Zhangyin","year":"2020","unstructured":"Zhangyin Feng, Daya Guo, Duyu Tang, Nan Duan, Xiaocheng Feng, Ming Gong, Linjun Shou, Bing Qin, Ting Liu, Daxin Jiang, et al. 2020. Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155 (2020)."},{"key":"e_1_3_2_1_18_1","volume-title":"Constructing Effective In-Context Demonstration for Code Intelligence Tasks: An Empirical Study. arXiv preprint arXiv:2304.07575","author":"Gao Shuzheng","year":"2023","unstructured":"Shuzheng Gao, Xin-Cheng Wen, Cuiyun Gao, Wenxuan Wang, and Michael R Lyu. 2023. Constructing Effective In-Context Demonstration for Code Intelligence Tasks: An Empirical Study. arXiv preprint arXiv:2304.07575 (2023)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Mingyang Geng Shangwen Wang Dezun Dong Haotian Wang Ge Li Zhi Jin Xiaoguang Mao and Xiangke Liao. 2024. Large Language Models are Few-Shot Summarizers: Multi-Intent Comment Generation via In-Context Learning. (2024).","DOI":"10.1145\/3597503.3608134"},{"key":"e_1_3_2_1_20_1","unstructured":"Mingsheng Jiao Tingrui Yu Xuan Li Guanjie Qiu Xiaodong Gu and Beijun Shen. [n. d.]. On the Evaluation of Neural Code Translation: Taxonomy and Benchmark. ([n. d.])."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661136.2661148"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2309.06180"},{"key":"e_1_3_2_1_23_1","first-page":"14967","article-title":"DOBF: A deobfuscation pre-training objective for programming languages","volume":"34","author":"Lachaux Marie-Anne","year":"2021","unstructured":"Marie-Anne Lachaux, Baptiste Roziere, Marc Szafraniec, and Guillaume Lample. 2021. DOBF: A deobfuscation pre-training objective for programming languages. Advances in Neural Information Processing Systems 34 (2021), 14967--14979.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"Raymond Li Loubna Ben Allal Yangtian Zi Niklas Muennighoff Denis Kocetkov Chenghao Mou Marc Marone Christopher Akiki Jia Li Jenny Chim Qian Liu Evgenii Zheltonozhskii Terry Yue Zhuo Thomas Wang Olivier Dehaene Mishig Davaadorj Joel Lamy-Poirier Jo\u00e3o Monteiro Oleh Shliazhko Nicolas Gontier Nicholas Meade Armel Zebaze Ming-Ho Yee Logesh Kumar Umapathi Jian Zhu Benjamin Lipkin Muhtasham Oblokulov Zhiruo Wang Rudra Murthy Jason Stillerman Siva Sankalp Patel Dmitry Abulkhanov Marco Zocca Manan Dey Zhihan Zhang Nour Fahmy Urvashi Bhattacharyya Wenhao Yu Swayam Singh Sasha Luccioni Paulo Villegas Maxim Kunakov Fedor Zhdanov Manuel Romero Tony Lee Nadav Timor Jennifer Ding Claire Schlesinger Hailey Schoelkopf Jan Ebert Tri Dao Mayank Mishra Alex Gu Jennifer Robinson Carolyn Jane Anderson Brendan Dolan-Gavitt Danish Contractor Siva Reddy Daniel Fried Dzmitry Bahdanau Yacine Jernite Carlos Mu\u00f1oz Ferrandis Sean Hughes Thomas Wolf Arjun Guha Leandro von Werra and Harm de Vries. 2023. StarCoder: may the source be with you! arXiv:2305.06161 [cs]. 10.48550\/arXiv.2305.06161","DOI":"10.48550\/arXiv.2305.06161"},{"key":"e_1_3_2_1_25_1","volume-title":"2023 38th IEEE\/ACM International Conference on Automated Software Engineering (ASE). IEEE, 14--26","author":"Li Tsz-On","year":"2023","unstructured":"Tsz-On Li, Wenxi Zong, Yibo Wang, Haoye Tian, Ying Wang, Shing-Chi Cheung, and Jeff Kramer. 2023. Nuances are the Key: Unlocking ChatGPT to Find Failure-Inducing Tests with Differential Prompting. In 2023 38th IEEE\/ACM International Conference on Automated Software Engineering (ASE). IEEE, 14--26."},{"key":"e_1_3_2_1_26_1","volume-title":"Codexglue: A machine learning benchmark dataset for code understanding and generation. arXiv preprint arXiv:2102.04664","author":"Lu Shuai","year":"2021","unstructured":"Shuai Lu, Daya Guo, Shuo Ren, Junjie Huang, Alexey Svyatkovskiy, Ambrosio Blanco, Colin Clement, Dawn Drain, Daxin Jiang, Duyu Tang, et al. 2021. Codexglue: A machine learning benchmark dataset for code understanding and generation. arXiv preprint arXiv:2102.04664 (2021)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"Ziyang Luo Can Xu Pu Zhao Qingfeng Sun Xiubo Geng Wenxiang Hu Chongyang Tao Jing Ma Qingwei Lin and Daxin Jiang. 2023. WizardCoder: Empowering Code Large Language Models with Evol-Instruct. arXiv:2306.08568 [cs]. 10.48550\/arXiv.2306.08568","DOI":"10.48550\/arXiv.2306.08568"},{"key":"e_1_3_2_1_28_1","unstructured":"Mistral AI Team. 2023. Mixtral of Experts. https:\/\/mistral.ai\/news\/mixtral-of-experts\/ Accessed: 2024-01-13."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2491411.2494584"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2591062.2591072"},{"key":"e_1_3_2_1_31_1","volume-title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In The Eleventh International Conference on Learning Representations.","author":"Nijkamp Erik","year":"2022","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, and Caiming Xiong. 2022. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In The Eleventh International Conference on Learning Representations."},{"key":"e_1_3_2_1_32_1","volume-title":"Rahul Krishna, Divya Sankar, Lambert Pouguem Wassi, Michele Merler, Boris Sobolev, Raju Pavuluri, Saurabh Sinha, and Reyhaneh Jabbarvand.","author":"Pan Rangeet","year":"2023","unstructured":"Rangeet Pan, Ali Reza Ibrahimzada, Rahul Krishna, Divya Sankar, Lambert Pouguem Wassi, Michele Merler, Boris Sobolev, Raju Pavuluri, Saurabh Sinha, and Reyhaneh Jabbarvand. 2023. Understanding the Effectiveness of Large Language Models in Code Translation. http:\/\/arxiv.org\/abs\/2308.03109 arXiv:2308.03109 [cs] (Accepted by ICSE 2024)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W15-3049"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","unstructured":"Ruchir Puri David S. Kung Geert Janssen Wei Zhang Giacomo Domeniconi Vladimir Zolotov Julian Dolby Jie Chen Mihir Choudhury Lindsey Decker Veronika Thost Luca Buratti Saurabh Pujar Shyam Ramji Ulrich Finkler Susan Malaika and Frederick Reiss. 2021. CodeNet: A Large-Scale AI for Code Dataset for Learning a Diversity of Coding Tasks. arXiv:2105.12655 [cs]. 10.48550\/arXiv.2105.12655","DOI":"10.48550\/arXiv.2105.12655"},{"key":"e_1_3_2_1_36_1","unstructured":"Shuo Ren Daya Guo Shuai Lu Long Zhou Shujie Liu Duyu Tang Neel Sundaresan Ming Zhou Ambrosio Blanco and Shuai Ma. 2020. CodeBLEU: a Method for Automatic Evaluation of Code Synthesis. http:\/\/arxiv.org\/abs\/2009.10297 arXiv:2009.10297 [cs]."},{"key":"e_1_3_2_1_37_1","first-page":"20601","article-title":"Unsupervised translation of programming languages","volume":"33","author":"Roziere Baptiste","year":"2020","unstructured":"Baptiste Roziere, Marie-Anne Lachaux, Lowik Chanussot, and Guillaume Lample. 2020. Unsupervised translation of programming languages. Advances in Neural Information Processing Systems 33 (2020), 20601--20611.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_38_1","volume-title":"Leveraging automated unit tests for unsupervised code translation. arXiv preprint arXiv:2110.06773","author":"Roziere Baptiste","year":"2021","unstructured":"Baptiste Roziere, Jie M Zhang, Francois Charton, Mark Harman, Gabriel Synnaeve, and Guillaume Lample. 2021. Leveraging automated unit tests for unsupervised code translation. arXiv preprint arXiv:2110.06773 (2021)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2023. Code Llama: Open Foundation Models for Code. arXiv:2308.12950 [cs]. 10.48550\/arXiv.2308.12950","DOI":"10.48550\/arXiv.2308.12950"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884833"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","unstructured":"Marc Szafraniec Baptiste Roziere Hugh Leather Francois Charton Patrick Labatut and Gabriel Synnaeve. 2023. Code Translation with Compiler Representations. arXiv:2207.03578 [cs]. 10.48550\/arXiv.2207.03578","DOI":"10.48550\/arXiv.2207.03578"},{"key":"e_1_3_2_1_42_1","volume-title":"Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv preprint arXiv:2109.00859","author":"Wang Yue","year":"2021","unstructured":"Yue Wang, Weishi Wang, Shafiq Joty, and Steven CH Hoi. 2021. Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv preprint arXiv:2109.00859 (2021)."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2312.02120"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450656"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1910.03771"},{"key":"e_1_3_2_1_46_1","volume-title":"Codegeex: A pre-trained model for code generation with multilingual evaluations on humaneval-x. arXiv preprint arXiv:2303.17568","author":"Zheng Qinkai","year":"2023","unstructured":"Qinkai Zheng, Xiao Xia, Xu Zou, Yuxiao Dong, Shan Wang, Yufei Xue, Zihan Wang, Lei Shen, Andi Wang, Yang Li, et al. 2023. Codegeex: A pre-trained model for code generation with multilingual evaluations on humaneval-x. arXiv preprint arXiv:2303.17568 (2023)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Shuyan Zhou Uri Alon Sumit Agarwal and Graham Neubig. 2023. Code-BERTScore: Evaluating Code Generation with Pretrained Models of Code. http:\/\/arxiv.org\/abs\/2302.05527 arXiv:2302.05527 [cs].","DOI":"10.18653\/v1\/2023.emnlp-main.859"},{"key":"e_1_3_2_1_48_1","volume-title":"Xlcost: A benchmark dataset for cross-lingual code intelligence. arXiv preprint arXiv:2206.08474","author":"Zhu Ming","year":"2022","unstructured":"Ming Zhu, Aneesh Jain, Karthik Suresh, Roshan Ravindran, Sindhu Tipirneni, and Chandan K Reddy. 2022. Xlcost: A benchmark dataset for cross-lingual code intelligence. arXiv preprint arXiv:2206.08474 (2022)."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21434"}],"event":{"name":"FORGE '24: 2024 IEEE\/ACM First International Conference on AI Foundation Models and Software Engineering","location":"Lisbon Portugal","acronym":"FORGE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 2024 IEEE\/ACM First International Conference on AI Foundation Models and Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650105.3652301","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650105.3652301","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:03:43Z","timestamp":1750291423000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650105.3652301"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,14]]},"references-count":47,"alternative-id":["10.1145\/3650105.3652301","10.1145\/3650105"],"URL":"https:\/\/doi.org\/10.1145\/3650105.3652301","relation":{},"subject":[],"published":{"date-parts":[[2024,4,14]]},"assertion":[{"value":"2024-06-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}