{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T11:36:25Z","timestamp":1791459385590,"version":"4.3.4"},"publisher-location":"New York, NY, USA","reference-count":41,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Ministry of Education - Singapore","award":["MOE-MOET32021-0001"],"award-info":[{"award-number":["MOE-MOET32021-0001"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,11]]},"DOI":"10.1145\/3650212.3680308","type":"proceedings-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T11:44:25Z","timestamp":1726055065000},"page":"628-640","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["Oracle-Guided Program Selection from Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8165-9493","authenticated-orcid":false,"given":"Zhiyu","family":"Fan","sequence":"first","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1080-4770","authenticated-orcid":false,"given":"Haifeng","family":"Ruan","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6088-4993","authenticated-orcid":false,"given":"Sergey","family":"Mechtaev","sequence":"additional","affiliation":[{"name":"Peking University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7127-1137","authenticated-orcid":false,"given":"Abhik","family":"Roychoudhury","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2023. Amazon CodeWhisperer. https:\/\/aws.amazon.com\/codewhisperer\/"},{"key":"e_1_3_2_1_2_1","unstructured":"2023. CrossHair: An analysis tool for Python that blurs the line between testing and type systems.. https:\/\/github.com\/pschanely\/CrossHair"},{"key":"e_1_3_2_1_3_1","unstructured":"2024. Introducing ChatGPT. https:\/\/openai.com\/blog\/chatgpt. Accessed: 2023-11-21"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/fmcad.2013.6679385"},{"key":"e_1_3_2_1_5_1","volume-title":"Program Synthesis with Large Language Models.. arXiv.org, abs\/2108.07732","author":"Austin Jacob","year":"2021","unstructured":"Jacob Austin, Augustus Odena, Maxwell I. Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie J. Cai, Michael Terry, Quoc V. Le, and Charles Sutton. 2021. Program Synthesis with Large Language Models.. arXiv.org, abs\/2108.07732 (2021), 8, issn:2331-8422 arxiv:2108.07732"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-39176-7_7"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409757"},{"key":"e_1_3_2_1_8_1","volume-title":"The oracle problem in software testing: A survey","author":"Barr Earl T","year":"2014","unstructured":"Earl T Barr, Mark Harman, Phil McMinn, Muzammil Shahbaz, and Shin Yoo. 2014. The oracle problem in software testing: A survey. IEEE transactions on software engineering, 41, 5 (2014), 507\u2013525."},{"key":"e_1_3_2_1_9_1","volume-title":"Language Models are Few-Shot Learners.. Neural Information Processing Systems, abs\/2005.14165","author":"Brown Tom B.","year":"2020","unstructured":"Tom B. Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel M. Ziegler, Jeffrey Wu, Clemens Winter, Christopher Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners.. Neural Information Processing Systems, abs\/2005.14165 (2020), 5, 1877\u20131901. https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html"},{"key":"e_1_3_2_1_10_1","volume-title":"The Eleventh International Conference on Learning Representations, ICLR 2023","author":"Chen Bei","year":"2023","unstructured":"Bei Chen, Fengji Zhang, Anh Nguyen, Daoguang Zan, Zeqi Lin, Jian-Guang Lou, and Weizhu Chen. 2023. CodeT: Code Generation with Generated Tests.. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1-5, 2023. OpenReview.net. https:\/\/openreview.net\/pdf?id=ktrw68Cmu9c"},{"key":"e_1_3_2_1_11_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Pond\u00e9 de Oliveira Pinto Jared Kaplan Harrison Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Joshua Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code.. arXiv.org abs\/2107.03374 (2021) 7 issn:2331-8422 arXiv:2107.03374. arxiv:2107.03374"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2304.05128"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2908080.2908095"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"e_1_3_2_1_15_1","volume-title":"Automating string processing in spreadsheets using input-output examples.. ACM SIGPLAN Notices, 46, 1","author":"Gulwani Sumit","year":"2011","unstructured":"Sumit Gulwani. 2011. Automating string processing in spreadsheets using input-output examples.. ACM SIGPLAN Notices, 46, 1 (2011), 1, 317\u2013330. issn:0362-1340"},{"key":"e_1_3_2_1_16_1","volume-title":"Spreadsheet data manipulation using examples.. Commun. ACM, 55, 8","author":"Gulwani Sumit","year":"2012","unstructured":"Sumit Gulwani, William R. Harris, and Rishabh Singh. 2012. Spreadsheet data manipulation using examples.. Commun. ACM, 55, 8 (2012), 8, 97\u2013105. issn:0001-0782"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1806799.1806833"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2306.02907"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00194"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2210.00848"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00085"},{"key":"e_1_3_2_1_22_1","first-page":"3843","article-title":"Solving quantitative reasoning problems with language models","volume":"35","author":"Lewkowycz Aitor","year":"2022","unstructured":"Aitor Lewkowycz, Anders Andreassen, David Dohan, Ethan Dyer, Henryk Michalewski, Vinay Ramasesh, Ambrose Slone, Cem Anil, Imanol Schlag, and Theo Gutman-Solo. 2022. Solving quantitative reasoning problems with language models. Advances in Neural Information Processing Systems, 35 (2022), 3843\u20133857.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"Jia Li Ge Li Yongming Li and Zhi Jin. 2023. Structured Chain-of-Thought Prompting for Code Generation. arxiv:2305.06599.","DOI":"10.1145\/3690635"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"Yujia Li David H. Choi Junyoung Chung Nate Kushman Julian Schrittwieser R\u00e9mi Leblond Tom Eccles James Keeling Felix Gimeno Agustin Dal Lago Thomas Hubert Peter Choy Cyprien de Masson d\u2019Autume Igor Babuschkin Xinyun Chen Po-Sen Huang Johannes Welbl Sven Gowal Alexey Cherepanov James Molloy Daniel J. Mankowitz Esme Sutherland Robson Pushmeet Kohli Nando de Freitas Koray Kavukcuoglu and Oriol Vinyals. 2022. Competition-Level Code Generation with AlphaCode.. Science abs\/2203.07814 6624 (2022) 12 1092\u20131097. issn:0036-8075 https:\/\/doi.org\/10.48550\/arxiv.2203.07814 10.48550\/arxiv.2203.07814","DOI":"10.48550\/arxiv.2203.07814"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2305.01210"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.21105\/joss.01891"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2303.17651"},{"key":"e_1_3_2_1_29_1","volume-title":"Type-and-example-directed program synthesis.. ACM SIGPLAN Notices, 50, 6","author":"Osera Peter-Michael","year":"2015","unstructured":"Peter-Michael Osera and Steve Zdancewic. 2015. Type-and-example-directed program synthesis.. ACM SIGPLAN Notices, 50, 6 (2015), 8, 619\u2013630. issn:0362-1340"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"Shuyin Ouyang Jie M Zhang Mark Harman and Meng Wang. 2023. LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation. arXiv preprint arXiv:2308.02828.","DOI":"10.1145\/3697010"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","unstructured":"Brayan Stiven Torrres Ovalle. 2023. GitHub Copilot. https:\/\/doi.org\/10.26507\/paper.2300 10.26507\/paper.2300","DOI":"10.26507\/paper.2300"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1453101.1453131"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton-Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2023. Code Llama: Open Foundation Models for Code.. arXiv.org abs\/2308.12950 (2023) 8 issn:2331-8422 https:\/\/doi.org\/10.48550\/arxiv.2308.12950 10.48550\/arxiv.2308.12950","DOI":"10.48550\/arxiv.2308.12950"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017055"},{"key":"e_1_3_2_1_35_1","volume-title":"Shao Kun Deng, and Neel Sundaresan","author":"Tufano Michele","year":"2020","unstructured":"Michele Tufano, Dawn Drain, Alexey Svyatkovskiy, Shao Kun Deng, and Neel Sundaresan. 2020. Unit Test Case Generation with Transformers and Focal Context. arXiv preprint arXiv:2009.05617, 9."},{"key":"e_1_3_2_1_36_1","volume-title":"Quoc Le, and Denny Zhou.","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Ed H. Chi, Quoc Le, and Denny Zhou. 2022. Chain of Thought Prompting Elicits Reasoning in Large Language Models.. Neural Information Processing Systems, abs\/2201.11903 (2022), 1, arxiv:2201.11903"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2308.04748"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","unstructured":"Zhuokui Xie Yinghao Chen Chen Zhi Shuiguang Deng and Jianwei Yin. 2023. ChatUniTest: a ChatGPT-based automated unit test generation tool.. issn:2331-8422 https:\/\/doi.org\/10.48550\/arxiv.2305.04764 arxiv:2305.04764. 10.48550\/arxiv.2305.04764","DOI":"10.48550\/arxiv.2305.04764"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","unstructured":"Pengcheng Yin and Graham Neubig. 2017. A Syntactic Neural Model for General-Purpose Code Generation.. arXiv preprint arXiv:1704.01696 440\u2013450. https:\/\/doi.org\/10.18653\/v1\/p17-1041 10.18653\/v1\/p17-1041","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623316"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2305.04207"},{"key":"e_1_3_2_1_42_1","volume-title":"Autocoderover: Autonomous program improvement. arXiv preprint arXiv:2404.05427.","author":"Zhang Yuntong","year":"2024","unstructured":"Yuntong Zhang, Haifeng Ruan, Zhiyu Fan, and Abhik Roychoudhury. 2024. Autocoderover: Autonomous program improvement. arXiv preprint arXiv:2404.05427."}],"event":{"name":"ISSTA '24: 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Vienna Austria","acronym":"ISSTA '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","AITO"]},"container-title":["Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680308","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650212.3680308","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:50:07Z","timestamp":1750287007000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680308"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":41,"alternative-id":["10.1145\/3650212.3680308","10.1145\/3650212"],"URL":"https:\/\/doi.org\/10.1145\/3650212.3680308","relation":{},"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"2024-09-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}