{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,30]],"date-time":"2026-01-30T02:23:45Z","timestamp":1769739825664,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":49,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62202420"],"award-info":[{"award-number":["62202420"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3691620.3695536","type":"proceedings-article","created":{"date-parts":[[2024,10,18]],"date-time":"2024-10-18T15:39:19Z","timestamp":1729265959000},"page":"1693-1705","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["B4: Towards Optimal Assessment of Plausible Code Solutions with Plausible Tests"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8341-1467","authenticated-orcid":false,"given":"Mouxiang","family":"Chen","sequence":"first","affiliation":[{"name":"The State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, Zhejiang, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1981-1626","authenticated-orcid":false,"given":"Zhongxin","family":"Liu","sequence":"additional","affiliation":[{"name":"The State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, Zhejiang, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-8572-1346","authenticated-orcid":false,"given":"He","family":"Tao","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, Zhejiang, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9418-3095","authenticated-orcid":false,"given":"Yusu","family":"Hong","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, Zhejiang, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4367-7201","authenticated-orcid":false,"given":"David","family":"Lo","sequence":"additional","affiliation":[{"name":"Singapore Management University, Singapore, Singapore"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6302-3256","authenticated-orcid":false,"given":"Xin","family":"Xia","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, Zhejiang, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8799-6020","authenticated-orcid":false,"given":"Jianling","family":"Sun","sequence":"additional","affiliation":[{"name":"The State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, Zhejiang, China"}]}],"member":"320","published-online":{"date-parts":[[2024,10,27]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Saranya Alagarsamy Chakkrit Tantithamthavorn and Aldeida Aleti. 2023. A3Test: Assertion-Augmented Automated Test Case Generation. arXiv:2302.10352 [cs.SE]","DOI":"10.2139\/ssrn.4724885"},{"key":"e_1_3_2_1_2_1","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le and Charles Sutton. 2021. Program Synthesis with Large Language Models. arXiv:2108.07732 [cs.PL]"},{"key":"e_1_3_2_1_3_1","volume-title":"The oracle problem in software testing: A survey","author":"Barr Earl T","year":"2014","unstructured":"Earl T Barr, Mark Harman, Phil McMinn, Muzammil Shahbaz, and Shin Yoo. 2014. The oracle problem in software testing: A survey. IEEE transactions on software engineering 41, 5 (2014), 507--525."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Thomas Bayes. 1763. LII. An essay towards solving a problem in the doctrine of chances. By the late Rev. Mr. Bayes FRS communicated by Mr. Price in a letter to John Canton AMFR S. Philosophical transactions of the Royal Society of London 53 (1763) 370--418.","DOI":"10.1098\/rstl.1763.0053"},{"key":"e_1_3_2_1_5_1","volume-title":"CodeT: Code Generation with Generated Tests. In The Eleventh International Conference on Learning Representations, ICLR 2023","author":"Chen Bei","year":"2023","unstructured":"Bei Chen, Fengji Zhang, Anh Nguyen, Daoguang Zan, Zeqi Lin, Jian-Guang Lou, and Weizhu Chen. 2023. CodeT: Code Generation with Generated Tests. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1--5, 2023. OpenReview.net. https:\/\/openreview.net\/pdf?id=ktrw68Cmu9c"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"Mouxiang Chen Zhongxin Liu He Tao Yusu Hong David Lo Xin Xia and Jianling Sun. 2024. B4: Towards Optimal Assessment of Plausible Code Solutions with Plausible Tests. 10.5281\/zenodo.13737381","DOI":"10.5281\/zenodo.13737381"},{"key":"e_1_3_2_1_7_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. arXiv:2107.03374 [cs.LG]"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Yinghao Chen Zehao Hu Chen Zhi Junxiao Han Shuiguang Deng and Jianwei Yin. 2024. ChatUniTest: A Framework for LLM-Based Test Generation. arXiv:2305.04764 [cs.SE]","DOI":"10.1145\/3663529.3663801"},{"key":"e_1_3_2_1_9_1","volume-title":"Desmarais","author":"Dakhel Arghavan Moradi","year":"2023","unstructured":"Arghavan Moradi Dakhel, Amin Nikanjam, Vahid Majdinasab, Foutse Khomh, and Michel C. Desmarais. 2023. Effective Test Generation Using Pre-trained Large Language Models and Mutation Testing. arXiv:2308.16557 [cs.SE]"},{"key":"e_1_3_2_1_10_1","volume-title":"Gamma function and related functions. Handbook of mathematical functions 256","author":"Davis Philip J","year":"1972","unstructured":"Philip J Davis. 1972. Gamma function and related functions. Handbook of mathematical functions 256 (1972)."},{"key":"e_1_3_2_1_11_1","volume-title":"Optimal statistical decisions","author":"DeGroot Morris H","unstructured":"Morris H DeGroot. 2005. Optimal statistical decisions. John Wiley & Sons."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2025113.2025179"},{"key":"e_1_3_2_1_13_1","unstructured":"Daya Guo Qihao Zhu Dejian Yang Zhenda Xie Kai Dong Wentao Zhang Guanting Chen Xiao Bi Y Wu YK Li et al. 2024. DeepSeek-Coder: When the Large Language Model Meets Programming-The Rise of Code Intelligence. arXiv preprint arXiv:2401.14196 (2024)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-54444-5_100"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2009.71"},{"key":"e_1_3_2_1_16_1","unstructured":"Dan Hendrycks Steven Basart Saurav Kadavath Mantas Mazeika Akul Arora Ethan Guo Collin Burns Samir Puranik Horace He Dawn Song and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2). https:\/\/openreview.net\/forum?id=sD93GOzH3i5"},{"key":"e_1_3_2_1_17_1","volume-title":"Oh (Eds.)","volume":"35","author":"Inala Jeevana Priya","year":"2022","unstructured":"Jeevana Priya Inala, Chenglong Wang, Mei Yang, Andres Codas, Mark Encarnaci\u00f3n, Shuvendu Lahiri, Madanlal Musuvathi, and Jianfeng Gao. 2022. Fault-Aware Neural Code Rankers. In Advances in Neural Information Processing Systems, S. Koyejo, S. Mohamed, A. Agarwal, D. Belgrave, K. Cho, and A. Oh (Eds.), Vol. 35. Curran Associates, Inc., 13419--13432. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2022\/file\/5762c579d09811b7639be2389b3d07be-Paper-Conference.pdf"},{"key":"e_1_3_2_1_18_1","volume-title":"Chenglong Wang, and Jianfeng Gao.","author":"Lahiri Shuvendu K.","year":"2023","unstructured":"Shuvendu K. Lahiri, Sarah Fakhoury, Aaditya Naik, Georgios Sakkas, Saikat Chakraborty, Madanlal Musuvathi, Piali Choudhury, Curtis von Veh, Jeevana Priya Inala, Chenglong Wang, and Jianfeng Gao. 2023. Interactive Code Generation via Test-Driven User-Intent Formalization. arXiv:2208.05950 [cs.SE]"},{"key":"e_1_3_2_1_19_1","first-page":"21314","article-title":"Coderl: Mastering code generation through pretrained models and deep reinforcement learning","volume":"35","author":"Le Hung","year":"2022","unstructured":"Hung Le, Yue Wang, Akhilesh Deepak Gotmare, Silvio Savarese, and Steven Chu Hong Hoi. 2022. Coderl: Mastering code generation through pretrained models and deep reinforcement learning. Advances in Neural Information Processing Systems 35 (2022), 21314--21328.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00085"},{"key":"e_1_3_2_1_21_1","unstructured":"Raymond Li Loubna Ben allal Yangtian Zi Niklas Muennighoff Denis Kocetkov Chenghao Mou Marc Marone Christopher Akiki Jia LI Jenny Chim Qian Liu Evgenii Zheltonozhskii Terry Yue Zhuo Thomas Wang Olivier Dehaene Joel Lamy-Poirier Joao Monteiro Nicolas Gontier Ming-Ho Yee Logesh Kumar Umapathi Jian Zhu Ben Lipkin Muhtasham Oblokulov Zhiruo Wang Rudra Murthy Jason T Stillerman Siva Sankalp Patel Dmitry Abulkhanov Marco Zocca Manan Dey Zhihan Zhang Urvashi Bhattacharyya Wenhao Yu Sasha Luccioni Paulo Villegas Fedor Zhdanov Tony Lee Nadav Timor Jennifer Ding Claire S Schlesinger Hailey Schoelkopf Jan Ebert Tri Dao Mayank Mishra Alex Gu Carolyn Jane Anderson Brendan Dolan-Gavitt Danish Contractor Siva Reddy Daniel Fried Dzmitry Bahdanau Yacine Jernite Carlos Mu\u00f1oz Ferrandis Sean Hughes Thomas Wolf Arjun Guha Leandro Von Werra and Harm de Vries. 2023. StarCoder: may the source be with you! Transactions on Machine Learning Research (2023). https:\/\/openreview.net\/forum?id=KoFOg41haE Reproducibility Certification."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","unstructured":"Yujia Li David Choi Junyoung Chung Nate Kushman Julian Schrittwieser R\u00e9mi Leblond Tom Eccles James Keeling Felix Gimeno Agustin Dal Lago Thomas Hubert Peter Choy Cyprien de Masson d'Autume Igor Babuschkin Xinyun Chen Po-Sen Huang Johannes Welbl Sven Gowal Alexey Cherepanov James Molloy Daniel J. Mankowitz Esme Sutherland Robson Pushmeet Kohli Nando de Freitas Koray Kavukcuoglu and Oriol Vinyals. 2022. Competition-level code generation with AlphaCode. Science 378 6624 (2022) 1092--1097. arXiv:https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.abq1158 10.1126\/science.abq1158","DOI":"10.1126\/science.abq1158"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2020.3018481"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3516829"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2022.3183297"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00041"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00205"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00178"},{"key":"e_1_3_2_1_29_1","volume-title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=iaYcJKpY2B_","author":"Nijkamp Erik","year":"2023","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, and Caiming Xiong. 2023. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=iaYcJKpY2B_"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2007.37"},{"key":"e_1_3_2_1_31_1","volume-title":"Applied statistical decision theory","author":"Raiffa Howard","unstructured":"Howard Raiffa and Robert Schlaifer. 2000. Applied statistical decision theory. Vol. 78. John Wiley & Sons."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE56229.2023.00193"},{"key":"e_1_3_2_1_33_1","volume-title":"Leveraging Automated Unit Tests for Unsupervised Code Translation. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=cmt-6KtR4c4","author":"Roziere Baptiste","year":"2022","unstructured":"Baptiste Roziere, Jie Zhang, Francois Charton, Mark Harman, Gabriel Synnaeve, and Guillaume Lample. 2022. Leveraging Automated Unit Tests for Unsupervised Code Translation. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=cmt-6KtR4c4"},{"key":"e_1_3_2_1_34_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Romain Sauvestre Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2024. Code Llama: Open Foundation Models for Code. arXiv:2308.12950 [cs.CL]"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2023.3334955"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.231"},{"key":"e_1_3_2_1_37_1","volume-title":"On products of Gaussian random variables. arXiv preprint arXiv:1711.10516","author":"Stojanac \u017deljka","year":"2017","unstructured":"\u017deljka Stojanac, Daniel Suess, and Martin Kliesch. 2017. On products of Gaussian random variables. arXiv preprint arXiv:1711.10516 (2017)."},{"key":"e_1_3_2_1_38_1","volume-title":"Introduction to Nonparametric Estimation","author":"Tsybakov A.B.","unstructured":"A.B. Tsybakov. 2008. Introduction to Nonparametric Estimation. Springer New York. https:\/\/books.google.com.hk\/books?id=mwB8rUBsbqoC"},{"key":"e_1_3_2_1_39_1","volume-title":"Shao Kun Deng, and Neel Sundaresan","author":"Tufano Michele","year":"2021","unstructured":"Michele Tufano, Dawn Drain, Alexey Svyatkovskiy, Shao Kun Deng, and Neel Sundaresan. 2021. Unit Test Case Generation with Transformers and Focal Context. arXiv:2009.05617 [cs.SE]"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3524481.3527220"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41592-019-0686-2"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/337180.337615"},{"key":"e_1_3_2_1_43_1","volume-title":"Breakthroughs in statistics: Methodology and distribution","author":"Wilcoxon Frank","unstructured":"Frank Wilcoxon. 1992. Individual comparisons by ranking methods. In Breakthroughs in statistics: Methodology and distribution. Springer, 196--202."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2013.6693084"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/11785477_23"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106274"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Zhiqiang Yuan Yiling Lou Mingwei Liu Shiji Ding Kaixin Wang Yixuan Chen and Xin Peng. 2024. No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation. arXiv:2305.04207 [cs.SE]","DOI":"10.1145\/3660783"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.411"},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the 40th International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"41846","author":"Zhang Tianyi","year":"2023","unstructured":"Tianyi Zhang, Tao Yu, Tatsunori Hashimoto, Mike Lewis, Wen-Tau Yih, Daniel Fried, and Sida Wang. 2023. Coder Reviewer Reranking for Code Generation. In Proceedings of the 40th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, 41832--41846. https:\/\/proceedings.mlr.press\/v202\/zhang23av.html"}],"event":{"name":"ASE '24: 39th IEEE\/ACM International Conference on Automated Software Engineering","location":"Sacramento CA USA","acronym":"ASE '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695536","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3691620.3695536","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:09:39Z","timestamp":1750295379000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695536"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":49,"alternative-id":["10.1145\/3691620.3695536","10.1145\/3691620"],"URL":"https:\/\/doi.org\/10.1145\/3691620.3695536","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2024-10-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}