{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,5]],"date-time":"2026-02-05T09:08:30Z","timestamp":1770282510299,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":85,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CCF-2131943 and CCF-2141474"],"award-info":[{"award-number":["CCF-2131943 and CCF-2141474"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["No.62002125 and No.61932021"],"award-info":[{"award-number":["No.62002125 and No.61932021"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Young Elite Scientists Sponsorship Program by CAST","award":["No. 2021QNRC001"],"award-info":[{"award-number":["No. 2021QNRC001"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3616331","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"1203-1215","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["A Large-Scale Empirical Review of Patch Correctness Checking Approaches"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-3657-6169","authenticated-orcid":false,"given":"Jun","family":"Yang","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-0988-9835","authenticated-orcid":false,"given":"Yuehan","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4066-3365","authenticated-orcid":false,"given":"Yiling","family":"Lou","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5588-9618","authenticated-orcid":false,"given":"Ming","family":"Wen","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology, Wuhan, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5175-2702","authenticated-orcid":false,"given":"Lingming","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"2023. JD-Core. https:\/\/github.com\/java-decompiler\/jd-core"},{"key":"e_1_3_2_2_2_1","unstructured":"2023. Patch manual inspection rules. https:\/\/github.com\/claudeyj\/patch_correctness\/blob\/master\/semantic_equivalence_rules.md"},{"key":"e_1_3_2_2_3_1","unstructured":"2023. Pitest tool. https:\/\/github.com\/hcoles\/pitest"},{"key":"e_1_3_2_2_4_1","volume-title":"Niklas Muennighoff, Mayank Mishra, Alex Gu, and Manan Dey.","author":"Allal Loubna Ben","year":"2023","unstructured":"Loubna Ben Allal, Raymond Li, Denis Kocetkov, Chenghao Mou, Christopher Akiki, Carlos Munoz Ferrandis, Niklas Muennighoff, Mayank Mishra, Alex Gu, and Manan Dey. 2023. SantaCoder: don\u2019t reach for the stars!. arXiv preprint arXiv:2301.03988."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2019.00050"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416566"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2628058"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2017.8115674"},{"key":"e_1_3_2_2_9_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, and Greg Brockman.","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, and Greg Brockman. 2021. Evaluating Large Language Models Trained on Code. CoRR, abs\/2107.03374 (2021), arXiv:2107.03374. arxiv:2107.03374"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","unstructured":"Z. Chen S. Kommrusch M. Tufano L. Pouchet D. Poshyvanyk and M. Monperrus. 2021. SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair. TSE 47 09 (2021) sep 1943\u20131959. issn:1939-3520 https:\/\/doi.org\/10.1109\/TSE.2019.2940179 10.1109\/TSE.2019.2940179","DOI":"10.1109\/TSE.2019.2940179"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/IBF50092.2020.9034714"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/APR52552.2021.00010"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1143844.1143874"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10479-005-5724-z"},{"key":"e_1_3_2_2_15_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. CoRR, abs\/1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. CoRR, abs\/1810.04805 (2018), arxiv:1810.04805. arxiv:1810.04805"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338911"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2896921.2896931"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2025113.2025179"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330559"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/2337223.2337322"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380361"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180245"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213871"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00069"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2628055"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379597.3387491"},{"key":"e_1_3_2_2_27_1","volume-title":"Patch Generation with Language Models: Feasibility and Scaling Behavior. In Deep Learning for Code Workshop. https:\/\/openreview.net\/forum?id=rHlzJh_b1-5","author":"Kolak Sophia D","year":"2022","unstructured":"Sophia D Kolak, Ruben Martins, Claire Le Goues, and Vincent Josua Hellendoorn. 2022. Patch Generation with Language Models: Feasibility and Scaling Behavior. In Deep Learning for Code Workshop. https:\/\/openreview.net\/forum?id=rHlzJh_b1-5"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-019-09780-z"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00064"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106309"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2016.76"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2011.104"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380345"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3505247"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2019.00020"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2019.8667970"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330577"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380338"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2018.8330202"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786811"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397351"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397369"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"crossref","unstructured":"Paula Maddigan and Teo Susnjak. 2023. Chat2VIS: Generating Data Visualisations via Natural Language using ChatGPT Codex and GPT-3 Large Language Models. arxiv:2302.02094.","DOI":"10.1109\/ACCESS.2023.3274199"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","unstructured":"A. Marginean J. Bader S. Chandra M. Harman Y. Jia K. Mao A. Mols and A. Scott. 2019. SapFix: Automated End-to-End Repair at Scale. In ICSE-SEIP 2019. IEEE Press 269\u2013278. https:\/\/doi.org\/10.1109\/ICSE-SEIP.2019.00039 10.1109\/ICSE-SEIP.2019.00039","DOI":"10.1109\/ICSE-SEIP.2019.00039"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2931037.2948705"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99241-9_3"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884807"},{"key":"e_1_3_2_2_49_1","unstructured":"Martin Monperrus. 2020. The living review on automated program repair."},{"key":"e_1_3_2_2_50_1","unstructured":"Erik Nijkamp Bo Pang Hiroaki Hayashi Lifu Tu Huan Wang Yingbo Zhou Silvio Savarese and Caiming Xiong. 2023. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. arxiv:2203.13474."},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510040"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/1297846.1297902"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.1509"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568254"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2771783.2771791"},{"key":"e_1_3_2_2_56_1","volume-title":"Counterexample-Guided Quantifier Instantiation for Synthesis in SMT","author":"Reynolds Andrew","unstructured":"Andrew Reynolds, Morgan Deters, Viktor Kuncak, Cesare Tinelli, and Clark Barrett. 2015. Counterexample-Guided Quantifier Instantiation for Synthesis in SMT. In Computer Aided Verification, Daniel Kroening and Corina S. P\u0103s\u0103reanu (Eds.). Springer International Publishing, Cham. 198\u2013216. isbn:978-3-319-21668-3"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2017.8115675"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00020"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0118432"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/1595696.1595750"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786825"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/2950290.2950295"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416532"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.14778\/3551793.3551841"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180250"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416590"},{"key":"e_1_3_2_2_67_1","volume-title":"Hoi","author":"Wang Yue","year":"2021","unstructured":"Yue Wang, Weishi Wang, Shafiq R. Joty, and Steven C. H. Hoi. 2021. CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation. CoRR, abs\/2109.00859 (2021), arXiv:2109.00859. arxiv:2109.00859"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2013.6693094"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2013.6693094"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180233"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00129"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549101"},{"key":"e_1_3_2_2_73_1","unstructured":"Chunqiu Steven Xia and Lingming Zhang. 2023. Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT. arxiv:2304.00385."},{"key":"e_1_3_2_2_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3092703.3092718"},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.1109\/GI.2019.00012"},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180182"},{"key":"e_1_3_2_2_77_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2017.45"},{"key":"e_1_3_2_2_78_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2560811"},{"key":"e_1_3_2_2_79_1","doi-asserted-by":"publisher","unstructured":"Jun Yang Yuehan Wang Yiling Lou Ming Wen and Lingming Zhang. 2023. ESEC\/FSE\u201923 Artifact for \"A Large-Scale Empirical Review of Patch Correctness Checking Approaches\". https:\/\/doi.org\/10.5281\/zenodo.8267827 10.5281\/zenodo.8267827","DOI":"10.5281\/zenodo.8267827"},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","unstructured":"Jinqiu Yang Alexey Zhikhartsev Yuefei Liu and Lin Tan. 2017. Better test cases for better automated program repair. 831\u2013841. isbn:978-1-4503-5105-8 https:\/\/doi.org\/10.1145\/3106237.3106274 10.1145\/3106237.3106274","DOI":"10.1145\/3106237.3106274"},{"key":"e_1_3_2_2_81_1","volume-title":"Automated Classification of Overfitting Patches with Statically Extracted Code Features. CoRR, abs\/1910.12057","author":"Ye He","year":"2019","unstructured":"He Ye, Jian Gu, Matias Martinez, Thomas Durieux, and Martin Monperrus. 2019. Automated Classification of Overfitting Patches with Statically Extracted Code Features. CoRR, abs\/1910.12057 (2019), arxiv:1910.12057. arxiv:1910.12057"},{"key":"e_1_3_2_2_82_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09920-w"},{"key":"e_1_3_2_2_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3182517"},{"key":"e_1_3_2_2_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2874648"},{"key":"e_1_3_2_2_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468544"}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616331","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3616331","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:11Z","timestamp":1750178171000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616331"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":85,"alternative-id":["10.1145\/3611643.3616331","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3616331","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}