{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T12:21:48Z","timestamp":1783513308955,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T00:00:00Z","timestamp":1775952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"NSF","award":["CCF-1846467"],"award-info":[{"award-number":["CCF-1846467"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,12]]},"DOI":"10.1145\/3786582.3786809","type":"proceedings-article","created":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T11:52:47Z","timestamp":1783511567000},"page":"51-55","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Automatic Validation of LLM-Generated Code with Prompt Paraphrasing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9079-5534","authenticated-orcid":false,"given":"Xiaoyin","family":"Wang","sequence":"first","affiliation":[{"name":"The University of Texas at San Antonio, San Antonio, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1938-9947","authenticated-orcid":false,"given":"Dakai","family":"Zhu","sequence":"additional","affiliation":[{"name":"The University of Texas at San Antonio, San Antonio, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,8]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"crossref","unstructured":"Gerardo Adesso. 2022. GPT4: The ultimate brain. Authorea Preprints (2022).","DOI":"10.22541\/au.167052124.41804127\/v1"},{"key":"e_1_3_3_2_3_2","first-page":"140","volume-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","author":"Ahlgren John","year":"2021","unstructured":"John Ahlgren, Maria Berezin, Kinga Bojarczuk, Elena Dulskyte, Inna Dvortsova, Johann George, Natalija Gucevska, Mark Harman, Maria Lomeli, Erik Meijer, et\u00a0al. 2021. Testing web enabled simulation at scale using metamorphic testing. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP). IEEE, 140\u2013149."},{"key":"e_1_3_3_2_4_2","unstructured":"AI Anthropic. 2024. Claude 3.5 sonnet model card addendum. Claude-3.5 Model Card 3 (2024) 1\u20138."},{"key":"e_1_3_3_2_5_2","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le et\u00a0al. 2021. Program synthesis with large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2108.07732 (2021)."},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134020"},{"key":"e_1_3_3_2_7_2","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de\u00a0Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman et\u00a0al. 2021. Evaluating large language models trained on code. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2107.03374 (2021)."},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"crossref","unstructured":"Tsong\u00a0Yueh Chen Fei-Ching Kuo Huai Liu Pak-Lok Poon Dave Towey TH Tse and Zhi\u00a0Quan Zhou. 2018. Metamorphic testing: A review of challenges and opportunities. ACM Computing Surveys (CSUR) 51 1 (2018) 1\u201327.","DOI":"10.1145\/3143561"},{"key":"e_1_3_3_2_9_2","first-page":"174","volume-title":"2025 IEEE International Conference on Software Maintenance and Evolution (ICSME)","author":"Cho Steven","year":"2025","unstructured":"Steven Cho, Stefano Ruberto, and Valerio Terragni. 2025. Metamorphic testing of large language models for natural language processing. In 2025 IEEE International Conference on Software Maintenance and Evolution (ICSME). IEEE, 174\u2013186."},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"crossref","unstructured":"Juan Cruz-Benito Sanjay Vishwakarma Francisco Martin-Fernandez and Ismael Faro. 2021. Automated source code generation and auto-completion using deep learning: Comparing and discussing current language model-related approaches. AI 2 1 (2021) 1\u201316.","DOI":"10.3390\/ai2010001"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/2896971.2896978"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"crossref","unstructured":"Andrea Fioraldi Alessandro Mantovani Dominik Maier and Davide Balzarotti. 2023. Dissecting American fuzzy lop: a FuzzBench evaluation. ACM Transactions on Software Engineering and Methodology 32 2 (2023) 1\u201326.","DOI":"10.1145\/3580596"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"crossref","first-page":"404","DOI":"10.1109\/QSIC.2007.4385527","volume-title":"Seventh International Conference on Quality Software (QSIC 2007)","author":"Guderlei Ralph","year":"2007","unstructured":"Ralph Guderlei and Johannes Mayer. 2007. Statistical metamorphic testing testing programs with random output by means of statistical hypothesis tests and metamorphic testing. In Seventh International Conference on Quality Software (QSIC 2007). IEEE, 404\u2013409."},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Abram Hindle Earl\u00a0T Barr Mark Gabel Zhendong Su and Premkumar Devanbu. 2016. On the naturalness of software. Commun. ACM 59 5 (2016) 122\u2013131.","DOI":"10.1145\/2902362"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238191"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICST60714.2024.00019"},{"key":"e_1_3_3_2_17_2","volume-title":"International Conference on Automated Software Engineering (NIER Track)","author":"Jahan Sadia","year":"2025","unstructured":"Sadia Jahan and Xiaoyin Wang. 2025. How Does ChatGPT Make Assumptions When Creating Erroneous Programs?. In International Conference on Automated Software Engineering (NIER Track). IEEE."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"crossref","unstructured":"Vu Le Mehrdad Afshari and Zhendong Su. 2014. Compiler validation via equivalence modulo inputs. ACM Sigplan Notices 49 6 (2014) 216\u2013226.","DOI":"10.1145\/2666356.2594334"},{"key":"e_1_3_3_2_19_2","unstructured":"Jiawei Liu Chunqiu\u00a0Steven Xia Yuyao Wang and Lingming Zhang. 2023. Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2305.01210 (2023)."},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238216"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/64"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2011.100"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2018.8330220"},{"key":"e_1_3_3_2_24_2","first-page":"26106","volume-title":"International Conference on Machine Learning","author":"Ni Ansong","year":"2023","unstructured":"Ansong Ni, Srini Iyer, Dragomir Radev, Veselin Stoyanov, Wen-tau Yih, Sida Wang, and Xi\u00a0Victoria Lin. 2023. Lever: Learning to verify language-to-code generation with execution. In International Conference on Machine Learning. PMLR, 26106\u201326128."},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"crossref","unstructured":"Xintao Niu Changhai Nie Hareton Leung Yu Lei Xiaoyin Wang Jiaxi Xu and Yan Wang. 2018. An interleaving approach to combinatorial testing and failure-inducing interaction identification. IEEE Transactions on Software Engineering 46 6 (2018) 584\u2013615.","DOI":"10.1109\/TSE.2018.2865772"},{"key":"e_1_3_3_2_26_2","unstructured":"Shuyin Ouyang Jie\u00a0M Zhang Mark Harman and Meng Wang. 2023. LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2308.02828 (2023)."},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"crossref","unstructured":"Qiwei Peng Yekun Chai and Xuhong Li. 2024. Humaneval-xl: A multilingual code generation benchmark for cross-lingual natural language generalization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.16694 (2024).","DOI":"10.63317\/2zjsm6sdd5yo"},{"key":"e_1_3_3_2_28_2","unstructured":"Python-AFL. [n. d.]. Python-AFL. https:\/\/github.com\/jwilk\/python-afl."},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"crossref","unstructured":"Hassan Shahmohammadi MirHossein Dezfoulian and Muharram Mansoorizadeh. 2021. Paraphrase detection using LSTM networks and handcrafted features. Multimedia Tools and Applications 80 (2021) 6479\u20136492.","DOI":"10.1007\/s11042-020-09996-y"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.5555\/1667583.1667644"},{"key":"e_1_3_3_2_32_2","unstructured":"Sam Witteveen and Martin Andrews. 2019. Paraphrasing with large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1911.09661 (2019)."},{"key":"e_1_3_3_2_33_2","first-page":"17","volume-title":"2019 IEEE\/ACM 4th International Workshop on Metamorphic Testing (MET)","author":"Wu Chaohua","year":"2019","unstructured":"Chaohua Wu, Liqun Sun, and Zhi\u00a0Quan Zhou. 2019. The impact of a dot: Case studies of a noise metamorphic relation pattern. In 2019 IEEE\/ACM 4th International Workshop on Metamorphic Testing (MET). IEEE, 17\u201323."},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"crossref","unstructured":"Xin Xia Lingfeng Bao David Lo Zhenchang Xing Ahmed\u00a0E Hassan and Shanping Li. 2017. Measuring program comprehension: A large-scale field study with professionals. IEEE Transactions on Software Engineering 44 10 (2017) 951\u2013976.","DOI":"10.1109\/TSE.2017.2734091"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3561157"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"crossref","unstructured":"Xueling Zhang John Heaps Rocky Slavin Jianwei Niu Travis Breaux and Xiaoyin Wang. 2023. Daisy: Dynamic-analysis-induced source discovery for sensitive data. ACM Transactions on Software Engineering and Methodology 32 4 (2023) 1\u201334.","DOI":"10.1145\/3569936"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"crossref","first-page":"796","DOI":"10.1109\/SP40001.2021.00040","volume-title":"2021 IEEE Symposium on Security and Privacy (SP)","author":"Zhang Xueling","year":"2021","unstructured":"Xueling Zhang, Xiaoyin Wang, Rocky Slavin, and Jianwei Niu. 2021. Condysta: Context-aware dynamic supplement to static taint analysis. In 2021 IEEE Symposium on Security and Privacy (SP). IEEE, 796\u2013812."},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00043"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"crossref","unstructured":"Zhi\u00a0Quan Zhou Liqun Sun Tsong\u00a0Yueh Chen and Dave Towey. 2018. Metamorphic relations for enhancing system understanding and use. IEEE Transactions on Software Engineering 46 10 (2018) 1120\u20131154.","DOI":"10.1109\/TSE.2018.2876433"}],"event":{"name":"ICSE-NIER '26: 2026 IEEE\/ACM 48th International Conference on Software Engineering","location":"Rio de Janeiro Brazil","acronym":"ICSE-NIER '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the IEEE\/ACM 48th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786582.3786809","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T11:54:26Z","timestamp":1783511666000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786582.3786809"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,12]]},"references-count":38,"alternative-id":["10.1145\/3786582.3786809","10.1145\/3786582"],"URL":"https:\/\/doi.org\/10.1145\/3786582.3786809","relation":{},"subject":[],"published":{"date-parts":[[2026,4,12]]},"assertion":[{"value":"2026-07-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}