{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,16]],"date-time":"2026-09-16T08:31:31Z","timestamp":1789547491012,"version":"build-2803163510"},"reference-count":69,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61972290"],"award-info":[{"award-number":["61972290"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007834","name":"Ningbo Natural Science Foundation","doi-asserted-by":"publisher","award":["2023J292"],"award-info":[{"award-number":["2023J292"]}],"id":[{"id":"10.13039\/100007834","id-type":"DOI","asserted-by":"publisher"}]},{"name":"General Research Fund of the Research Grants Council of Hong Kong and the research funds of the City University of Hong Kong","award":["6000796"],"award-info":[{"award-number":["6000796"]}]},{"name":"General Research Fund of the Research Grants Council of Hong Kong and the research funds of the City University of Hong Kong","award":["9229109"],"award-info":[{"award-number":["9229109"]}]},{"name":"General Research Fund of the Research Grants Council of Hong Kong and the research funds of the City University of Hong Kong","award":["9229098"],"award-info":[{"award-number":["9229098"]}]},{"name":"General Research Fund of the Research Grants Council of Hong Kong and the research funds of the City University of Hong Kong","award":["9220103"],"award-info":[{"award-number":["9220103"]}]},{"name":"General Research Fund of the Research Grants Council of Hong Kong and the research funds of the City University of Hong Kong","award":["9229029"],"award-info":[{"award-number":["9229029"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1109\/tse.2024.3492204","type":"journal-article","created":{"date-parts":[[2024,11,5]],"date-time":"2024-11-05T18:31:06Z","timestamp":1730831466000},"page":"3435-3453","source":"Crossref","is-referenced-by-count":27,"title":["Fight Fire With Fire: How Much Can We Trust ChatGPT on Source Code-Related Tasks?"],"prefix":"10.1109","volume":"50","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4473-3068","authenticated-orcid":false,"given":"Xiao","family":"Yu","sequence":"first","affiliation":[{"name":"State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, Zhejiang, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-9632-4678","authenticated-orcid":false,"given":"Lei","family":"Liu","sequence":"additional","affiliation":[{"name":"Faculty of Electronic and Information Engineering, Xi’an Jiaotong University, Xi’an,, Shanxi, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0093-3292","authenticated-orcid":false,"given":"Xing","family":"Hu","sequence":"additional","affiliation":[{"name":"State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, Zhejiang, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3803-9600","authenticated-orcid":false,"given":"Jacky Wai","family":"Keung","sequence":"additional","affiliation":[{"name":"Department of Computer Science, City University of Hong Kong, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0359-0248","authenticated-orcid":false,"given":"Jin","family":"Liu","sequence":"additional","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6302-3256","authenticated-orcid":false,"given":"Xin","family":"Xia","sequence":"additional","affiliation":[{"name":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3672459"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/COMPSAC57700.2023.00117"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.810"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"ref5","article-title":"Can OpenAI codex and other large language models help us fix security bugs","author":"Pearce","year":"2021"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.151"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3650212.3680323"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/APR59189.2023.00012"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"ref10","article-title":"Security implications of large language model code assistants: A user study","author":"Sandoval","year":"2022"},{"key":"ref11","article-title":"Self-contradictory hallucinations of large language models: Evaluation, detection and mitigation","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","author":"M\u00fcndler","year":"2024"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2023.3267446"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-C.2017.42"},{"key":"ref14","article-title":"CodeQL documentation","year":"2024"},{"key":"ref15","article-title":"Multi-lingual evaluation of code generation models","volume-title":"Proc. 11th Int. Conf. Learn. Represent.","author":"Athiwaratkun","year":"2023"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599790"},{"key":"ref17","article-title":"2021 CWE top 25 most dangerous software weaknesses","author":"(MITRE)","year":"2024"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/3135932.3135941"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00125"},{"key":"ref20","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/SMC53992.2023.10394237"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00181"},{"key":"ref23","article-title":"Exploring the robustness of large language models for solving programming problems","author":"Shirafuji","year":"2023"},{"key":"ref24","article-title":"COCO: Testing code generation systems via concretized instructions","author":"Yan","year":"2023"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.eacl-main.77"},{"key":"ref26","article-title":"Adversarial GLUE: A multi-task benchmark for robustness evaluation of language models","volume-title":"Proc. Neural Inf. Process. Syst. Track Datasets Benchmarks 1","author":"Wang","year":"2021"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678670"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556953"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409756"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"961","DOI":"10.1145\/3377811.3380339","article-title":"Structure-invariant testing for machine translation","volume-title":"Proc. 42nd ACM\/IEEE Int. Conf. Softw. Eng.","author":"He","year":"2020"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2628055"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2454513"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106432"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/s11390-019-1959-z"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/3639476.3639762"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC60848.2023.00085"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW60843.2023.00058"},{"key":"ref39","article-title":"CodeT: Code generation with generated tests","volume-title":"Proc. 11th Int. Conf. Learn. Represent.","author":"Chen","year":"2023"},{"key":"ref40","article-title":"Learning performance-improving code edits","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","author":"Madaan","year":"2024"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/3672456"},{"key":"ref42","first-page":"26106","article-title":"LEVER: Learning to verify language-to-code generation with execution","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ni","year":"2023"},{"key":"ref43","first-page":"1","article-title":"Memory efficient with parameter efficient fine-tuning for code generation using quantization","volume-title":"Proc. 18th Int. Conf. Ubiquitous Inf. Manage. Commun.","author":"Ali","year":"2024"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/3643681"},{"key":"ref45","article-title":"RepoBench: Benchmarking repository-level code auto-completion systems","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","author":"Liu","year":"2024"},{"key":"ref46","article-title":"DeepSeek-Coder: When the large language model meets programming\u2014The rise of code intelligence","author":"Guo","year":"2024"},{"key":"ref47","article-title":"Teaching large language models to self-debug","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","author":"Chen","year":"2024"},{"key":"ref48","article-title":"Explainable automated debugging via large language model-driven scientific debugging","author":"Kang","year":"2023"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/3611643.3613892"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00129"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"ref52","article-title":"DeepCode AI fix: Fixing security vulnerabilities with large language models","author":"Berabi","year":"2024"},{"key":"ref53","article-title":"A survey of hallucination in large foundation models","author":"Rawte","year":"2023"},{"key":"ref54","article-title":"In ChatGPT we trust? Measuring and characterizing the reliability of ChatGPT","author":"Shen","year":"2023"},{"key":"ref55","article-title":"Exploring and evaluating hallucinations in LLM-powered code generation","author":"Liu","year":"2024"},{"key":"ref56","article-title":"CodeHalu: Code hallucinations in LLMs driven by execution-based verification","author":"Tian","year":"2024"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.991"},{"key":"ref58","article-title":"The scope of ChatGPT in software engineering: A thorough investigation","author":"Ma","year":"2023"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-55642-5_4"},{"key":"ref60","article-title":"Large language models in fault localisation","author":"Wu","year":"2023"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3660771"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3608137"},{"key":"ref63","article-title":"LLM-powered test case generation for detecting tricky bugs","author":"Liu","year":"2024"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1007\/s11704-024-40231-1"},{"key":"ref65","first-page":"24950","article-title":"DetectGPT: Zero-shot machine-generated text detection using probability curvature","volume-title":"Proc. Int. Conf. Mach. Learn.","volume":"202","author":"Mitchell","year":"2023"},{"key":"ref66","article-title":"How close is ChatGPT to human experts? Comparison corpus, evaluation, detection","author":"Guo","year":"2023"},{"key":"ref67","article-title":"ArguGPT: Evaluating, understanding and identifying argumentative essays generated by GPT models","author":"Liu","year":"2023"},{"key":"ref68","article-title":"Differentiate ChatGPT-generated and human-written medical texts","author":"Liao","year":"2023"},{"key":"ref69","article-title":"Evaluating AIGC detectors on code content","author":"Wang","year":"2023"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/10794440\/10745266.pdf?arnumber=10745266","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,12]],"date-time":"2024-12-12T20:49:29Z","timestamp":1734036569000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10745266\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12]]},"references-count":69,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tse.2024.3492204","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,12]]}}}