{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T21:19:44Z","timestamp":1783804784127,"version":"3.55.0"},"reference-count":58,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"10","license":[{"start":{"date-parts":[[2024,10,1]],"date-time":"2024-10-01T00:00:00Z","timestamp":1727740800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,10,1]],"date-time":"2024-10-01T00:00:00Z","timestamp":1727740800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,10,1]],"date-time":"2024-10-01T00:00:00Z","timestamp":1727740800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100003725","name":"National Research Foundation of Korea","doi-asserted-by":"publisher","award":["RS-2023-00208998"],"award-info":[{"award-number":["RS-2023-00208998"]}],"id":[{"id":"10.13039\/501100003725","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Engineering Research Center Program","award":["2021R1A5A1021944"],"award-info":[{"award-number":["2021R1A5A1021944"]}]},{"DOI":"10.13039\/501100010418","name":"Institute for Information and Communications Technology Promotion","doi-asserted-by":"publisher","award":["IITP2022-0-00995"],"award-info":[{"award-number":["IITP2022-0-00995"]}],"id":[{"id":"10.13039\/501100010418","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2024,10]]},"DOI":"10.1109\/tse.2024.3450837","type":"journal-article","created":{"date-parts":[[2024,9,4]],"date-time":"2024-09-04T17:47:44Z","timestamp":1725472064000},"page":"2677-2694","source":"Crossref","is-referenced-by-count":24,"title":["Evaluating Diverse Large Language Models for Automatic and General Bug Reproduction"],"prefix":"10.1109","volume":"50","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0298-5320","authenticated-orcid":false,"given":"Sungmin","family":"Kang","sequence":"first","affiliation":[{"name":"Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2706-1156","authenticated-orcid":false,"given":"Juyeon","family":"Yoon","sequence":"additional","affiliation":[{"name":"Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-7824-5881","authenticated-orcid":false,"given":"Nargiz","family":"Askarbekkyzy","sequence":"additional","affiliation":[{"name":"Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0836-6993","authenticated-orcid":false,"given":"Shin","family":"Yoo","sequence":"additional","affiliation":[{"name":"Korea Advanced Institute of Science and Technology (KAIST), Daejeon, South Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/1117696.1117704"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213870"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180175"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338935"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00030"},{"key":"ref6","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Brown","year":"2020"},{"key":"ref7","article-title":"Evaluating large language models trained on code","author":"Chen"},{"key":"ref8","article-title":"Formal reasoning and the hacker way","volume-title":"Proc. IEEE\/ACM 42nd Int. Conf. Softw. Eng. (ICSE)","author":"O\u2019Hearn"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2628055"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/icse48619.2023.00194"},{"key":"ref11","article-title":"Self-consistency improves chain of thought reasoning in language models","volume-title":"Proc. 11th Int. Conf. Learn. Representations, (ICLR)","author":"Wang","year":"2023"},{"key":"ref12","article-title":"StarCoder: May the source be with you!","volume-title":"Transactions on Machine Learning Research","author":"Li","year":"2023"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/icse-seip.2019.00015"},{"key":"ref14","article-title":"code2seq: Generating sequences from structured representations of code","volume-title":"Proc. 7th Int. Conf. Learn. Representations, (ICLR)","author":"Alon","year":"2019"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-019-09762-1"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2015.7081820"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00085"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.5317da47"},{"key":"ref19","article-title":"OpenAI GPT-3.5 model documentation.","year":"2023"},{"key":"ref20","article-title":"BLOOM: A 176B-parameter open-access multilingual language model","author":"Workshop","year":"2023"},{"key":"ref21","article-title":"Large language models are zero-shot reasoners","author":"Kojima","year":"2022"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3411763.3451760"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/2931037.2931051"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510040"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99241-9_18"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICST60714.2024.00049"},{"key":"ref27","article-title":"Evaluating large language models trained on code","author":"Chen"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.891"},{"key":"ref29","article-title":"InCoder: A generative model for code infilling and synthesis","author":"Fried","year":"2023","journal-title":"Proc. 11th Int. Conf. Learn. Representations, (ICLR)"},{"key":"ref30","article-title":"CodeGen2: Lessons for training LLMs on programming and natural languages","author":"Nijkamp","year":"2023","journal-title":"ICLR"},{"key":"ref31","first-page":"10764","article-title":"PAL: Program-aided language models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Gao","year":"2023"},{"key":"ref32","article-title":"Javalang: Pure Python Java parser and tools","author":"Thunes"},{"key":"ref33","first-page":"27","article-title":"Extracting structural information from bug reports","volume-title":"Proc. Int. Workshop Mining Softw. Repositories - (MSR)","author":"Premraj","year":"2008"},{"key":"ref34","article-title":"Unit test case generation with transformers and focal context","author":"Tufano","year":"2020"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-016-9470-4"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00125"},{"key":"ref37","article-title":"Code Llama: Open foundation models for code","author":"Roziere"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1016\/0736-5853(85)90099-1"},{"key":"ref39","article-title":"GPT-4 technical report","year":"2023"},{"key":"ref40","article-title":"Reproducible outputs with openAIs text generation API,\u201d OpenAI.","year":"2024"},{"key":"ref41","article-title":"Is GPT-4 getting worse over time?","author":"Narayanan","year":"2023"},{"key":"ref42","article-title":"Emergent abilities of large language models","volume-title":"TMLR","author":"Wei","year":"2022"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1976.233818"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/1297846.1297902"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2012.14"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"ref47","article-title":"Unit test case generation with transformers","author":"Tufano","year":"2020"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00035"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2363469"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2803206"},{"key":"ref51","first-page":"1278","article-title":"Botsing, a search-based crash reproduction framework for Java","volume-title":"Proc. 35th IEEE\/ACM Int. Conf. Automated Softw. Eng. (ASE)","author":"Derakhshanfar","year":"2020"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213869"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417928"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/icse48619.2023.00119"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3608137"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/memcod.2015.7340460"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1126\/science.abq1158"},{"key":"ref58","article-title":"CodeT: Code generation with generated tests","author":"Chen"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/10721226\/10664637.pdf?arnumber=10664637","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,18]],"date-time":"2024-10-18T17:29:39Z","timestamp":1729272579000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10664637\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10]]},"references-count":58,"journal-issue":{"issue":"10"},"URL":"https:\/\/doi.org\/10.1109\/tse.2024.3450837","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,10]]}}}