{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,13]],"date-time":"2026-06-13T16:05:03Z","timestamp":1781366703336,"version":"3.54.1"},"reference-count":51,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,3,31]],"date-time":"2025-03-31T00:00:00Z","timestamp":1743379200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,3,31]],"date-time":"2025-03-31T00:00:00Z","timestamp":1743379200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,3,31]]},"DOI":"10.1109\/icst62969.2025.10988968","type":"proceedings-article","created":{"date-parts":[[2025,5,20]],"date-time":"2025-05-20T17:05:21Z","timestamp":1747760721000},"page":"103-114","source":"Crossref","is-referenced-by-count":30,"title":["Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities"],"prefix":"10.1109","author":[{"given":"Avishree","family":"Khare","sequence":"first","affiliation":[{"name":"University of Pennsylvania,Philadelphia,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Saikat","family":"Dutta","sequence":"additional","affiliation":[{"name":"Cornell University,Ithaca,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ziyang","family":"Li","sequence":"additional","affiliation":[{"name":"University of Pennsylvania,Philadelphia,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alaia","family":"Solko-Breslin","sequence":"additional","affiliation":[{"name":"University of Pennsylvania,Philadelphia,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rajeev","family":"Alur","sequence":"additional","affiliation":[{"name":"University of Pennsylvania,Philadelphia,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mayur","family":"Naik","sequence":"additional","affiliation":[{"name":"University of Pennsylvania,Philadelphia,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref3","volume-title":"Microsoft: 70 percent of all security bugs are memory safety issues","author":"Miller","year":"2019"},{"key":"ref4","author":"Manes","year":"2018","journal-title":"Fuzzing: Art, science, and engineering"},{"key":"ref5","article-title":"Ql: Object-oriented queries on relational data","volume-title":"European Conference on Object-Oriented Programming","author":"Avgustinov","year":"2016"},{"key":"ref6","article-title":"The semgrep platform","volume-title":"Semgrep","year":"2023"},{"key":"ref7","article-title":"Vulnerabilities discovered by CodeQL","volume-title":"Semmle","year":"2023"},{"key":"ref8","volume-title":"Mindshare: When mysql cluster encounters taint analysis","author":"Leong","year":"2022"},{"key":"ref9","article-title":"The bug slayer","volume-title":"GitHub","year":"2023"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623345"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3528452"},{"key":"ref12","author":"Achiam","year":"2023","journal-title":"Gpt-4 technical report"},{"key":"ref13","author":"Roziere","year":"2023","journal-title":"Code llama: Open foundation models for code"},{"key":"ref14","volume":"abs\/2107.03374","author":"Chen","year":"2021","journal-title":"Evaluating large language models trained on code"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00129"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i4.25642"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549101"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00085"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3597926.3598067"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3611643.3616350"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623342"},{"key":"ref22","article-title":"Emergent abilities of large language models","volume":"2022","author":"Wei","year":"2022","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref23","author":"Bubeck","year":"2023","journal-title":"Sparks of artificial general intelligence: Early experiments with gpt-4"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1108\/ws.2000.07949fab.004"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/3639476.3639762"},{"key":"ref26","author":"Gao","year":"2023","journal-title":"How far have we gone in vulnerability detection using large language models"},{"key":"ref27","author":"Steenhoek","year":"2024","journal-title":"A comprehensive study of the capabilities of large language models for vulnerability detection"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/sp54263.2024.00210"},{"key":"ref29","author":"Ding","year":"2024","journal-title":"Vulnerability detection with code language models: How far are we?"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3475960.3475985"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2012.345"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.6028\/nist.tn.1995"},{"key":"ref36","doi-asserted-by":"crossref","first-page":"508","DOI":"10.1145\/3379597.3387501","article-title":"A c\/c++ code vulnerability dataset with code changes and eve summaries","volume-title":"Proceedings of the 17th International Conference on Mining Software Repositories, ser. MSR \u201920","author":"Fan","year":"2020"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3087402"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/3607199.3607242"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00189"},{"key":"ref40","author":"Team","year":"2024","journal-title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context"},{"key":"ref41","author":"Dubey","year":"2024","journal-title":"The llama 3 herd of models"},{"key":"ref43","author":"Guo","year":"2024","journal-title":"Deepseek-coder: When the large language model meets programming-the rise of code intelligence"},{"key":"ref44","author":"Zhu","year":"2024","journal-title":"Deepseek-coder-v2: Breaking the barrier of closed-source models in code intelligence"},{"key":"ref45","author":"Yang","year":"2024","journal-title":"Qwen2 technical report"},{"key":"ref46","author":"Hui","year":"2024","journal-title":"Qwen2. 5-coder technical report"},{"key":"ref48","article-title":"In-context impersonation reveals large language models\u2019 strengths and biases","volume":"36","author":"Salewski","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref49","first-page":"24 824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Wei","year":"2022","journal-title":"Advances in neural information processing systems"},{"key":"ref56","article-title":"Devign: Effective vul-nerability identification by learning comprehensive program semantics via graph neural networks","author":"Zhou","year":"2019","journal-title":"Neural Information Processing Systems"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3087402"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3527949"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468597"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2021.3076142"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2021.3051525"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534371"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1145\/3564625.3567985"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/3649828"}],"event":{"name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","location":"Napoli, Italy","start":{"date-parts":[[2025,3,31]]},"end":{"date-parts":[[2025,4,4]]}},"container-title":["2025 IEEE Conference on Software Testing, Verification and Validation (ICST)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/10988917\/10988918\/10988968.pdf?arnumber=10988968","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,21]],"date-time":"2025-05-21T05:10:44Z","timestamp":1747804244000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10988968\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,31]]},"references-count":51,"URL":"https:\/\/doi.org\/10.1109\/icst62969.2025.10988968","relation":{},"subject":[],"published":{"date-parts":[[2025,3,31]]}}}