{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,14]],"date-time":"2026-02-14T03:10:22Z","timestamp":1771038622004,"version":"3.50.1"},"reference-count":112,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Research Grants Council of the Hong Kong Special Administrative Region, China","award":["General Research Fund under Project CUHK 14206921"],"award-info":[{"award-number":["General Research Fund under Project CUHK 14206921"]}]},{"DOI":"10.13039\/501100021171","name":"Guangdong Basic and Applied Basic Research Foundation","doi-asserted-by":"publisher","award":["2024A1515010145"],"award-info":[{"award-number":["2024A1515010145"]}],"id":[{"id":"10.13039\/501100021171","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Canada Research Chair and Discovery Grant programs of the Natural Sciences and Engineering Research Council of Canada"},{"DOI":"10.13039\/501100001602","name":"Science Foundation Ireland","doi-asserted-by":"publisher","award":["13\/RC\/2094-2"],"award-info":[{"award-number":["13\/RC\/2094-2"]}],"id":[{"id":"10.13039\/501100001602","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1109\/tse.2024.3475375","type":"journal-article","created":{"date-parts":[[2024,10,8]],"date-time":"2024-10-08T17:37:35Z","timestamp":1728409055000},"page":"3188-3207","source":"Crossref","is-referenced-by-count":23,"title":["Exploring the Effectiveness of LLMs in Automated Logging Statement Generation: An Empirical Study"],"prefix":"10.1109","volume":"50","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-8370-644X","authenticated-orcid":false,"given":"Yichen","family":"Li","sequence":"first","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-8798-5667","authenticated-orcid":false,"given":"Yintong","family":"Huo","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-1988-6219","authenticated-orcid":false,"given":"Zhihan","family":"Jiang","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6626-4437","authenticated-orcid":false,"given":"Renyi","family":"Zhong","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3377-8129","authenticated-orcid":false,"given":"Pinjia","family":"He","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3338-8561","authenticated-orcid":false,"given":"Yuxin","family":"Su","sequence":"additional","affiliation":[{"name":"School of Software Engineering, Sun Yat-sen University, Zhuhai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1393-1010","authenticated-orcid":false,"given":"Lionel C.","family":"Briand","sequence":"additional","affiliation":[{"name":"University of Ottawa, Ottawa, ON, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3666-5798","authenticated-orcid":false,"given":"Michael R.","family":"Lyu","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong, China"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00080"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/1629575.1629587"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00082"},{"key":"ref4","article-title":"Evlog: Evolving log analyzer for anomalous logs identification","author":"Huo","year":"2023"},{"key":"ref5","article-title":"Scalable and adaptive log-based anomaly detection with expert in the loop","author":"Liu","year":"2023"},{"key":"ref6","article-title":"Impact of log parsing on log-based anomaly detection","author":"Khan","year":"2023"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3460345"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678596"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2012.6227202"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238193"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00131"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534379"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2941943"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-016-9456-2"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3511561"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1007\/s11023-020-09548-1"},{"key":"ref17","article-title":"RoBERTa: A robustly optimized bert pretraining approach","author":"Liu","year":"2019"},{"key":"ref18","article-title":"InCoder: A generative model for code infilling and synthesis","author":"Fried","year":"2022"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.499"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00129"},{"key":"ref21","article-title":"GitHub Copilot: Parrot or crow? A first look at rote learning in GitHub Copilot suggestions","volume-title":"Mar.","year":"2023"},{"key":"ref22","article-title":"GitHub Copilot: Your AI pair programmer","year":"2023"},{"key":"ref23","article-title":"CodeWhisperer","year":"2023"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534862"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.107066"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00125"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/2591062.2591175"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.60"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-018-9595-8"},{"key":"ref30","first-page":"361","article-title":"Where shall we log? Studying and suggesting logging locations in code blocks","volume-title":"Proc. 35th IEEE\/ACM Int. Conf. Automated Softw. Eng. (ASE)","author":"Li","year":"2020"},{"key":"ref31","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Ouyang","year":"2022"},{"key":"ref32","article-title":"GPT-3.5","year":"2022"},{"key":"ref33","first-page":"4302","article-title":"Deep reinforcement learning from human preferences","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"30","author":"Christiano","year":"2017"},{"key":"ref34","article-title":"ChatGPT","year":"2023"},{"key":"ref35","article-title":"GPT-4o","year":"2024"},{"key":"ref36","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"ref37","article-title":"The llama 3 herd of models","author":"Dubey","year":"2024"},{"key":"ref38","article-title":"Replication package of lance","year":"2022"},{"key":"ref39","article-title":"CodeGeeX","year":"2023"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1108\/ws.2000.07949fab.004"},{"key":"ref41","article-title":"Code llama: Open foundation models for code","author":"Roziere","year":"2023"},{"key":"ref42","article-title":"Tabnine","year":"2023"},{"key":"ref43","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/3624740"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2019.00079"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623326"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/3624740"},{"key":"ref48","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref49","article-title":"Measuring massive multitask language understanding","author":"Hendrycks","year":"2020"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/ASE56229.2023.00133"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.bigscience-1.9"},{"key":"ref52","article-title":"GPT-J","year":"2022"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-47240-4_19"},{"key":"ref54","article-title":"News summarization and evaluation in the era of GPT-3","author":"Goyal","year":"2022"},{"key":"ref55","article-title":"A comprehensive capability analysis of GPT-3 and GPT-3.5 series models","author":"Ye","year":"2023"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.411"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380408"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238214"},{"key":"ref59","article-title":"The pile: An 800GB dataset of diverse text for language modeling","author":"Gao","year":"2020"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3639476.3639764"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3597926.3598135"},{"key":"ref62","article-title":"Top leaderboard ranking= top coding proficiency, always? Evoeval: Evolving coding benchmarks via LLM","author":"Xia","year":"2024"},{"key":"ref63","first-page":"479","article-title":"Misleading authorship attribution of source code using adversarial learning","volume-title":"Proc. USENIX Secur. Symp. (USENIX Secur.)","author":"Quiring","year":"2019"},{"key":"ref64","article-title":"A closer look into transformer-based code intelligence through code transformation: Challenges and opportunities","author":"Li","year":"2022"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/icse48619.2023.00110"},{"key":"ref66","article-title":"Towards making deep learning-based vulnerability detectors robust","author":"Li","year":"2021"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3133917"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/ICSESS47205.2019.9040853"},{"key":"ref69","article-title":"Code obfuscation literature survey","volume":"19","author":"Balakrishnan","year":"2005","journal-title":"CS701 Construction Compilers"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/3611643.3616272"},{"key":"ref71","article-title":"JavaParser","year":"2019"},{"key":"ref72","article-title":"Deepinfra","year":"2024"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510101"},{"key":"ref74","article-title":"Prompting for automatic log template extraction","author":"Xu","year":"2023"},{"key":"ref75","article-title":"Do not give away my secrets: Uncovering the privacy issue of neural code completion tools","author":"Huang","year":"2023"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/SANER53432.2022.00051"},{"key":"ref77","first-page":"311","article-title":"BLEU: A method for automatic evaluation of machine translation","volume-title":"Proc. 40th Annu. Meeting Assoc. Comput. Linguistics (ACL)","author":"Papineni","year":"2002"},{"key":"ref78","first-page":"74","article-title":"ROUGE: A package for automatic evaluation of summaries","volume-title":"Text summarization Branches Out","author":"Lin","year":"2004"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/ase56229.2023.00109"},{"key":"ref80","article-title":"CrossCodeEval: A diverse and multilingual benchmark for cross-file code completion","author":"Ding","year":"2023"},{"key":"ref81","article-title":"OpenAI embeddings","year":"2023"},{"key":"ref82","article-title":"Exploring the capabilities of LLMs for code change related tasks","author":"Fan","year":"2024"},{"key":"ref83","article-title":"Evaluating instruction-tuned large language models on code comprehension and generation","author":"Yuan","year":"2023"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00181"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238206"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-30826-0_14"},{"key":"ref87","article-title":"Beyond BLEU: Training neural machine translation with semantic similarity","author":"Wieting","year":"2019"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00151"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236068"},{"key":"ref90","first-page":"31210","article-title":"Large language models can be easily distracted by irrelevant context","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Shi","year":"2023"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00638"},{"key":"ref92","article-title":"Learning to retrieve prompts for in-context learning","author":"Rubin","year":"2021"},{"key":"ref93","article-title":"Chain of thought prompting elicits reasoning in large language models","author":"Wei","year":"2022"},{"key":"ref94","article-title":"Large language models are zero-shot reasoners","author":"Kojima","year":"2022"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/TNSM.2022.3209317"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1145\/2110356.2110360"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132778"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/COMPSAC.2016.149"},{"key":"ref100","first-page":"293","article-title":"Be conservative: Enhancing failure diagnosis with proactive logging","volume-title":"Proc. 10th USENIX Symp. Oper. Syst. Des. Implementation (OSDI)","author":"Yuan","year":"2012"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1145\/3448976"},{"key":"ref102","first-page":"139","article-title":"Log2: A cost-aware logging mechanism for performance diagnosis","volume-title":"Proc. USENIX Annu. Tech. Conf. (USENIX ATC)","author":"Ding","year":"2015"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2017.15"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.145"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-017-9518-0"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1002\/smr.1579"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.728"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref110","article-title":"GraphcodeBERT: Pre-training code representations with data flow","author":"Guo","year":"2020"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678927"},{"key":"ref112","article-title":"AIxcoder","year":"2023"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/10794440\/10707668.pdf?arnumber=10707668","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,15]],"date-time":"2025-01-15T19:53:53Z","timestamp":1736970833000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10707668\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12]]},"references-count":112,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tse.2024.3475375","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,12]]}}}