{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,6]],"date-time":"2026-10-06T04:23:08Z","timestamp":1791260588470,"version":"4.1.0"},"reference-count":55,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,10,1]],"date-time":"2024-10-01T00:00:00Z","timestamp":1727740800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,10,1]],"date-time":"2024-10-01T00:00:00Z","timestamp":1727740800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"crossref","award":["RGPIN-2019-05071"],"award-info":[{"award-number":["RGPIN-2019-05071"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,11]]},"DOI":"10.1007\/s10664-024-10548-3","type":"journal-article","created":{"date-parts":[[2024,10,1]],"date-time":"2024-10-01T08:02:21Z","timestamp":1727769741000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":32,"title":["An empirical study on the effectiveness of large language models for SATD identification and classification"],"prefix":"10.1007","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4219-642X","authenticated-orcid":false,"given":"Mohammad Sadegh","family":"Sheikhaei","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuan","family":"Tian","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shaowei","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bowen","family":"Xu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,10,1]]},"reference":[{"key":"10548_CR1","doi-asserted-by":"crossref","unstructured":"Aiken W, Mvula PK, Branco P, Jourdan G-V, Sabetzadeh M, Viktor H (2023) Measuring improvement of f 1-scores in detection of self-admitted technical debt. In: 2023 ACM\/IEEE international conference on technical debt (TechDebt), pp 37\u201341. IEEE","DOI":"10.1109\/TechDebt59074.2023.00011"},{"key":"10548_CR2","doi-asserted-by":"crossref","unstructured":"Bavota G, Russo B (2016) A large-scale empirical study on self-admitted technical debt. In: 2016 IEEE\/ACM 13th Working conference on Mining Software Repositories (MSR), pp 315\u2013326","DOI":"10.1145\/2901739.2901742"},{"key":"10548_CR3","unstructured":"Bhatia A, Khomh F, Adams B, Hassan AE (2023) An empirical study of self-admitted technical debt in machine learning software"},{"key":"10548_CR4","unstructured":"Brown T, Mann B, Ryder N, Subbiah M, Kaplan JD, Dhariwal P, Neelakantan A, Shyam P, Sastry G, Askell A, Agarwal S, Herbert-Voss A, Krueger G, Henighan T, Child R, Ramesh A, Ziegler D, Wu J, Winter C, Hesse C, Chen M, Sigler E, Litwin M, Gray S, Chess B, Clark J, Berner C, McCandlish S, Radford A, Sutskever I, Amodei D (2020) Language models are few-shot learners. In: Larochelle H, Ranzato M, Hadsell R, Balcan M, Lin H (eds) Adv Neural Inf Process Syst, vol 33. Curran Associates Inc, pp 1877\u20131901"},{"issue":"6","key":"10548_CR5","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1109\/MS.2011.150","volume":"28","author":"F Buschmann","year":"2011","unstructured":"Buschmann F (2011) To pay or not to pay technical debt. IEEE Softw 28(6):29\u201331","journal-title":"IEEE Softw"},{"key":"10548_CR6","doi-asserted-by":"crossref","unstructured":"Cassee N, Zampetti F, Novielli N, Serebrenik A, Di\u00a0Penta M (2022) Self-admitted technical debt and comments\u2019 polarity: an empirical study. Empirical Softw Engg 27(6)","DOI":"10.1007\/s10664-022-10183-w"},{"key":"10548_CR7","doi-asserted-by":"crossref","unstructured":"Chang Y, Wang X, Wang J, Wu Y, Yang L, Zhu K, Chen H, Yi X, Wang C, Wang Y, Ye W, Zhang Y, Chang Y, Yu PS, Yang Q, Xie X (2024) A survey on evaluation of large language models. ACM Trans Intell Syst Technol 15(3)","DOI":"10.1145\/3641289"},{"issue":"3","key":"10548_CR8","doi-asserted-by":"publisher","first-page":"1309","DOI":"10.1109\/TR.2021.3087864","volume":"71","author":"X Chen","year":"2022","unstructured":"Chen X, Yu D, Fan X, Wang L, Chen J (2022) Multiclass classification for self-admitted technical debt based on xgboost. IEEE Trans Reliab 71(3):1309\u20131324","journal-title":"IEEE Trans Reliab"},{"key":"10548_CR9","unstructured":"Chung HW, Hou L, Longpre S, Zoph B, Tay Y, Fedus W, Li Y, Wang X, Dehghani M, Brahma S, Webson A, Gu SS, Dai Z, Suzgun M, Chen X, Chowdhery A, Castro-Ros A, Pellat M, Robinson K, Valter D, Narang S, Mishra G, Yu A, Zhao V, Huang Y, Dai A, Yu H, Petrov S, Chi EH, Dean J, Devlin J, Roberts A, Zhou D, Le QV, Wei J (2022) Scaling instruction-finetuned language models"},{"key":"10548_CR10","doi-asserted-by":"crossref","unstructured":"Cunningham W (1992) The wycash portfolio management system. In: Addendum to the Proceedings on Object-Oriented Programming Systems, Languages, and Applications (Addendum), OOPSLA \u201992, pp 29\u201330, New York, NY, USA. Association for Computing Machinery","DOI":"10.1145\/157709.157715"},{"key":"10548_CR11","unstructured":"Devlin J, Chang M-W, Lee K, Toutanova K (2019) Bert: Pre-training of deep bidirectional transformers for language understanding. In: North American Chapter of the association for computational linguistics"},{"key":"10548_CR12","doi-asserted-by":"publisher","first-page":"1536","DOI":"10.18653\/v1\/2020.findings-emnlp.139","volume-title":"Findings of the association for computational linguistics: EMNLP 2020","author":"Z Feng","year":"2020","unstructured":"Feng Z, Guo D, Tang D, Duan N, Feng X, Gong M, Shou L, Qin B, Liu T, Jiang D, Zhou M (2020) CodeBERT: A pre-trained model for programming and natural languages. In: Cohn T, He Y, Liu Y (eds) Findings of the association for computational linguistics: EMNLP 2020. Online. Association for Computational Linguistics, pp 1536\u20131547"},{"key":"10548_CR13","doi-asserted-by":"crossref","unstructured":"Fucci G, Cassee N, Zampetti F, Novielli N, Serebrenik A, Di\u00a0Penta M (2021) Waiting around or job half-done? sentiment in self-admitted technical debt. In: 2021 IEEE\/ACM 18th International conference on Mining Software Repositories (MSR), pp 403\u2013414","DOI":"10.1109\/MSR52588.2021.00052"},{"key":"10548_CR14","doi-asserted-by":"crossref","unstructured":"Gao S, Wen X, Gao C, Wang W, Zhang H, Lyu MR (2023) What makes good in-context demonstrations for code intelligence tasks with llms? In: 2023 38th IEEE\/ACM International conference on Automated Software Engineering (ASE), pp 761\u2013773, Los Alamitos, CA, USA. IEEE Computer Society","DOI":"10.1109\/ASE56229.2023.00109"},{"key":"10548_CR15","volume-title":"Deep Learning","author":"I Goodfellow","year":"2016","unstructured":"Goodfellow I, Bengio Y, Courville A (2016) Deep Learning. MIT Press"},{"key":"10548_CR16","doi-asserted-by":"crossref","unstructured":"Guo Z, Liu S, Liu J, Li Y, Chen L, Lu H, Zhou Y (2021) How far have we progressed in identifying self-admitted technical debts? A comprehensive empirical study. ACM Trans Softw Eng Methodol 30(4)","DOI":"10.1145\/3447247"},{"key":"10548_CR17","doi-asserted-by":"crossref","unstructured":"Hou X, Zhao Y, Liu Y, Yang Z, Wang K, Li L, Luo X, Lo D, Grundy JC, Wang H (2023) Large language models for software engineering: a systematic literature review. arxiv:2308.10620","DOI":"10.1145\/3695988"},{"key":"10548_CR18","doi-asserted-by":"publisher","first-page":"418","DOI":"10.1007\/s10664-017-9522-4","volume":"23","author":"Q Huang","year":"2017","unstructured":"Huang Q, Shihab E, Xia X, Lo D, Li S (2017) Identifying self-admitted technical debt in open source projects using text mining. Empir Softw Eng 23:418\u2013451","journal-title":"Empir Softw Eng"},{"key":"10548_CR19","unstructured":"Jiang Z, Liu J, Chen Z, Li Y, Huang J, Huo Y, He P, Gu J, Lyu MR (2023) Llmparser: a llm-based log parsing framework"},{"key":"10548_CR20","doi-asserted-by":"crossref","unstructured":"Jin M, Shahriar S, Tufano M, Shi X, Lu S, Sundaresan N, Svyatkovskiy A (2023) Inferfix: end-to-end program repair with llms. arXiv preprint arXiv:2303.07263","DOI":"10.1145\/3611643.3613892"},{"key":"10548_CR21","doi-asserted-by":"crossref","unstructured":"Li Y, Soliman M, Avgeriou P (2022) Identifying self-admitted technical debt in issue tracking systems using machine learning. Empirical Softw Engg 27(6)","DOI":"10.1007\/s10664-022-10128-3"},{"key":"10548_CR22","doi-asserted-by":"crossref","unstructured":"Li Y, Soliman M, Avgeriou P (2023) Automatic identification of self-admitted technical debt from four different sources. Empirical Softw Engg 28(65)","DOI":"10.1007\/s10664-023-10297-9"},{"key":"10548_CR23","doi-asserted-by":"crossref","unstructured":"Liu J, Huang Q, Xia X, Shihab E, Lo D, Li S (2020) Is using deep learning frameworks free? Characterizing technical debt in deep learning frameworks. In: 2020 IEEE\/ACM 42nd International Conference on Software Engineering: Software Engineering in Society (ICSE-SEIS), pp 1\u201310","DOI":"10.1145\/3377815.3381377"},{"key":"10548_CR24","unstructured":"Liu Y, Ott M, Goyal N, Du J, Joshi M, Chen D, Levy O, Lewis M, Zettlemoyer L, Stoyanov V (2019) Roberta: a robustly optimized bert pretraining approach"},{"key":"10548_CR25","doi-asserted-by":"crossref","unstructured":"Liu Z, Huang Q, Xia X, Shihab E, Lo D, Li S (2018) Satd detector: a text-mining-based self-admitted technical debt detection tool. In: 2018 IEEE\/ACM 40th International Conference on Software Engineering: Companion (ICSE-Companion), pp 9\u201312","DOI":"10.1145\/3183440.3183478"},{"key":"10548_CR26","unstructured":"Loshchilov I, Hutter F (2019) Decoupled weight decay regularization. In: International conference on learning representations"},{"key":"10548_CR27","doi-asserted-by":"crossref","unstructured":"Maldonado EdS, Shihab E (2015) Detecting and quantifying different types of self-admitted technical debt. In: 2015 IEEE 7th International workshop on Managing Technical Debt (MTD), pp 9\u201315","DOI":"10.1109\/MTD.2015.7332619"},{"key":"10548_CR28","doi-asserted-by":"crossref","unstructured":"Maldonado E, d S, Shihab E, Tsantalis N (2017) Using natural language processing to automatically detect self-admitted technical debt. IEEE Trans Software Eng 43(11):1044\u20131062","DOI":"10.1109\/TSE.2017.2654244"},{"key":"10548_CR29","doi-asserted-by":"crossref","unstructured":"Manning C, Klein D (2003) Optimization, maxent models, and conditional estimation without magic. In: Companion volume of the proceedings of HLT-NAACL 2003 - Tutorial Abstracts, pp 8\u20138","DOI":"10.3115\/1075168.1075176"},{"key":"10548_CR30","unstructured":"Minaee S, Mikolov T, Nikzad N, Chenaghlu MA, Socher R, Amatriain X, Gao J (2024) Large language models: a survey. arxiv:2402.06196"},{"key":"10548_CR31","unstructured":"Naveed H, Khan AU, Qiu S, Saqib M, Anwar S, Usman M, Barnes N, Mian AS (2023) A comprehensive overview of large language models. arxiv:2307.06435"},{"key":"10548_CR32","doi-asserted-by":"crossref","unstructured":"OBrien D, Biswas S, Imtiaz S, Abdalkareem R, Shihab E, Rajan H (2022) 23 shades of self-admitted technical debt: an empirical study on machine learning software. In: Proceedings of the 30th ACM joint European software engineering conference and symposium on the foundations of software engineering, ESEC\/FSE 2022, pp 734\u2013746, New York, USA. Association for Computing Machinery","DOI":"10.1145\/3540250.3549088"},{"key":"10548_CR33","doi-asserted-by":"crossref","unstructured":"Pinna A, Lunesu MI, Orr\u00f9 S, Tonelli R (2023) Investigation on self-admitted technical debt in open-source blockchain projects. Future Internet, 15(7)","DOI":"10.3390\/fi15070232"},{"key":"10548_CR34","doi-asserted-by":"crossref","unstructured":"Potdar A, Shihab E (2014) An exploratory study on self-admitted technical debt. In: 2014 IEEE International conference on software maintenance and evolution, pp 91\u2013100","DOI":"10.1109\/ICSME.2014.31"},{"issue":"12","key":"10548_CR35","first-page":"5050","volume":"48","author":"J Prenner","year":"2022","unstructured":"Prenner J, Robbes R (2022) Making the most of small software engineering datasets with modern machine learning. IEEE Trans Software Eng 48(12):5050\u20135067","journal-title":"IEEE Trans Software Eng"},{"key":"10548_CR36","unstructured":"Radford A, Wu J, Child R, Luan D, Amodei D, Sutskever I (2019) Language models are unsupervised multitask learners"},{"key":"10548_CR37","first-page":"140:1","volume":"21","author":"C Raffel","year":"2019","unstructured":"Raffel C, Shazeer NM, Roberts A, Lee K, Narang S, Matena M, Zhou Y, Li W, Liu PJ (2019) Exploring the limits of transfer learning with a unified text-to-text transformer. J Mach Learn Res 21:140:1-140:67","journal-title":"J Mach Learn Res"},{"key":"10548_CR38","doi-asserted-by":"crossref","unstructured":"Rantala L, M\u00e4ntyl\u00e4 M, Lo D (2020) Prevalence, contents and automatic detection of kl-satd. In: 2020 46th Euromicro conference on Software Engineering and Advanced Applications (SEAA), pp 385\u2013388","DOI":"10.1109\/SEAA51224.2020.00069"},{"key":"10548_CR39","doi-asserted-by":"crossref","unstructured":"Ren X, Xing Z, Xia X, Lo D, Wang X, Grundy J (2019) Neural network-based detection of self-admitted technical debt: from performance to explainability. 28(3)","DOI":"10.1145\/3324916"},{"key":"10548_CR40","doi-asserted-by":"crossref","unstructured":"Sheikhaei M S, Tian Y (2023) Automated self-admitted technical debt tracking at commit-level: a language-independent approach. In: 2023 ACM\/IEEE International conference on Technical Debt (TechDebt), pp 22\u201326","DOI":"10.1109\/TechDebt59074.2023.00009"},{"key":"10548_CR41","doi-asserted-by":"crossref","unstructured":"Sridharan M, Rantala L, M\u00e4ntyl\u00e4 M (2023) Pentacet data-23 million contextual code comments and 250,000 satd comments. In: 2023 IEEE\/ACM 20th International conference on Mining Software Repositories (MSR), pp 412\u2013416. IEEE","DOI":"10.1109\/MSR59073.2023.00063"},{"key":"10548_CR42","doi-asserted-by":"crossref","unstructured":"Tam D, Mascarenhas A, Zhang S, Kwan S, Bansal M, Raffel C (2023) Evaluating the factual consistency of large language models through news summarization. In: Rogers A, Boyd-Graber J, Okazaki N (eds) Findings of the Association for Computational Linguistics: ACL 2023, pp 5220\u20135255, Toronto, Canada. Association for Computational Linguistics","DOI":"10.18653\/v1\/2023.findings-acl.322"},{"key":"10548_CR43","unstructured":"Tate K (2005) Sustainable software development: an agile perspective. Addison-Wesley Professional"},{"key":"10548_CR44","doi-asserted-by":"publisher","first-page":"2298","DOI":"10.1007\/s10664-015-9409-1","volume":"21","author":"Y Tian","year":"2016","unstructured":"Tian Y, Ali N, Lo D, Hassan AE (2016) On the unreliability of bug severity data. Empir Softw Eng 21:2298\u20132323","journal-title":"Empir Softw Eng"},{"key":"10548_CR45","unstructured":"Touvron H, Lavril T, Izacard G, Martinet X, Lachaux M-A, Lacroix T, Rozi\u00e8re B, Goyal N, Hambro E, Azhar F, Rodriguez A, Joulin A, Grave E, Lample G (2023) Llama: open and efficient foundation language models"},{"key":"10548_CR46","unstructured":"Trask A, Michalak P, Liu JC (2015) sense2vec - a fast and accurate method for word sense disambiguation in neural word embeddings. arxiv:1511.06388"},{"key":"10548_CR47","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser L u, Polosukhin I (2017) Attention is all you need. In: Guyon I, Luxburg UV, Bengio S, Wallach H, Fergus R, Vishwanathan S, Garnett R (eds) Advances in Neural Information Processing Systems, vol 30. Curran Associates Inc"},{"key":"10548_CR48","doi-asserted-by":"crossref","unstructured":"Wehaibi S, Shihab E, Guerrouj L (2016) Examining the impact of self-admitted technical debt on software quality. In: 2016 IEEE 23Rd international conference on software analysis, evolution, and reengineering (SANER), vol 1, pp 179\u2013188. IEEE","DOI":"10.1109\/SANER.2016.72"},{"key":"10548_CR49","unstructured":"Wei Y, Wang Z, Liu J, Ding Y, Zhang L (2023) Magicoder: Source code is all you need. arXiv preprint arXiv:2312.02120"},{"key":"10548_CR50","doi-asserted-by":"crossref","unstructured":"Xavier L, Ferreira F, Brito R, Valente MT (2020) Beyond the code: mining self-admitted technical debt in issue tracker systems. In: Proceedings of the 17th international conference on mining software repositories, MSR \u201920, pp 137\u2013146, New York, USA. Association for Computing Machinery","DOI":"10.1145\/3379597.3387459"},{"key":"10548_CR51","doi-asserted-by":"crossref","unstructured":"Xiao T, Wang D, McIntosh S, Hata H, Kula RG, Ishio T, ichi Matsumoto K (2021) Characterizing and mitigating self-admitted technical debt in build systems. IEEE Trans Software Eng 48:4214\u20134228","DOI":"10.1109\/TSE.2021.3115772"},{"issue":"4","key":"10548_CR52","doi-asserted-by":"publisher","first-page":"154208","DOI":"10.1007\/s11704-020-9281-z","volume":"15","author":"D Yu","year":"2021","unstructured":"Yu D, Wang L, Chen X, Chen J (2021) Using bilstm with attention mechanism to automatically detect self-admitted technical debt. Front Comp Sci 15(4):154208","journal-title":"Front Comp Sci"},{"issue":"5","key":"10548_CR53","doi-asserted-by":"publisher","first-page":"1676","DOI":"10.1109\/TSE.2020.3031401","volume":"48","author":"Z Yu","year":"2022","unstructured":"Yu Z, Fahid FM, Tu H, Menzies T (2022) Identifying self-admitted technical debts with jitterbug: a two-step approach. IEEE Trans Softw Eng 48(5):1676\u20131691","journal-title":"IEEE Trans Softw Eng"},{"key":"10548_CR54","unstructured":"Yuan Z, Liu J, Zi Q, Liu M, Peng X, Lou Y (2023) Evaluating instruction-tuned large language models on code comprehension and generation. arXiv preprint arXiv:2308.01240"},{"key":"10548_CR55","doi-asserted-by":"crossref","unstructured":"Zhang W, Deng Y, Liu B, Pan SJ, Bing L (2023) Sentiment analysis in the era of large language models: a reality check","DOI":"10.18653\/v1\/2024.findings-naacl.246"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10548-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10548-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10548-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,5]],"date-time":"2024-11-05T23:54:45Z","timestamp":1730850885000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10548-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,1]]},"references-count":55,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,11]]}},"alternative-id":["10548"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10548-3","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,10,1]]},"assertion":[{"value":"19 September 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 October 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}}],"article-number":"159"}}