{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T17:48:46Z","timestamp":1782928126296,"version":"3.54.5"},"reference-count":48,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T00:00:00Z","timestamp":1780531200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information and Software Technology"],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1016\/j.infsof.2026.108204","type":"journal-article","created":{"date-parts":[[2026,6,8]],"date-time":"2026-06-08T16:37:04Z","timestamp":1780936624000},"page":"108204","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"special_numbering":"C","title":["LLM4SCREENLIT: Recommendations on assessing the performance of large language models for screening literature in systematic reviews"],"prefix":"10.1016","volume":"198","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3907-3357","authenticated-orcid":false,"given":"Lech","family":"Madeyski","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6134-8460","authenticated-orcid":false,"given":"Barbara","family":"Kitchenham","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1874-6145","authenticated-orcid":false,"given":"Martin","family":"Shepperd","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.infsof.2026.108204_b1","series-title":"Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering","first-page":"262","article-title":"The promise and challenges of using LLMs to accelerate the screening process of systematic reviews","author":"Huotala","year":"2024"},{"key":"10.1016\/j.infsof.2026.108204_b2","series-title":"2025 ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement","first-page":"01","article-title":"SESR-eval: Dataset for evaluating LLMs in the title-abstract screening of systematic reviews","author":"Huotala","year":"2025"},{"key":"10.1016\/j.infsof.2026.108204_b3","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2025.107757","article-title":"Exploring the use of LLMs for the selection phase in systematic literature studies","volume":"184","author":"Thode","year":"2025","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2026.108204_b4","series-title":"Proceedings of the 18th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement","first-page":"25","article-title":"ChatGPT application in systematic literature reviews in software engineering: an evaluation of its accuracy to support the selection activity","author":"Felizardo","year":"2024"},{"key":"10.1016\/j.infsof.2026.108204_b5","series-title":"Assessing the ability of ChatGPT to screen articles for systematic reviews","author":"Syriani","year":"2023"},{"key":"10.1016\/j.infsof.2026.108204_b6","article-title":"Screening articles for systematic reviews with ChatGPT","volume":"80","author":"Syriani","year":"2024","journal-title":"J. Comput. Lang."},{"key":"10.1016\/j.infsof.2026.108204_b7","doi-asserted-by":"crossref","DOI":"10.1073\/pnas.2411962122","article-title":"Transforming literature screening: The emerging role of large language models in systematic reviews","volume":"122","author":"Delgado-Chaves","year":"2025","journal-title":"Proc. Natl. Acad. Sci."},{"key":"10.1016\/j.infsof.2026.108204_b8","doi-asserted-by":"crossref","first-page":"158","DOI":"10.1186\/s13643-024-02575-4","article-title":"Title and abstract screening for literature reviews using large language models: an exploratory study in the biomedical domain","volume":"13","author":"Dennst\u00e4dt","year":"2024","journal-title":"Syst. Rev."},{"key":"10.1016\/j.infsof.2026.108204_b9","doi-asserted-by":"crossref","first-page":"616","DOI":"10.1002\/jrsm.1715","article-title":"Can large language models replace humans in systematic reviews? Evaluating GPT-4\u2019s efficacy in screening and extracting data from peer-reviewed and grey literature in multiple languages","volume":"15","author":"Khraisha","year":"2024","journal-title":"Res Synth Methods"},{"key":"10.1016\/j.infsof.2026.108204_b10","doi-asserted-by":"crossref","first-page":"130","DOI":"10.1186\/s12874-025-02583-5","article-title":"Streamlining systematic reviews with large language models using prompt engineering and retrieval augmented generation","volume":"25","author":"Trad","year":"2025","journal-title":"BMC Med. Res. Methodol."},{"key":"10.1016\/j.infsof.2026.108204_b11","doi-asserted-by":"crossref","first-page":"893","DOI":"10.1093\/jamia\/ocaf050","article-title":"High-performance automated abstract screening with large language model ensembles","volume":"32","author":"Sanghera","year":"2025","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.infsof.2026.108204_b12","doi-asserted-by":"crossref","first-page":"1071","DOI":"10.1093\/jamia\/ocaf063","article-title":"The emergence of large language models as tools in literature reviews: a large language model-assisted systematic review","volume":"32","author":"Scherbakov","year":"2025","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.infsof.2026.108204_b13","series-title":"Advances in Information Retrieval","first-page":"403","article-title":"Zero-shot generative large language models for systematic review screening automation","author":"Wang","year":"2024"},{"key":"10.1016\/j.infsof.2026.108204_b14","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2024.20496","article-title":"Performance of a large language model in screening citations","volume":"7","author":"Oami","year":"2024","journal-title":"JAMA Netw. Open"},{"key":"10.1016\/j.infsof.2026.108204_b15","doi-asserted-by":"crossref","DOI":"10.21037\/jmai-24-408","article-title":"Evaluating large language models for title\/abstract screening: A systematic review and meta-analysis & development of new tool","author":"Kim","year":"2025","journal-title":"J. Med. Artif. Intell."},{"key":"10.1016\/j.infsof.2026.108204_b16","series-title":"Proceedings of the 14th International Conference on Data Science, Technology and Applications - Volume 1: DATA, INSTICC","first-page":"508","article-title":"Evaluating large language models for literature screening: A systematic review of sensitivity and workload reduction","author":"Sandner","year":"2025"},{"key":"10.1016\/j.infsof.2026.108204_b17","article-title":"Evaluation: from precision, recall and F-measure to ROC, informedness, markedness and correlation","volume":"2","author":"Powers","year":"2011","journal-title":"Int. J. Mach. Learn. Technol."},{"key":"10.1016\/j.infsof.2026.108204_b18","doi-asserted-by":"crossref","first-page":"423","DOI":"10.1016\/0895-4356(93)90018-V","article-title":"Bias, prevalence and kappa","volume":"46","author":"Byrt","year":"1993","journal-title":"J. Clin. Epidemiol."},{"key":"10.1016\/j.infsof.2026.108204_b19","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/1471-2288-9-5","article-title":"Measuring agreement of administrative data with chart data using prevalence unadjusted and adjusted kappa","volume":"9","author":"Chen","year":"2009","journal-title":"BMC Med. Res. Methodol."},{"key":"10.1016\/j.infsof.2026.108204_b20","doi-asserted-by":"crossref","first-page":"442","DOI":"10.1016\/0005-2795(75)90109-9","article-title":"Comparison of the predicted and observed secondary structure of T4 phage lysozyme","volume":"405","author":"Matthews","year":"1975","journal-title":"Biochim. Biophys. Acta (BBA)-Protein Struct."},{"key":"10.1016\/j.infsof.2026.108204_b21","doi-asserted-by":"crossref","first-page":"216","DOI":"10.1016\/j.patcog.2019.02.023","article-title":"The impact of class imbalance in classification performance metrics based on the binary confusion matrix","volume":"91","author":"Luque","year":"2019","journal-title":"Pattern Recognit."},{"key":"10.1016\/j.infsof.2026.108204_b22","doi-asserted-by":"crossref","first-page":"1273","DOI":"10.1109\/TSE.2022.3174092","article-title":"SEGRESS: Software engineering guidelines for REporting secondary studies","volume":"49","author":"Kitchenham","year":"2023","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2026.108204_b23","series-title":"Evidence-Based Software Engineering and Systematic Reviews","author":"Kitchenham","year":"2016"},{"key":"10.1016\/j.infsof.2026.108204_b24","doi-asserted-by":"crossref","DOI":"10.1016\/j.jclinepi.2024.111533","article-title":"Benchmarking human\u2013AI collaboration for common evidence appraisal tools","volume":"175","author":"Woelfle","year":"2024","journal-title":"J. Clin. Epidemiol."},{"key":"10.1016\/j.infsof.2026.108204_b25","doi-asserted-by":"crossref","first-page":"1939","DOI":"10.1093\/jamia\/ocae166","article-title":"A question-answering framework for automated abstract screening using large language models","volume":"31","author":"Akinseloyin","year":"2024","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.infsof.2026.108204_b26","doi-asserted-by":"crossref","first-page":"S270","DOI":"10.1016\/j.jval.2024.03.1490","article-title":"MSR57 transforming systematic literature reviews: unleashing the potential of GPT-4: a cutting-edge large language model, to elevate research synthesis","volume":"27","author":"Attri","year":"2024","journal-title":"Value Health"},{"key":"10.1016\/j.infsof.2026.108204_b27","article-title":"Utilizing chatgpt to select literature for meta-analysis shows workload reduction while maintaining a similar recall level as manual curation","author":"Cai","year":"2023","journal-title":"MedRxiv"},{"key":"10.1016\/j.infsof.2026.108204_b28","article-title":"Prompting is all you need: LLMs for systematic review screening","author":"Cao","year":"2024","journal-title":"MedRxiv"},{"key":"10.1016\/j.infsof.2026.108204_b29","series-title":"2023 World Engineering Education Forum-Global Engineering Deans Council","first-page":"1","article-title":"Leveraging the potential of generative AI to accelerate systematic literature reviews: an example in the area of educational technology","author":"Castillo-Segura","year":"2023"},{"key":"10.1016\/j.infsof.2026.108204_b30","doi-asserted-by":"crossref","first-page":"S279","DOI":"10.1016\/j.jval.2024.03.2418","article-title":"MSR103 optimizing systematic literature reviews in endometrial cancer: Leveraging AI for real-time article screening and data extraction in clinical trials","volume":"27","author":"Datta","year":"2024","journal-title":"Value Health"},{"key":"10.1016\/j.infsof.2026.108204_b31","doi-asserted-by":"crossref","first-page":"108","DOI":"10.1186\/s12874-024-02224-3","article-title":"Machine learning models for abstract screening task-a systematic literature review application for health economics and outcome research","volume":"24","author":"Du","year":"2024","journal-title":"BMC Med. Res. Methodol."},{"key":"10.1016\/j.infsof.2026.108204_b32","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1136\/bmjebm-2023-112678","article-title":"Enhancing title and abstract screening for systematic reviews with GPT-3.5 turbo","volume":"29","author":"Gargari","year":"2024","journal-title":"BMJ Evidence-Based Med."},{"key":"10.1016\/j.infsof.2026.108204_b33","doi-asserted-by":"crossref","DOI":"10.2196\/48996","article-title":"Automated paper screening for clinical reviews using large language models: data analysis study","volume":"26","author":"Guo","year":"2024","journal-title":"J. Med. Internet Res."},{"key":"10.1016\/j.infsof.2026.108204_b34","doi-asserted-by":"crossref","first-page":"78","DOI":"10.1186\/s12874-024-02203-8","article-title":"Methodological insights into ChatGPT\u2019s screening performance in systematic reviews","volume":"24","author":"Issaiy","year":"2024","journal-title":"BMC Med. Res. Methodol."},{"key":"10.1016\/j.infsof.2026.108204_b35","doi-asserted-by":"crossref","first-page":"S262","DOI":"10.1016\/j.jval.2024.03.1448","article-title":"MSR15 revolutionizing systematic literature reviews: harnessing the power of large language model (GPT-4) for enhanced research synthesis","volume":"27","author":"Kaur","year":"2024","journal-title":"Value Health"},{"key":"10.1016\/j.infsof.2026.108204_b36","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1186\/s13643-024-02609-x","article-title":"Evaluating the effectiveness of large language models in abstract screening: a comparative analysis","volume":"13","author":"Li","year":"2024","journal-title":"Syst. Rev."},{"key":"10.1016\/j.infsof.2026.108204_b37","doi-asserted-by":"crossref","first-page":"247","DOI":"10.1186\/s12911-023-02328-8","article-title":"Automatic literature screening using the PAJO deep-learning model for clinical practice guidelines","volume":"23","author":"Lin","year":"2023","journal-title":"BMC Med. Inform. Decis. Mak."},{"key":"10.1016\/j.infsof.2026.108204_b38","doi-asserted-by":"crossref","first-page":"S270","DOI":"10.1016\/j.jval.2024.03.1492","article-title":"MSR59 advancing systematic literature reviews: The integration of AI-powered NLP models in data collection processes","volume":"27","author":"Rai","year":"2024","journal-title":"Value Health"},{"key":"10.1016\/j.infsof.2026.108204_b39","series-title":"Bio-sieve: exploring instruction tuning large language models for systematic review automation","author":"Robinson","year":"2023"},{"key":"10.1016\/j.infsof.2026.108204_b40","doi-asserted-by":"crossref","first-page":"S418","DOI":"10.1016\/j.jval.2023.09.2190","article-title":"MSR131 prospects for automation of systemic literature reviews (SLRs) with artificial intelligence and natural language processing","volume":"26","author":"Royer","year":"2023","journal-title":"Value Health"},{"key":"10.1016\/j.infsof.2026.108204_b41","article-title":"Human-AI collaboration to identify literature for evidence synthesis","volume":"1","author":"Spillias","year":"2024","journal-title":"Cell Rep. Sustain."},{"key":"10.1016\/j.infsof.2026.108204_b42","article-title":"Sensitivity, specificity and avoidable workload of using a large language models for title and abstract screening in systematic reviews and meta-analyses","author":"Tran","year":"2023","journal-title":"MedRxiv"},{"key":"10.1016\/j.infsof.2026.108204_b43","series-title":"Automated title and abstract screening for scoping reviews using the GPT-4 large language model","author":"Wilkins","year":"2023"},{"key":"10.1016\/j.infsof.2026.108204_b44","doi-asserted-by":"crossref","first-page":"603","DOI":"10.1109\/TSE.2014.2322358","article-title":"Researcher bias: The use of machine learning in software defect prediction","volume":"40","author":"Shepperd","year":"2014","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2026.108204_b45","article-title":"The PRISMA 2020 statement: an updated guideline for reporting systematic reviews","volume":"372","author":"Page","year":"2021","journal-title":"BMJ"},{"key":"10.1016\/j.infsof.2026.108204_b46","doi-asserted-by":"crossref","first-page":"383","DOI":"10.3233\/JND-200546","article-title":"Targeted therapies for hereditary peripheral neuropathies: Systematic review and steps towards a treatabolome","volume":"8","author":"Jennings","year":"2021","journal-title":"J. Neuromuscul. Dis."},{"key":"10.1016\/j.infsof.2026.108204_b47","doi-asserted-by":"crossref","first-page":"367","DOI":"10.1016\/j.compbiolchem.2004.09.006","article-title":"Comparing two K-category assignments by a K-category correlation coefficient","volume":"28","author":"Gorodkin","year":"2004","journal-title":"Comput. Biol. Chem."},{"key":"10.1016\/j.infsof.2026.108204_b48","series-title":"A systematic review of effect size in software engineering experiments","author":"Kampenes","year":"2007"}],"container-title":["Information and Software Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S095058492600193X?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S095058492600193X?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T17:12:45Z","timestamp":1782925965000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S095058492600193X"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":48,"alternative-id":["S095058492600193X"],"URL":"https:\/\/doi.org\/10.1016\/j.infsof.2026.108204","relation":{},"ISSN":["0950-5849"],"issn-type":[{"value":"0950-5849","type":"print"}],"subject":[],"published":{"date-parts":[[2026,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"LLM4SCREENLIT: Recommendations on assessing the performance of large language models for screening literature in systematic reviews","name":"articletitle","label":"Article Title"},{"value":"Information and Software Technology","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.infsof.2026.108204","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Author(s). Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"108204"}}