{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T18:11:21Z","timestamp":1793211081904,"version":"4.5.0"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,7,21]],"date-time":"2026-07-21T00:00:00Z","timestamp":1784592000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2026,7,23]],"date-time":"2026-07-23T00:00:00Z","timestamp":1784764800000},"content-version":"vor","delay-in-days":2,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci Rep"],"DOI":"10.1038\/s41598-026-61195-1","type":"journal-article","created":{"date-parts":[[2026,7,21]],"date-time":"2026-07-21T07:23:14Z","timestamp":1784618594000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["A multidimensional benchmarking framework for large language models in oncologic decision making"],"prefix":"10.1038","volume":"16","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1778-6607","authenticated-orcid":false,"given":"Mehmet","family":"Halici","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3555-3687","authenticated-orcid":false,"given":"Serkan","family":"Salturk","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Irem","family":"Sayin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Burak","family":"Ertan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ibrahim Cem","family":"Balci","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kimia","family":"Cepni","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tanju","family":"Kapagan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Cumhur","family":"Yildirim","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gokmen Umut","family":"Erdem","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Huriye Senay","family":"Kiziltan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Muhammed Tayyip","family":"Kocak","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Huseyin","family":"Uvet","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,7,21]]},"reference":[{"key":"61195_CR1","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1038\/s41746-024-01079-8","volume":"7","author":"J Huang","year":"2024","unstructured":"Huang, J. et al. A critical assessment of using ChatGPT for extracting structured data from clinical notes. NPJ Digit. Med. 7, 106 (2024).","journal-title":"NPJ Digit. Med."},{"key":"61195_CR2","doi-asserted-by":"publisher","first-page":"405","DOI":"10.1186\/s12859-023-05480-0","volume":"24","author":"M Gholipour","year":"2023","unstructured":"Gholipour, M., Khajouei, R., Amiri, P., Hajesmaeel Gohari, S. & Ahmadian, L. Extracting cancer concepts from clinical notes using natural language processing: A systematic review. BMC Bioinform. 24, 405 (2023).","journal-title":"BMC Bioinform."},{"key":"61195_CR3","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1038\/s41746-024-01315-1","volume":"7","author":"E Klang","year":"2024","unstructured":"Klang, E. et al. A strategy for cost-effective large language model use at health system-scale. NPJ Digit. Med. 7, 320 (2024).","journal-title":"NPJ Digit. Med."},{"key":"61195_CR4","doi-asserted-by":"crossref","unstructured":"Gaber, F. et al. Evaluating large language model workflows in clinical decision support for triage and referral and diagnosis. npj Digit. Med. 8, 263 (2025).","DOI":"10.1038\/s41746-025-01684-1"},{"key":"61195_CR5","doi-asserted-by":"publisher","first-page":"e2343689","DOI":"10.1001\/jamanetworkopen.2023.43689","volume":"6","author":"M Benary","year":"2023","unstructured":"Benary, M. et al. Leveraging large language models for decision support in personalized oncology. JAMA Netw. Open 6, e2343689\u2013e2343689 (2023).","journal-title":"JAMA Netw. Open"},{"key":"61195_CR6","doi-asserted-by":"publisher","first-page":"1379297","DOI":"10.3389\/frai.2024.1379297","volume":"7","author":"MP Khan","year":"2024","unstructured":"Khan, M. P. & O\u2019Sullivan, E. D. A comparison of the diagnostic ability of large language models in challenging clinical cases. Front. Artif. Intell. 7, 1379297 (2024).","journal-title":"Front. Artif. Intell."},{"key":"61195_CR7","doi-asserted-by":"crossref","unstructured":"Rydzewski, N.\u00a0R. et al. Comparative evaluation of LLMS in clinical oncology. NEJM AI 1, AIoa2300151 (2024).","DOI":"10.1056\/AIoa2300151"},{"key":"61195_CR8","doi-asserted-by":"crossref","unstructured":"Wong, E. ESMO guidance on the use of large language models in clinical practice (ELCAP). Ann. Oncol. (2025).","DOI":"10.1016\/j.annonc.2025.09.001"},{"key":"61195_CR9","doi-asserted-by":"crossref","unstructured":"Abbas, A., Rehman, M.\u00a0S. & Rehman, S.\u00a0S. Comparing the performance of popular large language models on the national board of medical examiners sample questions. Cureus 16 (2024).","DOI":"10.7759\/cureus.55991"},{"key":"61195_CR10","doi-asserted-by":"publisher","first-page":"115261","DOI":"10.52202\/079017-3660","volume":"37","author":"J Dong","year":"2024","unstructured":"Dong, J. et al. Cost-efficient knowledge-based question answering with large language models. Adv. Neural Inf. Process. Syst. 37, 115261\u2013115281 (2024).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"61195_CR11","doi-asserted-by":"publisher","first-page":"258","DOI":"10.1038\/s41746-024-01258-7","volume":"7","author":"TYC Tam","year":"2024","unstructured":"Tam, T. Y. C. et al. A framework for human evaluation of large language models in healthcare derived from literature review. NPJ Digit. Med. 7, 258 (2024).","journal-title":"NPJ Digit. Med."},{"key":"61195_CR12","doi-asserted-by":"crossref","unstructured":"Brown, E. D. Precision oncology in non-small cell lung cancer: A comparative study of contextualized ChatGPT models. Cureus 17 (2025).","DOI":"10.7759\/cureus.81097"},{"key":"61195_CR13","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1038\/s41591-024-03425-5","volume":"31","author":"J Gallifant","year":"2025","unstructured":"Gallifant, J. et al. The tripod-LLM reporting guideline for studies using large language models. Nat. Med. 31, 60\u201369 (2025).","journal-title":"Nat. Med."},{"key":"61195_CR14","unstructured":"Society, B.\u00a0T. et al. BTS guidelines for the investigation and management of pulmonary nodules. https:\/\/www. brit-thoracic. org.uk\/quality-improvement\/guidelines\/pulmonary-nodules\/. Accessed 1 Dec 2021 (2015)."},{"key":"61195_CR15","doi-asserted-by":"crossref","unstructured":"Riely, G. J. NCCN guidelines\u00ae insights: Non-small cell lung cancer, version 7.2025: Featured updates to the NCCN guidelines\u00ae. J. Natl. Comprehens. Cancer Netw. 23, 354\u2013362 (2025).","DOI":"10.6004\/jnccn.2025.0043"},{"key":"61195_CR16","doi-asserted-by":"publisher","first-page":"1332","DOI":"10.1016\/j.annonc.2021.07.016","volume":"32","author":"E Le Rhun","year":"2021","unstructured":"Le Rhun, E. et al. EANO-ESMO clinical practice guidelines for diagnosis, treatment and follow-up of patients with brain metastasis from solid tumours. Ann. Oncol. 32, 1332\u20131347 (2021).","journal-title":"Ann. Oncol."},{"key":"61195_CR17","doi-asserted-by":"publisher","first-page":"265","DOI":"10.1016\/j.prro.2022.02.003","volume":"12","author":"V Gondi","year":"2022","unstructured":"Gondi, V. et al. Radiation therapy for brain metastases: An astro clinical practice guideline. Pract. Radiat. Oncol. 12, 265\u2013282 (2022).","journal-title":"Pract. Radiat. Oncol."},{"key":"61195_CR18","doi-asserted-by":"crossref","unstructured":"Postmus, P. et al. Early and locally advanced non-small-cell lung cancer (NSCLC): ESMO clinical practice guidelines for diagnosis, treatment and follow-up. Ann. Oncol. 28, iv1\u2013iv21 (2017).","DOI":"10.1093\/annonc\/mdx222"},{"key":"61195_CR19","doi-asserted-by":"crossref","unstructured":"Zer, A. Early and locally advanced non-small-cell lung cancer: ESMO clinical practice guideline for diagnosis, treatment and follow-up. Ann. Oncol. (2025).","DOI":"10.1016\/j.annonc.2025.08.003"},{"key":"61195_CR20","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.radonc.2018.02.023","volume":"127","author":"U Nestle","year":"2018","unstructured":"Nestle, U. et al. ESTRO ACROP guidelines for target volume definition in the treatment of locally advanced non-small cell lung cancer. Radiother. Oncol. 127, 1\u20135 (2018).","journal-title":"Radiother. Oncol."},{"key":"61195_CR21","doi-asserted-by":"publisher","first-page":"1356","DOI":"10.1200\/JCO.21.02528","volume":"40","author":"ME Daly","year":"2022","unstructured":"Daly, M. E. et al. Management of stage III non-small-cell lung cancer: Asco guideline. J. Clin. Oncol. 40, 1356\u20131384 (2022).","journal-title":"J. Clin. Oncol."},{"key":"61195_CR22","unstructured":"Google Cloud. Gemini 2.5 Pro Model Documentation. Vertex AI Documentation for Gemini 2.5 Pro. https:\/\/docs.cloud.google.com\/vertex-ai\/generative-ai\/docs\/models\/gemini\/2-5-pro (2025)."},{"key":"61195_CR23","unstructured":"OpenAI. GPT-5 System Card. System Card for GPT-5 (API and Reasoning Variants). https:\/\/openai.com\/tr-TR\/index\/gpt-5-system-card\/. Accessed Dec 2025 (2025)."},{"key":"61195_CR24","unstructured":"Anthropic. Claude Opus 4.1. Model Announcement and Details for Claude Opus 4.1. https:\/\/www.anthropic.com\/news\/claude-opus-4-1 (2025)."},{"key":"61195_CR25","unstructured":"https:\/\/openai.com\/tr-tr\/api\/pricing\/. Aug-Nov 2025 (2025)."},{"key":"61195_CR26","unstructured":"https:\/\/ai.google.dev\/gemini-api\/docs\/models?hl=tr#gemini-2.5-pro. Aug-Nov 2025 (2025)."},{"key":"61195_CR27","unstructured":"https:\/\/claude.com\/pricing#api. Aug-Nov 2025 (2025)."},{"key":"61195_CR28","doi-asserted-by":"publisher","first-page":"175","DOI":"10.3758\/BF03193146","volume":"39","author":"F Faul","year":"2007","unstructured":"Faul, F., Erdfelder, E., Lang, A.-G. & Buchner, A. G* power 3: A flexible statistical power analysis program for the social, behavioral, and biomedical sciences. Behav. Res. Methods 39, 175\u2013191 (2007).","journal-title":"Behav. Res. Methods"},{"key":"61195_CR29","doi-asserted-by":"publisher","first-page":"450","DOI":"10.1038\/s41746-025-01824-7","volume":"8","author":"Y Hao","year":"2025","unstructured":"Hao, Y. et al. Large language model integrations in cancer decision-making: A systematic review and meta-analysis. NPJ Digit. Med. 8, 450 (2025).","journal-title":"NPJ Digit. Med."},{"key":"61195_CR30","doi-asserted-by":"publisher","first-page":"1800","DOI":"10.1016\/j.jacr.2024.07.011","volume":"21","author":"R Chow","year":"2024","unstructured":"Chow, R. et al. The accuracy of artificial intelligence ChatGPT in oncology examination questions. J. Am. Coll. Radiol. 21, 1800\u20131804 (2024).","journal-title":"J. Am. Coll. Radiol."},{"key":"61195_CR31","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1186\/s12885-025-13596-0","volume":"25","author":"EC Erdat","year":"2025","unstructured":"Erdat, E. C. & Kavak, E. E. Benchmarking LLM chatbots\u2019 oncological knowledge with the Turkish Society of Medical Oncology\u2019s annual board examination questions. BMC Cancer 25, 197 (2025).","journal-title":"BMC Cancer"},{"key":"61195_CR32","doi-asserted-by":"crossref","unstructured":"Kasagga, A. Performance of ChatGPT and large language models on medical licensing exams worldwide: A systematic review and network meta-analysis with meta-regression. Cureus 17 (2025).","DOI":"10.7759\/cureus.94300"},{"key":"61195_CR33","doi-asserted-by":"crossref","unstructured":"Chen, D., Avison, K., Alnassar, S., Huang, R.\u00a0S. & Raman, S. Medical accuracy of artificial intelligence chatbots in oncology: A scoping review. Oncologist 30, oyaf038 (2025).","DOI":"10.1093\/oncolo\/oyaf038"},{"key":"61195_CR34","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1038\/s41746-024-01010-1","volume":"7","author":"T Savage","year":"2024","unstructured":"Savage, T., Nayak, A., Gallo, R., Rangan, E. & Chen, J. H. Diagnostic reasoning prompts reveal the potential for large language model interpretability in medicine. NPJ Digit. Med. 7, 20 (2024).","journal-title":"NPJ Digit. Med."},{"key":"61195_CR35","doi-asserted-by":"publisher","first-page":"128","DOI":"10.1038\/s44172-025-00453-y","volume":"4","author":"M Mesinovic","year":"2025","unstructured":"Mesinovic, M., Watkinson, P. & Zhu, T. Explainability in the age of large language models for healthcare. Commun. Eng. 4, 128 (2025).","journal-title":"Commun. Eng."},{"key":"61195_CR36","doi-asserted-by":"publisher","first-page":"579","DOI":"10.1038\/s41746-025-01971-x","volume":"8","author":"ML Burns","year":"2025","unstructured":"Burns, M. L. et al. Generative AI costs in large healthcare systems, an example in revenue cycle. NPJ Digit. Med. 8, 579 (2025).","journal-title":"NPJ Digit. Med."},{"key":"61195_CR37","unstructured":"Nori, H. Sequential diagnosis with language models. arXiv preprint arXiv:2506.22405 (2025)."},{"key":"61195_CR38","doi-asserted-by":"crossref","unstructured":"Chitty-Venkata, K. T. LLM-inference-bench: Inference benchmarking of large language models on AI accelerators. In SC24-W: Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis. 1362\u20131379 (IEEE, 2024).","DOI":"10.1109\/SCW63240.2024.00178"},{"key":"61195_CR39","unstructured":"Du, Z., Kang, H., Han, S., Krishna, T. & Zhu, L. Ockbench: Measuring the efficiency of LLM reasoning. arXiv preprint arXiv:2511.05722 (2025)."},{"key":"61195_CR40","unstructured":"Liang, P. Holistic evaluation of language models. arXiv preprint: arXiv:2211.09110 (2022)."},{"key":"61195_CR41","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1348\/000711006X126600","volume":"61","author":"KL Gwet","year":"2008","unstructured":"Gwet, K. L. Computing inter-rater reliability and its variance in the presence of high agreement. Br. J. Math. Stat. Psychol. 61, 29\u201348 (2008).","journal-title":"Br. J. Math. Stat. Psychol."},{"key":"61195_CR42","doi-asserted-by":"publisher","first-page":"e244630","DOI":"10.1001\/jamanetworkopen.2024.4630","volume":"7","author":"A Yalamanchili","year":"2024","unstructured":"Yalamanchili, A. et al. Quality of large language model responses to radiation oncology patient care questions. JAMA Netw. Open 7, e244630\u2013e244630 (2024).","journal-title":"JAMA Netw. Open"},{"key":"61195_CR43","doi-asserted-by":"publisher","first-page":"e230922","DOI":"10.1148\/radiol.230922","volume":"307","author":"AA Rahsepar","year":"2023","unstructured":"Rahsepar, A. A. et al. How AI responds to common lung cancer questions: ChatGPT versus google bard. Radiology 307, e230922 (2023).","journal-title":"Radiology"}],"container-title":["Scientific Reports"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41598-026-61195-1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41598-026-61195-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41598-026-61195-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,23]],"date-time":"2026-07-23T13:00:04Z","timestamp":1784811604000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41598-026-61195-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,21]]},"references-count":43,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2026,12]]}},"alternative-id":["61195"],"URL":"https:\/\/doi.org\/10.1038\/s41598-026-61195-1","relation":{},"ISSN":["2045-2322"],"issn-type":[{"value":"2045-2322","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,7,21]]},"assertion":[{"value":"28 December 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 July 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 July 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","label":"Competing interests","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The study was based entirely on fictional clinical scenarios and did not involve real patient data; therefore, formal ethical approval was not required.","order":2,"name":"Ethics","label":"Ethical approval","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Documented informed consent was obtained from all oncology specialists who contributed as domain experts to the preparation of the study questions and the evaluation of the Large Language Models\u2019 responses, in accordance with the journal\u2019s policy.","order":3,"name":"Ethics","label":"Informed consent","group":{"name":"EthicsHeading","label":"Declarations"}}],"article-number":"23086"}}