{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,7]],"date-time":"2026-05-07T15:22:03Z","timestamp":1778167323177,"version":"3.51.4"},"reference-count":30,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,3,31]],"date-time":"2025-03-31T00:00:00Z","timestamp":1743379200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,3,31]],"date-time":"2025-03-31T00:00:00Z","timestamp":1743379200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100000005","name":"U.S. Department of Defense","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000005","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,3,31]]},"DOI":"10.1109\/icstw64639.2025.10962520","type":"proceedings-article","created":{"date-parts":[[2025,4,16]],"date-time":"2025-04-16T17:47:42Z","timestamp":1744825662000},"page":"300-309","source":"Crossref","is-referenced-by-count":3,"title":["Evaluating Large Language Model Robustness using Combinatorial Testing"],"prefix":"10.1109","author":[{"given":"Jaganmohan","family":"Chandrasekaran","sequence":"first","affiliation":[{"name":"Virginia Tech,Sanghani Center for Artificial Intelligence & Data Analytics,Arlington,VA,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ankita Ramjibhai","family":"Patel","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Erin","family":"Lanus","sequence":"additional","affiliation":[{"name":"Virginia Tech,National Security Institute,Arlington,VA,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Laura J.","family":"Freeman","sequence":"additional","affiliation":[{"name":"Virginia Tech,National Security Institute,Arlington,VA,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3641289"},{"key":"ref2","article-title":"Evaluating large language models: A comprehensive survey","author":"Guo","year":"2023"},{"key":"ref3","first-page":"20 166","article-title":"Position: Trustllm: Trustworthiness in large language models","volume-title":"International Conference on Machine Learning","author":"Huang"},{"key":"ref4","article-title":"Measuring massive multitask language understanding","volume-title":"International Conference on Learning Representations","author":"Hendrycks"},{"key":"ref5","article-title":"Gemini","year":"2025"},{"key":"ref6","article-title":"Llama","year":"2025"},{"key":"ref7","article-title":"Helm Leaderboard","year":"2025"},{"key":"ref8","article-title":"Helm MMLU","year":"2024"},{"key":"ref9","article-title":"Changing answer order can decrease mmlu accuracy","author":"Gupta","year":"2024"},{"key":"ref10","article-title":"Large language models are not robust multiple choice selectors","volume-title":"The Twelfth International Conference on Learning Representations","author":"Zheng"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-naacl.130"},{"key":"ref12","article-title":"Can multiple-choice questions really be useful in detecting the abilities of llms?","author":"Li","year":"2024"},{"key":"ref13","article-title":"Multiple-choice questions are efficient and robust llm evaluators","author":"Zhang","year":"2024"},{"issue":"2","key":"ref14","first-page":"111","article-title":"Mitigating order sensitivity in large language models for multiple-choice question tasks","volume":"2","author":"Parlapalli","year":"2024","journal-title":"International Journal of Artificial Intelligence Research and Development (IJAIRD)"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.800-142"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/1883612.1883618"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2013.77"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2012.147"},{"key":"ref19","article-title":"Testing event sequences","year":"2025"},{"key":"ref20","article-title":"GPT-3.5 Turbo","year":"2024"},{"key":"ref21","article-title":"Openai rate limit","year":"2025"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW52544.2021.00022"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW50498.2020.00170"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW52544.2021.00025"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW55395.2022.00030"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW55395.2022.00031"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW60967.2024.00050"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2024.3366142"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-43240-8_16"},{"key":"ref30","first-page":"930","article-title":"Using combinatorial testing for prompt engineering of llms in medicine","volume-title":"27th International Multiconference Information Society\u2013IS 2024","author":"Perko"}],"event":{"name":"2025 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)","location":"Naples, Italy","start":{"date-parts":[[2025,3,31]]},"end":{"date-parts":[[2025,4,4]]}},"container-title":["2025 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/10962452\/10962453\/10962520.pdf?arnumber=10962520","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,17]],"date-time":"2025-04-17T05:57:03Z","timestamp":1744869423000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10962520\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,31]]},"references-count":30,"URL":"https:\/\/doi.org\/10.1109\/icstw64639.2025.10962520","relation":{},"subject":[],"published":{"date-parts":[[2025,3,31]]}}}