{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T04:42:04Z","timestamp":1773376924519,"version":"3.50.1"},"reference-count":23,"publisher":"IEEE","license":[{"start":{"date-parts":[[2024,7,7]],"date-time":"2024-07-07T00:00:00Z","timestamp":1720310400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,7,7]],"date-time":"2024-07-07T00:00:00Z","timestamp":1720310400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,7,7]]},"DOI":"10.1109\/isit57864.2024.10619136","type":"proceedings-article","created":{"date-parts":[[2024,8,19]],"date-time":"2024-08-19T13:25:01Z","timestamp":1724073901000},"page":"2033-2037","source":"Crossref","is-referenced-by-count":0,"title":["Predicting Uncertainty of Generative LLMs with MARS: Meaning-Aware Response Scoring"],"prefix":"10.1109","author":[{"given":"Yavuz Faruk","family":"Bakman","sequence":"first","affiliation":[{"name":"University of Southern California"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Duygu Nur","family":"Yaldiz","sequence":"additional","affiliation":[{"name":"University of Southern California"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Baturalp","family":"Buyukates","sequence":"additional","affiliation":[{"name":"University of Southern California"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Salman","family":"Avestimehr","sequence":"additional","affiliation":[{"name":"University of Southern California"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chenyang","family":"Tao","sequence":"additional","affiliation":[{"name":"Amazon AI"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dimitrios","family":"Dimitriadis","sequence":"additional","affiliation":[{"name":"Amazon AI"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"A comprehensive capability analysis of GPT-3 and GPT-3.5 series models","author":"Ye","year":"2023"},{"key":"ref2","volume-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"ref3","volume-title":"Benchmarking large language models as AI research agents","author":"Huang","year":"2023"},{"key":"ref4","article-title":"Simple and scalable predictive uncertainty estimation using deep ensembles","volume-title":"Advances in Neural Information Processing Systems","volume":"30","author":"Lakshminarayanan","year":"2017"},{"key":"ref5","first-page":"1050","article-title":"Dropout as a Bayesian approximation: Representing model uncertainty in deep learning","volume-title":"Proceedings of The 33rd International Conference on Machine Learning","volume":"48","author":"Gal","year":"2016"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/WACV48630.2021.00075"},{"key":"ref7","volume-title":"Uncertainty in natural language processing: Sources, quantification, and applications","author":"Hu","year":"2023"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017322"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.566"},{"key":"ref10","article-title":"Uncertainty estimation in autoregressive structured prediction","volume-title":"International Conference on Learning Representations","author":"Malinin","year":"2021"},{"key":"ref11","article-title":"Semantic uncertainty: Linguistic invari-ances for uncertainty estimation in natural language generation","volume-title":"The Eleventh International Conference on Learning Representations","author":"Kuhn","year":"2023"},{"key":"ref12","volume-title":"Generating with confidence: Uncertainty quantification for black-box large language models","author":"Lin","year":"2023"},{"key":"ref13","volume-title":"Quantifying uncertainty in answers from any language model and enhancing their trustworthiness","author":"Chen","year":"2023"},{"key":"ref14","first-page":"4171","article-title":"BERT: Pretraining of deep bidirectional transformers for language understanding","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)","author":"Devlin","year":"2019"},{"key":"ref15","article-title":"Benchmarking Bayesian deep learning on diabetic retinopathy detection tasks","volume-title":"Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)","author":"Band","year":"2021"},{"key":"ref16","doi-asserted-by":"crossref","first-page":"7273","DOI":"10.18653\/v1\/2022.findings-emnlp.538","article-title":"Uncertainty quantification with pretrained language models: A large-scale empirical analysis","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2022","author":"Xiao","year":"2022"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"291","DOI":"10.18653\/v1\/2022.emnlp-main.20","article-title":"Tomayto, tomahto. beyond token-level answer equivalence for question answering evaluation","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing","author":"Bulian","year":"2022"},{"key":"ref18","first-page":"1638","article-title":"Contextual string embeddings for sequence labeling","volume-title":"COLING 2018, 27th International Conference on Computational Linguistics","author":"Akbik","year":"2018"},{"key":"ref19","doi-asserted-by":"crossref","first-page":"1601","DOI":"10.18653\/v1\/P17-1147","article-title":"TriviaQA: A large scale distantly supervised challenge dataset for reading comprehension","volume-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"Joshi","year":"2017"},{"key":"ref20","first-page":"452","article-title":"Natural questions: A benchmark for question answering research","volume-title":"Transactions of the Association for Computational Linguistics","volume":"7","author":"Kwiatkowski","year":"2019"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01600"},{"key":"ref22","volume-title":"Mistral 7b","author":"Jiang","year":"2023"},{"key":"ref23","volume-title":"The Falcon series of open language models","author":"Almazrouei","year":"2023"}],"event":{"name":"2024 IEEE International Symposium on Information Theory (ISIT)","location":"Athens, Greece","start":{"date-parts":[[2024,7,7]]},"end":{"date-parts":[[2024,7,12]]}},"container-title":["2024 IEEE International Symposium on Information Theory (ISIT)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/10619013\/10619074\/10619136.pdf?arnumber=10619136","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,12]],"date-time":"2026-03-12T20:29:16Z","timestamp":1773347356000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10619136\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,7]]},"references-count":23,"URL":"https:\/\/doi.org\/10.1109\/isit57864.2024.10619136","relation":{},"subject":[],"published":{"date-parts":[[2024,7,7]]}}}