{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,3]],"date-time":"2026-04-03T05:05:02Z","timestamp":1775192702815,"version":"3.50.1"},"reference-count":122,"publisher":"MIT Press","license":[{"start":{"date-parts":[[2024,11,21]],"date-time":"2024-11-21T00:00:00Z","timestamp":1732147200000},"content-version":"vor","delay-in-days":325,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["direct.mit.edu"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,11,18]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>The rapid proliferation of large language models and natural language processing (NLP) applications creates a crucial need for uncertainty quantification to mitigate risks such as Hallucinations and to enhance decision-making reliability in critical applications. Conformal prediction is emerging as a theoretically sound and practically useful framework, combining flexibility with strong statistical guarantees. Its model-agnostic and distribution-free nature makes it particularly promising to address the current shortcomings of NLP systems that stem from the absence of uncertainty quantification. This paper provides a comprehensive survey of conformal prediction techniques, their guarantees, and existing applications in NLP, pointing to directions for future research and open challenges.<\/jats:p>","DOI":"10.1162\/tacl_a_00715","type":"journal-article","created":{"date-parts":[[2024,11,21]],"date-time":"2024-11-21T19:15:55Z","timestamp":1732216555000},"page":"1497-1516","update-policy":"https:\/\/doi.org\/10.1162\/mitpressjournals.corrections.policy","source":"Crossref","is-referenced-by-count":17,"title":["Conformal Prediction for Natural Language Processing: A Survey"],"prefix":"10.1162","volume":"12","author":[{"given":"Margarida","family":"Campos","sequence":"first","affiliation":[{"name":"Instituto de Telecomunica\u00e7\u00f5es, Portugal"},{"name":"Instituto Superior T\u00e9cnico, Universidade de Lisboa, Portugal. margarida.campos@tecnico.ulisboa.pt"}]},{"given":"Ant\u00f3nio","family":"Farinhas","sequence":"additional","affiliation":[{"name":"Instituto de Telecomunica\u00e7\u00f5es, Portugal"},{"name":"Instituto Superior T\u00e9cnico, Universidade de Lisboa, Portugal"}]},{"given":"Chrysoula","family":"Zerva","sequence":"additional","affiliation":[{"name":"Instituto de Telecomunica\u00e7\u00f5es, Portugal"},{"name":"Instituto Superior T\u00e9cnico, Universidade de Lisboa, Portugal"},{"name":"ELLIS Unit Lisbon, Portugal"}]},{"given":"M\u00e1rio A. T.","family":"Figueiredo","sequence":"additional","affiliation":[{"name":"Instituto de Telecomunica\u00e7\u00f5es, Portugal"},{"name":"Instituto Superior T\u00e9cnico, Universidade de Lisboa, Portugal"},{"name":"ELLIS Unit Lisbon, Portugal"}]},{"given":"Andr\u00e9 F. T.","family":"Martins","sequence":"additional","affiliation":[{"name":"Instituto de Telecomunica\u00e7\u00f5es, Portugal"},{"name":"Instituto Superior T\u00e9cnico, Universidade de Lisboa, Portugal"},{"name":"ELLIS Unit Lisbon, Portugal"},{"name":"Unbabel, Portugal"}]}],"member":"281","published-online":{"date-parts":[[2024,11,18]]},"reference":[{"key":"2024112119154842400_bib1","first-page":"23047","article-title":"Conformal pid control for time series prediction","volume-title":"Advances in Neural Information Processing Systems","author":"Angelopoulos","year":"2023"},{"issue":"4","key":"2024112119154842400_bib2","doi-asserted-by":"publisher","first-page":"494","DOI":"10.1561\/2200000101","article-title":"Conformal prediction: A gentle introduction","volume":"16","author":"Angelopoulos","year":"2023","journal-title":"Foundations and Trends in Machine Learning"},{"key":"2024112119154842400_bib3","article-title":"Learn then test: Calibrating predictive algorithms to achieve risk control","author":"Angelopoulos","year":"2022","journal-title":"arXiv preprint arXiv:2110.01052"},{"key":"2024112119154842400_bib4","article-title":"Conformal risk control","volume-title":"The Twelfth International Conference on Learning Representations","author":"Angelopoulos","year":"2024"},{"key":"2024112119154842400_bib5","doi-asserted-by":"publisher","first-page":"1892","DOI":"10.18653\/v1\/2022.emnlp-main.124","article-title":"Stop measuring calibration when humans disagree","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing","author":"Baan","year":"2022"},{"key":"2024112119154842400_bib6","article-title":"Uncertainty in natural language generation: From theory to applications","author":"Baan","year":"2023","journal-title":"arXiv preprint arXiv:2307.15703"},{"key":"2024112119154842400_bib7","first-page":"268","article-title":"Interpreting predictive probabilities: Model confidence or human label variation?","volume-title":"Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)","author":"Baan","year":"2024"},{"issue":"1","key":"2024112119154842400_bib8","doi-asserted-by":"publisher","first-page":"455","DOI":"10.1093\/imaiai\/iaaa017","article-title":"The limits of distribution-free conditional predictive inference","volume":"10","author":"Barber","year":"2020","journal-title":"Information and Inference: A Journal of the IMA"},{"issue":"1","key":"2024112119154842400_bib9","doi-asserted-by":"publisher","DOI":"10.1214\/20-aos1965","article-title":"Predictive inference with the jackknife+","volume":"49","author":"Barber","year":"2021","journal-title":"The Annals of Statistics"},{"issue":"2","key":"2024112119154842400_bib10","doi-asserted-by":"publisher","first-page":"486","DOI":"10.1214\/23-aos2276","article-title":"Conformal prediction beyond exchangeability","volume":"51","author":"Barber","year":"2023","journal-title":"The Annals of Statistics"},{"issue":"6","key":"2024112119154842400_bib11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3478535","article-title":"Distribution-free, risk-controlling prediction sets","volume":"68","author":"Bates","year":"2021","journal-title":"Journal of the ACM"},{"key":"2024112119154842400_bib12","doi-asserted-by":"publisher","first-page":"1727","DOI":"10.1145\/3564246.3585182","article-title":"A unifying theory of distance from calibration","volume-title":"Proceedings of the 55th Annual ACM Symposium on Theory of Computing","author":"B\u0142asiok","year":"2023"},{"key":"2024112119154842400_bib13","doi-asserted-by":"publisher","first-page":"5454","DOI":"10.18653\/v1\/2020.acl-main.485","article-title":"Language (technology) is power: A critical survey of \u201cbias\u201d in NLP","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Lin Blodgett","year":"2020"},{"key":"2024112119154842400_bib14","article-title":"Robust validation: Confident predictions even when distributions shift","author":"Cauchois","year":"2020","journal-title":"arXiv preprint arXiv:2008.04267"},{"key":"2024112119154842400_bib15","doi-asserted-by":"publisher","first-page":"732","DOI":"10.1920\/wp.cem.2018.1618","article-title":"Exact and robust conformal inference methods for predictive machine learning with dependent data","volume-title":"Proceedings of the 31st Conference On Learning Theory","author":"Chernozhukov","year":"2018"},{"issue":"48","key":"2024112119154842400_bib16","doi-asserted-by":"publisher","first-page":"e2107794118","DOI":"10.1073\/pnas.2107794118","article-title":"Distributional conformal prediction","volume":"118","author":"Chernozhukov","year":"2021","journal-title":"Proceedings of the National Academy of Sciences"},{"issue":"536","key":"2024112119154842400_bib17","doi-asserted-by":"publisher","first-page":"1849","DOI":"10.1080\/01621459.2021.1920957","article-title":"An exact and robust conformal inference method for counterfactual and synthetic controls","volume":"116","author":"Chernozhukov","year":"2021","journal-title":"Journal of the American Statistical Association"},{"key":"2024112119154842400_bib18","article-title":"How flawed is ece? An analysis via logit smoothing","author":"Chidambaram","year":"2024","journal-title":"arXiv preprint arXiv:2402.10046"},{"key":"2024112119154842400_bib19","doi-asserted-by":"publisher","first-page":"3027","DOI":"10.18653\/v1\/2022.emnlp-main.196","article-title":"Conformal predictor for improving zero-shot text classification efficiency","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing","author":"Choubey","year":"2022"},{"key":"2024112119154842400_bib20","article-title":"Conformal prediction sets improve human decision making","author":"Cresswell","year":"2024","journal-title":"arXiv preprint arXiv:2401.13744"},{"key":"2024112119154842400_bib21","volume-title":"Multi-Objective Optimization Using Evolutionary Algorithms","author":"Deb","year":"2001"},{"issue":"4","key":"2024112119154842400_bib22","doi-asserted-by":"publisher","first-page":"485","DOI":"10.1109\/JPROC.2020.2976475","article-title":"Model compression and hardware acceleration for neural networks: A comprehensive survey","volume":"108","author":"Deng","year":"2020","journal-title":"Proceedings of the IEEE"},{"key":"2024112119154842400_bib23","doi-asserted-by":"publisher","first-page":"295","DOI":"10.18653\/v1\/2020.emnlp-main.21","article-title":"Calibration of pre-trained transformers","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Desai","year":"2020"},{"key":"2024112119154842400_bib24","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)","author":"Devlin","year":"2019"},{"issue":"1","key":"2024112119154842400_bib25","doi-asserted-by":"publisher","first-page":"69","DOI":"10.51387\/22-NEJSDS8","article-title":"Conformal prediction for text infilling and part-of-speech prediction","volume":"1","author":"Dey","year":"2022","journal-title":"The New England Journal of Statistics in Data Science"},{"key":"2024112119154842400_bib26","article-title":"Estimating uncertainty in multimodal foundation models using public internet data","volume-title":"R0-FoMo:Robustness of Few-shot and Zero-shot Learning in Large Foundation Models","author":"Dutta","year":"2023"},{"key":"2024112119154842400_bib27","doi-asserted-by":"publisher","first-page":"4506","DOI":"10.18653\/v1\/2020.coling-main.398","article-title":"Is MAP decoding all you need? The inadequacy of the mode in neural machine translation","volume-title":"Proceedings of the 28th International Conference on Computational Linguistics","author":"Eikema","year":"2020"},{"key":"2024112119154842400_bib28","first-page":"16","article-title":"Applying the conformal prediction paradigm for the uncertainty quantification of an end-to-end automatic speech recognition model (wav2vec 2.0)","volume-title":"Proceedings of the Twelfth Symposium on Conformal and Probabilistic Prediction with Applications","author":"Ernez","year":"2023"},{"key":"2024112119154842400_bib29","doi-asserted-by":"publisher","first-page":"11956","DOI":"10.18653\/v1\/2023.emnlp-main.733","article-title":"An empirical study of translation hypothesis ensembling with large language models","volume-title":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","author":"Farinhas","year":"2023"},{"key":"2024112119154842400_bib30","article-title":"Non-exchangeable conformal risk control","volume-title":"The Twelfth International Conference on Learning Representations","author":"Farinhas","year":"2024"},{"key":"2024112119154842400_bib31","article-title":"Improving conditional coverage via orthogonal quantile regression","volume-title":"Advances in Neural Information Processing Systems","author":"Feldman","year":"2021"},{"key":"2024112119154842400_bib32","doi-asserted-by":"publisher","first-page":"1396","DOI":"10.18653\/v1\/2022.naacl-main.100","article-title":"Quality- aware decoding for neural machine translation","volume-title":"Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Fernandes","year":"2022"},{"key":"2024112119154842400_bib33","doi-asserted-by":"publisher","first-page":"1643","DOI":"10.1162\/tacl_a_00626","article-title":"Bridging the gap: A survey on integrating (human) feedback for natural language generation","volume":"11","author":"Fernandes","year":"2023","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024112119154842400_bib34","first-page":"3329","article-title":"Few-shot conformal prediction with auxiliary tasks","volume-title":"Proceedings of the 38th International Conference on Machine Learning","author":"Fisch","year":"2021"},{"key":"2024112119154842400_bib35","first-page":"6514","article-title":"Conformal prediction sets with limited false positives","volume-title":"Proceedings of the 39th International Conference on Machine Learning","author":"Fisch","year":"2022"},{"key":"2024112119154842400_bib36","article-title":"Efficient conformal prediction via cascaded inference with expanded admission","volume-title":"International Conference on Learning Representations","author":"Fisch","year":"2021"},{"key":"2024112119154842400_bib37","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00524","article-title":"Bias and fairness in large language models: A survey","author":"Gallegos","year":"2024","journal-title":"arXiv preprint arXiv:2309.00770"},{"key":"2024112119154842400_bib38","first-page":"1660","article-title":"Adaptive conformal inference under distribution shift","volume-title":"Advances in Neural Information Processing Systems","author":"Gibbs","year":"2021"},{"key":"2024112119154842400_bib39","article-title":"Conformal inference for online prediction with arbitrary distribution shifts","author":"Gibbs","year":"2022"},{"key":"2024112119154842400_bib40","article-title":"Conformal prediction with conditional guarantees","author":"Gibbs","year":"2023","journal-title":"arXiv preprint arXiv:2305.12616"},{"key":"2024112119154842400_bib41","first-page":"55","article-title":"Calibration of natural language understanding models with venn\u2013abers predictors","volume-title":"Proceedings of the Eleventh Symposium on Conformal and Probabilistic Prediction with Applications","author":"Giovannotti","year":"2022"},{"key":"2024112119154842400_bib42","first-page":"413","article-title":"Evaluating machine translation quality with conformal predictive distributions","volume-title":"Proceedings of the Twelfth Symposium on Conformal and Probabilistic Prediction with Applications","author":"Giovannotti","year":"2023"},{"key":"2024112119154842400_bib43","first-page":"243","article-title":"Transformer-based conformal predictors for paraphrase detection","volume-title":"Proceedings of the Tenth Symposium on Conformal and Probabilistic Prediction and Applications","author":"Giovannotti","year":"2021"},{"key":"2024112119154842400_bib44","doi-asserted-by":"publisher","first-page":"3920","DOI":"10.18653\/v1\/2021.findings-emnlp.330","article-title":"Uncertainty-aware machine translation evaluation","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2021","author":"Glushkova","year":"2021"},{"key":"2024112119154842400_bib45","article-title":"Time travel in LLMs: Tracing data contamination in large language models","volume-title":"The Twelfth International Conference on Learning Representations","author":"Golchin","year":"2024"},{"key":"2024112119154842400_bib46","doi-asserted-by":"publisher","first-page":"439","DOI":"10.1145\/2470654.2470718","article-title":"The efficacy of human post-editing for language translation","volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems","author":"Green","year":"2013"},{"key":"2024112119154842400_bib47","doi-asserted-by":"publisher","first-page":"1500","DOI":"10.1162\/tacl_a_00615","article-title":"Hallucinations in Large Multilingual Translation Models","volume":"11","author":"Guerreiro","year":"2023","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024112119154842400_bib48","article-title":"Detoxify","author":"Hanu","year":"2020"},{"key":"2024112119154842400_bib49","doi-asserted-by":"publisher","first-page":"8362","DOI":"10.18653\/v1\/2020.emnlp-main.671","article-title":"Towards more accurate uncertainty estimation in text classification","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"He","year":"2020"},{"key":"2024112119154842400_bib50","article-title":"Cautious deep learning","author":"Hechtlinger","year":"2019","journal-title":"arXiv preprint arXiv:1805.09460"},{"key":"2024112119154842400_bib51","article-title":"The curious case of neural text degeneration","volume-title":"International Conference on Learning Representations","author":"Holtzman","year":"2020"},{"key":"2024112119154842400_bib52","article-title":"Uncertainty in natural language processing: Sources, quantification, and applications","author":"Mengting","year":"2023","journal-title":"arXiv preprint arXiv:2306.04459"},{"key":"2024112119154842400_bib53","article-title":"A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions","author":"Huang","year":"2023","journal-title":"arXiv preprint arXiv:2311.05232"},{"issue":"12","key":"2024112119154842400_bib54","doi-asserted-by":"publisher","DOI":"10.1145\/3571730","article-title":"Survey of hallucination in natural language generation","volume":"55","author":"Ji","year":"2023","journal-title":"ACM Computing Surveys"},{"key":"2024112119154842400_bib55","article-title":"Confidence on the focal: Conformal prediction with selection-conditional coverage","author":"Jin","year":"2024","journal-title":"arXiv preprint arXiv:2403.03868"},{"key":"2024112119154842400_bib56","article-title":"Well-calibrated probabilistic predictive maintenance using venn-abers","author":"Johansson","year":"2023","journal-title":"arXiv preprint arXiv:2306.06642"},{"key":"2024112119154842400_bib57","article-title":"Spoc: Search-based pseudocode to code","volume-title":"Advances in Neural Information Processing Systems","author":"Kulal","year":"2019"},{"key":"2024112119154842400_bib58","first-page":"2796","article-title":"Accurate uncertainties for deep learning using calibrated regression","volume-title":"Proceedings of the 35th International Conference on Machine Learning","author":"Kuleshov","year":"2018"},{"key":"2024112119154842400_bib59","article-title":"Conformal prediction with large language models for multi-choice question answering","author":"Kumar","year":"2023","journal-title":"arXiv preprint arXiv:2305.18404"},{"key":"2024112119154842400_bib60","first-page":"169","article-title":"Minimum Bayes-risk decoding for statistical machine translation","volume-title":"Proceedings of the Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics: HLT-NAACL 2004","author":"Kumar","year":"2004"},{"key":"2024112119154842400_bib61","doi-asserted-by":"publisher","DOI":"10.1007\/s10472-014-9420-z","article-title":"Inductive venn prediction","volume":"74","author":"Lambrou","year":"2014","journal-title":"Annals of Mathematics and Artificial Intelligence"},{"key":"2024112119154842400_bib62","article-title":"Efficiently controlling multiple risks with pareto testing","volume-title":"The Eleventh International Conference on Learning Representations","author":"Laufer-Goldshtein","year":"2023"},{"key":"2024112119154842400_bib63","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4419-1428-6_1802","volume-title":"Introspective Learning and Reasoning","author":"Leake","year":"2012"},{"issue":"523","key":"2024112119154842400_bib64","doi-asserted-by":"publisher","first-page":"1094","DOI":"10.1080\/01621459.2017.1307116","article-title":"Distribution-free predictive inference for regression","volume":"113","author":"Lei","year":"2018","journal-title":"Journal of the American Statistical Association"},{"key":"2024112119154842400_bib65","first-page":"1253","article-title":"CUE: An uncertainty interpretation framework for text classifiers built on pre-trained language models","volume-title":"Proceedings of the Thirty-Ninth Conference on Uncertainty in Artificial Intelligence","author":"Li","year":"2023"},{"key":"2024112119154842400_bib66","article-title":"Introspective planning: Guiding language-enabled agents to refine their own uncertainty","author":"Liang","year":"2024","journal-title":"arXiv preprint arXiv:2402.06529"},{"key":"2024112119154842400_bib67","first-page":"74","article-title":"ROUGE: A package for automatic evaluation of summaries","volume-title":"Text Summarization Branches Out","author":"Lin","year":"2004"},{"key":"2024112119154842400_bib68","article-title":"Roberta: A robustly optimized bert pretraining approach","author":"Liu","year":"2019","journal-title":"arXiv preprint arXiv:1907.11692"},{"key":"2024112119154842400_bib69","doi-asserted-by":"publisher","first-page":"108271","DOI":"10.1016\/j.patcog.2021.108271","article-title":"Well-calibrated confidence measures for multi-label text classification with a large number of labels","volume":"122","author":"Maltoudoglou","year":"2022","journal-title":"Pattern Recognition"},{"key":"2024112119154842400_bib70","first-page":"269","article-title":"Bert-based conformal predictor for sentiment analysis","volume-title":"Proceedings of the Ninth Symposium on Conformal and Probabilistic Prediction and Applications","author":"Maltoudoglou","year":"2020"},{"key":"2024112119154842400_bib71","first-page":"228","article-title":"Multi-class probabilistic classification using inductive and cross Venn\u2013Abers predictors","volume-title":"Proceedings of the Sixth Workshop on Conformal and Probabilistic Prediction and Applications","author":"Manokhin","year":"2017"},{"key":"2024112119154842400_bib72","article-title":"When less is more: Investigating data pruning for pretraining LLMs at scale","volume-title":"NeurIPS Workshop on Attributing Model Behavior at Scale","author":"Marion","year":"2023"},{"key":"2024112119154842400_bib73","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-50146-4_39","volume-title":"Deep Conformal Prediction for Robust Models","author":"Messoudi","year":"2020"},{"issue":"2","key":"2024112119154842400_bib74","doi-asserted-by":"publisher","DOI":"10.1145\/3605943","article-title":"Recent advances in natural language processing via large pre-trained language models: A survey","volume":"56","author":"Min","year":"2023","journal-title":"ACM Computing Surveys"},{"key":"2024112119154842400_bib75","first-page":"15682","article-title":"Revisiting the calibration of modern neural networks","volume":"34","author":"Minderer","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"issue":"1","key":"2024112119154842400_bib76","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1080\/23270012.2022.2031324","article-title":"Predicting amazon customer reviews with deep confidence using deep learning and conformal prediction","volume":"9","author":"Norinder","year":"2022","journal-title":"Journal of Management Analytics"},{"key":"2024112119154842400_bib77","first-page":"228","article-title":"A deep neural network conformal predictor for multi-label text classification","volume-title":"Proceedings of the Eighth Symposium on Conformal and Probabilistic Prediction and Applications","author":"Paisios","year":"2019"},{"key":"2024112119154842400_bib78","doi-asserted-by":"publisher","first-page":"677","DOI":"10.1162\/tacl_a_00293","article-title":"Inherent disagreements in human textual inferences","volume":"7","author":"Pavlick","year":"2019","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024112119154842400_bib79","doi-asserted-by":"publisher","first-page":"10671","DOI":"10.18653\/v1\/2022.emnlp-main.731","article-title":"The \u201cproblem\u201d of human label variation: On ground truth in data, modeling and evaluation","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing","author":"Plank","year":"2022"},{"key":"2024112119154842400_bib80","first-page":"844","article-title":"Distribution-free uncertainty quantification for classification under label shift","volume-title":"Proceedings of the Thirty-Seventh Conference on Uncertainty in Artificial Intelligence","author":"Podkopaev","year":"2021"},{"key":"2024112119154842400_bib81","article-title":"Conformal language modeling","volume-title":"The Twelfth International Conference on Learning Representations","author":"Quach","year":"2024"},{"key":"2024112119154842400_bib82","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proceedings of the 38th International Conference on Machine Learning","author":"Radford","year":"2021"},{"key":"2024112119154842400_bib83","doi-asserted-by":"publisher","first-page":"27","DOI":"10.18653\/v1\/2023.findings-acl.3","article-title":"Conformal nucleus sampling","volume-title":"Findings of the Association for Computational Linguistics: ACL 2023","author":"Ravfogel","year":"2023"},{"key":"2024112119154842400_bib84","article-title":"Robots that ask for help: Uncertainty alignment for large language model planners","volume-title":"7th Annual Conference on Robot Learning","author":"Ren","year":"2023"},{"key":"2024112119154842400_bib85","first-page":"4036","article-title":"Mitigating bias in calibration error estimation","volume-title":"International Conference on Artificial Intelligence and Statistics","author":"Roelofs","year":"2022"},{"issue":"2","key":"2024112119154842400_bib86","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.03f00592","article-title":"With Malice Toward None: Assessing Uncertainty via Equalized Coverage","volume":"2","author":"Romano","year":"2020","journal-title":"Harvard Data Science Review"},{"key":"2024112119154842400_bib87","doi-asserted-by":"publisher","first-page":"10776","DOI":"10.18653\/v1\/2023.findings-emnlp.722","article-title":"NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2023","author":"Sainz","year":"2023"},{"key":"2024112119154842400_bib88","article-title":"Confident adaptive language modeling","volume-title":"Advances in Neural Information Processing Systems","author":"Schuster","year":"2022"},{"key":"2024112119154842400_bib89","article-title":"Consistent accelerated inference via confident adaptive transformers","author":"Schuster","year":"2021","journal-title":"arXiv preprint arXiv:2104.08803"},{"key":"2024112119154842400_bib90","doi-asserted-by":"publisher","first-page":"6640","DOI":"10.18653\/v1\/2020.acl-main.593","article-title":"The right tool for the job: Matching model and instance complexities","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Schwartz","year":"2020"},{"key":"2024112119154842400_bib91","article-title":"Active learning literature survey","author":"Settles","year":"2009"},{"key":"2024112119154842400_bib92","first-page":"371","article-title":"A tutorial on conformal prediction","volume":"9","author":"Shafer","year":"2008","journal-title":"Journal of Machine Learning Research"},{"key":"2024112119154842400_bib93","first-page":"6216","article-title":"Conformal time-series forecasting","volume":"34","author":"Stankeviciute","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024112119154842400_bib94","first-page":"3008","article-title":"Learning to summarize with human feedback","volume":"33","author":"Stiennon","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024112119154842400_bib95","doi-asserted-by":"publisher","first-page":"4265","DOI":"10.18653\/v1\/2023.findings-acl.262","article-title":"Follow the wisdom of the crowd: Effective text generation via minimum Bayes risk decoding","volume-title":"Findings of the Association for Computational Linguistics: ACL 2023","author":"Suzgun","year":"2023"},{"key":"2024112119154842400_bib96","article-title":"Conformal prediction under covariate shift","volume-title":"Advances in Neural Information Processing Systems","author":"Tibshirani","year":"2019"},{"key":"2024112119154842400_bib97","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-09823-4_34","volume-title":"Mining Multi-label Data","author":"Tsoumakas","year":"2010"},{"key":"2024112119154842400_bib98","first-page":"1909","article-title":"Non-exchangeable conformal language generation with nearest neighbors","volume-title":"Findings of the Association for Computational Linguistics: EACL 2024","author":"Ulmer","year":"2024"},{"key":"2024112119154842400_bib99","doi-asserted-by":"publisher","first-page":"7335","DOI":"10.1109\/ICASSP.2019.8683359","article-title":"Towards better confidence estimation for neural models","volume-title":"ICASSP 2019 \u2013 2019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","author":"Vasudevan","year":"2019"},{"key":"2024112119154842400_bib100","first-page":"475","article-title":"Conditional validity of inductive conformal predictors","volume-title":"Proceedings of the Asian Conference on Machine Learning","author":"Vovk","year":"2012"},{"issue":"1","key":"2024112119154842400_bib101","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1007\/s10472-013-9368-4","article-title":"Cross-conformal predictors","volume":"74","author":"Vovk","year":"2015","journal-title":"Annals of Mathematics and Artificial Intelligence"},{"key":"2024112119154842400_bib102","volume-title":"Algorithmic Learning in a Random World","author":"Vovk","year":"2005"},{"key":"2024112119154842400_bib103","first-page":"37","article-title":"Cross-conformal predictive distributions","volume-title":"Proceedings of the Seventh Workshop on Conformal and Probabilistic Prediction and Applications","author":"Vovk","year":"2018"},{"key":"2024112119154842400_bib104","first-page":"829","article-title":"Venn-abers predictors","volume-title":"Proceedings of the Thirtieth Conference on Uncertainty in Artificial Intelligence","author":"Vovk","year":"2014"},{"key":"2024112119154842400_bib105","first-page":"82","article-title":"Nonparametric predictive distributions based on conformal prediction","volume-title":"Proceedings of the Sixth Workshop on Conformal and Probabilistic Prediction and Applications","author":"Vovk","year":"2017"},{"key":"2024112119154842400_bib106","article-title":"Safe task planning for language-instructed multi-robot systems using conformal prediction","author":"Wang","year":"2024","journal-title":"arXiv preprint arXiv:2402.15368"},{"key":"2024112119154842400_bib107","article-title":"Self-consistency improves chain of thought reasoning in language models","volume-title":"The Eleventh International Conference on Learning Representations","author":"Wang","year":"2023"},{"key":"2024112119154842400_bib108","doi-asserted-by":"publisher","first-page":"680","DOI":"10.1162\/tacl_a_00483","article-title":"Uncertainty estimation and reduction of pre-trained models for text regression","volume":"10","author":"Wang","year":"2022","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024112119154842400_bib109","first-page":"47","article-title":"Putting humans in the natural language processing loop: A survey","volume-title":"Proceedings of the First Workshop on Bridging Human\u2013Computer Interaction and Natural Language Processing","author":"Wang","year":"2021"},{"key":"2024112119154842400_bib110","doi-asserted-by":"publisher","first-page":"11","DOI":"10.18653\/v1\/D19-1002","article-title":"Attention is not not explanation","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Wiegreffe","year":"2019"},{"issue":"01","key":"2024112119154842400_bib111","doi-asserted-by":"publisher","first-page":"7322","DOI":"10.1609\/aaai.v33i01.33017322","article-title":"Quantifying uncertainties in natural language processing tasks","volume":"33","author":"Xiao","year":"2019","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"2024112119154842400_bib112","first-page":"11559","article-title":"Conformal prediction interval for dynamic time-series","volume-title":"Proceedings of the 38th International Conference on Machine Learning","author":"Chen","year":"2021"},{"key":"2024112119154842400_bib113","article-title":"Mitigating llm hallucinations via conformal abstention","author":"Yadkori","year":"2024"},{"key":"2024112119154842400_bib114","article-title":"Benchmarking llms via uncertainty quantification","author":"Ye","year":"2024","journal-title":"arXiv preprint arXiv:2401.12794"},{"key":"2024112119154842400_bib115","first-page":"25834","article-title":"Adaptive conformal predictions for time series","volume-title":"Proceedings of the 39th International Conference on Machine Learning","author":"Zaffran","year":"2022"},{"key":"2024112119154842400_bib116","doi-asserted-by":"publisher","first-page":"8622","DOI":"10.18653\/v1\/2022.emnlp-main.591","article-title":"Disentangling uncertainty in machine translation evaluation","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing","author":"Zerva","year":"2022"},{"key":"2024112119154842400_bib117","article-title":"Conformalizing machine translation evaluation","author":"Zerva","year":"2023","journal-title":"arXiv preprint arXiv:2306.06221"},{"key":"2024112119154842400_bib118","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/JBHI.2022.3193365","article-title":"Reliably filter drug-induced liver injury literature with natural language processing and conformal prediction","volume":"PP","author":"Zhan","year":"2022","journal-title":"IEEE Journal of Biomedical and Health Informatics"},{"key":"2024112119154842400_bib119","article-title":"Reliability-based cleaning of noisy training labels with inductive conformal prediction in multi-modal biomedical data mining","author":"Zhan","year":"2023","journal-title":"arXiv preprint arXiv:2309.07332"},{"issue":"2","key":"2024112119154842400_bib120","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3639372","article-title":"Explainability for large language models: A survey","volume":"15","author":"Zhao","year":"2024","journal-title":"ACM Transactions on Intelligent Systems and Technology"},{"key":"2024112119154842400_bib121","article-title":"Batch calibration: Rethinking calibration for in-context learning and prompt engineering","author":"Zhou","year":"2024"},{"key":"2024112119154842400_bib122","article-title":"Prompt risk control: A rigorous framework for responsible deployment of large language models","volume-title":"Socially Responsible Language Modelling Research","author":"Zollo","year":"2023"}],"container-title":["Transactions of the Association for Computational Linguistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00715\/2480372\/tacl_a_00715.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00715\/2480372\/tacl_a_00715.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,21]],"date-time":"2024-11-21T19:16:16Z","timestamp":1732216576000},"score":1,"resource":{"primary":{"URL":"https:\/\/direct.mit.edu\/tacl\/article\/doi\/10.1162\/tacl_a_00715\/125278\/Conformal-Prediction-for-Natural-Language"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":122,"URL":"https:\/\/doi.org\/10.1162\/tacl_a_00715","relation":{},"ISSN":["2307-387X"],"issn-type":[{"value":"2307-387X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024]]},"published":{"date-parts":[[2024]]}}}