{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T11:37:47Z","timestamp":1791459467181,"version":"4.3.4"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3663529.3663858","type":"proceedings-article","created":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T15:43:13Z","timestamp":1720626193000},"page":"388-398","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["LM-PACE: Confidence Estimation by Large Language Models for Effective Root Causing of Cloud Incidents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-8974-3356","authenticated-orcid":false,"given":"Dylan","family":"Zhang","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Champaign, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5344-456X","authenticated-orcid":false,"given":"Xuchao","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0102-8139","authenticated-orcid":false,"given":"Chetan","family":"Bansal","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7590-3736","authenticated-orcid":false,"given":"Pedro","family":"Las-Casas","sequence":"additional","affiliation":[{"name":"Azure Research, Redmond, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9662-2661","authenticated-orcid":false,"given":"Rodrigo","family":"Fonseca","sequence":"additional","affiliation":[{"name":"Azure Research, Redmond, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2019-213X","authenticated-orcid":false,"given":"Saravan","family":"Rajmohan","sequence":"additional","affiliation":[{"name":"Microsoft 365, Redmond, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Toufique Ahmed Supriyo Ghosh Chetan Bansal Thomas Zimmermann Xuchao Zhang and Saravan Rajmohan. 2023. Recommending Root-Cause and Mitigation Steps for Cloud Incidents using Large Language Models. arxiv:2301.03797.","DOI":"10.1109\/ICSE48619.2023.00149"},{"key":"e_1_3_2_1_2_1","unstructured":"Amar Prakash Azad Supriyo Ghosh Ajay Gupta Harshit Kumar and Prateeti Mohapatra. 2021. Picking Pearl From Seabed: Extracting Artefacts from Noisy Issue Triaging Collaborative Conversations for Hybrid Cloud Services. arxiv:2105.15065."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Yejin Bang Samuel Cahyawijaya Nayeon Lee Wenliang Dai Dan Su Bryan Wilie Holy Lovenia Ziwei Ji Tiezheng Yu and Willy Chung. 2023. A multitask multilingual multimodal evaluation of chatgpt on reasoning hallucination and interactivity. arXiv preprint arXiv:2302.04023.","DOI":"10.18653\/v1\/2023.ijcnlp-main.45"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377813.3381353"},{"key":"e_1_3_2_1_5_1","volume-title":"Yuanzhi Li, Scott Lundberg, Harsha Nori, Hamid Palangi, Marco Tulio Ribeiro, and Yi Zhang.","author":"Bubeck S\u00e9bastien","year":"2023","unstructured":"S\u00e9bastien Bubeck, Varun Chandrasekaran, Ronen Eldan, Johannes Gehrke, Eric Horvitz, Ece Kamar, Peter Lee, Yin Tat Lee, Yuanzhi Li, Scott Lundberg, Harsha Nori, Hamid Palangi, Marco Tulio Ribeiro, and Yi Zhang. 2023. Sparks of Artificial General Intelligence: Early experiments with GPT-4. arxiv:2303.12712."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00020"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00042"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Yinfang Chen Huaibing Xie Minghua Ma Yu Kang Xin Gao Liu Shi Yunjie Cao Xuedong Gao Hao Fan Ming Wen Jun Zeng Supriyo Ghosh Xuchao Zhang Chaoyun Zhang Qingwei Lin Saravan Rajmohan and Dongmei Zhang. 2023. Empowering Practical Root Cause Analysis by Large Language Models for Cloud Incidents. arxiv:2305.15778.","DOI":"10.1145\/3627703.3629553"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/0010-0277(95)00664-8"},{"key":"e_1_3_2_1_10_1","volume-title":"Chenyuan Yang, Shizhuo Dylan Zhang, Shujing Yang, and Lingming Zhang.","author":"Deng Yinlin","year":"2023","unstructured":"Yinlin Deng, Chunqiu Steven Xia, Chenyuan Yang, Shizhuo Dylan Zhang, Shujing Yang, and Lingming Zhang. 2023. Large language models are edge-case fuzzers: Testing deep learning libraries via fuzzgpt. arXiv preprint arXiv:2304.02014."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Shehzaad Dhuliawala Mojtaba Komeili Jing Xu Roberta Raileanu Xian Li Asli Celikyilmaz and Jason Weston. 2023. Chain-of-Verification Reduces Hallucination in Large Language Models. arxiv:2309.11495.","DOI":"10.18653\/v1\/2024.findings-acl.212"},{"key":"e_1_3_2_1_12_1","volume-title":"International conference on machine learning. 3929\u20133938","author":"Guu Kelvin","year":"2020","unstructured":"Kelvin Guu, Kenton Lee, Zora Tung, Panupong Pasupat, and Mingwei Chang. 2020. Retrieval augmented language model pre-training. In International conference on machine learning. 3929\u20133938."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Lei Huang Weijiang Yu Weitao Ma Weihong Zhong Zhangyin Feng Haotian Wang Qianglong Chen Weihua Peng Xiaocheng Feng Bing Qin and Ting Liu. 2023. A Survey on Hallucination in Large Language Models: Principles Taxonomy Challenges and Open Questions. arxiv:2311.05232.","DOI":"10.1145\/3703155"},{"key":"e_1_3_2_1_14_1","unstructured":"Gautier Izacard Patrick Lewis Maria Lomeli Lucas Hosseini Fabio Petroni Timo Schick Jane Dwivedi-Yu Armand Joulin Sebastian Riedel and Edouard Grave. 2022. Few-shot learning with retrieval augmented language models. arXiv preprint arXiv:2208.03299."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571730"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417054"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00407"},{"key":"e_1_3_2_1_18_1","unstructured":"Pengxiang Jin Shenglin Zhang Minghua Ma Haozhe Li Yu Kang Liqun Li Yudong Liu Bo Qiao Chaoyun Zhang Pu Zhao Shilin He Federica Sarro Yingnong Dang Saravan Rajmohan Qingwei Lin and Dongmei Zhang. 2023. Assess and Summarize: Improve Outage Understanding with Large Language Models. arxiv:2305.18084."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Vladimir Karpukhin Barlas O\u011fuz Sewon Min Patrick Lewis Ledell Wu Sergey Edunov Danqi Chen and Wen-tau Yih. 2020. Dense passage retrieval for open-domain question answering. arXiv preprint arXiv:2004.04906.","DOI":"10.18653\/v1\/2020.emnlp-main.550"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_21_1","unstructured":"Stephanie Lin Jacob Hilton and Owain Evans. 2022. Teaching Models to Express Their Uncertainty in Words. arxiv:2205.14334."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623374"},{"key":"e_1_3_2_1_23_1","unstructured":"Gr\u00e9goire Mialon Roberto Dessi Maria Lomeli Christoforos Nalmpantis Ramakanth Pasunuru Roberta Raileanu Baptiste Roziere Timo Schick Jane Dwivedi-Yu Asli Celikyilmaz Edouard Grave Yann LeCun and Thomas Scialom. 2023. Augmented Language Models: a Survey. Transactions on Machine Learning Research issn:2835-8856 https:\/\/openreview.net\/forum?id=jh7wH2AzKK Survey Certification"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00494"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788624"},{"key":"e_1_3_2_1_26_1","volume-title":"Guy Gur-Ari, Henryk Michalewski, Jacob Austin, David Bieber, David Dohan, Aitor Lewkowycz, Maarten Bosma, and David Luan.","author":"Nye Maxwell","year":"2021","unstructured":"Maxwell Nye, Anders Johan Andreassen, Guy Gur-Ari, Henryk Michalewski, Jacob Austin, David Bieber, David Dohan, Aitor Lewkowycz, Maarten Bosma, and David Luan. 2021. Show your work: Scratchpads for intermediate computation with language models. arXiv preprint arXiv:2112.00114."},{"key":"e_1_3_2_1_27_1","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, and Alex Ray. 2022. Training language models to follow instructions with human feedback. Advances in Neural Information Processing Systems, 35 (2022), 27730\u201327744.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_28_1","unstructured":"Vipula Rawte Amit Sheth and Amitava Das. 2023. A Survey of Hallucination in Large Foundation Models. arXiv preprint arXiv:2309.05922."},{"key":"e_1_3_2_1_29_1","volume-title":"Replug: Retrieval-augmented black-box language models. arXiv preprint arXiv:2301.12652.","author":"Shi Weijia","year":"2023","unstructured":"Weijia Shi, Sewon Min, Michihiro Yasunaga, Minjoon Seo, Rich James, Mike Lewis, Luke Zettlemoyer, and Wen-tau Yih. 2023. Replug: Retrieval-augmented black-box language models. arXiv preprint arXiv:2301.12652."},{"key":"e_1_3_2_1_30_1","volume-title":"Manning","author":"Tian Katherine","year":"2023","unstructured":"Katherine Tian, Eric Mitchell, Allan Zhou, Archit Sharma, Rafael Rafailov, Huaxiu Yao, Chelsea Finn, and Christopher D. Manning. 2023. Just Ask for Calibration: Strategies for Eliciting Calibrated Confidence Scores from Language Models Fine-Tuned with Human Feedback. arxiv:2305.14975."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Zezhong Wang Fangkai Yang Pu Zhao Lu Wang Jue Zhang Mohit Garg Qingwei Lin and Dongmei Zhang. 2023. Empower Large Language Model to Perform Better on Industrial Domain-Specific Question Answering. arxiv:2305.11541.","DOI":"10.18653\/v1\/2023.emnlp-industry.29"},{"key":"e_1_3_2_1_32_1","unstructured":"Miao Xiong Zhiyuan Hu Xinyang Lu Yifei Li Jie Fu Junxian He and Bryan Hooi. 2023. Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs. arXiv preprint arXiv:2306.13063."},{"key":"e_1_3_2_1_33_1","volume-title":"Cognitive Mirage: A Review of Hallucinations in Large Language Models. arxiv:2309.06794.","author":"Ye Hongbin","year":"2023","unstructured":"Hongbin Ye, Tong Liu, Aijia Zhang, Wei Hua, and Weiqiang Jia. 2023. Cognitive Mirage: A Review of Hallucinations in Large Language Models. arxiv:2309.06794."},{"key":"e_1_3_2_1_34_1","volume-title":"Smith","author":"Zhang Muru","year":"2023","unstructured":"Muru Zhang, Ofir Press, William Merrill, Alisa Liu, and Noah A. Smith. 2023. How Language Model Hallucinations Can Snowball. arxiv:2305.13534."},{"key":"e_1_3_2_1_35_1","unstructured":"Shizhuo Dylan Zhang Talia Ringer and Emily First. 2023. Getting More out of Large Language Models for Proofs. arxiv:2305.04369."},{"key":"e_1_3_2_1_36_1","volume-title":"Wei Bi, Freda Shi, and Shuming Shi.","author":"Zhang Yue","year":"2023","unstructured":"Yue Zhang, Yafu Li, Leyang Cui, Deng Cai, Lemao Liu, Tingchen Fu, Xinting Huang, Enbo Zhao, Yu Zhang, Yulong Chen, Longyue Wang, Anh Tuan Luu, Wei Bi, Freda Shi, and Shuming Shi. 2023. Siren\u2019s Song in the AI Ocean: A Survey on Hallucination in Large Language Models. arxiv:2309.01219."},{"key":"e_1_3_2_1_37_1","unstructured":"Theodore Zhao Mu Wei J Samuel Preston and Hoifung Poon. 2023. Automatic Calibration and Error Correction for Large Language Models via Pareto Optimal Self-Supervision. arXiv preprint arXiv:2306.16564."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Kaitlyn Zhou Dan Jurafsky and Tatsunori Hashimoto. 2023. Navigating the Grey Area: Expressions of Overconfidence and Uncertainty in Language Models. arxiv:2302.13439.","DOI":"10.18653\/v1\/2023.emnlp-main.335"},{"key":"e_1_3_2_1_39_1","volume-title":"Fine-Tuning Language Models from Human Preferences. ArXiv, abs\/1909.08593","author":"Ziegler Daniel M.","year":"2019","unstructured":"Daniel M. Ziegler, Nisan Stiennon, Jeff Wu, Tom B. Brown, Alec Radford, Dario Amodei, Paul Christiano, and Geoffrey Irving. 2019. Fine-Tuning Language Models from Human Preferences. ArXiv, abs\/1909.08593 (2019), https:\/\/api.semanticscholar.org\/CorpusID:202660943"}],"event":{"name":"FSE '24: 32nd ACM International Conference on the Foundations of Software Engineering","location":"Porto de Galinhas Brazil","acronym":"FSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663858","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3663529.3663858","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:44:22Z","timestamp":1750275862000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663858"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":39,"alternative-id":["10.1145\/3663529.3663858","10.1145\/3663529"],"URL":"https:\/\/doi.org\/10.1145\/3663529.3663858","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}