{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T13:40:43Z","timestamp":1768570843807,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,7,18]],"date-time":"2019-07-18T00:00:00Z","timestamp":1563408000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Natural Science Foundation of China (NSFC)","award":["Grants No. 61425016, 61722211, 61773362, and 61872338"],"award-info":[{"award-number":["Grants No. 61425016, 61722211, 61773362, and 61872338"]}]},{"name":"Youth Innovation Promotion Association CAS","award":["Grants No. 20144310, and 2016102"],"award-info":[{"award-number":["Grants No. 20144310, and 2016102"]}]},{"name":"National Key R&D Program of China","award":["Grants No. 2016QY02D0405"],"award-info":[{"award-number":["Grants No. 2016QY02D0405"]}]},{"name":"Foundation and Frontier Research Key Program of Chongqing Science and Technology Commission","award":["No. cstc2017jcyjBX0059"],"award-info":[{"award-number":["No. cstc2017jcyjBX0059"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,7,18]]},"DOI":"10.1145\/3331184.3331261","type":"proceedings-article","created":{"date-parts":[[2019,7,19]],"date-time":"2019-07-19T13:40:26Z","timestamp":1563543626000},"page":"115-124","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Controlling Risk of Web Question Answering"],"prefix":"10.1145","author":[{"given":"Lixin","family":"Su","sequence":"first","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiafeng","family":"Guo","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yixin","family":"Fan","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yanyan","family":"Lan","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xueqi","family":"Cheng","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences & University of Chinese Academy of Sciences, beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2019,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Understanding intermediate layers using linear classifier probes. arXiv preprint arXiv:1610.01644","author":"Alain Guillaume","year":"2016","unstructured":"Guillaume Alain and Yoshua Bengio. 2016. Understanding intermediate layers using linear classifier probes. arXiv preprint arXiv:1610.01644 (2016)."},{"key":"e_1_3_2_2_2_1","volume-title":"Confidence from Invariance to Image Transformations. arXiv preprint arXiv:1804.00657","author":"Bahat Yuval","year":"2018","unstructured":"Yuval Bahat and Gregory Shakhnarovich. 2018. Confidence from Invariance to Image Transformations. arXiv preprint arXiv:1804.00657 (2018)."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","unstructured":"John Blatz Erin Fitzgerald George F. Foster Simona Gandrabur Cyril Goutte Alex Kulesza Alberto Sanchis and Nicola Ueffing. 2004. Confidence estimation for machine translation. (2004) 315. 10.3115\/1220355.1220401","DOI":"10.3115\/1220355.1220401"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.3115\/1118693.1118726"},{"key":"e_1_3_2_2_5_1","unstructured":"Danqi Chen. 2018. Neural Reading Comprehension and Beyond. Ph.D. Dissertation. Stanford University."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1171"},{"key":"e_1_3_2_2_7_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","unstructured":"Akshay Raj Dhamija Manuel G\u00fcnther and Terrance Boult. 2018. Reducing Network Agnostophobia. In Advances in Neural Information Processing Systems. 9175--9186.","DOI":"10.5555\/3327546.3327590"},{"key":"e_1_3_2_2_9_1","volume-title":"Cohen","author":"Dhingra Bhuwan","year":"2017","unstructured":"Bhuwan Dhingra, Kathryn Mazaitis, and William W. Cohen. 2017. Quasar: Datasets for Question Answering by Search and Reading. arXiv preprint arXiv:1707.03904 (2017)."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1069"},{"key":"e_1_3_2_2_11_1","volume-title":"Searchqa: A new q&a dataset augmented with context from a search engine. arXiv preprint arXiv:1704.05179","author":"Dunn Matthew","year":"2017","unstructured":"Matthew Dunn, Levent Sagun, Mike Higgins, Ugur Guney, Volkan Cirik, and Kyunghyun Cho. 2017. Searchqa: A new q&a dataset augmented with context from a search engine. arXiv preprint arXiv:1704.05179 (2017)."},{"key":"e_1_3_2_2_12_1","volume-title":"Building Watson: An overview of the DeepQA project. AI magazine","author":"Ferrucci David","year":"2010","unstructured":"David Ferrucci, Eric Brown, Jennifer Chu-Carroll, James Fan, David Gondek, Aditya A. Kalyanpur, Adam Lally, J. William Murdock, Eric Nyberg, John Prager, et al. 2010. Building Watson: An overview of the DeepQA project. AI magazine, Vol. 31, 3 (2010), 59--79."},{"key":"e_1_3_2_2_13_1","volume-title":"Uncertainty in deep learning","author":"Gal Yarin","unstructured":"Yarin Gal. 2016. Uncertainty in deep learning. University of Cambridge (2016)."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.5555\/3045390.3045502"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","unstructured":"Yonatan Geifman and Ran El-Yaniv. 2017. Selective classification for deep neural networks. In Advances in neural information processing systems. 4878--4887.","DOI":"10.5555\/3295222.3295241"},{"key":"e_1_3_2_2_16_1","volume-title":"Bias-Reduced Uncertainty Estimation for Deep Neural Classifiers. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=SJfb5jCqKm","author":"Geifman Yonatan","year":"2019","unstructured":"Yonatan Geifman, Guy Uziel, and Ran El-Yaniv. 2019. Bias-Reduced Uncertainty Estimation for Deep Neural Classifiers. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=SJfb5jCqKm"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.5555\/3305381.3305518"},{"key":"e_1_3_2_2_18_1","volume-title":"A baseline for detecting misclassified and out-of-distribution examples in neural networks. arXiv preprint arXiv:1610.02136","author":"Hendrycks Dan","year":"2016","unstructured":"Dan Hendrycks and Kevin Gimpel. 2016. A baseline for detecting misclassified and out-of-distribution examples in neural networks. arXiv preprint arXiv:1610.02136 (2016)."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","unstructured":"Karl Moritz Hermann Tomas Kocisky Edward Grefenstette Lasse Espeholt Will Kay Mustafa Suleyman and Phil Blunsom. 2015. Teaching machines to read and comprehend. In Advances in Neural Information Processing Systems. 1693--1701.","DOI":"10.5555\/2969239.2969428"},{"key":"e_1_3_2_2_20_1","volume-title":"et almbox","author":"Hu Minghao","year":"2018","unstructured":"Minghao Hu, Yuxing Peng, Zhen Huang, Nan Yang, Ming Zhou, et almbox. 2018. Read+verify: Machine reading comprehension with unanswerable questions. arXiv preprint arXiv:1808.05759 (2018)."},{"key":"e_1_3_2_2_21_1","volume-title":"Mnemonic reader for machine comprehension. arXiv preprint arXiv:1705.02798","author":"Hu Minghao","year":"2017","unstructured":"Minghao Hu, Yuxing Peng, and Xipeng Qiu. 2017. Mnemonic reader for machine comprehension. arXiv preprint arXiv:1705.02798 (2017)."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1147"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1456"},{"key":"e_1_3_2_2_24_1","volume-title":"Race: Large-scale reading comprehension dataset from examinations. arXiv preprint arXiv:1704.04683","author":"Lai Guokun","year":"2017","unstructured":"Guokun Lai, Qizhe Xie, Hanxiao Liu, Yiming Yang, and Eduard Hovy. 2017. Race: Large-scale reading comprehension dataset from examinations. arXiv preprint arXiv:1704.04683 (2017)."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","unstructured":"Balaji Lakshminarayanan Alexander Pritzel and Charles Blundell. 2017. Simple and scalable predictive uncertainty estimation using deep ensembles. In Advances in Neural Information Processing Systems. 6402--6413.","DOI":"10.5555\/3295222.3295387"},{"key":"e_1_3_2_2_26_1","volume-title":"conference on computational natural language learning","author":"Levy Omer","year":"2017","unstructured":"Omer Levy, Minjoon Seo, Eunsol Choi, and Luke S. Zettlemoyer. 2017. Zero-Shot Relation Extraction via Reading Comprehension. conference on computational natural language learning (2017), 333--342."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1161"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1992.4.3.448"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/3327757.3327808"},{"key":"e_1_3_2_2_30_1","volume-title":"Distance-based Confidence Score for Neural Network Classifiers. arXiv preprint arXiv:1709.09844","author":"Mandelbaum Amit","year":"2017","unstructured":"Amit Mandelbaum and Daphna Weinshall. 2017. Distance-based Confidence Score for Neural Network Classifiers. arXiv preprint arXiv:1709.09844 (2017)."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","unstructured":"Dan Moldovan Sanda Harabagiu Marius Pasca Rada Mihalcea Roxana Girju Richard Goodrum and Vasile Rus. 2000. The structure and performance of an open-domain question answering system. 563--570. 10.3115\/1075218.1075289","DOI":"10.3115\/1075218.1075289"},{"key":"e_1_3_2_2_32_1","volume-title":"MS MARCO: A human generated machine reading comprehension dataset. arXiv preprint arXiv:1611.09268","author":"Nguyen Tri","year":"2016","unstructured":"Tri Nguyen, Mir Rosenberg, Xia Song, Jianfeng Gao, Saurabh Tiwary, Rangan Majumder, and Li Deng. 2016. MS MARCO: A human generated machine reading comprehension dataset. arXiv preprint arXiv:1611.09268 (2016)."},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271702"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-2124"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D13-1020"},{"key":"e_1_3_2_2_37_1","volume-title":"Bidirectional attention flow for machine comprehension. arXiv preprint arXiv:1611.01603","author":"Seo Minjoon","year":"2016","unstructured":"Minjoon Seo, Aniruddha Kembhavi, Ali Farhadi, and Hannaneh Hajishirzi. 2016. Bidirectional attention flow for machine comprehension. arXiv preprint arXiv:1611.01603 (2016)."},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99495-6_8"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.5555\/3327345.3327399"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.5555\/2969442.2969540"},{"key":"e_1_3_2_2_41_1","first-page":"82","article-title":"The TREC-8 Question Answering Track Evaluation","volume":"1999","author":"Voorhees Ellen M.","year":"1999","unstructured":"Ellen M. Voorhees and Dawn M. Tice. 1999. The TREC-8 Question Answering Track Evaluation. In TREC, Vol. 1999. 82.","journal-title":"TREC"},{"key":"e_1_3_2_2_42_1","volume-title":"Machine comprehension using match-lstm and answer pointer. arXiv preprint arXiv:1608.07905","author":"Wang Shuohang","year":"2016","unstructured":"Shuohang Wang and Jing Jiang. 2016. Machine comprehension using match-lstm and answer pointer. arXiv preprint arXiv:1608.07905 (2016)."},{"key":"e_1_3_2_2_43_1","volume-title":"R3: Reinforced Reader-Ranker for Open-Domain Question Answering. arXiv preprint arXiv:1709.00023","author":"Wang Shuohang","year":"2017","unstructured":"Shuohang Wang, Mo Yu, Xiaoxiao Guo, Zhiguo Wang, Tim Klinger, Wei Zhang, Shiyu Chang, Gerald Tesauro, Bowen Zhou, and Jing Jiang. 2017. R3: Reinforced Reader-Ranker for Open-Domain Question Answering. arXiv preprint arXiv:1709.00023 (2017)."},{"key":"e_1_3_2_2_44_1","volume-title":"Evidence Aggregation for Answer Re-Ranking in Open-Domain Question Answering. In International Conference on Learning Representations.","author":"Wang Shuohang","year":"2018","unstructured":"Shuohang Wang, Mo Yu, Jing Jiang, Wei Zhang, Xiaoxiao Guo, Shiyu Chang, Zhiguo Wang, Tim Klinger, Gerald Tesauro, and Murray Campbell. 2018. Evidence Aggregation for Answer Re-Ranking in Open-Domain Question Answering. In International Conference on Learning Representations."},{"key":"e_1_3_2_2_45_1","volume-title":"Multi-granularity hierarchical attention fusion networks for reading comprehension and question answering. arXiv preprint arXiv:1811.11934","author":"Wang Wei","year":"2018","unstructured":"Wei Wang, Ming Yan, and Chen Wu. 2018. Multi-granularity hierarchical attention fusion networks for reading comprehension and question answering. arXiv preprint arXiv:1811.11934 (2018)."},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1018"},{"key":"e_1_3_2_2_47_1","volume-title":"Dynamic coattention networks for question answering. arXiv preprint arXiv:1611.01604","author":"Xiong Caiming","year":"2016","unstructured":"Caiming Xiong, Victor Zhong, and Richard Socher. 2016. Dynamic coattention networks for question answering. arXiv preprint arXiv:1611.01604 (2016)."}],"event":{"name":"SIGIR '19: The 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Paris France","acronym":"SIGIR '19","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331261","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3331184.3331261","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T00:52:45Z","timestamp":1768524765000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331261"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,7,18]]},"references-count":47,"alternative-id":["10.1145\/3331184.3331261","10.1145\/3331184"],"URL":"https:\/\/doi.org\/10.1145\/3331184.3331261","relation":{},"subject":[],"published":{"date-parts":[[2019,7,18]]},"assertion":[{"value":"2019-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}