{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:15:42Z","timestamp":1784178942497,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":60,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NSF","award":["IIS-1956221"],"award-info":[{"award-number":["IIS-1956221"]}]},{"name":"IARPA","award":["2019-19051600004"],"award-info":[{"award-number":["2019-19051600004"]}]},{"name":"ODNI","award":["2019-19051600004"],"award-info":[{"award-number":["2019-19051600004"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3404835.3462951","type":"proceedings-article","created":{"date-parts":[[2021,7,12]],"date-time":"2021-07-12T02:41:52Z","timestamp":1626057712000},"page":"654-664","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["Not All Relevance Scores are Equal: Efficient Uncertainty and Calibration Modeling for Deep Retrieval Models"],"prefix":"10.1145","author":[{"given":"Daniel","family":"Cohen","sequence":"first","affiliation":[{"name":"Brown University, Providence, RI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bhaskar","family":"Mitra","sequence":"additional","affiliation":[{"name":"Microsoft, Montreal, PQ, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Oleg","family":"Lesota","sequence":"additional","affiliation":[{"name":"Johannes Kepler University, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Navid","family":"Rekabsaz","sequence":"additional","affiliation":[{"name":"Johannes Kepler University, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Carsten","family":"Eickhoff","sequence":"additional","affiliation":[{"name":"Brown University, Providence, RI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,7,11]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_2_1","volume-title":"Proceedings of the 29th European Conference on IR Research","author":"Javed","unstructured":"Javed A. Aslam and Virgil Pavlu. 2007. Query Hardness Estimation Using Jensen-Shannon Divergence among Multiple Scoring Functions. In Proceedings of the 29th European Conference on IR Research (Rome, Italy) (ECIR'07). Springer-Verlag, Berlin, Heidelberg, 198--209."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401188"},{"key":"e_1_3_2_2_4_1","volume-title":"Garnett (Eds.)","volume":"28","author":"Chow Yinlam","year":"2015","unstructured":"Yinlam Chow, Aviv Tamar, Shie Mannor, and Marco Pavone. 2015. Risk-Sensitive and Robust Decision-Making: a CVaR Optimization Approach. In Advances in Neural Information Processing Systems, C. Cortes, N. Lawrence, D. Lee, M. Sugiyama, and R. Garnett (Eds.), Vol. 28. Curran Associates, Inc., 1522--1530. https:\/\/proceedings.neurips.cc\/paper\/2015\/file\/64223ccf70bbb65a3a4aceac37e21016-Paper.pdf"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390446"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210141"},{"key":"e_1_3_2_2_7_1","volume-title":"Overview of the TREC 2019 deep learning track.","author":"Craswell Nick","year":"2019","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, and Daniel Campos. 2019. Overview of the TREC 2019 deep learning track."},{"key":"e_1_3_2_2_8_1","volume-title":"Proceedings of the 21st Australasian Document Computing Symposium, ADCS 2016, Caulfield, VIC, Australia, December 5--7","author":"Culpepper J. Shane","year":"2016","unstructured":"J. Shane Culpepper, Charles L. A. Clarke, and Jimmy J. Lin. 2016. Dynamic Cutoff Prediction in Multi-Stage Retrieval Systems. In Proceedings of the 21st Australasian Document Computing Symposium, ADCS 2016, Caulfield, VIC, Australia, December 5--7, 2016, Sarvnaz Karimi and Mark James Carman (Eds.). ACM, 17--24. http:\/\/dl.acm.org\/citation.cfm?id=3015026"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010063"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331303"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159659"},{"key":"e_1_3_2_2_12_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. CoRR","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. CoRR, Vol. abs\/1810.04805 (2018). arxiv: 1810.04805 http:\/\/arxiv.org\/abs\/1810.04805"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411962"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.134"},{"key":"e_1_3_2_2_15_1","volume-title":"Proceedings of The 33rd International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"1059","author":"Gal Yarin","year":"2016","unstructured":"Yarin Gal and Zoubin Ghahramani. 2016. Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning. In Proceedings of The 33rd International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 48), Maria Florina Balcan and Kilian Q. Weinberger (Eds.). PMLR, New York, New York, USA, 1050--1059. http:\/\/proceedings.mlr.press\/v48\/gal16.html"},{"key":"e_1_3_2_2_16_1","volume-title":"Concrete Dropout. In Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017","author":"Gal Yarin","year":"2017","unstructured":"Yarin Gal, Jiri Hron, and Alex Kendall. 2017. Concrete Dropout. In Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, December 4--9, 2017, Long Beach, CA, USA, Isabelle Guyon, Ulrike von Luxburg, Samy Bengio, Hanna M. Wallach, Rob Fergus, S. V. N. Vishwanathan, and Roman Garnett (Eds.). 3581--3590. https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/84ddfb34126fc3a48ee38d7044e87276-Abstract.html"},{"key":"e_1_3_2_2_17_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning -","volume":"70","author":"Guo Chuan","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian Q. Weinberger. 2017. On Calibration of Modern Neural Networks. In Proceedings of the 34th International Conference on Machine Learning - Volume 70 (Sydney, NSW, Australia) (ICML'17). JMLR.org, 1321--1330."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341981.3344249"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341981.3344249"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401061"},{"key":"e_1_3_2_2_21_1","first-page":"1","article-title":"Stochastic Variational Inference","volume":"14","author":"Hoffman Matthew D.","year":"2013","unstructured":"Matthew D. Hoffman, David M. Blei, Chong Wang, and John Paisley. 2013. Stochastic Variational Inference. J. Mach. Learn. Res., Vol. 14, 1 (May 2013), 1303--1347.","journal-title":"J. Mach. Learn. Res."},{"key":"e_1_3_2_2_22_1","volume-title":"Proc. of the ACM SIGIR Conference on Research and Development in Information Retrieval .","author":"Sebastian","year":"2019","unstructured":"Sebastian Hofst\"atter, Navid Rekabsaz, Carsten Eickhoff, and Allan Hanbury. 2019. On the Effect of Low-Frequency Terms on Neural-IR Models. In Proc. of the ACM SIGIR Conference on Research and Development in Information Retrieval ."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401224"},{"key":"e_1_3_2_2_24_1","volume-title":"Leveraging passage retrieval with generative models for open domain question answering. arXiv preprint arXiv:2007.01282","author":"Izacard Gautier","year":"2020","unstructured":"Gautier Izacard and Edouard Grave. 2020. Leveraging passage retrieval with generative models for open domain question answering. arXiv preprint arXiv:2007.01282 (2020)."},{"key":"e_1_3_2_2_25_1","volume-title":"Proceedings of the 37th International Conference on Machine Learning, ICML 2020, 13--18","volume":"4973","author":"Jordan Scott M.","year":"2020","unstructured":"Scott M. Jordan, Yash Chandak, Daniel Cohen, Mengxue Zhang, and Philip S. Thomas. 2020. Evaluating the Performance of Reinforcement Learning Algorithms. In Proceedings of the 37th International Conference on Machine Learning, ICML 2020, 13--18 July 2020, Virtual Event (Proceedings of Machine Learning Research, Vol. 119). PMLR, 4962--4973. http:\/\/proceedings.mlr.press\/v119\/jordan20a.html"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/3295222.3295309"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401075"},{"key":"e_1_3_2_2_29_1","volume-title":"Kingma and Jimmy Ba","author":"Diederik","year":"2015","unstructured":"Diederik P. Kingma and Jimmy Ba. 2015. Adam: A Method for Stochastic Optimization. In 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7--9, 2015, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.). http:\/\/arxiv.org\/abs\/1412.6980"},{"key":"e_1_3_2_2_30_1","volume-title":"Fixes Overconfidence in ReLU Networks. In Proceedings of the 37th International Conference on Machine Learning (Proceedings of Machine Learning Research","author":"Kristiadi Agustinus","year":"2020","unstructured":"Agustinus Kristiadi, Matthias Hein, and Philipp Hennig. 2020. Being Bayesian, Even Just a Bit, Fixes Overconfidence in ReLU Networks. In Proceedings of the 37th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 119), Hal Daum\u00e9 III and Aarti Singh (Eds.). PMLR, Virtual, 5436--5446."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341981.3344234"},{"key":"e_1_3_2_2_32_1","volume-title":"Pretrained Transformers for Text Ranking: BERT and Beyond. arxiv","author":"Lin Jimmy","year":"2010","unstructured":"Jimmy Lin, Rodrigo Nogueira, and Andrew Yates. 2020. Pretrained Transformers for Text Ranking: BERT and Beyond. arxiv: 2010.06467 [cs.IR]"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911473"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1387"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"crossref","unstructured":"Bhaskar Mitra Sebastian Hofst\"atter Hamed Zamani and Nick Craswell. 2020. Conformer-Kernel with Query Term Independence at TREC 2020 Deep Learning Track.","DOI":"10.6028\/NIST.SP.1266.deep-MSAI"},{"key":"e_1_3_2_2_36_1","volume-title":"Proceedings for the 27th AIAI Irish Conference on Artificial Intelligence and Cognitive Science, Galway, Ireland, December 5--6, 2019 (CEUR Workshop Proceedings","volume":"151","author":"Mitros John","year":"2019","unstructured":"John Mitros and Brian Mac Namee. 2019. On the Validity of Bayesian Neural Networks for Uncertainty Estimation. In Proceedings for the 27th AIAI Irish Conference on Artificial Intelligence and Cognitive Science, Galway, Ireland, December 5--6, 2019 (CEUR Workshop Proceedings, Vol. 2563), Edward Curry, Mark T. Keane, Adegboyega Ojo, and Dhaval Salwala (Eds.). CEUR-WS.org, 140--151. http:\/\/ceur-ws.org\/Vol-2563\/aics_15.pdf"},{"key":"e_1_3_2_2_37_1","volume-title":"Proceedings of the 27th International Conference on International Conference on Machine Learning","author":"Nair Vinod","unstructured":"Vinod Nair and Geoffrey E. Hinton. 2010. Rectified Linear Units Improve Restricted Boltzmann Machines. In Proceedings of the 27th International Conference on International Conference on Machine Learning (Haifa, Israel) (ICML'10). Omnipress, Madison, WI, USA, 807--814."},{"key":"e_1_3_2_2_38_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=H1xwNhCcYm","author":"Nalisnick Eric","year":"2019","unstructured":"Eric Nalisnick, Akihiro Matsukawa, Yee Whye Teh, Dilan Gorur, and Balaji Lakshminarayanan. 2019. Do Deep Generative Models Know What They Don't Know?. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=H1xwNhCcYm"},{"key":"e_1_3_2_2_39_1","volume-title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. CoRR","author":"Nguyen Tri","year":"2016","unstructured":"Tri Nguyen, Mir Rosenberg, Xia Song, Jianfeng Gao, Saurabh Tiwary, Rangan Majumder, and Li Deng. 2016. MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. CoRR, Vol. abs\/1611.09268 (2016). arxiv: 1611.09268 http:\/\/arxiv.org\/abs\/1611.09268"},{"key":"e_1_3_2_2_40_1","volume-title":"Passage Re-ranking with BERT. CoRR","author":"Nogueira Rodrigo","year":"2019","unstructured":"Rodrigo Nogueira and Kyunghyun Cho. 2019. Passage Re-ranking with BERT. CoRR, Vol. abs\/1901.04085 (2019). arxiv: 1901.04085 http:\/\/arxiv.org\/abs\/1901.04085"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_42_1","volume-title":"Document Expansion by Query Prediction. arXiv preprint arXiv:1904.08375","author":"Nogueira Rodrigo","year":"2019","unstructured":"Rodrigo Nogueira, Wei Yang, Jimmy Lin, and Kyunghyun Cho. 2019. Document Expansion by Query Prediction. arXiv preprint arXiv:1904.08375 (2019)."},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"crossref","unstructured":"Gustavo Penha and Claudia Hauff. 2021. On the Calibration and Uncertainty of Neural Learning to Rank Models. arxiv: 2101.04356 [cs.IR]","DOI":"10.18653\/v1\/2021.eacl-main.12"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291008"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401279"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-56608-5_31"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.21314\/JOR.2000.038"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121051"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121087"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2180868.2180873"},{"key":"e_1_3_2_2_51_1","first-page":"1","article-title":"Dropout: A Simple Way to Prevent Neural Networks from Overfitting","volume":"15","author":"Srivastava Nitish","year":"2014","unstructured":"Nitish Srivastava, Geoffrey Hinton, Alex Krizhevsky, Ilya Sutskever, and Ruslan Salakhutdinov. 2014. Dropout: A Simple Way to Prevent Neural Networks from Overfitting. J. Mach. Learn. Res., Vol. 15, 1 (Jan. 2014), 1929--1958.","journal-title":"J. Mach. Learn. Res."},{"key":"e_1_3_2_2_52_1","volume-title":"International 1989 Joint Conference on Neural Networks","volume":"2","author":"Tishby Naftali","year":"1989","unstructured":"Naftali Tishby, E. Levin, and S. Solla. 1989. Consistent inference of probabilities in layered networks: predictions and generalizations. International 1989 Joint Conference on Neural Networks (1989), 403--409 vol.2."},{"key":"e_1_3_2_2_53_1","volume-title":"Well-Read Students Learn Better: The Impact of Student Initialization on Knowledge Distillation. CoRR","author":"Turc Iulia","year":"2019","unstructured":"Iulia Turc, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. Well-Read Students Learn Better: The Impact of Student Initialization on Knowledge Distillation. CoRR, Vol. abs\/1908.08962 (2019). arxiv: 1908.08962 http:\/\/arxiv.org\/abs\/1908.08962"},{"key":"e_1_3_2_2_54_1","volume-title":"Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020","author":"Wilson Andrew Gordon","year":"2020","unstructured":"Andrew Gordon Wilson and Pavel Izmailov. 2020. Bayesian Deep Learning and a Probabilistic Perspective of Generalization. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6--12, 2020, virtual, Hugo Larochelle, Marc'Aurelio Ranzato, Raia Hadsell, Maria-Florina Balcan, and Hsuan-Tien Lin (Eds.). https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/322f62469c5e3c7dc3e58f5a4d1ea399-Abstract.html"},{"key":"e_1_3_2_2_55_1","volume-title":"Approximate Nearest Neighbor Negative Contrastive Learning for Dense Text Retrieval. arxiv","author":"Xiong Lee","year":"2007","unstructured":"Lee Xiong, Chenyan Xiong, Ye Li, Kwok-Fung Tang, Jialin Liu, Paul Bennett, Junaid Ahmed, and Arnold Overwijk. 2020. Approximate Nearest Neighbor Negative Contrastive Learning for Dense Text Retrieval. arxiv: 2007.00808 [cs.IR]"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210041"},{"key":"e_1_3_2_2_57_1","volume-title":"2020 a. Learning To Retrieve: How to Train a Dense Retrieval Model Effectively and Efficiently. arxiv","author":"Zhan Jingtao","year":"2010","unstructured":"Jingtao Zhan, Jiaxin Mao, Yiqun Liu, Min Zhang, and Shaoping Ma. 2020 a. Learning To Retrieve: How to Train a Dense Retrieval Model Effectively and Efficiently. arxiv: 2010.10469 [cs.IR]"},{"key":"e_1_3_2_2_58_1","volume-title":"2020 b. RepBERT: Contextualized Text Embeddings for First-Stage Retrieval. arxiv","author":"Zhan Jingtao","year":"2006","unstructured":"Jingtao Zhan, Jiaxin Mao, Yiqun Liu, Min Zhang, and Shaoping Ma. 2020 b. RepBERT: Contextualized Text Embeddings for First-Stage Retrieval. arxiv: 2006.15498 [cs.IR]"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183696"},{"key":"e_1_3_2_2_60_1","volume-title":"Cox","author":"Zhu Jianhan","year":"2009","unstructured":"Jianhan Zhu, Jun Wang, Michael Taylor, and Ingemar J. Cox. 2009. Risk-Aware Information Retrieval. In Advances in Information Retrieval, Mohand Boughanem, Catherine Berrut, Josiane Mothe, and Chantal Soule-Dupuy (Eds.). Springer Berlin Heidelberg, Berlin, Heidelberg, 17--28."}],"event":{"name":"SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Virtual Event Canada","acronym":"SIGIR '21","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3462951","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3462951","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3462951","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:18:20Z","timestamp":1750191500000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3462951"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":60,"alternative-id":["10.1145\/3404835.3462951","10.1145\/3404835"],"URL":"https:\/\/doi.org\/10.1145\/3404835.3462951","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-07-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}