{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:14:54Z","timestamp":1775283294743,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Australian Research Council","award":["DP200103136"],"award-info":[{"award-number":["DP200103136"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3471158.3472239","type":"proceedings-article","created":{"date-parts":[[2021,8,31]],"date-time":"2021-08-31T20:38:17Z","timestamp":1630442297000},"page":"231-237","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["ERR is not C\/W\/L: Exploring the Relationship Between Expected Reciprocal Rank and Other Metrics"],"prefix":"10.1145","author":[{"given":"Leif","family":"Azzopardi","sequence":"first","affiliation":[{"name":"University of Strathclyde, Glasgow, United Kingdom"}]},{"given":"Joel","family":"Mackenzie","sequence":"additional","affiliation":[{"name":"The University of Melbourne, Melbourne, Australia"}]},{"given":"Alistair","family":"Moffat","sequence":"additional","affiliation":[{"name":"The University of Melbourne, Melbourne, Australia"}]}],"member":"320","published-online":{"date-parts":[[2021,8,31]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210027"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331398"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3343413.3377959"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401033"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_2_7_1","volume-title":"Proc. TREC","author":"Clarke C. L. A.","year":"2010","unstructured":"C. L. A. Clarke , N. Craswell , I. Soboroff , and G. V. Cormack . Overview of the TREC 2010 Web Track . In Proc. TREC , 2010 . C. L. A. Clarke, N. Craswell, I. Soboroff, and G. V. Cormack. Overview of the TREC 2010 Web Track. In Proc. TREC, 2010."},{"key":"e_1_3_2_2_8_1","first-page":"27","volume-title":"Proc. SIGIR","author":"Collins-Thompson K.","year":"2009","unstructured":"K. Collins-Thompson . Accounting for stability of retrieval algorithms using risk-reward curves . In Proc. SIGIR , pages 27 -- 28 , 2009 . K. Collins-Thompson. Accounting for stability of retrieval algorithms using risk-reward curves. In Proc. SIGIR, pages 27--28, 2009."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_2_10_1","first-page":"23","volume-title":"Proc. SIGIR","author":"B. T.","year":"2014","unstructured":"B. T. Dincc er, C. Macdonald , and I. Ounis . Hypothesis testing for the risk-sensitive evaluation of retrieval systems . In Proc. SIGIR , pages 23 -- 32 , 2014 . B. T. Dincc er, C. Macdonald, and I. Ounis. Hypothesis testing for the risk-sensitive evaluation of retrieval systems. In Proc. SIGIR, pages 23--32, 2014."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409832"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809452"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1_14"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.2307\/2332226"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401085"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3052768"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-7152(98)00006-6"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348300"},{"key":"e_1_3_2_2_23_1","volume-title":"Proc. TREC","author":"Soboroff I.","year":"2011","unstructured":"I. Soboroff , N. Craswell , C. L. A. Clarke , and G. V. Cormack . Overview of the TREC 2011 Web Track . In Proc. TREC , 2011 . I. Soboroff, N. Craswell, C. L. A. Clarke, and G. V. Cormack. Overview of the TREC 2011 Web Track. In Proc. TREC, 2011."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741088"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3269242"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371799"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102601"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080841"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-009-9099-7"}],"event":{"name":"ICTIR '21: The 2021 ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Virtual Event Canada","acronym":"ICTIR '21","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 2021 ACM SIGIR International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3471158.3472239","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3471158.3472239","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T17:49:29Z","timestamp":1750268969000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3471158.3472239"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":29,"alternative-id":["10.1145\/3471158.3472239","10.1145\/3471158"],"URL":"https:\/\/doi.org\/10.1145\/3471158.3472239","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-08-31","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}