{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:15:05Z","timestamp":1775283305348,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,6]],"date-time":"2022-07-06T00:00:00Z","timestamp":1657065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP190101113,DP200103136"],"award-info":[{"award-number":["DP190101113,DP200103136"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,6]]},"DOI":"10.1145\/3477495.3531924","type":"proceedings-article","created":{"date-parts":[[2022,7,7]],"date-time":"2022-07-07T15:12:08Z","timestamp":1657206728000},"page":"578-587","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":19,"title":["A Flexible Framework for Offline Effectiveness Metrics"],"prefix":"10.1145","author":[{"given":"Alistair","family":"Moffat","sequence":"first","affiliation":[{"name":"The University of Melbourne, Melbourne, VIC, Australia"}]},{"given":"Joel","family":"Mackenzie","sequence":"additional","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}]},{"given":"Paul","family":"Thomas","sequence":"additional","affiliation":[{"name":"Microsoft, Canberra, Australia"}]},{"given":"Leif","family":"Azzopardi","sequence":"additional","affiliation":[{"name":"University of Strathclyde, Glasgow, United Kingdom"}]}],"member":"320","published-online":{"date-parts":[[2022,7,7]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210027"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331398"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3343413.3377959"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3471158.3472239"},{"key":"e_1_3_2_2_5_1","first-page":"53","volume-title":"Voorhees and Harman [39]","author":"Buckley C.","unstructured":"C. Buckley and E. M. Voorhees . Retrieval system evaluation . In Voorhees and Harman [39] , chapter 3, pages 53 -- 78 . C. Buckley and E. M. Voorhees. Retrieval system evaluation. In Voorhees and Harman [39], chapter 3, pages 53--78."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2396782"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450127"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080804"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb049778"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835538"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-019-09362-z"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.65.1.45"},{"issue":"3","key":"e_1_3_2_2_16_1","first-page":"114","article-title":"Rating the rating scales","volume":"9","author":"Friedman H. H.","year":"1999","unstructured":"H. H. Friedman and T. Amoo . Rating the rating scales . J. Marketing Manag. , 9 ( 3 ): 114 -- 123 , 1999 . H. H. Friedman and T. Amoo. Rating the rating scales. J. Marketing Manag., 9(3): 114--123, 1999.","journal-title":"J. Marketing Manag."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3190580.3190586"},{"key":"e_1_3_2_2_18_1","first-page":"21","volume-title":"Voorhees and Harman [39]","author":"Harman D. K.","unstructured":"D. K. Harman . The TREC test collections. In Voorhees and Harman [39] , chapter 2, pages 21 -- 52 . D. K. Harman. The TREC test collections. In Voorhees and Harman [39], chapter 2, pages 21--52."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.5555\/1611321"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3178876.3186065"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911507"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-45068-6_1"},{"issue":"1","key":"e_1_3_2_2_25_1","first-page":"2","article-title":"Rank-biased precision for measurement of retrieval effectiveness","volume":"27","author":"Moffat A.","unstructured":"A. Moffat and J. Zobel . Rank-biased precision for measurement of retrieval effectiveness . ACM Trans. Inf. Sys. , 27 ( 1 ): 2 .1--2.27, 2008. A. Moffat and J. Zobel. Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Sys., 27(1):2.1--2.27, 2008.","journal-title":"ACM Trans. Inf. Sys."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3052768"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390453"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835550"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"crossref","unstructured":"T. Sakai. On Fuhr's guideline for IR evaluation. SIGIR Forum 54(1):12:1--12:8 2020.  T. Sakai. On Fuhr's guideline for IR evaluation. SIGIR Forum 54(1):12:1--12:8 2020.","DOI":"10.1145\/3451964.3451976"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484031"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331215"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3431813"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835542"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2398568"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348300"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-45068-6_36"},{"key":"e_1_3_2_2_39_1","volume-title":"TREC: Experiment and Evaluation in Information Retrieval","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees and D. K. Harman , editors . TREC: Experiment and Evaluation in Information Retrieval . MIT Press , 2005 . E. M. Voorhees and D. K. Harman, editors. TREC: Experiment and Evaluation in Information Retrieval. MIT Press, 2005."},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009934"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3269242"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102601"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/1871437.1871672"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080841"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2021.02.003"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401162"}],"event":{"name":"SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Madrid Spain","acronym":"SIGIR '22","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3531924","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477495.3531924","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:19Z","timestamp":1750183819000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3531924"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,6]]},"references-count":46,"alternative-id":["10.1145\/3477495.3531924","10.1145\/3477495"],"URL":"https:\/\/doi.org\/10.1145\/3477495.3531924","relation":{},"subject":[],"published":{"date-parts":[[2022,7,6]]},"assertion":[{"value":"2022-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}