{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,30]],"date-time":"2025-08-30T16:49:41Z","timestamp":1756572581285,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,6]],"date-time":"2022-07-06T00:00:00Z","timestamp":1657065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Spanish Ministry of Economic Affairs and Digital Transformation","award":["C039\/21-OT"],"award-info":[{"award-number":["C039\/21-OT"]}]},{"name":"Australian Research Council Centre of Excellence for Automated Decision-Making and Society","award":["CE200100005"],"award-info":[{"award-number":["CE200100005"]}]},{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DE200100064"],"award-info":[{"award-number":["DE200100064"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,6]]},"DOI":"10.1145\/3477495.3532051","type":"proceedings-article","created":{"date-parts":[[2022,7,7]],"date-time":"2022-07-07T15:12:08Z","timestamp":1657206728000},"page":"588-598","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Ranking Interruptus"],"prefix":"10.1145","author":[{"given":"Enrique","family":"Amig\u00f3","sequence":"first","affiliation":[{"name":"UNED NLP & IR Group, Madrid, Spain"}]},{"given":"Stefano","family":"Mizzaro","sequence":"additional","affiliation":[{"name":"University of Udine, Udine, Italy"}]},{"given":"Damiano","family":"Spina","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, VIC, Australia"}]}],"member":"320","published-online":{"date-parts":[[2022,7,7]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-15712-8_39"},{"key":"e_1_3_2_1_2_1","article-title":"Component-Based Analysis of Dynamic Search Performance","volume":"40","author":"Albahem Ameer","year":"2021","unstructured":"Ameer Albahem , Damiano Spina , Falk Scholer , and Lawrence Cavedon . 2021 . Component-Based Analysis of Dynamic Search Performance . ACM Trans. Inf. Syst. 40 , 3, Article 61 (nov 2021), 47 pages. https:\/\/doi.org\/10.1145\/3483237 10.1145\/3483237 Ameer Albahem, Damiano Spina, Falk Scholer, and Lawrence Cavedon. 2021. Component-Based Analysis of Dynamic Search Performance. ACM Trans. Inf. Syst. 40, 3, Article 61 (nov 2021), 47 pages. https:\/\/doi.org\/10.1145\/3483237","journal-title":"ACM Trans. Inf. Syst."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210131"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-020-09375-z"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3234944.3234958"},{"key":"e_1_3_2_1_6_1","first-page":"1","article-title":"Combining Evaluation Metrics via the Unanimous Improvement Ratio and Its Application to Clustering Tasks","volume":"42","author":"Amig\u00f3 Enrique","year":"2011","unstructured":"Enrique Amig\u00f3 , Julio Gonzalo , Javier Artiles , and Felisa Verdejo . 2011 . Combining Evaluation Metrics via the Unanimous Improvement Ratio and Its Application to Clustering Tasks . J. Artif. Int. Res. 42 , 1 (sep 2011), 689--718. https:\/\/doi.org\/10. 5555\/2208436.2208454 Enrique Amig\u00f3, Julio Gonzalo, Javier Artiles, and Felisa Verdejo. 2011. Combining Evaluation Metrics via the Unanimous Improvement Ratio and Its Application to Clustering Tasks. J. Artif. Int. Res. 42, 1 (sep 2011), 689--718. https:\/\/doi.org\/10. 5555\/2208436.2208454","journal-title":"J. Artif. Int. Res."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.363"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484081"},{"key":"e_1_3_2_1_9_1","volume-title":"The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '18)","author":"Amig\u00f3 Enrique","year":"2099","unstructured":"Enrique Amig\u00f3 , Damiano Spina , and Jorge Carrillo- de Albornoz . 2018. An Axiomatic Analysis of Diversity Evaluation Metrics: Introducing the Rank-Biased Utility Metric . In The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '18) . Association for Computing Machinery , New York, NY, USA , 625--634. https:\/\/doi.org\/10.1145\/3 2099 78.3210024 10.1145\/3209978.3210024 Enrique Amig\u00f3, Damiano Spina, and Jorge Carrillo-de Albornoz. 2018. An Axiomatic Analysis of Diversity Evaluation Metrics: Introducing the Rank-Biased Utility Metric. In The 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '18). Association for Computing Machinery, New York, NY, USA, 625--634. https:\/\/doi.org\/10.1145\/3209978.3210024"},{"key":"e_1_3_2_1_10_1","first-page":"34","article-title":"Conversational Search (Dagstuhl Seminar 19461)","volume":"9","author":"Anand Avishek","year":"2020","unstructured":"Avishek Anand , Lawrence Cavedon , Hideo Joho , Mark Sanderson , and Benno Stein . 2020 . Conversational Search (Dagstuhl Seminar 19461) . Dagstuhl Reports 9 , 11 (2020), 34 -- 83 . https:\/\/doi.org\/10.4230\/DagRep.9.11.34 10.4230\/DagRep.9.11.34 Avishek Anand, Lawrence Cavedon, Hideo Joho, Mark Sanderson, and Benno Stein. 2020. Conversational Search (Dagstuhl Seminar 19461). Dagstuhl Reports 9, 11 (2020), 34--83. https:\/\/doi.org\/10.4230\/DagRep.9.11.34","journal-title":"Dagstuhl Reports"},{"volume-title":"Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '04)","author":"Buckley Chris","key":"e_1_3_2_1_11_1","unstructured":"Chris Buckley and Ellen M. Voorhees . 2004. Retrieval Evaluation with Incomplete Information . In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '04) . Association for Computing Machinery, New York, NY, USA, 25--32. https:\/\/doi.org\/10.1145\/ 1008992.1009000 Chris Buckley and Ellen M. Voorhees. 2004. Retrieval Evaluation with Incomplete Information. In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '04). Association for Computing Machinery, New York, NY, USA, 25--32. https:\/\/doi.org\/10.1145\/ 1008992.1009000"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371780"},{"volume-title":"Proceedings of the IR Research, 30th European Conference on Advances in Information Retrieval (ECIR'08)","author":"Carterette Ben","key":"e_1_3_2_1_13_1","unstructured":"Ben Carterette , Paul N. Bennett , David Maxwell Chickering , and Susan T. Dumais . 2008. Here or There: Preference Judgments for Relevance . In Proceedings of the IR Research, 30th European Conference on Advances in Information Retrieval (ECIR'08) . Springer-Verlag, Berlin, Heidelberg, 16--27. https:\/\/doi.org\/10.5555\/ 1793274.1793281 Ben Carterette, Paul N. Bennett, David Maxwell Chickering, and Susan T. Dumais. 2008. Here or There: Preference Judgments for Relevance. In Proceedings of the IR Research, 30th European Conference on Advances in Information Retrieval (ECIR'08). Springer-Verlag, Berlin, Heidelberg, 16--27. https:\/\/doi.org\/10.5555\/ 1793274.1793281"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953"},{"volume-title":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '21)","author":"Clarke Charles L.A.","key":"e_1_3_2_1_15_1","unstructured":"Charles L.A. Clarke , Chengxi Luo , and Mark D. Smucker . 2021. Evaluation Measures Based on Preference Graphs . In Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '21) . Association for Computing Machinery, New York, NY, USA, 1534--1543. https:\/\/doi.org\/10.1145\/3404835.3462947 10.1145\/3404835.3462947 Charles L.A. Clarke, Chengxi Luo, and Mark D. Smucker. 2021. Evaluation Measures Based on Preference Graphs. In Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '21). Association for Computing Machinery, New York, NY, USA, 1534--1543. https:\/\/doi.org\/10.1145\/3404835.3462947"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411915"},{"volume-title":"Aslib proceedings","author":"Cleverdon Cyril","key":"e_1_3_2_1_17_1","unstructured":"Cyril Cleverdon . 1967. The Cranfield Tests on Index Language Devices . In Aslib proceedings , Vol. 19 . MCB UP Ltd, Emerald Publishing Ltd , 173--194. https:\/\/doi.org\/10.1108\/eb050097 10.1108\/eb050097 Cyril Cleverdon. 1967. The Cranfield Tests on Index Language Devices. In Aslib proceedings, Vol. 19. MCB UP Ltd, Emerald Publishing Ltd, 173--194. https:\/\/doi.org\/10.1108\/eb050097"},{"volume-title":"Mobile Information Retrieval","author":"Crestani Fabio","key":"e_1_3_2_1_18_1","unstructured":"Fabio Crestani , Stefano Mizzaro , and Ivan Scagnetto . 2017. Mobile Information Retrieval . Springer . https:\/\/doi.org\/10.1007\/978--3--319--60777--1 10.1007\/978--3--319--60777--1 Fabio Crestani, Stefano Mizzaro, and Ivan Scagnetto. 2017. Mobile Information Retrieval. Springer. https:\/\/doi.org\/10.1007\/978--3--319--60777--1"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3116857"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809452"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(91)90046-O"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-019-01383-w"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3469595.3469615"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914737"},{"key":"#cr-split#-e_1_3_2_1_26_1.1","doi-asserted-by":"crossref","unstructured":"Simon Mason and N.E. Graham. 2002. Areas Beneath the Relative Operating Characteristics (ROC) and Relative Operating Levels (ROL) Curves: Statistical Significance and Interpretation. Quarterly Journal of the Royal Meteorological Society 128 (07 2002) 2145 -- 2166. https:\/\/doi.org\/10.1256\/003590002320603584 10.1256\/003590002320603584","DOI":"10.1256\/003590002320603584"},{"key":"#cr-split#-e_1_3_2_1_26_1.2","doi-asserted-by":"crossref","unstructured":"Simon Mason and N.E. Graham. 2002. Areas Beneath the Relative Operating Characteristics (ROC) and Relative Operating Levels (ROL) Curves: Statistical Significance and Interpretation. Quarterly Journal of the Royal Meteorological Society 128 (07 2002) 2145 -- 2166. https:\/\/doi.org\/10.1256\/003590002320603584","DOI":"10.1256\/003590002320603584"},{"volume-title":"Information Retrieval Technology (AIRS'13)","author":"Moffat Alistair","key":"e_1_3_2_1_27_1","unstructured":"Alistair Moffat . 2013. Seven Numeric Properties of Effectiveness Metrics . In Information Retrieval Technology (AIRS'13) . Springer Berlin Heidelberg , Berlin, Heidelberg , 1--12. https:\/\/doi.org\/10.1007\/978--3--642--45068--6_1 10.1007\/978--3--642--45068--6_1 Alistair Moffat. 2013. Seven Numeric Properties of Effectiveness Metrics. In Information Retrieval Technology (AIRS'13). Springer Berlin Heidelberg, Berlin, Heidelberg, 1--12. https:\/\/doi.org\/10.1007\/978--3--642--45068--6_1"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"volume-title":"Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies (ACL '11)","author":"Anselmo","key":"e_1_3_2_1_29_1","unstructured":"Anselmo Pe\u00f1as and \u00c1lvaro Rodrigo. 2011. A Simple Measure to Assess Nonresponse . In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies (ACL '11) . Association for Computational Linguistics, Portland, Oregon, USA, 1415--1424. https: \/\/aclanthology.org\/P11--1142 Anselmo Pe\u00f1as and \u00c1lvaro Rodrigo. 2011. A Simple Measure to Assess Nonresponse. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies (ACL '11). Association for Computational Linguistics, Portland, Oregon, USA, 1415--1424. https: \/\/aclanthology.org\/P11--1142"},{"key":"e_1_3_2_1_30_1","volume-title":"Proceedings of the Fourth NTCIR Workshop on Research in Information Access Technologies Information Retrieval, Question Answering and Summarization (NTCIR-4). National Institute of Informatics (NII)","author":"Sakai Tetsuya","year":"2004","unstructured":"Tetsuya Sakai . 2004 . New Performance Metrics Based on Multigrade Relevance: Their Application to Question Answering . In Proceedings of the Fourth NTCIR Workshop on Research in Information Access Technologies Information Retrieval, Question Answering and Summarization (NTCIR-4). National Institute of Informatics (NII) , Tokyo, Japan, 17 pages. http:\/\/research.nii.ac.jp\/ntcir\/workshop\/ OnlineProceedings4\/OPEN\/NTCIR4-OPEN-SakaiTrev.pdf Tetsuya Sakai. 2004. New Performance Metrics Based on Multigrade Relevance: Their Application to Question Answering. In Proceedings of the Fourth NTCIR Workshop on Research in Information Access Technologies Information Retrieval, Question Answering and Summarization (NTCIR-4). National Institute of Informatics (NII), Tokyo, Japan, 17 pages. http:\/\/research.nii.ac.jp\/ntcir\/workshop\/ OnlineProceedings4\/OPEN\/NTCIR4-OPEN-SakaiTrev.pdf"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.2197\/ipsjjip.17.156"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331215"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-016-9281-7"},{"volume-title":"Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '12)","author":"Mark","key":"e_1_3_2_1_34_1","unstructured":"Mark D. Smucker and Charles L.A. Clarke. 2012. Time-Based Calibration of Effectiveness Measures . In Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '12) . Association for Computing Machinery, New York, NY, USA, 95--104. https: \/\/doi.org\/10.1145\/2348283.2348300 10.1145\/2348283.2348300 Mark D. Smucker and Charles L.A. Clarke. 2012. Time-Based Calibration of Effectiveness Measures. In Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '12). Association for Computing Machinery, New York, NY, USA, 95--104. https: \/\/doi.org\/10.1145\/2348283.2348300"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026616"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476415.3476421"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2019.102162"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383963"},{"volume-title":"Evaluation of Cross-Language Information Retrieval Systems (CLEF '02)","author":"Voorhees Ellen M.","key":"e_1_3_2_1_39_1","unstructured":"Ellen M. Voorhees . 2002. The Philosophy of Information Retrieval Evaluation . In Evaluation of Cross-Language Information Retrieval Systems (CLEF '02) , Carol Peters, Martin Braschler, Julio Gonzalo, and Michael Kluck (Eds.). Springer Berlin Heidelberg, Berlin , Heidelberg , 355--370. https:\/\/doi.org\/10.1007\/3--540--45691- 0_34 10.1007\/3--540--45691- Ellen M. Voorhees. 2002. The Philosophy of Information Retrieval Evaluation. In Evaluation of Cross-Language Information Retrieval Systems (CLEF '02), Carol Peters, Martin Braschler, Julio Gonzalo, and Michael Kluck (Eds.). Springer Berlin Heidelberg, Berlin, Heidelberg, 355--370. https:\/\/doi.org\/10.1007\/3--540--45691- 0_34"},{"key":"e_1_3_2_1_40_1","volume-title":"Harman","author":"Voorhees Ellen M.","year":"2005","unstructured":"Ellen M. Voorhees and Donna K . Harman . 2005 . TREC : Experiment and Evaluation in Information Retrieval. Vol. 1 . MIT Press Cambridge . Ellen M. Voorhees and Donna K. Harman. 2005. TREC: Experiment and Evaluation in Information Retrieval. Vol. 1. MIT Press Cambridge."},{"key":"e_1_3_2_1_41_1","volume-title":"Conversational Information Seeking. arXiv preprint arXiv:2201.08808","author":"Zamani Hamed","year":"2022","unstructured":"Hamed Zamani , Johanne R Trippas , Jeff Dalton , and Filip Radlinski . 2022. Conversational Information Seeking. arXiv preprint arXiv:2201.08808 ( 2022 ). https:\/\/doi.org\/10.48550\/arXiv.2201.08808 10.48550\/arXiv.2201.08808 Hamed Zamani, Johanne R Trippas, Jeff Dalton, and Filip Radlinski. 2022. Conversational Information Seeking. arXiv preprint arXiv:2201.08808 (2022). https:\/\/doi.org\/10.48550\/arXiv.2201.08808"}],"event":{"name":"SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Madrid Spain","acronym":"SIGIR '22"},"container-title":["Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3532051","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477495.3532051","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:35Z","timestamp":1750183835000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3532051"}},"subtitle":["When Truncated Rankings Are Better and How to Measure That"],"short-title":[],"issued":{"date-parts":[[2022,7,6]]},"references-count":42,"alternative-id":["10.1145\/3477495.3532051","10.1145\/3477495"],"URL":"https:\/\/doi.org\/10.1145\/3477495.3532051","relation":{},"subject":[],"published":{"date-parts":[[2022,7,6]]},"assertion":[{"value":"2022-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}