{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T01:00:03Z","timestamp":1774400403410,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":49,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,10,26]],"date-time":"2021-10-26T00:00:00Z","timestamp":1635206400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"EU Horizon 2020 MSCA","award":["893667"],"award-info":[{"award-number":["893667"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,10,26]]},"DOI":"10.1145\/3459637.3482287","type":"proceedings-article","created":{"date-parts":[[2021,10,30]],"date-time":"2021-10-30T18:33:14Z","timestamp":1635618794000},"page":"1232-1242","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Principled Multi-Aspect Evaluation Measures of Rankings"],"prefix":"10.1145","author":[{"given":"Maria","family":"Maistro","sequence":"first","affiliation":[{"name":"University of Copenhagen, Copenhagen, Denmark"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lucas","family":"Chaves Lima","sequence":"additional","affiliation":[{"name":"University of Copenhagen, Copenhagen, Denmark"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jakob","family":"Grue Simonsen","sequence":"additional","affiliation":[{"name":"University of Copenhagen, Copenhagen, Denmark"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christina","family":"Lioma","sequence":"additional","affiliation":[{"name":"University of Copenhagen, Copenhagen, Denmark"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2021,10,30]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1498759.1498766"},{"key":"e_1_3_2_1_2_1","volume-title":"Proc. 41st European Conference on IR Research (ECIR","author":"Albahem A.","year":"2019"},{"key":"e_1_3_2_1_3_1","volume-title":"Proc. 4th ACM SIGIR International Conference on the Theory of Information Retrieval (ICTIR","author":"Amig\u00f3 E.","year":"2018"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-020-09374-0"},{"key":"e_1_3_2_1_5_1","volume-title":"Proc. 41th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR","author":"Amig\u00f3 E.","year":"2018"},{"key":"e_1_3_2_1_6_1","volume-title":"SIGIR 2005: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Aslam J. A.","year":"2005"},{"key":"e_1_3_2_1_7_1","unstructured":"C. Buckley and E. M. Voorhees. 2005. Retrieval System Evaluation. In TREC. Experiment and Evaluation in Information Retrieval D. K. Harman and E. M. Voorhees (Eds.). MIT Press Cambridge (MA) USA 53--78.  C. Buckley and E. M. Voorhees. 2005. Retrieval System Evaluation. In TREC. Experiment and Evaluation in Information Retrieval D. K. Harman and E. M. Voorhees (Eds.). MIT Press Cambridge (MA) USA 53--78."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1102351.1102363"},{"key":"e_1_3_2_1_9_1","volume-title":"Proc. 4th International Conference on the Theory of Information Retrieval (ICTIR","author":"Busin L.","year":"2013"},{"key":"e_1_3_2_1_10_1","volume-title":"Proc. 18th International Conference on Information and Knowledge Management (CIKM","author":"Chapelle O.","year":"2009"},{"key":"e_1_3_2_1_11_1","volume-title":"Overview of the TREC 2009 Web Track. In TREC. National Institute of Standards and Technology (NIST).","author":"Clarke C."},{"key":"e_1_3_2_1_12_1","volume-title":"Overview of the TREC 2020 Health Misinformation Track (to appear). In Proceedings of the Twenty-Nine Text REtrieval Conference, TREC 2020","author":"Clarke Charles","year":"2020"},{"key":"e_1_3_2_1_13_1","volume-title":"Overview of the TREC 2010 Web Track. In TREC. National Institute of Standards and Technology (NIST).","author":"Clarke C. L. A."},{"key":"e_1_3_2_1_14_1","volume-title":"Overview of the TREC 2011 Web Track. In TREC. National Institute of Standards and Technology (NIST).","author":"Clarke C. L. A."},{"key":"e_1_3_2_1_15_1","volume-title":"Overview of the TREC 2012 Web Track. In TREC. National Institute of Standards and Technology (NIST).","author":"Clarke C. L. A."},{"key":"e_1_3_2_1_16_1","volume-title":"Proc. 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR","author":"Clarke C. L. A.","year":"2008"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411915"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409816"},{"key":"e_1_3_2_1_19_1","unstructured":"C. L. A. Clarke A. Vtyurina and M. D. Smucker. 2021. Assessing Top-k Preferences. arxiv: cs.IR\/2007.11682  C. L. A. Clarke A. Vtyurina and M. D. Smucker. 2021. Assessing Top-k Preferences. arxiv: cs.IR\/2007.11682"},{"key":"e_1_3_2_1_20_1","volume-title":"TREC 2014 Web Track Overview. In The Twenty-Second Text REtrieval Conference Proceedings (TREC","author":"Collins-Thompson K.","year":"2013"},{"key":"e_1_3_2_1_21_1","volume-title":"TREC 2014 Web Track Overview. In The Twenty-Third Text REtrieval Conference Proceedings (TREC","author":"Collins-Thompson K.","year":"2014"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10260-010-0142-z"},{"key":"e_1_3_2_1_23_1","volume-title":"Towards Meaningful Statements in IR Evaluation. Mapping Evaluation Measures to Interval Scales. CoRR","author":"Ferrante M.","year":"2021"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"M. Ferrante N. Ferro and E. Losiouk. 2019a. How do Interval Scales Help us with Better Understanding IR Evaluation Measures? Information Retrieval Journal (2019). https:\/\/doi.org\/10.1007\/s10791-019-09362-z  M. Ferrante N. Ferro and E. Losiouk. 2019a. How do Interval Scales Help us with Better Understanding IR Evaluation Measures? Information Retrieval Journal (2019). https:\/\/doi.org\/10.1007\/s10791-019-09362-z","DOI":"10.1007\/s10791-019-09362-z"},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the 2015 International Conference on The Theory of Information Retrieval (ICTIR '15)","author":"Ferrante M."},{"key":"e_1_3_2_1_26_1","volume-title":"Proceedings of the ACM SIGIR International Conference on Theory of Information Retrieval (ICTIR '17)","author":"Ferrante M."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2840708"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","volume-title":"Naive Set Theory","author":"Halmos P. R.","DOI":"10.1007\/978-1-4757-1645-0"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_30_1","volume-title":"Proc. 26th European Conference on IR Research (ECIR","author":"Kazai G.","year":"2004"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10137"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/33.3.239"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772749"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.21475\/ajcs.19.13.03.p1462"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121072"},{"key":"e_1_3_2_1_36_1","volume-title":"Proc. 6th International Workshop on Evaluating Information Access (EVIA","author":"Maddalena E.","year":"2014"},{"key":"e_1_3_2_1_37_1","volume-title":"Proc. 27th International Conference on Information and Knowledge Management (CIKM","author":"Palotti J.","year":"2018"},{"key":"e_1_3_2_1_38_1","volume-title":"Proceedings of the 2015 International Conference on The Theory of Information Retrieval (ICTIR '15)","author":"Petersen C."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.2197\/ipsjdc.3.625"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187904"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809449"},{"key":"e_1_3_2_1_42_1","volume-title":"Proceedings of the ACM SIGIR International Conference on Theory of Information Retrieval (ICTIR '17)","author":"Tang Z."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383963"},{"key":"e_1_3_2_1_45_1","volume-title":"Proceedings of The Twenty-Fifth Text REtrieval Conference, TREC 2016","volume":"321","author":"Voorhees E. M.","year":"2016"},{"key":"e_1_3_2_1_46_1","volume-title":"Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '08)","author":"Webber W."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.5555\/208098.208105"},{"key":"e_1_3_2_1_48_1","volume-title":"Overview of the TREC 2015 Tasks Track. In The Twenty-Fourth Text REtrieval Conference Proceedings (TREC","author":"Yilmaz E.","year":"2015"},{"key":"e_1_3_2_1_49_1","volume-title":"Proc. 38th European Conference on IR Research (ECIR","author":"Zuccon G.","year":"2016"}],"event":{"name":"CIKM '21: The 30th ACM International Conference on Information and Knowledge Management","location":"Virtual Event Queensland Australia","acronym":"CIKM '21","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 30th ACM International Conference on Information & Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3459637.3482287","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3459637.3482287","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:13Z","timestamp":1750191133000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3459637.3482287"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,10,26]]},"references-count":49,"alternative-id":["10.1145\/3459637.3482287","10.1145\/3459637"],"URL":"https:\/\/doi.org\/10.1145\/3459637.3482287","relation":{},"subject":[],"published":{"date-parts":[[2021,10,26]]},"assertion":[{"value":"2021-10-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}