{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,19]],"date-time":"2025-08-19T10:26:27Z","timestamp":1755599187096,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3404835.3463236","type":"proceedings-article","created":{"date-parts":[[2021,7,12]],"date-time":"2021-07-12T02:41:54Z","timestamp":1626057714000},"page":"2376-2382","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["WWW3E8: 259,000 Relevance Labels for Studying the Effect of Document Presentation Order for Relevance Assessors"],"prefix":"10.1145","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[{"name":"Waseda University, Tokyo, Japan"}]},{"given":"Sijie","family":"Tao","sequence":"additional","affiliation":[{"name":"Waseda University, Tokyo, Japan"}]},{"given":"Zhaohao","family":"Zeng","sequence":"additional","affiliation":[{"name":"Waseda University, Tokyo, Japan"}]}],"member":"320","published-online":{"date-parts":[[2021,7,11]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"crossref","unstructured":"James Allan Ben Carterette Javed A. Aslam Virgil Pavlu Blagovest Dachev and Evangelos Kanoulas. 2008. Million Query Track 2007 Overview.","DOI":"10.21236\/ADA477388"},{"key":"e_1_3_2_2_2_1","volume-title":"Proceedings of TREC","author":"Allan James","year":"2018","unstructured":"James Allan, Donna Harman, Evangelos Kanoulas, Dan Li, Christophe Van Gysel, and Ellen Voorhees. 2018. TREC Common Core Track Overview. In Proceedings of TREC 2017."},{"key":"e_1_3_2_2_3_1","first-page":"625","article-title":"An Axiomatic Analysis of Diversity Evaluation Metrics","volume":"2018","author":"Amig\u00f3 Enrique","year":"2018","unstructured":"Enrique Amig\u00f3, Damiano Spina, and Jorge Carrillo de Albornoz. 2018. An Axiomatic Analysis of Diversity Evaluation Metrics: Introducting the Rank-Biased Utility Metric. In Proceedings of ACM SIGIR 2018. 625--634.","journal-title":"Introducting the Rank-Biased Utility Metric. In Proceedings of ACM SIGIR"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_2_5_1","volume-title":"Proceedings of TREC","author":"Carterette Ben","year":"2010","unstructured":"Ben Carterette, Virgil Pavlu, Hui Fang, and Evangelos Kanoulas. 2010. Million Query Track 2009 Overview. In Proceedings of TREC 2009."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433411"},{"key":"e_1_3_2_2_7_1","volume-title":"Proceedings of TREC","author":"Clarke Charles L.A.","year":"2010","unstructured":"Charles L.A. Clarke, Nick Craswell, and Ian Soboroff. 2010. Overview of the TREC 2009 Web Track. In Proceedings of TREC 2009."},{"volume-title":"Proceedings of ACM SIGIR '98","author":"Cormack Gordon V.","key":"e_1_3_2_2_8_1","unstructured":"Gordon V. Cormack, Christopher R. Palmer, and Charles L.A. Clarke. 1998. Efficient Construction of Large Test Collections. In Proceedings of ACM SIGIR '98. 282--289."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271750"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(198809)39:5<293::AID-ASI1>3.0.CO;2-I"},{"key":"e_1_3_2_2_11_1","volume-title":"Changing World: Lessons Learned from 20 Years of CLEF","author":"Ferro Nicola","year":"2019","unstructured":"Nicola Ferro and Carol Peters (Eds.). 2019. Information Retrieval in a Changing World: Lessons Learned from 20 Years of CLEF. Springer."},{"volume-title":"TREC: Experiment and Evaluation in Information Retrieval, Ellen M","author":"Harman Donna K.","key":"e_1_3_2_2_12_1","unstructured":"Donna K. Harman. 2005. The TREC Test Collections. In TREC: Experiment and Evaluation in Information Retrieval, Ellen M. Voorhees and Donna K. Harman (Eds.). The MIT Press, Chapter 2."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.20047"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-30222-3_4"},{"key":"e_1_3_2_2_16_1","volume-title":"Content Analysis: An Introduction to Its Methodology","author":"Krippendorff Klaus","year":"2018","unstructured":"Klaus Krippendorff. 2018. Content Analysis: An Introduction to Its Methodology (Fourth Edition) .SAGE Publications."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2017.04.005"},{"key":"e_1_3_2_2_18_1","volume-title":"When to Stop Making Relevance Judgments? A Study of Stopping Methods for Building Information Retrieval Test Collections. Journal of the Association for Information Science and Technology","author":"Losada David E.","year":"2018","unstructured":"David E. Losada, Javier Parapar, and \u00c1lvaro Barreiro. 2018. When to Stop Making Relevance Judgments? A Study of Stopping Methods for Building Information Retrieval Test Collections. Journal of the Association for Information Science and Technology (2018)."},{"key":"e_1_3_2_2_19_1","volume-title":"Proceedings of NTCIR-13","author":"Luo Cheng","year":"2017","unstructured":"Cheng Luo, Tetsuya Sakai, Yiqun Liu, Zhicheng Dou, Chenyan Xiong, and Jingfang Xu. 2017. Overview of the NTCIR-13 We Want Web Task. In Proceedings of NTCIR-13. 394--401."},{"key":"e_1_3_2_2_20_1","volume-title":"Proceedings of NTCIR-14","author":"Mao Jiaxin","year":"2019","unstructured":"Jiaxin Mao, Tetsuya Sakai, Cheng Luo, Peng Xiao, Yiqun Liu, and Zhicheng Dou. 2019. Overview of the NTCIR-14 We Want Web Task. In Proceedings of NTCIR-14. 455--467."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"crossref","unstructured":"Tetsuya Sakai. 2014. Metrics Statistics Tests. In PROMISE Winter School 2013: Bridging between Information Retrieval and Databases (LNCS 8173). 116--163.","DOI":"10.1007\/978-3-642-54798-0_6"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911492"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9273-z"},{"volume-title":"Effect Sizes, and Statistical Power","author":"Sakai Tetsuya","key":"e_1_3_2_2_25_1","unstructured":"Tetsuya Sakai. 2018. Laboratory Experiments in Information Retrieval: Sample Sizes, Effect Sizes, and Statistical Power. Springer."},{"volume-title":"Information Retrieval Evaluation in a Changing World","author":"Sakai Tetsuya","key":"e_1_3_2_2_26_1","unstructured":"Tetsuya Sakai. 2019. How to Run an Evaluation Task. In Information Retrieval Evaluation in a Changing World, Nicola Ferro and Carol Peters (Eds.). Springer."},{"key":"e_1_3_2_2_27_1","volume-title":"Proceedings of NTCIR-7. 77--114","author":"Sakai Tetsuya","year":"2008","unstructured":"Tetsuya Sakai, Noriko Kando, Chuan-Jie Lin, Teruko Mitamura, Hideki Shima, Donghong Ji, Kuang-Hua Chen, and Eric Nyberg. 2008. Overview of the NTCIR-7 ACLIA IR4QA Task. In Proceedings of NTCIR-7. 77--114."},{"volume-title":"2020 a. Evaluating Information Retrieval and Access Tasks: NTCIR's Legacy of Research Impact","author":"Sakai Tetsuya","key":"e_1_3_2_2_28_1","unstructured":"Tetsuya Sakai, Douglas W. Oard, and Noriko Kando (Eds.). 2020 a. Evaluating Information Retrieval and Access Tasks: NTCIR's Legacy of Research Impact .Springer."},{"key":"e_1_3_2_2_29_1","volume-title":"Proceedings of NTCIR-15","author":"Sakai Tetsuya","year":"2020","unstructured":"Tetsuya Sakai, Sijie Tao, Zhaohao Zeng, Yukun Zheng, Jiaxin Mao, Zhumin Chu, Yiqun Liu, Zhicheng Dou, Nicola Ferro, Maria Maistro, and Ian Soboroff. 2020 b. Overview of the NTCIR-15 We Want Web with CENTRE (WWW-3) Task. In Proceedings of NTCIR-15. 219--234."},{"key":"e_1_3_2_2_30_1","volume-title":"Proceedings of AIRS 2019 (LNCS 12004)","author":"Sakai Tetsuya","year":"2019","unstructured":"Tetsuya Sakai and Peng Xiao. 2019. Randomised vs. Prioritised Pools for Relevance Assessments: Sample Size Considerations. In Proceedings of AIRS 2019 (LNCS 12004). 94--105."},{"key":"e_1_3_2_2_31_1","first-page":"595","article-title":"Which Diversity Evaluation Measures are \u201cGood","volume":"2019","author":"Sakai Tetsuya","year":"2019","unstructured":"Tetsuya Sakai and Zhaohao Zeng. 2019. Which Diversity Evaluation Measures are \u201cGood\u201d?. In Proceedings of ACM SIGIR 2019. 595--604.","journal-title":". In Proceedings of ACM SIGIR"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401115"},{"key":"e_1_3_2_2_33_1","volume-title":"ACM TOIS","volume":"39","author":"Sakai Tetsuya","year":"2020","unstructured":"Tetsuya Sakai and Zhaohao Zeng. 2020 b. Retrieval Evaluation Measures that Agree with Users??SERP Preferences: Traditional, Preference-based, and Diversity Measures. ACM TOIS, Vol. 39, 2 (2020)."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-45691-0_34"},{"key":"e_1_3_2_2_35_1","volume-title":"TREC: Experiment and Evaluation in Information Retrieval","author":"Voorhees Ellen M.","year":"2005","unstructured":"Ellen M. Voorhees and Donna K. Harman (Eds.). 2005. TREC: Experiment and Evaluation in Information Retrieval. The MIT Press."},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Virtual Event Canada","acronym":"SIGIR '21"},"container-title":["Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463236","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3463236","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:17:44Z","timestamp":1750191464000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463236"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":36,"alternative-id":["10.1145\/3404835.3463236","10.1145\/3404835"],"URL":"https:\/\/doi.org\/10.1145\/3404835.3463236","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-07-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}