{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,3]],"date-time":"2025-12-03T18:02:04Z","timestamp":1764784924881,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,6]],"date-time":"2022-07-06T00:00:00Z","timestamp":1657065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,6]]},"DOI":"10.1145\/3477495.3531728","type":"proceedings-article","created":{"date-parts":[[2022,7,7]],"date-time":"2022-07-07T15:12:13Z","timestamp":1657206733000},"page":"2970-2980","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["Too Many Relevants"],"prefix":"10.1145","author":[{"given":"Ellen M.","family":"Voorhees","sequence":"first","affiliation":[{"name":"National Institute of Standards and Technology, Gaithersburg, MD, USA"}]},{"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, Bellevue, WA, USA"}]},{"given":"Jimmy","family":"Lin","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]}],"member":"320","published-online":{"date-parts":[[2022,7,7]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Clarke","author":"Arabzadeh Negar","year":"2021","unstructured":"Negar Arabzadeh , Alexandra Vtyurina , Xinyi Yan , and Charles L. A . Clarke . 2021 . Shallow pooling for sparse labels. arXiv:2109.00062 [cs.IR] Negar Arabzadeh, Alexandra Vtyurina, Xinyi Yan, and Charles L. A. Clarke. 2021. Shallow pooling for sparse labels. arXiv:2109.00062 [cs.IR]"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-007-9032-x"},{"volume-title":"Proceedings of the 23rd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '00)","author":"Buckley Chris","key":"e_1_3_2_2_3_1","unstructured":"Chris Buckley and Ellen M. Voorhees . 2000. Evaluating Evaluation Measure Stability . In Proceedings of the 23rd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '00) . 33--40. https: \/\/doi.org\/10.1145\/345508.345543 10.1145\/345508.345543 Chris Buckley and Ellen M. Voorhees. 2000. Evaluating Evaluation Measure Stability. In Proceedings of the 23rd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '00). 33--40. https: \/\/doi.org\/10.1145\/345508.345543"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148219"},{"key":"e_1_3_2_2_5_1","volume-title":"Grossman","author":"Cormack Gordon V.","year":"2015","unstructured":"Gordon V. Cormack and Maura R . Grossman . 2015 . Autonomy and Reliability of Continuous Active Learning for Technology-Assisted Review . arXiv:1504.06868 [cs.IR] Gordon V. Cormack and Maura R. Grossman. 2015. Autonomy and Reliability of Continuous Active Learning for Technology-Assisted Review. arXiv:1504.06868 [cs.IR]"},{"volume-title":"Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '16)","author":"Gordon","key":"e_1_3_2_2_6_1","unstructured":"Gordon V. Cormack and Maura R. Grossman. 2016. Engineering Quality and Reliability in Technology-Assisted Review . In Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '16) . 75--84. https:\/\/doi.org\/10.1145\/2911451.2911510 10.1145\/2911451.2911510 Gordon V. Cormack and Maura R. Grossman. 2016. Engineering Quality and Reliability in Technology-Assisted Review. In Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '16). 75--84. https:\/\/doi.org\/10.1145\/2911451.2911510"},{"volume-title":"Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '98)","author":"Cormack Gordon V.","key":"e_1_3_2_2_7_1","unstructured":"Gordon V. Cormack , Christopher R. Palmer , and Charles L. A. Clarke . 1998. Efficient Construction of Large Test Collections . In Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '98) . 282--289. https:\/\/doi.org\/10.1145\/290941.291009 10.1145\/290941.291009 Gordon V. Cormack, Christopher R. Palmer, and Charles L. A. Clarke. 1998. Efficient Construction of Large Test Collections. In Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '98). 282--289. https:\/\/doi.org\/10.1145\/290941.291009"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463249"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1022904715765"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_2_11_1","volume-title":"Active Sampling for Large-Scale Information Retrieval Evaluation (CIKM '17)","author":"Li Dan","year":"2017","unstructured":"Dan Li and Evangelos Kanoulas . 2017 . Active Sampling for Large-Scale Information Retrieval Evaluation (CIKM '17) . 49--58. https:\/\/doi.org\/10.1145\/3132847. 3133015 10.1145\/3132847 Dan Li and Evangelos Kanoulas. 2017. Active Sampling for Large-Scale Information Retrieval Evaluation (CIKM '17). 49--58. https:\/\/doi.org\/10.1145\/3132847. 3133015"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2851613.2851692"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277806"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_2_16_1","volume-title":"van Rijsbergen","author":"Jones Karen Sp\u00e4rck","year":"1975","unstructured":"Karen Sp\u00e4rck Jones and Cornelis J . van Rijsbergen . 1975 . Report on the Need for and Provision of an \"Ideal\" Information Retrieval Test Collection. British Library Research and Development Report 5266, Computer Laboratory, University of Cambridge . Karen Sp\u00e4rck Jones and Cornelis J. van Rijsbergen. 1975. Report on the Need for and Provision of an \"Ideal\" Information Retrieval Test Collection. British Library Research and Development Report 5266, Computer Laboratory, University of Cambridge."},{"volume-title":"Current Challenges in Patent Information Retrieval","author":"Tomlinson Stephen","key":"e_1_3_2_2_17_1","unstructured":"Stephen Tomlinson and Bruce Hedin . 2011. Measuring Effectiveness in the TREC Legal Track . In Current Challenges in Patent Information Retrieval , M. Lupu, K. Mayer, J. Tait, and A.J. Trippe (Eds.). The Information Retrieval Series, Vol . 29. Springer , 167--180. Stephen Tomlinson and Bruce Hedin. 2011. Measuring Effectiveness in the TREC Legal Track. In Current Challenges in Patent Information Retrieval, M. Lupu, K. Mayer, J. Tait, and A.J. Trippe (Eds.). The Information Retrieval Series, Vol. 29. Springer, 167--180."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383963"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271766"},{"volume-title":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2422--2428","author":"Ellen","key":"e_1_3_2_2_20_1","unstructured":"Ellen M. Voorhees and Kirk Roberts. 2021. On the Quality of the TREC-COVID IR Test Collections . In Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2422--2428 . https:\/\/doi. org\/10.1145\/3404835.3463244 Ellen M. Voorhees and Kirk Roberts. 2021. On the Quality of the TREC-COVID IR Test Collections. In Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2422--2428. https:\/\/doi. org\/10.1145\/3404835.3463244"},{"volume-title":"Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '08)","author":"Yilmaz Emine","key":"e_1_3_2_2_21_1","unstructured":"Emine Yilmaz , Evangelos Kanoulas , and Javed A. Aslam . 2008. A Simple and Efficient Sampling Method for Estimating AP and NDCG . In Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '08) . 603--610. https:\/\/doi.org\/10.1145\/1390334. 1390437 10.1145\/1390334 Emine Yilmaz, Evangelos Kanoulas, and Javed A. Aslam. 2008. A Simple and Efficient Sampling Method for Estimating AP and NDCG. In Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '08). 603--610. https:\/\/doi.org\/10.1145\/1390334. 1390437"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Madrid Spain","acronym":"SIGIR '22"},"container-title":["Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3531728","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477495.3531728","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:07Z","timestamp":1750186927000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3531728"}},"subtitle":["Whither Cranfield Test Collections?"],"short-title":[],"issued":{"date-parts":[[2022,7,6]]},"references-count":22,"alternative-id":["10.1145\/3477495.3531728","10.1145\/3477495"],"URL":"https:\/\/doi.org\/10.1145\/3477495.3531728","relation":{},"subject":[],"published":{"date-parts":[[2022,7,6]]},"assertion":[{"value":"2022-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}