{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,15]],"date-time":"2025-10-15T17:45:34Z","timestamp":1760550334216,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,8,7]],"date-time":"2017-08-07T00:00:00Z","timestamp":1502064000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP170102231, DP140103256"],"award-info":[{"award-number":["DP170102231, DP140103256"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,8,7]]},"DOI":"10.1145\/3077136.3080793","type":"proceedings-article","created":{"date-parts":[[2017,7,28]],"date-time":"2017-07-28T19:35:01Z","timestamp":1501270501000},"page":"35-44","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Can Deep Effectiveness Metrics Be Evaluated Using Shallow Judgment Pools?"],"prefix":"10.1145","author":[{"given":"Xiaolu","family":"Lu","sequence":"first","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]},{"given":"Alistair","family":"Moffat","sequence":"additional","affiliation":[{"name":"University of Melbourne, Melbourne, Australia"}]},{"given":"J. Shane","family":"Culpepper","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]}],"member":"320","published-online":{"date-parts":[[2017,8,7]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"J. A. Aslam V. Pavlu and E. Yilmaz 2006. A statistical method for system evaluation using incomplete judgments Proc. SIGIR. 541--548.  J. A. Aslam V. Pavlu and E. Yilmaz 2006. A statistical method for system evaluation using incomplete judgments Proc. SIGIR. 541--548.","DOI":"10.1145\/1148170.1148263"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-007-9032-x"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"S. B\u00fcttcher C. L. A. Clarke P. C. K. Yeung and I. Soboroff 2007. Reliable information retrieval evaluation with incomplete and biased judgements Proc. SIGIR. 63--70.  S. B\u00fcttcher C. L. A. Clarke P. C. K. Yeung and I. Soboroff 2007. Reliable information retrieval evaluation with incomplete and biased judgements Proc. SIGIR. 63--70.","DOI":"10.1145\/1277741.1277755"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1992.10475194"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_1_7_1","volume-title":"Aust. Doc. Comp. Symp. 2--9.","author":"Jayasinghe J. K.","year":"2014","unstructured":"J. K. Jayasinghe , W. Webber , M. Sanderson , and J. S. Culpepper 2014 . Improving test collection pools with machine learning Proc . Aust. Doc. Comp. Symp. 2--9. J. K. Jayasinghe, W. Webber, M. Sanderson, and J. S. Culpepper 2014. Improving test collection pools with machine learning Proc. Aust. Doc. Comp. Symp. 2--9."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772749"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"A. Lipani M. Lupu and A. Hanbury 2015. Splitting water: Precision and anti-precision to reduce pool bias Proc. SIGIR. 103--112.  A. Lipani M. Lupu and A. Hanbury 2015. Splitting water: Precision and anti-precision to reduce pool bias Proc. SIGIR. 103--112.","DOI":"10.1145\/2766462.2767749"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983891"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-016-9282-6"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"X. Lu A. Moffat and J. S. Culpepper 2016natexlabb. Modeling relevance as a function of retrieval rank Proc. AIRS. 3--15.  X. Lu A. Moffat and J. S. Culpepper 2016natexlabb. Modeling relevance as a function of retrieval rank Proc. AIRS. 3--15.","DOI":"10.1007\/978-3-319-48051-0_1"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"A. Moffat P. Thomas and F. Scholer 2013. Users versus models: What observation tells us about effectiveness metrics Proc. CIKM. 659--668.  A. Moffat P. Thomas and F. Scholer 2013. Users versus models: What observation tells us about effectiveness metrics Proc. CIKM. 659--668.","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"A. Moffat W. Webber and J. Zobel 2007. Strategic system comparisons via targeted relevance judgments Proc. SIGIR. 375--382.  A. Moffat W. Webber and J. Zobel 2007. Strategic system comparisons via targeted relevance judgments Proc. SIGIR. 375--382.","DOI":"10.1145\/1277741.1277806"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"S. D. Ravana and A. Moffat 2010. Score estimation incomplete judgments and significance testing in IR evaluation Proc. AIRS. 97--109.  S. D. Ravana and A. Moffat 2010. Score estimation incomplete judgments and significance testing in IR evaluation Proc. AIRS. 97--109.","DOI":"10.1007\/978-3-642-17187-1_9"},{"key":"e_1_3_2_1_17_1","unstructured":"S. E. Robertson. 2007. On document populations and measures of IR effectiveness Proc. ICTIR. 9--22.  S. E. Robertson. 2007. On document populations and measures of IR effectiveness Proc. ICTIR. 9--22."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277756"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"T. Sakai. 2008natexlaba. Comparing metrics across TREC and NTCIR: The robustness to pool depth bias Proc. SIGIR. 691--692.  T. Sakai. 2008natexlaba. Comparing metrics across TREC and NTCIR: The robustness to pool depth bias Proc. SIGIR. 691--692.","DOI":"10.1145\/1390334.1390454"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"T. Sakai. 2008natexlabb. Comparing metrics across TREC and NTCIR: The robustness to system bias Proc. CIKM. 581--590.  T. Sakai. 2008natexlabb. Comparing metrics across TREC and NTCIR: The robustness to system bias Proc. CIKM. 581--590.","DOI":"10.1145\/1458082.1458159"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-54798-0_6"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","unstructured":"T. Schnabel A. Swaminathan P. I. Frazier and T. Joachims 2016. Unbiased comparative evaluation of ranking functions Proc. ICTIR. 109--118.  T. Schnabel A. Swaminathan P. I. Frazier and T. Joachims 2016. Unbiased comparative evaluation of ranking functions Proc. ICTIR. 109--118.","DOI":"10.1145\/2970398.2970410"},{"volume-title":"Proc. WWW. 935--937","author":"Schnabel T.","key":"e_1_3_2_1_23_1","unstructured":"T. Schnabel , A. Swaminathan , and T. Joachims . 2015. Unbiased ranking evaluation on a budget . In Proc. WWW. 935--937 . T. Schnabel, A. Swaminathan, and T. Joachims. 2015. Unbiased ranking evaluation on a budget. In Proc. WWW. 935--937."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"E. M. Voorhees. 2014. The effect of sampling strategy on inferred measures Proc. SIGIR. 1119--1122.  E. M. Voorhees. 2014. The effect of sampling strategy on inferred measures Proc. SIGIR. 1119--1122.","DOI":"10.1145\/2600428.2609524"},{"key":"e_1_3_2_1_25_1","volume-title":"Harman","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees and D. K. Harman 2005 . TREC : Experiment and Evaluation in Information Retrieval. The MIT Press . E. M. Voorhees and D. K. Harman 2005. TREC: Experiment and Evaluation in Information Retrieval. The MIT Press."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572018"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-007-0101-7"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"E. Yilmaz E. Kanoulas and J. A. Aslam 2008. A simple and efficient sampling method for estimating AP and NDCG Proc. SIGIR. 603--610.  E. Yilmaz E. Kanoulas and J. A. Aslam 2008. A simple and efficient sampling method for estimating AP and NDCG Proc. SIGIR. 603--610.","DOI":"10.1145\/1390334.1390437"},{"volume-title":"Ensemble Methods: Foundations and Algorithms","author":"Z.","key":"e_1_3_2_1_29_1","unstructured":"Z. Zhou 2012. Ensemble Methods: Foundations and Algorithms . CRC press . Z. Zhou 2012. Ensemble Methods: Foundations and Algorithms. CRC press."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '17: The 40th International ACM SIGIR conference on research and development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Shinjuku Tokyo Japan","acronym":"SIGIR '17"},"container-title":["Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080793","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3080793","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:04:07Z","timestamp":1750273447000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080793"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,8,7]]},"references-count":30,"alternative-id":["10.1145\/3077136.3080793","10.1145\/3077136"],"URL":"https:\/\/doi.org\/10.1145\/3077136.3080793","relation":{},"subject":[],"published":{"date-parts":[[2017,8,7]]},"assertion":[{"value":"2017-08-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}