{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T15:38:28Z","timestamp":1726069108578},"publisher-location":"Cham","reference-count":22,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030428341"},{"type":"electronic","value":"9783030428358"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-42835-8_9","type":"book-chapter","created":{"date-parts":[[2020,2,26]],"date-time":"2020-02-26T07:02:53Z","timestamp":1582700573000},"page":"94-105","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Randomised vs. Prioritised Pools for Relevance Assessments: Sample Size Considerations"],"prefix":"10.1007","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peng","family":"Xiao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,2,27]]},"reference":[{"doi-asserted-by":"crossref","unstructured":"Allan, J., Carterette, B., Aslam, J.A., Pavlu, V., Dachev, B., Kanoulas, E.: Million query track 2007 overview (2008)","key":"9_CR1","DOI":"10.21236\/ADA477388"},{"doi-asserted-by":"crossref","unstructured":"Allan, J., Harman, D., Kanoulas, E., Li, D., Van Gysel, C., Voorhees, E.: TREC common core track overview. In: Proceedings of TREC 2017 (2018)","key":"9_CR2","DOI":"10.6028\/NIST.SP.500-324.core-overview"},{"doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Fang, H., Kanoulas, E.: Million query track 2009 overview. In: Proceedings of TREC 2009 (2010)","key":"9_CR3","DOI":"10.6028\/NIST.SP.500-278.million-query-overview"},{"doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Palmer, C.R., Clarke, C.L.: Efficient construction of large test collections. In: Proceedings of ACM SIGIR 1998, pp. 282\u2013289 (1998)","key":"9_CR4","DOI":"10.1145\/290941.291009"},{"doi-asserted-by":"crossref","unstructured":"Damessie, T.T., Culpepper, J.S., Kim, J., Scholer, F.: Presentation ordering effects on assessor agreement. In: Proceedings of ACM CIKM 2018, pp. 723\u2013732 (2018)","key":"9_CR5","DOI":"10.1145\/3269206.3271750"},{"issue":"5","key":"9_CR6","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1002\/(SICI)1097-4571(198809)39:5<293::AID-ASI1>3.0.CO;2-I","volume":"39","author":"M Eisenberg","year":"1988","unstructured":"Eisenberg, M., Barry, C.: Order effects: a study of the possible influence of presentation order on user judgments of document relevance. J. Am. Soc. Inf. Sci. 39(5), 293\u2013300 (1988)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"9_CR7","doi-asserted-by":"publisher","DOI":"10.4324\/9781315629049","volume-title":"What If There Were No Significance Tests? (Classic Edition)","author":"LL Harlow","year":"2016","unstructured":"Harlow, L.L., Mulaik, S.A., Steiger, J.H.: What If There Were No Significance Tests? (Classic Edition). Routledge, London (2016)"},{"key":"9_CR8","volume-title":"TREC: Experiment and Evaluation in Information Retrieval (Chapter 2)","author":"DK Harman","year":"2005","unstructured":"Harman, D.K.: The TREC test collections. In: Voorhees, E.M., Harman, D.K. (eds.) TREC: Experiment and Evaluation in Information Retrieval (Chapter 2). The MIT Press, Cambridge (2005)"},{"issue":"11","key":"9_CR9","doi-asserted-by":"publisher","first-page":"970","DOI":"10.1002\/asi.20047","volume":"55","author":"MH Huang","year":"2004","unstructured":"Huang, M.H., Wang, H.Y.: The influence of document presentation order and number of documents judged on users\u2019 judgments of relevance. J. Am. Soc. Inf. Sci. 55(11), 970\u2013979 (2004)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"9_CR10","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1007\/978-3-540-30222-3_4","volume-title":"Comparative Evaluation of Multilingual Information Access Systems","author":"N Kando","year":"2004","unstructured":"Kando, N.: Evaluation of information access technologies at the NTCIR workshop. In: Peters, C., Gonzalo, J., Braschler, M., Kluck, M. (eds.) CLEF 2003. LNCS, vol. 3237, pp. 29\u201343. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-30222-3_4"},{"issue":"3","key":"9_CR11","doi-asserted-by":"publisher","first-page":"1005","DOI":"10.1016\/j.ipm.2017.04.005","volume":"53","author":"DE Losada","year":"2017","unstructured":"Losada, D.E., Parapar, J., Barreiro, \u00c1.: Multi-armed bandits for ordering judgements in pooling-based evaluation. Inf. Process. Manag. 53(3), 1005\u20131025 (2017)","journal-title":"Inf. Process. Manag."},{"issue":"1","key":"9_CR12","doi-asserted-by":"publisher","first-page":"49","DOI":"10.1002\/asi.24077","volume":"70","author":"DE Losada","year":"2018","unstructured":"Losada, D.E., Parapar, J., Barreiro, \u00c1.: When to stop making relevance judgments? A study of stopping methods for building information retrieval test collections. J. Assoc. Inf. Sci. Technol. 70(1), 49\u201360 (2018)","journal-title":"J. Assoc. Inf. Sci. Technol."},{"unstructured":"Luo, C., Sakai, T., Liu, Y., Dou, Z., Xiong, C., Xu, J.: Overview of the NTCIR-13 we want web task. In: Proceedings of NTCIR-13, pp. 394\u2013401 (2017)","key":"9_CR13"},{"unstructured":"Mao, J., Sakai, T., Luo, C., Xiao, P., Liu, Y., Dou, Z.: Overview of the NTCIR-14 we want web task. In: Proceedings of NTCIR-14 (2019)","key":"9_CR14"},{"issue":"3","key":"9_CR15","doi-asserted-by":"publisher","first-page":"638","DOI":"10.1037\/0033-2909.86.3.638","volume":"86","author":"R Rosenthal","year":"1979","unstructured":"Rosenthal, R.: The \u201cfile drawer problem\u201d and tolerance for null results. Psychol. Bull. 86(3), 638\u2013641 (1979)","journal-title":"Psychol. Bull."},{"doi-asserted-by":"crossref","unstructured":"Sakai, T.: Statistical significance, power, and sample sizes: a systematic review of SIGIR and TOIS, 2006\u20132015. In: Proceedings of ACM SIGIR 2016, pp. 5\u201314 (2016)","key":"9_CR16","DOI":"10.1145\/2911451.2911492"},{"key":"9_CR17","series-title":"The Information Retrieval Series","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-13-1199-4","volume-title":"Laboratory Experiments in Information Retrieval: Sample Sizes, Effect Sizes, and Statistical Power","author":"T Sakai","year":"2018","unstructured":"Sakai, T.: Laboratory Experiments in Information Retrieval: Sample Sizes, Effect Sizes, and Statistical Power. TIRS, vol. 40. Springer, Singapore (2018). https:\/\/doi.org\/10.1007\/978-981-13-1199-4"},{"key":"9_CR18","series-title":"The Information Retrieval Series","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1007\/978-3-030-22948-1_3","volume-title":"Information Retrieval Evaluation in a Changing World","author":"T Sakai","year":"2019","unstructured":"Sakai, T.: How to run an evaluation task. Information Retrieval Evaluation in a Changing World. TIRS, vol. 41, pp. 71\u2013102. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-22948-1_3"},{"unstructured":"Sakai, T., et al.: Overview of the NTCIR-7 ACLIA IR4QA task. In: Proceedings of NTCIR-7, pp. 77\u2013114 (2008)","key":"9_CR19"},{"key":"9_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1007\/3-540-45691-0_34","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"EM Voorhees","year":"2002","unstructured":"Voorhees, E.M.: The philosophy of information retrieval evaluation. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol. 2406, pp. 355\u2013370. Springer, Heidelberg (2002). https:\/\/doi.org\/10.1007\/3-540-45691-0_34"},{"doi-asserted-by":"crossref","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Proceedings of ACM SIGIR 1998, pp. 307\u2013314 (1998)","key":"9_CR21","DOI":"10.1145\/290941.291014"},{"key":"9_CR22","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4471-6639-9","volume-title":"Writing for Computer Science","author":"J Zobel","year":"2014","unstructured":"Zobel, J.: Writing for Computer Science, 3rd edn. Springer, London (2014). https:\/\/doi.org\/10.1007\/978-1-4471-6639-9","edition":"3"}],"container-title":["Lecture Notes in Computer Science","Information Retrieval Technology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-42835-8_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T03:46:54Z","timestamp":1722484014000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-42835-8_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030428341","9783030428358"],"references-count":22,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-42835-8_9","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"27 February 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIRS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Asia Information Retrieval Symposium","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Hong Kong","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 November 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 November 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"airs2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/airs2019.ouhk.edu.hk\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}