{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,29]],"date-time":"2026-05-29T11:32:28Z","timestamp":1780054348113,"version":"3.54.0"},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2015,9,8]],"date-time":"2015-09-08T00:00:00Z","timestamp":1441670400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2015,9,8]],"date-time":"2015-09-08T00:00:00Z","timestamp":1441670400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2015,10]]},"DOI":"10.1007\/s10791-015-9266-y","type":"journal-article","created":{"date-parts":[[2015,9,8]],"date-time":"2015-09-08T09:11:07Z","timestamp":1441703467000},"page":"445-472","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":28,"title":["Pooling-based continuous evaluation of information retrieval systems"],"prefix":"10.1007","volume":"18","author":[{"given":"Alberto","family":"Tonon","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gianluca","family":"Demartini","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Philippe","family":"Cudr\u00e9-Mauroux","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2015,9,8]]},"reference":[{"key":"9266_CR1","doi-asserted-by":"crossref","unstructured":"Alonso, O., & Baeza-Yates, R.A. (2011). Design and implementation of relevance assessments using crowdsourcing. In European conference on information retrieval, ECIR (pp. 153\u2013164).","DOI":"10.1007\/978-3-642-20161-5_16"},{"issue":"6","key":"9266_CR2","doi-asserted-by":"publisher","first-page":"1053","DOI":"10.1016\/j.ipm.2012.01.004","volume":"48","author":"O Alonso","year":"2012","unstructured":"Alonso, O., & Mizzaro, S. (2012). Using crowdsourcing for trec relevance assessment. Information Processing & Management, 48(6), 1053\u20131066.","journal-title":"Information Processing & Management"},{"key":"9266_CR3","unstructured":"Aslam, J., & Pavlu, V. (2007). A practical sampling strategy for efficient retrieval evaluation. Working Draft. http:\/\/www.ccs.neu.edu\/home\/jaa\/papers\/drafts\/statAP.html."},{"key":"9266_CR4","doi-asserted-by":"crossref","unstructured":"Aslam, J. A., Pavlu, V., & Yilmaz, E. (2006). A statistical method for system evaluation using incomplete judgments. In Proceedings of the 29th annual international ACM SIGIR conference on research and development in information retrieval, SIGIR (pp. 541\u2013548). ACM.","DOI":"10.1145\/1148170.1148263"},{"key":"9266_CR5","doi-asserted-by":"crossref","unstructured":"Balog, K., Kelly, L., & Schuth, A. (2014). Head first: Living labs for ad-hoc search evaluation. In International conference on information and knowledge management, CIKM (pp. 1815\u20131818).","DOI":"10.1145\/2661829.2661962"},{"key":"9266_CR6","doi-asserted-by":"crossref","unstructured":"Blanco, R., Halpin, H., Herzig, D. M., Mika, P., Pound, J., Thompson, H. S., et al. (2011). Repeatable and reliable search system evaluation using crowdsourcing. In International ACM SIGIR conference on research and development in information retrieval, SIGIR (pp. 923\u2013932).","DOI":"10.1145\/2009916.2010039"},{"key":"9266_CR7","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1016\/j.websem.2013.05.005","volume":"21","author":"R Blanco","year":"2013","unstructured":"Blanco, R., Halpin, H., Herzig, D. M., Mika, P., Pound, J., Thompson, H. S., et al. (2013). Repeatable and reliable semantic search evaluation. Journal of Web Semantics, 21, 14\u201329.","journal-title":"Journal of Web Semantics"},{"issue":"6","key":"9266_CR8","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1007\/s10791-007-9032-x","volume":"10","author":"C Buckley","year":"2007","unstructured":"Buckley, C., Dimmick, D., Soboroff, I., & Voorhees, E. M. (2007). Bias and the limits of pooling for large collections. Information Retrieval, 10(6), 491\u2013508.","journal-title":"Information Retrieval"},{"key":"9266_CR9","doi-asserted-by":"crossref","unstructured":"Buckley, C., & Voorhees, E. (2004). Retrieval evaluation with incomplete information. In International ACM SIGIR conference on research and development in information retrieval, SIGIR (pp. 25\u201332). ACM.","DOI":"10.1145\/1008992.1009000"},{"key":"9266_CR10","doi-asserted-by":"crossref","unstructured":"B\u00fcttcher, S., Clarke, C. L. A., Yeung, P. C. K., & Soboroff, I. (2007). Reliable information retrieval evaluation with incomplete and biased judgements. In International ACM SIGIR conference on research and development in information retrieval, SIGIR (pp. 63\u201370).","DOI":"10.1145\/1277741.1277755"},{"key":"9266_CR11","doi-asserted-by":"crossref","unstructured":"Carterette, B., Allan, J., & Sitaraman, R.\u00a0K. (2006). Minimal test collections for retrieval evaluation. In Proceedings of the international ACM SIGIR conference on research and development in information retrieval, SIGIR  (pages 268\u2013275). New York: ACM.","DOI":"10.1145\/1148170.1148219"},{"key":"9266_CR12","doi-asserted-by":"crossref","unstructured":"Carterette, B., Kanoulas, E., Pavlu, V., & Fang, H. (2010). Reusable test collections through experimental design. In Proceedings of the international ACM SIGIR conference on research and development in information retrieval, SIGIR (pp. 547\u2013554).","DOI":"10.1145\/1835449.1835541"},{"key":"9266_CR13","volume-title":"Report on the testing and analysis of an investigation into the comparative efficiency of indexing systems","author":"C Cleverdon","year":"1962","unstructured":"Cleverdon, C. (1962). Report on the testing and analysis of an investigation into the comparative efficiency of indexing systems. Cranfield: College of Aeronautics."},{"key":"9266_CR14","doi-asserted-by":"crossref","unstructured":"Demartini, G., Iofciu, T., & de Vries, A. P. (2009). Overview of the INEX 2009 entity ranking track. In S. Geva, J. Kamps & A. Rotman (Eds.), Focused Retrieval and Evaluation, 8th International Workshop of the Initiative for the Evaluation of XML Retrieval, INEX 2009 Brisbane, Australia, December  2009, Revised and Selected Papers (pp. 254\u2013264).  Heidelberg: Springer","DOI":"10.1007\/978-3-642-14556-8_26"},{"key":"9266_CR15","doi-asserted-by":"crossref","unstructured":"Difallah, D. E., Demartini, G., & Cudr\u00e9-Mauroux, P. (2013). Pick-a-crowd: Tell me what you like, and i\u2019ll tell you what to do. In Proceedings of the 22nd international conference on World Wide Web (pp. 367\u2013374). International World Wide Web Conferences Steering Committee","DOI":"10.1145\/2488388.2488421"},{"key":"9266_CR16","unstructured":"Halpin, H., Herzig, D. M., Mika, P., Blanco, R., Pound, J., Thompson, H. S., et al. (2010). Evaluating ad-hoc object retrieval. In Evaluation of Semantic Technologies (IWEST 2010) at ISWC 2010."},{"key":"9266_CR17","doi-asserted-by":"crossref","unstructured":"Hosseini, M., Cox, I. J., Milic-Frayling, N., Kazai, G., & Vinay, V. (2012). On aggregating labels from multiple crowd workers to infer relevance of documents. In European conference on information retrieval, ECIR (pp. 182\u2013194)","DOI":"10.1007\/978-3-642-28997-2_16"},{"issue":"4","key":"9266_CR18","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of ir techniques. ACM Transactions on Information System, 20(4), 422\u2013446.","journal-title":"ACM Transactions on Information System"},{"key":"9266_CR19","unstructured":"Jones, K., & Van Rijsbergen, C. (1975). Report on the need for and provision of an ideal information retrieval test collection. British library research and development reports."},{"key":"9266_CR20","doi-asserted-by":"crossref","unstructured":"Kazai, G. (2011). In search of quality in crowdsourcing for search engine evaluation. In European conference on information retrieval, ECIR (pp. 165\u2013176).","DOI":"10.1007\/978-3-642-20161-5_17"},{"key":"9266_CR21","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Koolen, M., & Milic-Frayling, N. (2011). Crowdsourcing for book search evaluation: Impact of hit design on comparative system ranking. In International ACM SIGIR conference on research and development in\ninformation retrieval, SIGIR (pp. 205\u2013214).","DOI":"10.1145\/2009916.2009947"},{"key":"9266_CR22","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., & Milic-Frayling, N. (2011). Worker types and personality traits in crowdsourcing relevance labels. In Proceedings of the 20th ACM international conference on Information and knowledge management, CIKM (pp. 1941\u20131944).","DOI":"10.1145\/2063576.2063860"},{"issue":"4","key":"9266_CR23","doi-asserted-by":"publisher","first-page":"343","DOI":"10.1016\/0020-0271(68)90029-6","volume":"4","author":"M Lesk","year":"1968","unstructured":"Lesk, M., & Salton, G. (1968). Relevance assessments and retrieval system evaluation. Information Storage and Retrieval, 4(4), 343\u2013359.","journal-title":"Information Storage and Retrieval"},{"key":"9266_CR24","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809071","volume-title":"Introduction to information retrieval","author":"CD Manning","year":"2008","unstructured":"Manning, C. D., Raghavan, P., & Sch\u00fctze, H. (2008). Introduction to information retrieval (Vol. 1). Cambridge: Cambridge University Press."},{"key":"9266_CR25","unstructured":"McCreadie, R., Macdonald, C., & Ounis, I. (2011). Crowdsourcing blog track top news judgments at TREC. In Crowdsourcing for Search and Data Mining (CSDM) at WSDM 2011 (pp. 23\u201326)."},{"issue":"9","key":"9266_CR26","doi-asserted-by":"publisher","first-page":"810","DOI":"10.1002\/(SICI)1097-4571(199709)48:9<810::AID-ASI6>3.0.CO;2-U","volume":"48","author":"S Mizzaro","year":"1997","unstructured":"Mizzaro, S. (1997). Relevance: The whole history. JASIS, 48(9), 810\u2013832.","journal-title":"JASIS"},{"key":"9266_CR27","doi-asserted-by":"crossref","unstructured":"Moffat, A., Webber, W., & Zobel, J. (2007). Strategic system comparisons via targeted relevance judgments. In Proceedings of the international ACM SIGIR conference on research and development in information retrieval SIGIR, 07 (pp. 375\u2013382).","DOI":"10.1145\/1277741.1277806"},{"issue":"1","key":"9266_CR28","doi-asserted-by":"publisher","first-page":"2:1","DOI":"10.1145\/1416950.1416952","volume":"27","author":"A Moffat","year":"2008","unstructured":"Moffat, A., & Zobel, J. (2008). Rank-biased precision for measurement of retrieval effectiveness. ACM Transactions on Information Systems, 27(1), 2:1\u20132:27.","journal-title":"ACM Transactions on Information Systems"},{"key":"9266_CR29","doi-asserted-by":"crossref","unstructured":"Moshfeghi, Y., Matthews, M., Blanco, R., & Jose, J. M. (2013). Influence of timeline and named-entity components on user engagement. In Lecture notes in computer science (including subseries lecture notes in artificial intelligence and lecture notes in bioinformatics) (Vol. 7814) LNCS (pp. 305\u2013317).","DOI":"10.1007\/978-3-642-36973-5_26"},{"key":"9266_CR30","doi-asserted-by":"crossref","unstructured":"Pavlu, V., Rajput, S., Golbus, P. B., & Aslam, J. A. (2012). Ir system evaluation using nugget-based test collections. In Proceedings of the fifth ACM international conference on Web search and data mining, WSDM \u201912 (pp. 393\u2013402). New York, NY: ACM.","DOI":"10.1145\/2124295.2124343"},{"key":"9266_CR31","doi-asserted-by":"crossref","unstructured":"Sanderson, M., & Zobel, J. (2005). Information retrieval system evaluation: Effort, sensitivity, and reliability. In Proceedings of the 28th annual international ACM SIGIR conference on Research and development in information retrieval\u2014SIGIR \u201905 (p. 162). New York, NY: ACM Press, Aug. 2005.","DOI":"10.1145\/1076034.1076064"},{"key":"9266_CR32","doi-asserted-by":"crossref","unstructured":"Scholer, F., Kelly, D., Wu, W.-C., Lee, H.\u00a0S., & Webber, W. (2013). The effect of threshold priming and need for cognition on relevance calibration and assessment. In Proceedings of the 36th international ACM SIGIR conference on research and development in information retrieval (pp. 623\u2013632). ACM.","DOI":"10.1145\/2484028.2484090"},{"key":"9266_CR33","unstructured":"Smucker, M., Kazai, G., & Lease, M. (2013). Overview of the trec 2012 crowdsourcing track. In Proceedings of the 21st NIST text retrieval conference (TREC)."},{"issue":"11","key":"9266_CR34","doi-asserted-by":"publisher","first-page":"2224","DOI":"10.1002\/asi.22948","volume":"64","author":"S Verberne","year":"2013","unstructured":"Verberne, S., Heijden, M. V. D., Hinne, M., Sappelli, M., Koldijk, S., Hoenkamp, E., et al. (2013). Reliability and validity of query intent assessments. JASIST, 64(11), 2224\u20132237.","journal-title":"JASIST"},{"key":"9266_CR35","doi-asserted-by":"crossref","unstructured":"Voorhees, E., & Harman, D. (1998). Overview of the Seventh Text REtrieval Conference (TREC-7). NIST Special Publication 500-242.","DOI":"10.6028\/NIST.SP.500-242"},{"key":"9266_CR36","doi-asserted-by":"crossref","unstructured":"Voorhees, E., & Harman, D. (1999). Overview of the Eighth Text REtrieval Conference (TREC-8). NIST Special Publication 500-246.","DOI":"10.6028\/NIST.SP.500-246"},{"key":"9266_CR37","doi-asserted-by":"crossref","unstructured":"Voorhees, E.\u00a0M. (1998) Variations in relevance judgments and the measurement of retrieval effectiveness. In International ACM SIGIR conference on research and development in\ninformation retrieval, SIGIR (pp. 315\u2013323). New York, NY: ACM.","DOI":"10.1145\/290941.291017"},{"key":"9266_CR38","doi-asserted-by":"crossref","unstructured":"Webber, W., & Park, L. A. F. (2009). Score adjustment for correction of pooling bias. In Proceedings of the international ACM SIGIR conference on research and development in\ninformation retrieval, SIGIR (pp. 444\u2013451).","DOI":"10.1145\/1571941.1572018"},{"key":"9266_CR39","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., & Aslam, J. A. (2006). Estimating average precision with incomplete and imperfect judgments. In Proceedings of the 15th ACM international conference on information and knowledge management, CIKM (pp. 102\u2013111). Arlington, VA: ACM.","DOI":"10.1145\/1183614.1183633"},{"key":"9266_CR40","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Kanoulas, E., & Aslam, J. A. (2008). A simple and efficient sampling method for estimating AP and NDCG. In International ACM SIGIR conference on research and development in\ninformation retrieval, SIGIR (pp. 603\u2013610).","DOI":"10.1145\/1390334.1390437"},{"key":"9266_CR41","doi-asserted-by":"crossref","unstructured":"Zobel, J. (1998). How reliable are the results of large-scale information retrieval experiments? In International ACM SIGIR conference on research and development in\ninformation retrieval, SIGIR (pp. 307\u2013314). New York, NY: ACM.","DOI":"10.1145\/290941.291014"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9266-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-015-9266-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9266-y","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9266-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:20:40Z","timestamp":1704205240000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-015-9266-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,9,8]]},"references-count":41,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2015,10]]}},"alternative-id":["9266"],"URL":"https:\/\/doi.org\/10.1007\/s10791-015-9266-y","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,9,8]]},"assertion":[{"value":"2 July 2014","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 August 2015","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 September 2015","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}