{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,28]],"date-time":"2025-03-28T02:01:18Z","timestamp":1743127278109,"version":"3.40.3"},"publisher-location":"Berlin, Heidelberg","reference-count":19,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642047688"},{"type":"electronic","value":"9783642047695"}],"license":[{"start":{"date-parts":[[2009,1,1]],"date-time":"2009-01-01T00:00:00Z","timestamp":1230768000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2009]]},"DOI":"10.1007\/978-3-642-04769-5_31","type":"book-chapter","created":{"date-parts":[[2009,9,28]],"date-time":"2009-09-28T16:04:25Z","timestamp":1254153865000},"page":"351-362","source":"Crossref","is-referenced-by-count":7,"title":["Automatic Search Engine Performance Evaluation with the Wisdom of Crowds"],"prefix":"10.1007","author":[{"given":"Rongwei","family":"Cen","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yiqun","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Min","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liyun","family":"Ru","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shaoping","family":"Ma","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"31_CR1","first-page":"3","volume-title":"SIGIR 2006","author":"E. Agichtein","year":"2006","unstructured":"Agichtein, E., Brill, E., Dumais, S., Ragno, R.: Learning user interaction models for predicting web search result preferences. In: SIGIR 2006, pp. 3\u201310. ACM, New York (2006)"},{"issue":"2","key":"31_CR2","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1145\/792550.792552","volume":"36","author":"A. Broder","year":"2002","unstructured":"Broder, A.: A taxonomy of web search. SIGIR Forum\u00a036(2), 3\u201310 (2002)","journal-title":"SIGIR Forum"},{"issue":"6","key":"31_CR3","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1007\/s10791-007-9032-x","volume":"10","author":"C. Buckley","year":"2007","unstructured":"Buckley, C., Dimmick, D., Soboroff, I., Voorhees, E.: Bias and the limits of pooling for large collections. Inf. Retr.\u00a010(6), 491\u2013508 (2007)","journal-title":"Inf. Retr."},{"key":"31_CR4","unstructured":"Cleverdon, C., Mills, J., Keen, M.: Aslib Cranfield research project - Factors determining the performance of indexing systems; Design; Part 1, vol.\u00a01 (1966)"},{"key":"31_CR5","doi-asserted-by":"crossref","unstructured":"Craswell, M., Hawking, D.: Overview of the TREC 2003 Web track. In: Voorhees, E.M., Buckland, L.P. (eds.) NIST Special Publication 500-261: TREC 2004 (2004)","DOI":"10.6028\/NIST.SP.500-261.web-overview"},{"key":"31_CR6","first-page":"61","volume-title":"Proc. of WWW 2008","author":"A. Fuxman","year":"2008","unstructured":"Fuxman, A., Tsaparas, P., Achan, K., Agrawal, R.: Using the wisdom of the crowds for keyword generation. In: Proc. of WWW 2008, pp. 61\u201370. ACM, New York (2008)"},{"key":"31_CR7","doi-asserted-by":"crossref","unstructured":"Hawking, D., Craswell, N.: Overview of the TREC 2003 Web track. In: Voorhees, E.M., Buckland, L.P. (eds.) NIST Special Publication 500-255: TREC 2003 (2003)","DOI":"10.6028\/NIST.SP.500-255.web-overview"},{"key":"31_CR8","first-page":"154","volume-title":"SIGIR 2005","author":"T. Joachims","year":"2005","unstructured":"Joachims, T., Granka, L., Pan, B., Hembrooke, H., Gay, G.: Accurately interpreting clickthrough data as implicit feedback. In: SIGIR 2005, pp. 154\u2013161. ACM, New York (2005)"},{"key":"31_CR9","doi-asserted-by":"crossref","unstructured":"Dou, Z., Song, R., Yuan, X., Wen, J.R.: Are click-through data adequate for learning web search rankings? In: CIKM 2008, New York, NY, pp. 73\u201382 (2008)","DOI":"10.1145\/1458082.1458095"},{"issue":"2","key":"31_CR10","first-page":"93","volume":"6","author":"A. Kent","year":"1955","unstructured":"Kent, A., Berry, M., Leuhrs, F.U., Perry, J.W.: Machine literature searching VIII. Operational criteria for designing information retrieval systems. American Documentation\u00a06(2), 93\u2013101 (1955)","journal-title":"Operational criteria for designing information retrieval systems. American Documentation"},{"key":"31_CR11","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"593","DOI":"10.1007\/11880592_51","volume-title":"Information Retrieval Technology","author":"Y. Liu","year":"2006","unstructured":"Liu, Y., Zhang, M., Ru, L., Ma, S.: Automatic Query Type Identification Based on Click Through Information. In: Ng, H.T., Leong, M.-K., Kan, M.-Y., Ji, D. (eds.) AIRS 2006. LNCS, vol.\u00a04182, pp. 593\u2013600. Springer, Heidelberg (2006)"},{"key":"31_CR12","first-page":"379","volume-title":"Proc. of SIGIR 2003","author":"R. Nuray","year":"2003","unstructured":"Nuray, R., Can, F.: Automatic ranking of retrieval systems in imperfect environments. In: Proc. of SIGIR 2003, pp. 379\u2013380. ACM, New York (2003)"},{"key":"31_CR13","unstructured":"Oard, D.W., Kim, J.: Modeling information content using observable behavior. In: Proc. of ASIST 2001, Washington, D.C., USA, pp. 38\u201345 (2001)"},{"key":"31_CR14","first-page":"13","volume-title":"Proc. of WWW 2004","author":"D.E. Rose","year":"2004","unstructured":"Rose, D.E., Levinson, D.: Understanding user goals in web search. In: Proc. of WWW 2004, pp. 13\u201319. ACM, New York (2004)"},{"key":"31_CR15","first-page":"138","volume-title":"Proc. of SIGIR 1995","author":"T. Saracevic","year":"1995","unstructured":"Saracevic, T.: Evaluation of evaluation in information retrieval. In: Proc. of SIGIR 1995, pp. 138\u2013146. ACM, New York (1995)"},{"issue":"1","key":"31_CR16","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1145\/331403.331405","volume":"33","author":"C. Silverstein","year":"1999","unstructured":"Silverstein, C., Marais, H., Henzinger, M., Moricz, M.: Analysis of a very large web search engine query log. SIGIR Forum\u00a033(1), 6\u201312 (1999)","journal-title":"SIGIR Forum"},{"key":"31_CR17","first-page":"66","volume-title":"Proc. of SIGIR 2001","author":"I. Soboroff","year":"2001","unstructured":"Soboroff, I., Nicholas, C., Cahan, P.: Ranking retrieval systems without relevance judgments. In: Proc. of SIGIR 2001, pp. 66\u201373. ACM, New York (2001)"},{"issue":"2","key":"31_CR18","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1145\/959258.959267","volume":"37","author":"I. Soboroff","year":"2003","unstructured":"Soboroff, I., Voorhees, E., Craswell, N.: Summary of the SIGIR 2003 workshop on defining evaluation methodologies for terabyte-scale test collections. SIGIR Forum\u00a037(2), 55\u201358 (2003)","journal-title":"SIGIR Forum"},{"key":"31_CR19","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1007\/3-540-45691-0_34","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"E.M. Voorhees","year":"2002","unstructured":"Voorhees, E.M.: The Philosophy of Information Retrieval Evaluation. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol.\u00a02406, pp. 355\u2013370. Springer, Heidelberg (2002)"}],"container-title":["Lecture Notes in Computer Science","Information Retrieval Technology"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-04769-5_31","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,12]],"date-time":"2025-02-12T11:34:15Z","timestamp":1739360055000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-04769-5_31"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009]]},"ISBN":["9783642047688","9783642047695"],"references-count":19,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-04769-5_31","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2009]]}}}