{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,8]],"date-time":"2026-05-08T15:09:23Z","timestamp":1778252963356,"version":"3.51.4"},"publisher-location":"Berlin, Heidelberg","reference-count":11,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783540240174","type":"print"},{"value":"9783540302223","type":"electronic"}],"license":[{"start":{"date-parts":[[2004,1,1]],"date-time":"2004-01-01T00:00:00Z","timestamp":1072915200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2004,1,1]],"date-time":"2004-01-01T00:00:00Z","timestamp":1072915200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2004]]},"DOI":"10.1007\/978-3-540-30222-3_3","type":"book-chapter","created":{"date-parts":[[2009,11,20]],"date-time":"2009-11-20T04:05:35Z","timestamp":1258689935000},"page":"21-28","source":"Crossref","is-referenced-by-count":0,"title":["Analysis of the Reliability of the Multilingual Topic Set for the Cross Language Evaluation Forum"],"prefix":"10.1007","author":[{"given":"Thomas","family":"Mandl","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christa","family":"Womser-Hacker","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"issue":"3","key":"3_CR1","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1016\/0306-4573(94)00048-8","volume":"31","author":"K. Sparck Jones","year":"1995","unstructured":"Sparck Jones, K.: Reflections on TREC. Information Processing & Management\u00a031(3), 291\u2013314 (1995)","journal-title":"Information Processing & Management"},{"key":"3_CR2","doi-asserted-by":"crossref","unstructured":"Kluck, M., Womser-Hacker, C.: Inside the Evaluation Process of the Cross-Language Evaluation Forum (CLEF): Issues of Multilingual Topic Creation and Multilingual Relevance Assessment. In: Rodr\u00edguez, M.G., Araujo, C.P.S. (eds.) Proceedings of the Third International Conference on Language Resources and Evaluation, LREC 2002. Las Palmas de Gran Canaria, May 29-31, 2002, pp. 573\u2013576. ELRA, Paris (2002)","DOI":"10.63317\/2p7pbsd7fib6"},{"key":"3_CR3","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"389","DOI":"10.1007\/3-540-45691-0_36","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"C. Womser-Hacker","year":"2002","unstructured":"Womser-Hacker, C.: Multilingual Topic Generation within the CLEF 2001 Experiments. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol.\u00a02406, pp. 389\u2013393. Springer, Heidelberg (2002)"},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Zobel, J.: How Reliable are the Results of Large-Scale Information Retrieval Experiments? In: Proceedings of the Annual International ACM Conference on Research and Development in Information Retrieval (SIGIR 1998), Melbourne, pp. 307\u2013314 (1998)","DOI":"10.1145\/290941.291014"},{"key":"3_CR5","doi-asserted-by":"crossref","unstructured":"Voorhees, E., Buckley, C.: The Effect of Topic Set Size on Retrieval Experiment Error. In: Proceedings of the Annual International ACM Conference on Research and Development in Information Retrieval (SIGIR 2002), Tampere, Finland, pp. 316\u2013323 (2002)","DOI":"10.1145\/564376.564432"},{"key":"3_CR6","doi-asserted-by":"crossref","unstructured":"Soboroff, I., Nicholas, C., Cahan, P.: Ranking Retrieval Systems without Relevance Judgments. In: Proceedings of the Annual International ACM Conference on Research and Development in Information Retrieval (SIGIR 2001), New Orleans, pp. 66\u201373 (2001)","DOI":"10.1145\/383952.383961"},{"key":"3_CR7","doi-asserted-by":"crossref","unstructured":"Voorhees, E.: Variations in relevance judgments and the measurement of retrieval effectiveness. In: Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 1998), Melbourne, pp. 315\u2013223 (1998)","DOI":"10.1145\/290941.291017"},{"key":"3_CR8","doi-asserted-by":"crossref","unstructured":"Voorhees, E., Harman, D.: Overview of the Sixth Text REtrieval Conference. In: Voorhees, E., Harman, D. (eds.) The Sixth Text REtrieval Conference (TREC-6). National Institute of Standards and Technology, Gaithersburg, Maryland, NIST Special Publication (1997), \n                      http:\/\/trec.nist.gov\/pubs\/","DOI":"10.6028\/NIST.SP.500-240.overview"},{"key":"3_CR9","doi-asserted-by":"crossref","unstructured":"Eguchi, K., Kuriyama, K., Kando, N.: Sensitivity of IR Systems Evaluation to Topic Difficulty. In: Rodr\u00edguez, M.G., Araujo, C.P.S. (eds.) Proceedings of the Third International Conference on Language Resources and Evaluation, LREC 2002, Las Palmas de Gran Canaria, May 29-31, 2002, pp. 585\u2013589. ELRA, Paris (2002)","DOI":"10.63317\/3h4g2yfpxzrh"},{"key":"3_CR10","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"505","DOI":"10.1007\/978-3-540-45237-9_43","volume-title":"Advances in Cross-Language Information Retrieval","author":"T. Mandl","year":"2003","unstructured":"Mandl, T., Womser-Hacker, C.: Linguistic and Statistical Analysis of the CLEF Topics. In: Peters, C., Braschler, M., Gonzalo, J. (eds.) CLEF 2002. LNCS, vol.\u00a02785, pp. 505\u2013511. Springer, Heidelberg (2003)"},{"key":"3_CR11","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1145\/775152.775156","volume-title":"Proceedings of the Twelfth International World Wide Web Conference (WWW 2003)","author":"R. Lempel","year":"2003","unstructured":"Lempel, R., Moran, S.: Predictive Caching and Prefetching of Query Results in Search Engines. In: Proceedings of the Twelfth International World Wide Web Conference (WWW 2003), Budapest, pp. 19\u201328. ACM Press, New York (2003)"}],"container-title":["Lecture Notes in Computer Science","Comparative Evaluation of Multilingual Information Access Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-540-30222-3_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,8]],"date-time":"2026-05-08T14:09:37Z","timestamp":1778249377000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-540-30222-3_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2004]]},"ISBN":["9783540240174","9783540302223"],"references-count":11,"URL":"https:\/\/doi.org\/10.1007\/978-3-540-30222-3_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2004]]}}}