{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,31]],"date-time":"2026-10-31T00:20:39Z","timestamp":1793406039586,"version":"4.5.0"},"reference-count":52,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2010,1,1]],"date-time":"2010-01-01T00:00:00Z","timestamp":1262304000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2010,1,1]],"date-time":"2010-01-01T00:00:00Z","timestamp":1262304000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval"],"published-print":{"date-parts":[[2010,8]]},"DOI":"10.1007\/s10791-009-9123-y","type":"journal-article","created":{"date-parts":[[2009,12,31]],"date-time":"2009-12-31T10:16:55Z","timestamp":1262254615000},"page":"346-374","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":336,"title":["LETOR: A benchmark collection for research on learning to rank for information retrieval"],"prefix":"10.1007","volume":"13","author":[{"given":"Tao","family":"Qin","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tie-Yan","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jun","family":"Xu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hang","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2010,1,1]]},"reference":[{"key":"9123_CR1","unstructured":"Asuncion, A., & Newman, D. J. (2007). UCI machine learning repository. Irvine, CA: University of California, School of Information and Computer Science."},{"key":"9123_CR2","volume-title":"Modern information retrieval","author":"R. Baeza-Yates","year":"1999","unstructured":"Baeza-Yates, R., & Ribeiro-Neto, B. (1999). Modern information retrieval. New York: Addison Wesley."},{"issue":"1\u20137","key":"9123_CR3","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1016\/S0169-7552(98)00110-X","volume":"30","author":"S. Brin","year":"1998","unstructured":"Brin, S., & Page, L. (1998). The anatomy of a large-scale hypertextual web search engine. Computer Networks and ISDN Systems, 30(1\u20137), 107\u2013117.","journal-title":"Computer Networks and ISDN Systems"},{"key":"9123_CR4","doi-asserted-by":"crossref","unstructured":"Burges, C., Shaked, T., Renshaw, E., Lazier, A., Deeds, M., Hamilton, N., et al. (2005). Learning to rank using gradient descent. In ICML \u201905: Proceedings of the 22nd International Conference on Machine Learning (pp. 89\u201396). New York, NY: ACM Press.","DOI":"10.1145\/1102351.1102363"},{"key":"9123_CR5","doi-asserted-by":"crossref","unstructured":"Cao, Y., Xu, J., Liu, T.-Y., Li, H., Huang, Y., & Hon, H.-W. (2006). Adapting ranking svm to document retrieval. In SIGIR \u201906: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 186\u2013193). New York, NY: ACM Press.","DOI":"10.1145\/1148170.1148205"},{"key":"9123_CR6","doi-asserted-by":"crossref","unstructured":"Cao, Z., Qin, T., Liu, T.-Y., Tsai, M.-F., et al.  (2007). Learning to rank: From pairwise approach to listwise approach. In ICML \u201907: Proceedings of the 24th International Conference on Machine Learning (pp. 129\u2013136). New York, NY: ACM Press.","DOI":"10.1145\/1273496.1273513"},{"key":"9123_CR7","first-page":"1","volume":"9","author":"G. Chechik","year":"2008","unstructured":"Chechik, G., Heitz, G., Elidan, G., Abbeel, P., & Koller, D. (2008). Max-margin classification of data with absent features. Journal of Machine Learning Research, 9, 1\u201321.","journal-title":"Journal of Machine Learning Research"},{"key":"9123_CR8","doi-asserted-by":"crossref","unstructured":"Chirita, P., Diederich, J., & Nejdl, W. (2005). MailRank: Using ranking for spam detection. In Proceedings of the 14th ACM International Conference on Information and Knowledge Management (pp. 373\u2013380). New York, NY: ACM.","DOI":"10.1145\/1099554.1099671"},{"key":"9123_CR9","doi-asserted-by":"crossref","unstructured":"Collins, M. (2002). Ranking algorithms for named-entity extraction: Boosting and the voted perceptron. In Proceedings of the 40th Annual Meeting on Association for Computational Linguistics, July (pp. 07\u201312).","DOI":"10.3115\/1073083.1073165"},{"key":"9123_CR10","unstructured":"Craswell, N., & Hawking, D. (2004). Overview of the TREC 2004 Web track. In Proceedings of the 13th Text Retrieval Conference (TREC 2004). Gaithersburg, MD: NIST."},{"key":"9123_CR11","unstructured":"Craswell, N., Hawking, D., Wilkinson, R., & Wu, M. (2003). Overview of the TREC 2003 web track. In Proceedings of TREC 2003 (pp. 78\u201392)."},{"key":"9123_CR12","doi-asserted-by":"crossref","unstructured":"Dave, K., Lawrence, S., & Pennock, D. (2003). Mining the peanut gallery: Opinion extraction and semantic classification of product reviews. In Proceedings of the 12th International Conference on World Wide Web (pp. 519\u2013528). New York, NY: ACM Press.","DOI":"10.1145\/775152.775226"},{"key":"9123_CR13","doi-asserted-by":"publisher","first-page":"933","DOI":"10.1162\/jmlr.2003.4.6.933","volume":"4","author":"Y. Freund","year":"2003","unstructured":"Freund, Y., Iyer, R., Schapire, R. E., & Singer, Y. (2003). An efficient boosting algorithm for combining preferences. Journal of Machine Learning Research, 4, 933\u2013969.","journal-title":"Journal of Machine Learning Research"},{"key":"9123_CR14","doi-asserted-by":"crossref","unstructured":"Geng, X., Liu, T.-Y., Qin, T., Arnold, A., Li, H., & Shum, H.-Y. (2008). Query dependent ranking using k-nearest neighbor. In SIGIR \u201908: Proceedings of the 31th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 115\u2013122). New York, NY: ACM.","DOI":"10.1145\/1390334.1390356"},{"key":"9123_CR15","doi-asserted-by":"crossref","unstructured":"Gy\u00f6ngyi, Z., Garcia-Molina, H., & Pedersen, J. (2004). Combating web spam with trustrank. In Proceedings of the Thirtieth International Conference on Very Large Data Bases-Volume 30 (pp. 576\u2013587). VLDB Endowment.","DOI":"10.1016\/B978-012088469-8.50052-8"},{"key":"9123_CR16","unstructured":"Harrington, E. F. (2003). Online ranking\/collaborative filtering using the perceptron algorithm. In Proceedings of the 20th International Conference on Machine Learning (pp. 250\u2013257)."},{"key":"9123_CR17","doi-asserted-by":"crossref","unstructured":"Herbrich, R., Graepel, T., & Obermayer, K. (1999). Support vector learning for ordinal regression. In ICANN1999 (pp. 97\u2013102).","DOI":"10.1049\/cp:19991091"},{"key":"9123_CR18","doi-asserted-by":"crossref","unstructured":"Hersh, W., Buckley, C., Leone, T. J., & Hickam, D. (1994). Ohsumed: An interactive retrieval evaluation and new large test collection for research. In SIGIR \u201994 (pp. 192\u2013201). New York, NY: Springer.","DOI":"10.1007\/978-1-4471-2099-5_20"},{"key":"9123_CR19","doi-asserted-by":"crossref","unstructured":"Hu, Y., Xin, G., Song, R., Hu, G., Shi, S., Cao, Y., et al. (2005). Title extraction from bodies of html documents and its application to web page retrieval. In SIGIR \u201905: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 250\u2013257). New York, NY: ACM.","DOI":"10.1145\/1076034.1076079"},{"key":"9123_CR20","volume-title":"Advances in neural information processing systems 21","author":"J. C. Huang","year":"2009","unstructured":"Huang, J. C., & Frey, B. J. (2009). Structured ranking learning using cumulative distribution networks. In D. Koller, D. Schuurmans, Y. Bengio, & L. Bottou (Eds.), Advances in Neural Information Processing Systems 21 (pp. 697\u2013704). Cambridge: MIT Press."},{"issue":"4","key":"9123_CR21","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K. J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems, 20(4), 422\u2013446.","journal-title":"ACM Transactions on Information Systems"},{"key":"9123_CR22","doi-asserted-by":"crossref","unstructured":"Joachims, T. (2002). Optimizing search engines using clickthrough data. In KDD \u201902: Proceedings of the Eighth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (pp. 133\u2013142). New York, NY: ACM Press.","DOI":"10.1145\/775047.775067"},{"key":"9123_CR23","first-page":"361","volume":"5","author":"D. Lewis","year":"2004","unstructured":"Lewis, D., Yang, Y., Rose, T., & Li, F. (2004). RCV1: A new benchmark collection for text categorization research. The Journal of Machine Learning Research, 5, 361\u2013397.","journal-title":"The Journal of Machine Learning Research"},{"key":"9123_CR24","doi-asserted-by":"crossref","unstructured":"Li, L., & Lin, H.-T. (2006). Ordinal regression by extended binary classification. In NIPS (pp. 865\u2013872).","DOI":"10.7551\/mitpress\/7503.003.0113"},{"key":"9123_CR25","unstructured":"Li, P., Burges, C., & Wu, Q. (2008). Mcrank: Learning to rank using multiple classification and gradient boosting. In Advances in Neural Information Processing Systems 20  (pp. 897\u2013904). Cambridge, MA: MIT Press."},{"key":"9123_CR26","doi-asserted-by":"crossref","unstructured":"Matveeva, I., Burges, C., Burkard, T., Laucius, A., & Wong, L. (2006). High accuracy retrieval with multiple nested ranker. In SIGIR \u201906: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 437\u2013444). New York, NY: ACM.","DOI":"10.1145\/1148170.1148246"},{"key":"9123_CR27","unstructured":"Minka, T., & Robertson, S. (2008). Selection bias in the LETOR datasets. In Proceedings of SIGIR 2008 Workshop on Learning to Rank for Information Retrieval."},{"key":"9123_CR28","doi-asserted-by":"crossref","unstructured":"Nie, L., Davison, B. D., & Qi, X. (2006). Topical link analysis for web search. In SIGIR \u201906: Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 91\u201398). New York, NY: ACM.","DOI":"10.1145\/1148170.1148189"},{"key":"9123_CR29","doi-asserted-by":"crossref","unstructured":"Pang, B., & Lee, L. (2005). Seeing stars: Exploiting class relationships for sentiment categorization with respect to rating scales. In Proceedings of the 43rd Annual Meeting on Association for Computational Linguistics (pp. 115\u2013124). NJ: Association for Computational Linguistics Morristown.","DOI":"10.3115\/1219840.1219855"},{"key":"9123_CR30","unstructured":"Qin, T., Liu, T., Xu, J., & Li, H. (2008a). How to make LETOR more useful and reliable. In Proceedings of SIGIR 2008 Workshop on Learning to Rank for Information Retrieval."},{"key":"9123_CR31","unstructured":"Qin, T., Liu, T.-Y., & Li, H. (2008b). A general approximation framework for direct optimization of information retrieval measures. Technical Report MSR-TR-2008-164, Microsoft Corporation."},{"key":"9123_CR32","doi-asserted-by":"crossref","unstructured":"Qin, T., Liu, T.-Y., Zhang, X.-D., Chen, Z., & Ma, W.-Y. (2005). A study of relevance propagation for web search. In SIGIR \u201905 (pp. 408\u2013415). New York, NY: ACM Press.","DOI":"10.1145\/1076034.1076105"},{"key":"9123_CR33","unstructured":"Qin, T., Liu, T.-Y., Zhang, X.-D., Wang, D.-S., & Li, H. (2008c). Global ranking using continuous conditional random fields. In D. Koller, D. Schuurmans, Y. Bengio, & L. Bottou (Eds.), NIPS. MIT Press."},{"key":"9123_CR34","doi-asserted-by":"crossref","unstructured":"Qin, T., Liu, T.-Y., Zhang, X.-D., Wang, D.-S., Xiong, W.-Y., & Li, H. (2008d). Learning to rank relational objects and its application to web search. In WWW \u201908: Proceeding of the 17th International Conference on World Wide Web (pp. 407\u2013416). New York, NY: ACM.","DOI":"10.1145\/1367497.1367553"},{"issue":"2","key":"9123_CR35","doi-asserted-by":"publisher","first-page":"838","DOI":"10.1016\/j.ipm.2007.07.016","volume":"44","author":"T. Qin","year":"2008","unstructured":"Qin, T., Zhang, X.-D., Tsai, M.-F., Wang, D.-S., Liu, T.-Y., & Li, H. (2008e). Query-level loss functions for information retrieval. Information Processing & Management, 44(2):838\u2013855.","journal-title":"Information Processing & Management"},{"key":"9123_CR36","doi-asserted-by":"crossref","unstructured":"Qin, T., Zhang, X.-D., Wang, D.-S., Liu, T.-Y., Lai, W., & Li, H. (2007). Ranking with multiple hyperplanes. In SIGIR \u201907 (pp. 279\u2013286). New York, NY: ACM Press.","DOI":"10.1145\/1277741.1277791"},{"key":"9123_CR37","doi-asserted-by":"crossref","unstructured":"Robertson, S., Zaragoza, H., & Taylor, M. (2004). Simple bm25 extension to multiple weighted fields. In CIKM \u201904: Proceedings of the Thirteenth ACM International Conference on Information and Knowledge Management (pp. 42\u201349). New York, NY: ACM.","DOI":"10.1145\/1031171.1031181"},{"key":"9123_CR38","unstructured":"Robertson S. E., & Hull, D. A. (2000). The TREC-9 filtering track final report. In TREC (pp. 25\u201340)."},{"key":"9123_CR39","unstructured":"Shakery, A., & Zhai, C. (2003). Relevance propagation for topic distillation UIUC TREC-2003 Web track experiments. In Proceedings of TREC (pp. 673\u2013677)."},{"key":"9123_CR40","doi-asserted-by":"crossref","unstructured":"Taylor, M., Guiver, J., Robertson, S., & Minka, T. (2008). Softrank: Optimizing non-smooth rank metrics. In WSDM \u201908: Proceedings of the International Conference on Web Search and Web Data Mining (pp. 77\u201386). New York, NY: ACM.","DOI":"10.1145\/1341531.1341544"},{"key":"9123_CR41","unstructured":"TREC. (2004). Web track guideline. http:\/\/www.research.microsoft.com\/en-us\/um\/people\/nickcr\/guidelines_2004.html."},{"key":"9123_CR42","doi-asserted-by":"crossref","unstructured":"Tsai, M.-F., Liu, T.-Y., Qin, T., Chen, H.-H., & Ma, W.-Y. (2007). Frank: A ranking method with fidelity loss. In SIGIR \u201907 (pp. 383\u2013390). New York, NY: ACM Press","DOI":"10.1145\/1277741.1277808"},{"key":"9123_CR43","doi-asserted-by":"crossref","unstructured":"Volkovs, M. N., & Zemel, R. S. (2009). Boltzrank: Learning to maximize expected ranking gain. In ICML \u201909: Proceedings of the 26th Annual International Conference on Machine Learning (pp. 1089\u20131096). New York, NY: ACM","DOI":"10.1145\/1553374.1553513"},{"key":"9123_CR44","volume-title":"TREC: Experiment and evaluation in information retrieval","author":"E. Voorhees","year":"2005","unstructured":"Voorhees, E., & Harman, D. (2005). TREC: Experiment and evaluation in information retrieval. Cambridge, MA: MIT Press."},{"key":"9123_CR45","doi-asserted-by":"crossref","unstructured":"Xia, F., Liu, T.-Y., Wang, J., Zhang, W., & Li, H. (2008). Listwise approach to learning to rank\u2014Theory and algorithm. In ICML \u201908: Proceedings of the 25th International Conference on Machine Learning. New York, NY: ACM Press.","DOI":"10.1145\/1390156.1390306"},{"key":"9123_CR46","doi-asserted-by":"crossref","unstructured":"Xu, J., Cao, Y., Li, H., & Zhao, M. (2005). Ranking definitions with supervised learning methods. In International World Wide Web Conference (pp. 811\u2013819). New York, NY: ACM Press.","DOI":"10.1145\/1062745.1062761"},{"key":"9123_CR47","doi-asserted-by":"crossref","unstructured":"Xu, J., & Li, H. (2007). Adarank: A boosting algorithm for information retrieval. In SIGIR \u201907: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 391\u2013398). New York, NY: ACM Press.","DOI":"10.1145\/1277741.1277809"},{"key":"9123_CR48","doi-asserted-by":"crossref","unstructured":"Xu, J., Liu, T.-Y., Lu, M., Li, H., & Ma, W.-Y. (2008). Directly optimizing evaluation measures in learning to rank. In SIGIR \u201908: Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 107\u2013114). New York, NY: ACM.","DOI":"10.1145\/1390334.1390355"},{"key":"9123_CR49","doi-asserted-by":"crossref","unstructured":"Xue, G.-R., Yang, Q., Zeng, H.-J., Yu, Y., & Chen, Z. (2005). Exploiting the hierarchical structure for link analysis. In SIGIR \u201905: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 186\u2013193). New York, NY: ACM.","DOI":"10.1145\/1076034.1076068"},{"key":"9123_CR50","doi-asserted-by":"crossref","unstructured":"Yue, Y., Finley, T., Radlinski, F., & Joachims, T. (2007). A support vector method for optimizing average precision. In SIGIR \u201907: Proceedings of the 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 271\u2013278). New York, NY: ACM Press","DOI":"10.1145\/1277741.1277790"},{"key":"9123_CR51","doi-asserted-by":"crossref","unstructured":"Zhai, C., & Lafferty, J. (2001). A study of smoothing methods for language models applied to Ad Hoc information retrieval. In Proceedings of the 24th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (pp. 334\u2013342). New York, NY: ACM Press.","DOI":"10.1145\/383952.384019"},{"key":"9123_CR52","doi-asserted-by":"crossref","unstructured":"Zhai, C. X., Cohen, W. W., & Lafferty, J. (2003). Beyond independent relevance: Methods and evaluation metrics for subtopic retrieval. In SIGIR \u201903: Proceedings of the 26th Annual International ACM SIGIR Conference on Research and Development in Informaion Retrieval (pp. 10\u201317). New York, NY: ACM Press.","DOI":"10.1145\/860435.860440"}],"container-title":["Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-009-9123-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-009-9123-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-009-9123-y","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-009-9123-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:51:29Z","timestamp":1704207089000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-009-9123-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2010,1,1]]},"references-count":52,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2010,8]]}},"alternative-id":["9123"],"URL":"https:\/\/doi.org\/10.1007\/s10791-009-9123-y","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2010,1,1]]},"assertion":[{"value":"29 April 2009","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 December 2009","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 January 2010","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}