{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,3]],"date-time":"2025-05-03T04:13:59Z","timestamp":1746245639630,"version":"3.37.3"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2019,4,20]],"date-time":"2019-04-20T00:00:00Z","timestamp":1555718400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2019,4,20]],"date-time":"2019-04-20T00:00:00Z","timestamp":1555718400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61872370","61502501"],"award-info":[{"award-number":["61872370","61502501"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012166","name":"National Key R&D Program of China","doi-asserted-by":"crossref","award":["2018YFC0830703"],"award-info":[{"award-number":["2018YFC0830703"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Beijing Natural Science Foundation","award":["4162032"],"award-info":[{"award-number":["4162032"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2020,2]]},"DOI":"10.1007\/s10791-019-09356-x","type":"journal-article","created":{"date-parts":[[2019,4,20]],"date-time":"2019-04-20T16:04:01Z","timestamp":1555776241000},"page":"86-113","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Low-cost, bottom-up measures for evaluating search result diversification"],"prefix":"10.1007","volume":"23","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9781-948X","authenticated-orcid":false,"given":"Zhicheng","family":"Dou","sequence":"first","affiliation":[]},{"given":"Xue","family":"Yang","sequence":"additional","affiliation":[]},{"given":"Diya","family":"Li","sequence":"additional","affiliation":[]},{"given":"Ji-Rong","family":"Wen","sequence":"additional","affiliation":[]},{"given":"Tetsuya","family":"Sakai","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,4,20]]},"reference":[{"key":"9356_CR1","doi-asserted-by":"crossref","unstructured":"Agrawal, R., Gollapudi, S., Halverson, A., & Ieong, S. (2009). Diversifying search results. In Proceedings of WSDM \u201909 (pp. 5\u201314).","DOI":"10.1145\/1498759.1498766"},{"key":"9356_CR2","doi-asserted-by":"crossref","unstructured":"Carbonell, J., & Goldstein, J. (1998). The use of mmr, diversity-based reranking for reordering documents and producing summaries. In: Proceedings of the 21st annual international ACM SIGIR conference on Research and development in information retrieval (pp. 335\u2013336). ACM.","DOI":"10.1145\/290941.291025"},{"key":"9356_CR3","doi-asserted-by":"crossref","unstructured":"Charikar, M. S. (2002). Similarity estimation techniques from rounding algorithms. In STOC \u201902 (pp. 380\u2013388). ACM.","DOI":"10.1145\/509907.509965"},{"key":"9356_CR4","unstructured":"Clarke, C. L., Craswell, N., & Soboroff, I. (2009). Overview of the trec 2009 web track. DTIC Document, Technical Report."},{"key":"9356_CR5","doi-asserted-by":"crossref","unstructured":"Clarke, C. L., Kolla, M., Cormack, G. V., Vechtomova, O., Ashkan, A., B\u00fcttcher, S., & MacKinnon, I. (2008). Novelty and diversity in information retrieval evaluation. In Proceedings of SIGIR \u201908 (pp. 659\u2013666).","DOI":"10.1145\/1390334.1390446"},{"key":"9356_CR6","unstructured":"Collins-Thompson, K., Bennett, P., Diaz, F., Clarke, C., & Voorhees, E. (2013). Overview of the trec 2013 web track. In TREC 2013."},{"key":"9356_CR7","doi-asserted-by":"crossref","unstructured":"Dang, V., & Croft, B. W. (2013). Term level search result diversification. In Proceedings of SIGIR \u201913 (pp. 603\u2013612). ACM","DOI":"10.1145\/2484028.2484095"},{"key":"9356_CR8","doi-asserted-by":"crossref","unstructured":"Dang, V., & Croft, W. B. (2012). Diversity by proportionality: An election-based approach to search result diversification. In Proceedings of SIGIR \u201912 (pp. 65\u201374).","DOI":"10.1145\/2348283.2348296"},{"key":"9356_CR9","doi-asserted-by":"crossref","unstructured":"Dou, Z., Hu, S., Chen, K., Song, R., & Wen, J. R. (2011). Multi-dimensional search result diversification. In Proceedings of WSDM \u201911 (pp. 475\u2013484).","DOI":"10.1145\/1935826.1935897"},{"key":"9356_CR10","doi-asserted-by":"publisher","unstructured":"Dou, Z., Song, R., & Wen, J. (2007). A large-scale evaluation and analysis of personalized search strategies. In C. L. Williamson, M. E. Zurko, P. F. Patel-Schneider, & P. J. Shenoy (Eds.) Proceedings of the 16th international conference on world wide web, WWW 2007, Banff, Alberta, Canada, May 8\u201312, 2007 (pp. 581\u2013590). ACM. https:\/\/doi.org\/10.1145\/1242572.1242651.","DOI":"10.1145\/1242572.1242651"},{"issue":"8","key":"9356_CR11","doi-asserted-by":"publisher","first-page":"1178","DOI":"10.1109\/TKDE.2008.172","volume":"21","author":"Z Dou","year":"2009","unstructured":"Dou, Z., Song, R., Wen, J., & Yuan, X. (2009). Evaluating the effectiveness of personalized web search. IEEE Transactions on Knowledge and Data Engineering, 21(8), 1178\u20131190. https:\/\/doi.org\/10.1109\/TKDE.2008.172.","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"issue":"2","key":"9356_CR12","doi-asserted-by":"publisher","first-page":"147","DOI":"10.1002\/j.1538-7305.1950.tb00463.x","volume":"29","author":"RW Hamming","year":"1950","unstructured":"Hamming, R. W. (1950). Error detecting and error correcting codes. The Bell System Technical Journal, 29(2), 147\u2013160. https:\/\/doi.org\/10.1002\/j.1538-7305.1950.tb00463.x.","journal-title":"The Bell System Technical Journal"},{"key":"9356_CR13","doi-asserted-by":"crossref","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2000). Ir evaluation methods for retrieving highly relevant documents. In Proceedings of SIGIR \u201900 (pp. 41\u201348). ACM.","DOI":"10.1145\/345508.345545"},{"issue":"1\/2","key":"9356_CR14","doi-asserted-by":"publisher","first-page":"81","DOI":"10.2307\/2332226","volume":"30","author":"MG Kendall","year":"1938","unstructured":"Kendall, M. G. (1938). A new measure of rank correlation. Biometrika, 30(1\/2), 81\u201393.","journal-title":"Biometrika"},{"key":"9356_CR15","doi-asserted-by":"crossref","unstructured":"Radlinski, F., & Dumais, S. (2006). Improving personalized web search using result diversification. In Proceedings of SIGIR \u201906 (pp. 691\u2013692).","DOI":"10.1145\/1148170.1148320"},{"key":"9356_CR16","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2006a). Bootstrap-based comparisons of ir metrics for finding one relevant document. In Proceedings of AIRS \u201906 (pp. 374\u2013389). Springer.","DOI":"10.1007\/11880592_29"},{"key":"9356_CR17","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2006b). Evaluating evaluation metrics based on the bootstrap. In Proceedings of SIGIR \u201906 (pp. 525\u2013532). ACM.","DOI":"10.1145\/1148170.1148261"},{"key":"9356_CR18","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2012). Evaluation with informational and navigational intents. In Proceedings of WWW \u201912 (pp. 499\u2013508). ACM.","DOI":"10.1145\/2187836.2187904"},{"key":"9356_CR19","unstructured":"Sakai, T., Craswell, N., Song, R., Robertson, S., Dou, Z., & Lin, C. Y. (2010). Simple evaluation metrics for diversified search results. In Proceedings of EVIA \u201910 (pp. 42\u201350)."},{"key":"9356_CR20","doi-asserted-by":"publisher","unstructured":"Sakai, T., Dou, Z., & Clarke, C. L. (2013). The impact of intent selection on diversified search evaluation. In Proceedings of SIGIR \u201913, SIGIR \u201913 (pp. 921\u2013924). ACM, New York, NY, USA. https:\/\/doi.org\/10.1145\/2484028.2484105.","DOI":"10.1145\/2484028.2484105"},{"key":"9356_CR21","unstructured":"Sakai, T., & Robertson, S. (2008). Modelling a user population for designing information retrieval metrics. In Proceedings of EVIA \u201908."},{"key":"9356_CR22","doi-asserted-by":"crossref","unstructured":"Sakai, T., & Song, R. (2011). Evaluating diversified search results using per-intent graded relevance. In Proceedings of SIGIR \u201911 (pp. 1043\u20131052).","DOI":"10.1145\/2009916.2010055"},{"key":"9356_CR23","unstructured":"Salton, G., & McGill, M. J. (1986). Introduction to modern information retrieval."},{"key":"9356_CR24","doi-asserted-by":"crossref","unstructured":"Santos, R. L., Macdonald, C., & Ounis, I. (2010a). Exploiting query reformulations for web search result diversification. In Proceedings of WWW \u201910 (pp. 881\u2013890).","DOI":"10.1145\/1772690.1772780"},{"key":"9356_CR25","doi-asserted-by":"crossref","unstructured":"Santos, R. L., Macdonald, C., & Ounis, I. (2010b). Selectively diversifying web search results. In Proceedings of the 19th ACM international conference on information and knowledge management (pp. 1179\u20131188). ACM.","DOI":"10.1145\/1871437.1871586"},{"key":"9356_CR26","doi-asserted-by":"crossref","unstructured":"Santos, R. L., Macdonald, C., & Ounis, I. (2011). Intent-aware search result diversification. In Proceedings of SIGIR \u201911 (pp. 595\u2013604). ACM.","DOI":"10.1145\/2009916.2009997"},{"key":"9356_CR27","doi-asserted-by":"crossref","unstructured":"Santos, R. L., Peng, J., Macdonald, C., & Ounis, I. (2010). Explicit search result diversification through sub-queries. In European conference on information retrieval (pp. 87\u201399). Springer.","DOI":"10.1007\/978-3-642-12275-0_11"},{"key":"9356_CR28","unstructured":"Schiffman, B., & McKeown, K. (2004). Columbia university in the novelty track at trec 2004. In TREC."},{"key":"9356_CR29","unstructured":"Soboroff, I. (2004). Overview of the TREC 2004 novelty track. In E. M. Voorhees, & L. P. Buckland (Eds.) Proceedings of the thirteenth text retrieval conference, TREC 2004, Gaithersburg, Maryland, USA, November 16\u201319, 2004, vol. Special Publication 500-261. National Institute of Standards and Technology (NIST). http:\/\/trec.nist.gov\/pubs\/trec13\/papers\/NOVELTY.OVERVIEW.pdf."},{"issue":"4","key":"9356_CR30","doi-asserted-by":"publisher","first-page":"728","DOI":"10.1007\/s11390-010-9360-y","volume":"25","author":"R Song","year":"2010","unstructured":"Song, R., Dou, Z., Hon, H., & Yu, Y. (2010). Learning query ambiguity models by using search logs. Journal of Computer Science and Technology, 25(4), 728\u2013738. https:\/\/doi.org\/10.1007\/s11390-010-9360-y.","journal-title":"Journal of Computer Science and Technology"},{"key":"9356_CR31","unstructured":"The clueweb09 dataset. (2009). http:\/\/boston.lti.cs.cmu.edu\/Data\/clueweb09\/."},{"key":"9356_CR32","unstructured":"The clueweb12 dataset. (2012). http:\/\/boston.lti.cs.cmu.edu\/Data\/clueweb12\/."},{"key":"9356_CR33","doi-asserted-by":"crossref","unstructured":"Wang, X., Dou, Z., Sakai, T., & Wen, J. R. (2016). Evaluating search result diversity using intent hierarchies. In Proceedings of SIGIR \u201916.","DOI":"10.1145\/2911451.2911497"},{"key":"9356_CR34","first-page":"94","volume":"2016","author":"T Yamamoto","year":"2016","unstructured":"Yamamoto, T., Liu, Y., Zhang, M., Dou, Z., Zhou, K., Markov, I., et al. (2016). Overview of the ntcir-12 imine-2 task. NTCIR, 2016, 94\u2013123.","journal-title":"NTCIR"},{"key":"9356_CR35","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J. A., & Robertson, S. (2008). A new rank correlation coefficient for information retrieval. In Proceedings of SIGIR \u201908 (pp. 587\u2013594). ACM.","DOI":"10.1145\/1390334.1390435"},{"issue":"Oct","key":"9356_CR36","first-page":"1205","volume":"5","author":"L Yu","year":"2004","unstructured":"Yu, L., & Liu, H. (2004). Efficient feature selection via analysis of relevance and redundancy. Journal of Machine Learning Research, 5(Oct), 1205\u20131224.","journal-title":"Journal of Machine Learning Research"},{"key":"9356_CR37","doi-asserted-by":"crossref","unstructured":"Zhai, C. X., Cohen, W. W., & Lafferty, J. (2003). Beyond independent relevance: Methods and evaluation metrics for subtopic retrieval. In Proceedings of SIGIR \u201903 (pp. 10\u201317). ACM.","DOI":"10.1145\/860435.860440"},{"key":"9356_CR38","unstructured":"Zhu, X., Goldberg, A. B., Van\u00a0Gael, J., & Andrzejewski, D. (2007). Improving diversity in ranking using absorbing random walks. In Proceedings of HLT-NAACL \u201907 (pp. 97\u2013104)."}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-019-09356-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-019-09356-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-019-09356-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:48:24Z","timestamp":1704206904000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-019-09356-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,4,20]]},"references-count":38,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2020,2]]}},"alternative-id":["9356"],"URL":"https:\/\/doi.org\/10.1007\/s10791-019-09356-x","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"type":"print","value":"1386-4564"},{"type":"electronic","value":"1573-7659"}],"subject":[],"published":{"date-parts":[[2019,4,20]]},"assertion":[{"value":"13 October 2018","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 March 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 April 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}