{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T00:59:40Z","timestamp":1774400380216,"version":"3.50.1"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2022,7,20]],"date-time":"2022-07-20T00:00:00Z","timestamp":1658275200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,7,20]],"date-time":"2022-07-20T00:00:00Z","timestamp":1658275200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2022,12]]},"DOI":"10.1007\/s10791-022-09411-0","type":"journal-article","created":{"date-parts":[[2022,7,20]],"date-time":"2022-07-20T10:02:41Z","timestamp":1658311361000},"page":"365-385","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":30,"title":["Shallow pooling for sparse labels"],"prefix":"10.1007","volume":"25","author":[{"given":"Negar","family":"Arabzadeh","sequence":"first","affiliation":[]},{"given":"Alexandra","family":"Vtyurina","sequence":"additional","affiliation":[]},{"given":"Xinyi","family":"Yan","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8178-9194","authenticated-orcid":false,"given":"Charles L. A.","family":"Clarke","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,7,20]]},"reference":[{"issue":"2","key":"9411_CR1","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1145\/792550.792552","volume":"36","author":"A Broder","year":"2002","unstructured":"Broder, A. (2002). A taxonomy of Web search. SIGIR Forum, 36(2), 3\u201310.","journal-title":"SIGIR Forum"},{"key":"9411_CR2","doi-asserted-by":"crossref","unstructured":"Burges, C., Shaked, T., Renshaw, E., Lazier, A., Deeds, M., Hamilton, N., & Hullender, G. (2005). Learning to rank using gradient descent. In: 22nd International Conference on Machine Learning, pp. 89\u201396, Bonn, Germany, August.","DOI":"10.1145\/1102351.1102363"},{"key":"9411_CR3","unstructured":"Ben, C., & Bennett, P. N. (2008). Evaluation measures for preference judgments. In: 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 685\u2014686, Singapore."},{"key":"9411_CR4","doi-asserted-by":"crossref","unstructured":"Carterette, B., Bennett, P., Chapelle, O. (2008). A test collection of preference judgments. In: SIGIR 2008 Workshop on Beyond Binary Relevance: Preferences, Diversity, and Set-Level Judgments, Singapore. July 2008.","DOI":"10.1145\/1480506.1480516"},{"key":"9411_CR5","unstructured":"Carterette, B., Bennett, P.\u00a0N., Chickering, D. M., & Dumais, S.\u00a0T. (2008). Here or there: Preference judgments for relevance. Computer Science Department Faculty Publication Series\u00a046, University of Massachusetts Amherst."},{"key":"9411_CR6","first-page":"1","volume":"14","author":"O Chapelle","year":"2011","unstructured":"Chapelle, O., & Chang, Y. (2011). Yahoo! Learning to rank challenge overview. Proceedings of Machine Learning Research, 14, 1\u201324.","journal-title":"Proceedings of Machine Learning Research"},{"key":"9411_CR7","doi-asserted-by":"crossref","unstructured":"Choi, D., Shin, M., Kim, E., & Shin, D.\u00a0R. (2021). Adaptive batch scheduling for open-domain question answering. IEEE Access, 9, 112097\u2013112103.","DOI":"10.1109\/ACCESS.2021.3103963"},{"key":"9411_CR8","doi-asserted-by":"crossref","unstructured":"Clarke, C. L.\u00a0A., Vtyurina, A., & Smucker, M.\u00a0D. (2021). Assessing top-$$k$$ preferences. ACM Transactions on Information Systems, 39(3)","DOI":"10.1145\/3451161"},{"key":"9411_CR9","doi-asserted-by":"crossref","unstructured":"Clarke, C.\u00a0L. A., Vtyurina, A., & Smucker, M.\u00a0D. (2020). Offline evaluation without gain. In: ACM SIGIR International Conference on Theory of Information Retrieval, pp. 185\u2013192, Stavanger, Norway.","DOI":"10.1145\/3409256.3409816"},{"key":"9411_CR10","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., & Voorhees, E.\u00a0M. (2019). Overview of the TREC 2019 Deep Learning Track. In: 28th Text REtrieval Conference, Gaithersburg, Maryland."},{"key":"9411_CR11","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., & Campos, D. (2020). Overview of the TREC 2020 Deep Learning Track. In: 29th Text REtrieval Conference, Gaithersburg, Maryland."},{"key":"9411_CR12","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D & Lin, J. (2021). MS MARCO: Benchmarking ranking models in the large-data regime. In: 44th Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 1566\u20131576, Montreal, Canada.","DOI":"10.1145\/3404835.3462804"},{"key":"9411_CR13","doi-asserted-by":"publisher","first-page":"136182","DOI":"10.1109\/ACCESS.2021.3116857","volume":"9","author":"M Ferrante","year":"2021","unstructured":"Ferrante, M., Ferro, N., & Fuhr, N. (2021). Towards meaningful statements in IR evaluation. Mapping evaluation measures to interval scales. IEEE Access, 9, 136182\u2013136216.","journal-title":"IEEE Access"},{"key":"9411_CR14","unstructured":"Gao, L., & Callan, J. (2021). Unsupervised corpus aware language model pre-training for dense passage retrieval. CoRR, abs\/2108.05540, 2021."},{"key":"9411_CR15","unstructured":"Han, S., Wang, X., ike, Bendersky, M., & Najork, M. (2020). Learning-to-rank with BERT in TF-ranking. CoRR, abs\/2004.08476"},{"key":"9411_CR16","doi-asserted-by":"crossref","unstructured":"Hofst\u00e4tter, S., Rekabsaz, N., Eickhoff, C., & Hanbury, A. (2019). On the effect of low-frequency terms on neural-IR models. In: 42nd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1137\u20131140, Paris, France.","DOI":"10.1145\/3331184.3331344"},{"key":"9411_CR17","unstructured":"Hofst\u00e4tter, S., Althammer, S., Schr\u00f6der, M., Sertkan, M., & Hanbury, A. (2020). Improving efficient neural ranking models with cross-architecture knowledge distillation. CoRR, abs\/2010.02666."},{"issue":"8","key":"9411_CR18","doi-asserted-by":"publisher","first-page":"34","DOI":"10.1109\/MC.2007.289","volume":"40","author":"T Joachims","year":"2007","unstructured":"Joachims, T., & Radlinski, F. (2007). Search engines that learn from implicit feedback. IEEE Computer, 40(8), 34\u201340.","journal-title":"IEEE Computer"},{"key":"9411_CR19","doi-asserted-by":"crossref","unstructured":"Khattab, O., & Zaharia, M. (2020). ColBERT: Efficient and effective passage search via contextualized late interaction over BERT. In: 43rd Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 39\u201348, Xi\u2019an, China.","DOI":"10.1145\/3397271.3401075"},{"issue":"2","key":"9411_CR20","doi-asserted-by":"publisher","first-page":"40","DOI":"10.1145\/3308774.3308781","volume":"52","author":"J Lin","year":"2018","unstructured":"Lin, J. (2018). The neural hype and comparisons against weak baselines. SIGIR Forum, 52(2), 40\u201351.","journal-title":"SIGIR Forum"},{"key":"9411_CR21","doi-asserted-by":"crossref","unstructured":"Lin, J., Campos, D., Craswell, N., itra, B., & Yilmaz, E. (2021). Significant improvements over the state of the art? A case study of the MS MARCO document ranking leaderboard. In: 44th Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 2283\u20132287, Montreal, Canada.","DOI":"10.1145\/3404835.3463034"},{"key":"9411_CR22","doi-asserted-by":"crossref","unstructured":"Liu, Z., Zhang, K., Xiong, C., Liu, Z., & Sun, M. (2021). OpenMatch: An open source library for Neu-IR research. CoRR, abs\/2102.00166.","DOI":"10.1145\/3404835.3462789"},{"key":"9411_CR23","doi-asserted-by":"crossref","unstructured":"Luo, C., Clarke, C. L.\u00a0A., & Smucker, M. (2021). Evaluation measures based on preference graphs. In: 44th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1534-1543, Montreal, Canada.","DOI":"10.1145\/3404835.3462947"},{"key":"9411_CR24","doi-asserted-by":"crossref","unstructured":"Maddalena, E., Mizzaro, S., Scholer, F., & Turpin, A. (2017). On crowdsourcing relevance magnitudes for information retrieval evaluation. ACM Transactions on Information Systems, 35(3).","DOI":"10.1145\/3002172"},{"key":"9411_CR25","unstructured":"Metzler, D., Strohman, T., & Croft, W. B. (2006). Indri TREC notebook 2006: Lessons learned from three Terabyte Tracks. In: 15th Text REtrieval Conference, Gaithersburg, Maryland."},{"key":"9411_CR26","unstructured":"Nguyen, T., Rosenberg, M., Song, X., Gao, J., Tiwary, S., Majumder, R., & \u00a0Deng, L. (2016). MS MARCO: a human generated machine reading comprehension dataset. In: Workshop on Cognitive Computation: Integrating neural and symbolic approaches 2016 co-located with the 30th Annual Conference on Neural Information Processing Systems, Barcelona, Spain."},{"key":"9411_CR27","unstructured":"Nogueira, R., & Cho, K. (2019). Passage re-ranking with BERT. CoRR, abs\/1901.04085."},{"key":"9411_CR28","unstructured":"Nogueira, R., & Lin, J. (2019). From doc2query to docTTTTTquery. Online preprint.https:\/\/cs.uwaterloo.ca\/~jimmylin\/publications\/Nogueira_Lin_2019_docTTTTTquery-v2.pdf."},{"key":"9411_CR29","unstructured":"Nogueira, R., Yang, W., Cho, K., & Lin, J. (2019). Multi-stage document ranking with BERT. CoRR, abs\/1910.14424."},{"key":"9411_CR30","unstructured":"Nogueira, R., Yang, W., Lin, J., & Cho, K. (2019). Document expansion by query prediction. CoRR, abs\/1904.08375."},{"key":"9411_CR31","doi-asserted-by":"crossref","unstructured":"Pradeep, R., Nogueira, R., & Lin, J. (2021). The Expando-Mono-Duo design pattern for text ranking with pretrained sequence-to-sequence models. CoRR, abs\/2101.05667.","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"9411_CR32","doi-asserted-by":"crossref","unstructured":"Qu, Y., Ding, Y., Liu, J., Liu, K., Ren, R., Zhao, X., Dong, D., Wu, H., & Wang, H. (2020). RocketQA: An optimized training approach to dense passage retrieval for open-domain question answering. CoRR, abs\/2010.08191, 2020.","DOI":"10.18653\/v1\/2021.naacl-main.466"},{"key":"9411_CR33","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2007). Alternatives to bpref. In: 30th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 71\u201478, Amsterdam, Netherlands.","DOI":"10.1145\/1277741.1277756"},{"key":"9411_CR34","doi-asserted-by":"crossref","unstructured":"Sakai, T., & Zeng, Z. (2020). Good evaluation measures based on document preferences. In: 43rd Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 359\u2014368, Xi\u2019an, China.","DOI":"10.1145\/3397271.3401115"},{"key":"9411_CR35","doi-asserted-by":"crossref","unstructured":"Shao, Y., Liu, Y., Zhang, F., Zhang, M., & Ma, S. (2019). On annotation methodologies for image search evaluation. ACM Transactions on Information Systems, 37(3).","DOI":"10.1145\/3309994"},{"key":"9411_CR36","doi-asserted-by":"crossref","unstructured":"Svore, K.\u00a0M., & Burges, C.\u00a0J. C. (2009). A machine learning approach for improved BM25 retrieval. In: 18th ACM International Conference on Information and Knowledge Management, pp. 1811\u20131814, Hong Kong, China.","DOI":"10.1145\/1645953.1646237"},{"key":"9411_CR37","doi-asserted-by":"crossref","unstructured":"Xie, X., Mao, J., Liu, Y., de\u00a0Rijke, M., Chen, H., Zhang, M., & Ma, S. (2020). Preference-based evaluation metrics for web image search. In: 43th Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 2101\u20142104","DOI":"10.1145\/3397271.3401146"},{"key":"9411_CR38","unstructured":"Xiong, L., Xiong, C., Li, Y., Tang, K., Liu, J., Bennett, P., Ahmed, J., & Overwijk, A. (2020). Approximate nearest neighbor negative contrastive learning for dense text retrieval. CoRR, abs\/2007.00808, 2020."},{"key":"9411_CR39","doi-asserted-by":"crossref","unstructured":"Yang, P., Fang, H., & Lin, J. (2017). Anserini: Enabling the use of Lucene for information retrieval research. In: 40th Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 1253\u20131256, Tokyo, Japan.","DOI":"10.1145\/3077136.3080721"},{"issue":"4","key":"9411_CR40","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3239571","volume":"10","author":"P Yang","year":"2018","unstructured":"Yang, P., Fang, H., & Lin, J. (2018). Anserini: Reproducible ranking baselines using Lucene. Journal of Data and Information Quality, 10(4), 1\u201320.","journal-title":"Journal of Data and Information Quality"},{"key":"9411_CR41","doi-asserted-by":"crossref","unstructured":"Yang, Z., Moffat, A., & Turpin, A. (2018). Pairwise crowd judgments: Preference, absolute, and ratio. In: 23rd Australasian Document Computing Symposium, Dunedin, New Zealand.","DOI":"10.1145\/3291992.3291995"},{"key":"9411_CR42","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Craswell, N., Mitra, B., & Campos, D. (2020). On the reliability of test collections for evaluating systems of different types. In: 43th Annual International ACM SIGIR Conference on Research and Development in Infomation Retrieval, pp. 2101\u20142104, Xi\u2019an, China.","DOI":"10.1145\/3397271.3401317"},{"key":"9411_CR43","unstructured":"Zhan, J., Mao, J., Liu, Y., Zhang, M., & Ma, S. (2020). RepBERT: Contextualized text embeddings for first-stage retrieval. CoRR, abs\/2006.15498."},{"key":"9411_CR44","doi-asserted-by":"crossref","unstructured":"Zhan, J., Mao, J., Liu, Y., Guo, J., Zhang, M., & Ma, S. (2022) . Learning discrete representations via constrained clustering for effective and efficient dense retrieval. In: 15th ACM International Web Search and Data Mining Conference, Phoenix, Arizona, February 2022.","DOI":"10.1145\/3488560.3498443"},{"key":"9411_CR45","unstructured":"Zhu, D., & Carterette, B. (2010). An analysis of assessor behavior in crowdsourced preference judgments. In: SIGIR Workshop on Crowdsourcing for Search Evaluation (p. 2010). Geneva: Switzerland, July."}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-022-09411-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-022-09411-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-022-09411-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:14:05Z","timestamp":1704204845000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-022-09411-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,20]]},"references-count":45,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2022,12]]}},"alternative-id":["9411"],"URL":"https:\/\/doi.org\/10.1007\/s10791-022-09411-0","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,7,20]]},"assertion":[{"value":"31 August 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 May 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 July 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}