{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T18:03:43Z","timestamp":1768586623824,"version":"3.49.0"},"reference-count":32,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2016,3,9]],"date-time":"2016-03-09T00:00:00Z","timestamp":1457481600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2016,3,9]],"date-time":"2016-03-09T00:00:00Z","timestamp":1457481600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DE140100275"],"award-info":[{"award-number":["DE140100275"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP140103256"],"award-info":[{"award-number":["DP140103256"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP140101587"],"award-info":[{"award-number":["DP140101587"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2016,8]]},"DOI":"10.1007\/s10791-016-9279-1","type":"journal-article","created":{"date-parts":[[2016,3,9]],"date-time":"2016-03-09T06:49:40Z","timestamp":1457506180000},"page":"351-377","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":28,"title":["Assessing efficiency\u2013effectiveness tradeoffs in multi-stage retrieval systems without using relevance judgments"],"prefix":"10.1007","volume":"19","author":[{"given":"Charles L. A.","family":"Clarke","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1902-9087","authenticated-orcid":false,"given":"J. Shane","family":"Culpepper","sequence":"additional","affiliation":[]},{"given":"Alistair","family":"Moffat","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2016,3,9]]},"reference":[{"key":"9279_CR1","unstructured":"Al-akashi, F.\u00a0H., & Inkpen, D. (2011). Query-structure based web page indexing. In Proceedings of the text retrieval conference."},{"issue":"6","key":"9279_CR2","doi-asserted-by":"publisher","first-page":"747","DOI":"10.1007\/s10791-012-9217-9","volume":"16","author":"N Asadi","year":"2013","unstructured":"Asadi, N., & Lin, J. (2013a). Document vector representations for feature extraction in multi-stage document ranking. Information Retrieval Journal, 16(6), 747\u2013768.","journal-title":"Information Retrieval Journal"},{"key":"9279_CR3","doi-asserted-by":"crossref","unstructured":"Asadi, N., & Lin, J. (2013b). Effectiveness\/efficiency tradeoffs for candidate generation in multi-stage retrieval architectures. In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 997\u20131000).","DOI":"10.1145\/2484028.2484132"},{"key":"9279_CR4","doi-asserted-by":"crossref","unstructured":"Brin, S., & Page, L. (1998). The anatomy of a large-scale hypertextual web search engine. In Proceedings of the international conference on the world wide web (pp. 107\u2013117).","DOI":"10.1016\/S0169-7552(98)00110-X"},{"key":"9279_CR5","doi-asserted-by":"crossref","unstructured":"Broder, A.\u00a0Z., Carmel, D., Herscovici, M., Soffer, A., & Zien, J.\u00a0Y. (2003). Efficient query evaluation using a two-level retrieval process. In: Proceedings of the conference on information and knowledge management (pp. 426\u2013434).","DOI":"10.1145\/956863.956944"},{"key":"9279_CR6","volume-title":"Information retrieval: Implementing and evaluating search engines","author":"S B\u00fcttcher","year":"2010","unstructured":"B\u00fcttcher, S., Clarke, C. L. A., & Cormack, G. V. (2010). Information retrieval: Implementing and evaluating search engines. Cambridge, MA: MIT Press."},{"key":"9279_CR7","doi-asserted-by":"crossref","unstructured":"Chapelle, O., Metzler, D., Zhang, Y., & Grinspan, P. (2009). Expected reciprocal rank for graded relevance. In Proceedigns of the conference on information and knowledge management (pp. 621\u2013630).","DOI":"10.1145\/1645953.1646033"},{"issue":"5","key":"9279_CR8","doi-asserted-by":"publisher","first-page":"441","DOI":"10.1007\/s10791-011-9162-z","volume":"14","author":"GV Cormack","year":"2011","unstructured":"Cormack, G. V., Smucker, M. D., & Clarke, C. L. A. (2011). Efficient and effective spam filtering and re-ranking for large web datasets. Information Retrieval Journal, 14(5), 441\u2013465.","journal-title":"Information Retrieval Journal"},{"issue":"1","key":"9279_CR9","doi-asserted-by":"publisher","first-page":"1:1","DOI":"10.1145\/1877766.1877767","volume":"29","author":"J\u00a0S Culpepper","year":"2010","unstructured":"Culpepper, J. S., & Moffat, A. (2010). Compact set intersection for inverted indexing. ACM Transactions on Information Systems, 29(1), 1:1\u20131:25.","journal-title":"ACM Transactions on Information Systems"},{"key":"9279_CR10","unstructured":"Demaine, E.\u00a0D., L\u00f3pez-Ortiz, A., & Munro, J.\u00a0I. (2000). Adaptive set intersections, unions, and differences. In Proceedings of the ACM-SIAM symposium on discrete algorithms (pp. 743\u2013752)."},{"key":"9279_CR11","doi-asserted-by":"crossref","unstructured":"Demaine, E.\u00a0D., L\u00f3pez-Ortiz, A., & Munro, J.\u00a0I. (2001). Experiments on adaptive set intersections for text retrieval systems. In Proceedings of the workshop on algorithm engineering and experiments (pp. 91\u2013104).","DOI":"10.1007\/3-540-44808-X_7"},{"key":"9279_CR12","unstructured":"Eisayed, T., Asadi, N., Wang, L., Lin, J., & Metzler, D. (2011). UMD and USC\/ISI: TREC 2010 Web Track experiments with Ivory. In Proceedings of the text retrieval conference."},{"issue":"4","key":"9279_CR13","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems, 20(4), 422\u2013446.","journal-title":"ACM Transactions on Information Systems"},{"key":"9279_CR14","doi-asserted-by":"crossref","unstructured":"Kane, A., & Tompa, F. W. (2014). Skewed partial bitvectors for list intersection. In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 263\u2013272).","DOI":"10.1145\/2600428.2609609"},{"issue":"1","key":"9279_CR15","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1002\/spe.2203","volume":"45","author":"D Lemire","year":"2015","unstructured":"Lemire, D., & Boytsov, L. (2015). Decoding billions of integers per second through vectorization. Software Practice and Experience, 45(1), 1\u201329.","journal-title":"Software Practice and Experience"},{"issue":"5","key":"9279_CR16","doi-asserted-by":"publisher","first-page":"584","DOI":"10.1007\/s10791-012-9209-9","volume":"16","author":"C Macdonald","year":"2013","unstructured":"Macdonald, C., Santos, R. L. T., & Ounis, I. (2013a). The whens and hows of learning to rank for web search. Information Retrieval Journal, 16(5), 584\u2013628.","journal-title":"Information Retrieval Journal"},{"issue":"3","key":"9279_CR17","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1145\/2493175.2493176","volume":"31","author":"C Macdonald","year":"2013","unstructured":"Macdonald, C., Santos, R. L. T., Ounis, I., & He, B. (2013b). About learning models with multiple query-dependent features. ACM Transactions on Information Systems, 31(3), 11.","journal-title":"ACM Transactions on Information Systems"},{"issue":"1","key":"9279_CR18","doi-asserted-by":"publisher","first-page":"2.1","DOI":"10.1145\/1416950.1416952","volume":"27","author":"A Moffat","year":"2008","unstructured":"Moffat, A., & Zobel, J. (2008). Rank-biased precision for measurement of retrieval effectiveness. ACM Transactions on Information Systems, 27(1), 2.1\u20132.27.","journal-title":"ACM Transactions on Information Systems"},{"key":"9279_CR19","unstructured":"Page, L., Brin, S., Motwani, R., & Winograd, T. (1999). The PageRank citation ranking: Bringing order to the web. Technical Report 1999\u201366. Stanford InfoLab. http:\/\/ilpubs.stanford.edu:8090\/422\/."},{"key":"9279_CR20","doi-asserted-by":"crossref","unstructured":"Richardson, M., Prakash, A., & Brill, E. (2006). Beyond PageRank: Machine learning for static ranking. In Proceedings of the international conference on the world wide web (pp. 707\u2013715).","DOI":"10.1145\/1135777.1135881"},{"key":"9279_CR21","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2007). Alternatives to bpref. In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 71\u201378).","DOI":"10.1145\/1277741.1277756"},{"issue":"5","key":"9279_CR22","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., & Kando, N. (2008). On information retrieval metrics designed for evaluation with incomplete relevance assessments. Information Retrieval Journal, 11(5), 447\u2013470.","journal-title":"Information Retrieval Journal"},{"key":"9279_CR23","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., & Clarke, C. L. A. (2012). Time-based calibration of effectiveness measures. In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 95\u2013104).","DOI":"10.1145\/2348283.2348300"},{"key":"9279_CR24","doi-asserted-by":"crossref","unstructured":"Strohman, T., Turtle, H., & Croft, W. B. (2005). Optimization strategies for complex queries. In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 219\u2013225).","DOI":"10.1145\/1076034.1076074"},{"issue":"11","key":"9279_CR25","doi-asserted-by":"publisher","first-page":"2865","DOI":"10.1109\/TKDE.2015.2448541","volume":"27","author":"L Tan","year":"2015","unstructured":"Tan, L., & Clarke, C. L. A. (2015). A family of rank similarity measures based on maximized effectiveness difference. IEEE Transactions on Knowledge and Data Engineering, 27(11), 2865\u20132877.","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"key":"9279_CR26","unstructured":"Trotman, A., Jia, X.-F., & Crane, M. (2012). Towards an efficient and effective search engine. In Workshop on open source IR (pp. 40\u201347)."},{"issue":"6","key":"9279_CR27","doi-asserted-by":"publisher","first-page":"831","DOI":"10.1016\/0306-4573(95)00020-H","volume":"31","author":"HR Turtle","year":"1995","unstructured":"Turtle, H. R., & Flood, J. (1995). Query evaluation: Strategies and optimizations. Information Processing & Management, 31(6), 831\u2013850.","journal-title":"Information Processing & Management"},{"key":"9279_CR28","doi-asserted-by":"crossref","unstructured":"Wang, L., Lin, J., & Metzler, D. (2011). A cascade ranking model for efficient ranked retrieval. In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 105\u2013114).","DOI":"10.1145\/2009916.2009934"},{"issue":"4","key":"9279_CR29","doi-asserted-by":"publisher","first-page":"20.1","DOI":"10.1145\/1852102.1852106","volume":"28","author":"W Webber","year":"2010","unstructured":"Webber, W., Moffat, A., & Zobel, J. (2010). A similarity measure for indefinite rankings. ACM Transactions on Information Systems, 28(4), 20.1\u201320.38.","journal-title":"ACM Transactions on Information Systems"},{"key":"9279_CR30","doi-asserted-by":"crossref","unstructured":"Wu, H., & Fang, H. (2014). Document prioritization for scalable query processing. In Proceedings of the conference on information and knowledge management (pp. 1609\u20131618).","DOI":"10.1145\/2661829.2661914"},{"key":"9279_CR31","doi-asserted-by":"crossref","unstructured":"Zobel, J. (1998). How reliable are the results of large-scale information retrieval experiments? In Proceedings of the ACM-SIGIR international conference on research and development in information retrieval (pp. 307\u2013314).","DOI":"10.1145\/290941.291014"},{"issue":"2","key":"9279_CR32","doi-asserted-by":"publisher","first-page":"6.1","DOI":"10.1145\/1132956.1132959","volume":"38","author":"J Zobel","year":"2006","unstructured":"Zobel, J., & Moffat, A. (2006). Inverted files for text search engines. ACM Computing Surveys, 38(2), 6.1\u20136.56.","journal-title":"ACM Computing Surveys"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-016-9279-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-016-9279-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-016-9279-1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-016-9279-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:23:54Z","timestamp":1704205434000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-016-9279-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,3,9]]},"references-count":32,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2016,8]]}},"alternative-id":["9279"],"URL":"https:\/\/doi.org\/10.1007\/s10791-016-9279-1","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,3,9]]},"assertion":[{"value":"28 May 2015","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 February 2016","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 March 2016","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}