{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T09:13:31Z","timestamp":1766049211975,"version":"3.37.3"},"reference-count":60,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T00:00:00Z","timestamp":1565654400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T00:00:00Z","timestamp":1565654400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","award":["CRDPJ 468812-14","RGPIN-2017-04239","RGPIN-2014-03642"],"award-info":[{"award-number":["CRDPJ 468812-14","RGPIN-2017-04239","RGPIN-2014-03642"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2020,2]]},"DOI":"10.1007\/s10791-019-09361-0","type":"journal-article","created":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T19:02:18Z","timestamp":1565722938000},"page":"1-26","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":14,"title":["Evaluating sentence-level relevance feedback for high-recall information retrieval"],"prefix":"10.1007","volume":"23","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7786-389X","authenticated-orcid":false,"given":"Haotian","family":"Zhang","sequence":"first","affiliation":[]},{"given":"Gordon V.","family":"Cormack","sequence":"additional","affiliation":[]},{"given":"Maura R.","family":"Grossman","sequence":"additional","affiliation":[]},{"given":"Mark D.","family":"Smucker","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,8,13]]},"reference":[{"key":"9361_CR1","doi-asserted-by":"crossref","unstructured":"Aalbersberg, I. J. (1992). Incremental relevance feedback. In Proceedings of the 15th annual international ACM SIGIR conference on research and development in information retrieval (pp. 11\u201322). ACM.","DOI":"10.1145\/133160.133169"},{"key":"9361_CR2","doi-asserted-by":"crossref","unstructured":"Abualsaud, M., Cormack, G. V., Ghelani, N., Ghenai, A., Grossman, M. R., Rahbariasl, S., Smucker, M. D., & Zhang, H. (2018a). UWaterlooMDS at the TREC 2018 common core track. TREC.","DOI":"10.6028\/NIST.SP.500-331.core-UWaterlooMDS"},{"key":"9361_CR3","doi-asserted-by":"publisher","unstructured":"Abualsaud, M., Ghelani, N., Zhang, H., Smucker, M.D., Cormack, G.V., & Grossman, M.R. (2018b). A system for efficient high-recall retrieval. In The 41st international ACM SIGIR conference on research & development in information retrieval, SIGIR 2018, Ann Arbor, MI, USA, July 08\u201312, 2018 (pp. 1317\u20131320). https:\/\/doi.org\/10.1145\/3209978.3210176.","DOI":"10.1145\/3209978.3210176"},{"key":"9361_CR4","doi-asserted-by":"crossref","unstructured":"Allan, J. (2005). Hard track overview in TREC 2003 high accuracy retrieval from documents. Tech. rep., DTIC Document.","DOI":"10.21236\/ADA439446"},{"key":"9361_CR5","unstructured":"Baron, J. R., Lewis, D. D., & Oard, D. W. (2006). TREC 2006 legal track overview. In TREC."},{"key":"9361_CR6","doi-asserted-by":"crossref","unstructured":"Baruah, G., Zhang, H., Guttikonda, R., Lin, J., Smucker, M. D., & Vechtomova, O. (2016). Optimizing nugget annotations with active learning. In Proceedings of the 25th ACM international on conference on information and knowledge management (pp 2359\u20132364). ACM.","DOI":"10.1145\/2983323.2983694"},{"issue":"3","key":"9361_CR7","doi-asserted-by":"publisher","first-page":"289","DOI":"10.1145\/3166.3197","volume":"28","author":"DC Blair","year":"1985","unstructured":"Blair, D. C., & Maron, M. E. (1985). An evaluation of retrieval effectiveness for a full-text document-retrieval system. Communications of the ACM, 28(3), 289\u2013299.","journal-title":"Communications of the ACM"},{"key":"9361_CR8","unstructured":"B\u00fcttcher, S., Clarke, C. L., & Soboroff, I. (2006). The TREC 2006 terabyte track. In TREC (Vol. 6, p. 39)."},{"key":"9361_CR9","doi-asserted-by":"publisher","unstructured":"B\u00fcttcher, S., Clarke, C. L. A., & Yeung, P. C. K., Soboroff, I. (2007). Reliable information retrieval evaluation with incomplete and biased judgements. In Proceedings of the 30th annual international ACM SIGIR conference on research and development in information retrieval, ACM, New York, SIGIR \u201907 (pp. 63\u201370). https:\/\/doi.org\/10.1145\/1277741.1277755.","DOI":"10.1145\/1277741.1277755"},{"key":"9361_CR10","unstructured":"Clarke, C. L., Scholer, F., & Soboroff, I. (2005). The TREC 2005 terabyte track. In TREC."},{"key":"9361_CR11","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., & Grossman, M. R. (2014). Evaluation of machine-learning protocols for technology-assisted review in electronic discovery. In Proceedings of the 37th international ACM SIGIR conference on research and development in information retrieval, (pp. 153\u2013162). ACM.","DOI":"10.1145\/2600428.2609601"},{"key":"9361_CR12","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., & Grossman, M. R. (2015). Autonomy and reliability of continuous active learning for technology-assisted review. InCoRR. arxiv:abs\/1504.06868.","DOI":"10.1145\/2766462.2767771"},{"key":"9361_CR13","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., & Grossman, M. R. (2016). Scalability of continuous active learning for reliable high-recall text classification. In Proceedings of the 25th ACM international on conference on information and knowledge management (pp 1039\u20131048). ACM.","DOI":"10.1145\/2983323.2983776"},{"key":"9361_CR14","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., & Grossman, M. R. (2017a). Navigating imprecision in relevance assessments on the road to total recall: Roger and me. In Proceedings of the 40th international ACM SIGIR conference on research and development in information retrieval (pp. 5\u201314). ACM.","DOI":"10.1145\/3077136.3080812"},{"key":"9361_CR15","unstructured":"Cormack, G. V., & Grossman, M. R. (2017b). Technology-assisted review in empirical medicine: Waterloo participation in CLEF ehealth 2017. In Working Notes of CLEF 2017\u2014Conference and Labs of the Evaluation Forum, Dublin, Ireland, September 11\u201314, 2017."},{"key":"9361_CR16","unstructured":"Cormack, G. V., & Lynam, T. R. (2005a). Spam corpus creation for TREC. In CEAS."},{"key":"9361_CR17","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., & Lynam, T. R. (2005b). TREC 2005 spam track overview. In TREC (pp. 500\u2013274).","DOI":"10.6028\/NIST.SP.500-266.spam-overview"},{"key":"9361_CR18","unstructured":"Cormack, G. V., & Mojdeh, M. (2009). Machine learning for information retrieval: TREC 2009 web, relevance feedback and legal tracks. In TREC."},{"key":"9361_CR19","unstructured":"Cormack, G. V., Grossman, M. R., Hedin, B., & Oard, D. W. (2010). Overview of the TREC 2010 legal track. In Proceedings of 19th text REtrieval conference (p. 1)."},{"key":"9361_CR20","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., Palmer, C. R., & Clarke, C. L. (1998). Efficient construction of large test collections. In Proceedings of the 21st annual international ACM SIGIR conference on research and development in information retrieval (pp. 282\u2013289). ACM.","DOI":"10.1145\/290941.291009"},{"key":"9361_CR21","unstructured":"Drucker, H., Shahrary, B., & Gibbon, D. C. (2001). Relevance feedback using support vector machines. In ICML (pp. 122\u2013129)."},{"key":"9361_CR22","unstructured":"Dumais, S. (2005). The interactive TREC Track: Putting the user into search. MIT Press. https:\/\/www.microsoft.com\/en-us\/research\/publication\/interactive-trec-track-putting-user-search\/"},{"key":"9361_CR23","unstructured":"Grossman, M., Cormack, G., & Roegiest, A. (2016). TREC 2016 total recall track overview. In Proceedings of TREC-2016."},{"key":"9361_CR24","unstructured":"Grossman, M. R., Cormack, G. V., Hedin, B., & Oard, D. W. (2011). Overview of the TREC 2011 legal track. In TREC (Vol. 11)."},{"key":"9361_CR25","first-page":"1","volume":"17","author":"MR Grossman","year":"2011","unstructured":"Grossman, M. R., & Cormack, G. V. (2011). Technology-assisted review in e-discovery can be more effective and more efficient than exhaustive manual review. Richmond Journal of Law and Technology, 17, 1.","journal-title":"Richmond Journal of Law and Technology"},{"key":"9361_CR26","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781139644082","volume-title":"Search user interfaces","author":"M Hearst","year":"2009","unstructured":"Hearst, M. (2009). Search user interfaces. Cambridge: Cambridge University Press."},{"key":"9361_CR27","unstructured":"Hedin, B., Tomlinson, S., Baron, J. R., & Oard, D. W. (2009). Overview of the TREC 2009 legal track. Tech. rep., NATIONAL ARCHIVES AND RECORDS ADMINISTRATION COLLEGE PARK MD."},{"key":"9361_CR28","unstructured":"Hersh, W. R., & Bhupatiraju, R. T. (2003). TREC genomics track overview. In TREC (Vol. 2003, pp. 14\u201323)."},{"key":"9361_CR29","unstructured":"Hogan, C., Reinhart, J., Brassil, D., Gerber, M., Rugani, S. M., & Jade, T. (2008). H5 at TREC 2008 legal interactive: User modeling, assessment & measurement. Tech. rep., H5 SAN FRANCISCO CA."},{"key":"9361_CR30","unstructured":"Kanoulas, E., Li, D., Azzopardi, L., & Spijker, R. (2017). CLEF 2017 technologically assisted reviews in empirical medicine overview. Working Notes of CLEF (pp. 11\u201314)."},{"key":"9361_CR31","doi-asserted-by":"crossref","unstructured":"Kolcz, A., & Cormack, G. V. (2009). Genre-based decomposition of email class noise. In Proceedings of the 15th ACM SIGKDD international conference on Knowledge discovery and data mining (pp. 427\u2013436). ACM.","DOI":"10.1145\/1557019.1557070"},{"key":"9361_CR32","doi-asserted-by":"crossref","unstructured":"Liu, X., & Croft, W. B. (2002). Passage retrieval based on language models. In Proceedings of the eleventh international conference on Information and knowledge management (pp. 375\u2013382). ACM.","DOI":"10.1145\/584792.584854"},{"key":"9361_CR33","doi-asserted-by":"crossref","unstructured":"Maddalena, E., Basaldella, M., De\u00a0Nart, D., Degl\u2019Innocenti, D., Mizzaro, S., & Demartini, G. (2016). Crowdsourcing relevance assessments: The unexpected benefits of limiting the time to judge. In Fourth AAAI conference on human computation and crowdsourcing.","DOI":"10.1609\/hcomp.v4i1.13284"},{"key":"9361_CR34","unstructured":"Oard, D. W., Hedin, B., Tomlinson, S., & Baron, J. R. (2008). Overview of the TREC 2008 legal track. Tech. rep.: MARYLAND UNIV COLLEGE PARK COLL OF INFORMATION STUDIES."},{"issue":"3","key":"9361_CR35","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1016\/S0306-4573(00)00053-4","volume":"37","author":"P Over","year":"2001","unstructured":"Over, P. (2001). The trec interactive track: An annotated bibliography. Information Processing & Management, 37(3), 369\u2013381.","journal-title":"Information Processing & Management"},{"key":"9361_CR36","unstructured":"Pickens, J., Gricks, T., Hardi, B., & Noel, M. (2015). A constrained approach to manual total recall. In Proceedings of The twenty-fourth text REtrieval conference, TREC 2015, Gaithersburg, Maryland, November 17\u201320, 2015. http:\/\/trec.nist.gov\/pubs\/trec24\/papers\/catres-TR.pdf."},{"key":"9361_CR37","unstructured":"Rahbariasl, S. (2018). The effects of time constraints and document excerpts on relevance assessing behavior. Master\u2019s thesis. http:\/\/hdl.handle.net\/10012\/13678."},{"key":"9361_CR38","unstructured":"Robertson, S. E., & Soboroff, I. (2002). The TREC 2002 filtering track report. In TREC (Vol. 2002, p. 5)."},{"key":"9361_CR39","unstructured":"Roegiest, A., Cormack, G., Grossman, M., & Clarke, C. (2015). TREC 2015 total recall track overview. In Proceedings of TREC-2015."},{"issue":"2","key":"9361_CR40","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1017\/S0269888903000638","volume":"18","author":"I Ruthven","year":"2003","unstructured":"Ruthven, I., & Lalmas, M. (2003). A survey on the use of relevance feedback for information access systems. The Knowledge Engineering Review, 18(2), 95\u2013145.","journal-title":"The Knowledge Engineering Review"},{"key":"9361_CR41","doi-asserted-by":"crossref","unstructured":"Salton, G., Allan, J., & Buckley, C. (1993). Approaches to passage retrieval in full text information systems. In Proceedings of the 16th annual international ACM SIGIR conference on research and development in information retrieval (pp. 49\u201358). ACM.","DOI":"10.1145\/160688.160693"},{"key":"9361_CR42","doi-asserted-by":"crossref","unstructured":"Sanderson, M. (1998). Accurate user directed summarization from existing tools. In Proceedings of the seventh international conference on Information and knowledge management (pp. 45\u201351). ACM.","DOI":"10.1145\/288627.288640"},{"key":"9361_CR43","doi-asserted-by":"crossref","unstructured":"Sanderson, M., & Joho, H. (2004). Forming test collections with no system pooling. In Proceedings of the 27th annual international ACM SIGIR conference on research and development in information retrieval (pp. 33\u201340). ACM.","DOI":"10.1145\/1008992.1009001"},{"key":"9361_CR44","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., & Jethani, C. P. (2010). Human performance and retrieval precision revisited. In Proceedings of the 33rd international ACM SIGIR conference on research and development in information retrieval (pp. 595\u2013602). ACM.","DOI":"10.1145\/1835449.1835549"},{"key":"9361_CR45","doi-asserted-by":"crossref","unstructured":"Soboroff, I., & Robertson, S. (2003). Building a filtering test collection for TREC 2002. In Proceedings of the 26th annual international ACM SIGIR conference on research and development in information retrieval (pp. 243\u2013250). ACM.","DOI":"10.1145\/860435.860481"},{"key":"9361_CR46","unstructured":"Sparck Jones, K., & Van Rijsbergen, C. (1975). Report on the need for and provision of an \u2018ideal\u2019 information retrieval test collection. Computer Laboratory."},{"key":"9361_CR47","doi-asserted-by":"crossref","unstructured":"Tombros, A., & Sanderson, M. (1998). Advantages of query biased summaries in information retrieval. In Proceedings of the 21st annual international ACM SIGIR conference on research and development in information retrieval (pp. 2\u201310). ACM.","DOI":"10.1145\/290941.290947"},{"key":"9361_CR48","unstructured":"Tomlinson, S., Oard, D. W., Baron, J, R., & Thompson, P. (2007). Overview of the TREC 2007 legal track. In TREC."},{"key":"9361_CR49","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M., & Harman, D. (2000). Overview of the eighth text retrieval conference (TREC-8) (pp. 1\u201324).","DOI":"10.6028\/NIST.SP.500-246"},{"issue":"5","key":"9361_CR50","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"EM Voorhees","year":"2000","unstructured":"Voorhees, E. M. (2000). Variations in relevance judgments and the measurement of retrieval effectiveness. Information Processing & Management, 36(5), 697\u2013716.","journal-title":"Information Processing & Management"},{"key":"9361_CR51","volume-title":"TREC: Experiment and evaluation in information retrieval","author":"EM Voorhees","year":"2005","unstructured":"Voorhees, E. M., Harman, D. K., et al. (2005). TREC: Experiment and evaluation in information retrieval (Vol. 1). Cambridge: MIT Press."},{"key":"9361_CR52","doi-asserted-by":"crossref","unstructured":"Wallace, B. C., Small, K., Brodley, C. E., & Trikalinos, T. A. (2010). Active learning for biomedical citation screening. In Proceedings of the 16th ACM SIGKDD international conference on Knowledge discovery and data mining (pp. 173\u2013182). ACM.","DOI":"10.1145\/1835804.1835829"},{"issue":"3","key":"9361_CR53","doi-asserted-by":"publisher","first-page":"273","DOI":"10.2217\/cer.13.17","volume":"2","author":"BC Wallace","year":"2013","unstructured":"Wallace, B. C., Dahabreh, I. J., Schmid, C. H., Lau, J., & Trikalinos, T. A. (2013). Modernizing the systematic review process to inform comparative effectiveness: Tools and methods. Journal of Comparative Effectiveness Research, 2(3), 273\u2013282.","journal-title":"Journal of Comparative Effectiveness Research"},{"key":"9361_CR54","unstructured":"Wang, J. (2011). Accuracy, agreement, speed, and perceived difficulty of users\u2019 relevance judgments for e-discovery. In Proceedings of SIGIR information retrieval for e-discovery workshop (Vol. 1)."},{"key":"9361_CR55","doi-asserted-by":"crossref","unstructured":"Wang, J., & Soergel, D. (2010). A user study of relevance judgments for e-discovery. In Proceedings of the 73rd ASIS&T annual meeting on navigating streams in an information ecosystem (Vol. 47, p. 74). American Society for Information Science.","DOI":"10.1002\/meet.14504701157"},{"key":"9361_CR56","unstructured":"Yu, Z., Kraft, N. A., & Menzies, T. (2016). How to read less: Better machine assisted reading methods for systematic literature reviews. arXiv preprint arXiv:161203224."},{"key":"9361_CR57","unstructured":"Zhang, H., Abualsaud, M., Ghelani, N., Ghosh, A., Smucker, M. D., Cormack, G. V., & Grossman, M. R. (2017). UwaterlooMDS at the TREC 2017 common core track. In TREC."},{"key":"9361_CR58","doi-asserted-by":"publisher","unstructured":"Zhang, H., Abualsaud, M., Ghelani, N., Smucker, M. D., Cormack, G. V., & Grossman, M. R. (2018). Effective user interaction for high-recall retrieval: Less is more. In Proceedings of the 27th ACM international conference on information and knowledge management, CIKM \u201918 (pp. 187\u2013196). https:\/\/doi.org\/10.1145\/3269206.3271796.","DOI":"10.1145\/3269206.3271796"},{"key":"9361_CR59","doi-asserted-by":"crossref","unstructured":"Zhang, H., Lin, J., Cormack, G. V., & Smucker, M. D. (2016). Sampling strategies and active learning for volume estimation. In Proceedings of the 39th International ACM SIGIR conference on research and development in information retrieval (pp. 981\u2013984). ACM.","DOI":"10.1145\/2911451.2914685"},{"key":"9361_CR60","unstructured":"Zhang, H., Lin, W., Wang, Y., Clarke, C. L. A., & Smucker, M. D. (2015). Waterlooclarke: TREC 2015 total recall track. In Proceedings of The twenty-fourth text REtrieval conference, TREC 2015, Gaithersburg, Maryland, USA, November 17\u201320, 2015."}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-019-09361-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-019-09361-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-019-09361-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,21]],"date-time":"2024-07-21T23:55:23Z","timestamp":1721606123000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-019-09361-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,8,13]]},"references-count":60,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2020,2]]}},"alternative-id":["9361"],"URL":"https:\/\/doi.org\/10.1007\/s10791-019-09361-0","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"type":"print","value":"1386-4564"},{"type":"electronic","value":"1573-7659"}],"subject":[],"published":{"date-parts":[[2019,8,13]]},"assertion":[{"value":"9 May 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 August 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 August 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}