{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:08:20Z","timestamp":1755839300019,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030997380"},{"type":"electronic","value":"9783030997397"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99739-7_18","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T23:02:47Z","timestamp":1649113367000},"page":"153-162","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Less is Less: When are Snippets Insufficient for Human vs Machine Relevance Estimation?"],"prefix":"10.1007","author":[{"given":"Gabriella","family":"Kazai","sequence":"first","affiliation":[]},{"given":"Bhaskar","family":"Mitra","sequence":"additional","affiliation":[]},{"given":"Anlei","family":"Dong","sequence":"additional","affiliation":[]},{"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[]},{"given":"Linjun","family":"Yang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"18_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"162","DOI":"10.1007\/978-3-540-78646-7_17","volume-title":"Advances in Information Retrieval","author":"M Bendersky","year":"2008","unstructured":"Bendersky, M., Kurland, O.: Utilizing passage-based language models for document retrieval. In: Macdonald, C., Ounis, I., Plachouras, V., Ruthven, I., White, R.W. (eds.) ECIR 2008. LNCS, vol. 4956, pp. 162\u2013174. Springer, Heidelberg (2008). https:\/\/doi.org\/10.1007\/978-3-540-78646-7_17"},{"key":"18_CR2","doi-asserted-by":"publisher","unstructured":"Bolotova, V., Blinov, V., Zheng, Y., Croft, W.B., Scholer, F., Sanderson, M.: Do people and neural nets pay attention to the same words: studying eye-tracking data for non-factoid QA evaluation. In: Proceedings of CIKM, pp. 85\u201394 (2020). https:\/\/doi.org\/10.1145\/3340531.3412043","DOI":"10.1145\/3340531.3412043"},{"key":"18_CR3","doi-asserted-by":"crossref","unstructured":"Clarke, C.L., Agichtein, E., Dumais, S., White, R.W.: The influence of caption features on clickthrough patterns in web search. In: Proceedings of SIGIR, pp. 135\u2013142. ACM (2007)","DOI":"10.1145\/1277741.1277767"},{"key":"18_CR4","doi-asserted-by":"crossref","unstructured":"Cutrell, E., Guan, Z.: What are you looking for? an eye-tracking study of information usage in web search. In: Proceedings of the SIGCHI conference on Human factors in computing systems, pp. 407\u2013416 (2007)","DOI":"10.1145\/1240624.1240690"},{"key":"18_CR5","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"250","DOI":"10.1007\/978-3-642-35341-3_21","volume-title":"Information Retrieval Technology","author":"T Demeester","year":"2012","unstructured":"Demeester, T., Nguyen, D., Trieschnigg, D., Develder, C., Hiemstra, D.: What snippets say about pages in federated web search. In: Hou, Y., Nie, J.-Y., Sun, L., Wang, B., Zhang, P. (eds.) AIRS 2012. LNCS, vol. 7675, pp. 250\u2013261. Springer, Heidelberg (2012). https:\/\/doi.org\/10.1007\/978-3-642-35341-3_21"},{"key":"18_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1007\/978-3-642-36973-5_63","volume-title":"Advances in Information Retrieval","author":"T Demeester","year":"2013","unstructured":"Demeester, T., Nguyen, D., Trieschnigg, D., Develder, C., Hiemstra, D.: Snippet-based relevance predictions for federated web search. In: Serdyukov, P., et al. (eds.) ECIR 2013. LNCS, vol. 7814, pp. 697\u2013700. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-36973-5_63"},{"key":"18_CR7","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171\u20134186, Association for Computational Linguistics, Minneapolis, Minnesota, June 2019. https:\/\/doi.org\/10.18653\/v1\/N19-1423, https:\/\/www.aclweb.org\/anthology\/N19-1423","DOI":"10.18653\/v1\/N19-1423"},{"issue":"4","key":"18_CR8","doi-asserted-by":"publisher","first-page":"259","DOI":"10.1145\/364005.364088","volume":"7","author":"H Edmundson","year":"1964","unstructured":"Edmundson, H.: Problems in automatic abstracting. Commun. ACM 7(4), 259\u2013263 (1964)","journal-title":"Commun. ACM"},{"key":"18_CR9","doi-asserted-by":"crossref","unstructured":"Hofst\u00e4tter, S., Mitra, B., Zamani, H., Craswell, N., Hanbury, A.: Intra-document cascading: learning to select passages for neural document ranking. In: Proceedings of SIGIR, ACM. ACM (2021)","DOI":"10.1145\/3404835.3462889"},{"key":"18_CR10","doi-asserted-by":"crossref","unstructured":"Hofst\u00e4tter, S., Zamani, H., Mitra, B., Craswell, N., Hanbury, A.: Local self-attention over long text for efficient document retrieval. In: Proceedings of SIGIR. ACM (2020)","DOI":"10.1145\/3397271.3401224"},{"key":"18_CR11","unstructured":"Kaisser, M., Hearst, M.A., Lowe, J.B.: Improving search results quality by customizing summary lengths. In: Proceedings of ACL-08: HLT, pp. 701\u2013709 (2008)"},{"key":"18_CR12","doi-asserted-by":"crossref","unstructured":"Lagun, D., Agichtein, E.: Viewser: Enabling large-scale remote user studies of web search examination and interaction. In: Proceedings of SIGIR, pp. 365\u2013374. ACM (2011)","DOI":"10.1145\/2009916.2009967"},{"key":"18_CR13","doi-asserted-by":"crossref","unstructured":"Lagun, D., Agichtein, E.: Re-examining search result snippet examination time for relevance estimation. In: Proceedings of SIGIR, pp. 1141\u20131142. ACM (2012)","DOI":"10.1145\/2348283.2348509"},{"key":"18_CR14","unstructured":"Li, C., Yates, A., MacAvaney, S., He, B., Sun, Y.: Parade: Passage representation aggregation for document reranking. arXiv preprint arXiv:2008.09093 (2020)"},{"key":"18_CR15","doi-asserted-by":"crossref","unstructured":"Lin, J., Nogueira, R., Yates, A.: Pretrained transformers for text ranking: Bert and beyond. arXiv preprint arXiv:2010.06467 (2020)","DOI":"10.2200\/S01123ED1V01Y202108HLT053"},{"key":"18_CR16","doi-asserted-by":"crossref","unstructured":"Lou, Y., Caruana, R., Gehrke, J., Hooker, G.: Accurate intelligible models with pairwise interactions. In: Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 623\u2013631, KDD 2013. ACM (2013). https:\/\/doi.org\/10.1145\/2487575.2487579","DOI":"10.1145\/2487575.2487579"},{"issue":"2","key":"18_CR17","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1147\/rd.22.0159","volume":"2","author":"HP Luhn","year":"1958","unstructured":"Luhn, H.P.: The automatic creation of literature abstracts. IBM J. Res. Dev. 2(2), 159\u2013165 (1958)","journal-title":"IBM J. Res. Dev."},{"key":"18_CR18","doi-asserted-by":"crossref","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Syst. 27(1) (2008). https:\/\/doi.org\/10.1145\/1416950.1416952","DOI":"10.1145\/1416950.1416952"},{"key":"18_CR19","unstructured":"Nogueira, R., Cho, K.: Passage re-ranking with BERT. CoRR abs\/1901.04085 (2019). http:\/\/arxiv.org\/abs\/1901.04085"},{"key":"18_CR20","doi-asserted-by":"crossref","unstructured":"Robertson, S., Zaragoza, H., et al.: The probabilistic relevance framework: Bm25 and beyond. Found. Trends\u00ae Inf. Retrieval 3(4), 333\u2013389 (2009)","DOI":"10.1561\/1500000019"},{"key":"18_CR21","doi-asserted-by":"crossref","unstructured":"Salton, G., Allan, J., Buckley, C.: Approaches to passage retrieval in full text information systems. In: Proceedings of SIGIR. ACM (1993)","DOI":"10.1145\/160688.160693"},{"key":"18_CR22","doi-asserted-by":"crossref","unstructured":"Sanderson, M.: Accurate user directed summarization from existing tools. In: Proceedings of CIKM, pp. 45\u201351. ACM (1998). https:\/\/doi.org\/10.1145\/288627.288640","DOI":"10.1145\/288627.288640"},{"key":"18_CR23","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1007\/978-3-642-23708-9_4","volume-title":"Multilingual and Multimodal Information Access Evaluation","author":"D Savenkov","year":"2011","unstructured":"Savenkov, D., Braslavski, P., Lebedev, M.: Search snippet evaluation at yandex: lessons learned and future directions. In: Forner, P., Gonzalo, J., Kek\u00e4l\u00e4inen, J., Lalmas, M., de Rijke, M. (eds.) CLEF 2011. LNCS, vol. 6941, pp. 14\u201325. Springer, Heidelberg (2011). https:\/\/doi.org\/10.1007\/978-3-642-23708-9_4"},{"key":"18_CR24","doi-asserted-by":"crossref","unstructured":"Tombros, A., Sanderson, M.: Advantages of query biased summaries in information retrieval. In: Proceedings of SIGIR, pp. 2\u201310. ACM (1998)","DOI":"10.1145\/290941.290947"},{"key":"18_CR25","unstructured":"Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, \u0141., Polosukhin, I.: Attention is all you need. In: Proceedings of NeurIPS (2017)"},{"issue":"5","key":"18_CR26","doi-asserted-by":"publisher","first-page":"707","DOI":"10.1016\/S0306-4573(02)00033-X","volume":"39","author":"RW White","year":"2003","unstructured":"White, R.W., Jose, J.M., Ruthven, I.: A task-oriented study on the influencing effects of query-biased summarisation in web searching. Inf. Process. Manag. 39(5), 707\u2013733 (2003)","journal-title":"Inf. Process. Manag."},{"key":"18_CR27","doi-asserted-by":"crossref","unstructured":"Yan, M., et al.: IDST at TREC 2019 deep learning track: Deep cascade ranking with generation-based document expansion and pre-trained language modeling. In: TREC (2019)","DOI":"10.6028\/NIST.SP.1250.deep-IDST"},{"key":"18_CR28","doi-asserted-by":"crossref","unstructured":"Yan, M., et al.: IDST at TREC 2019 deep learning track: Deep cascade ranking with generation-based document expansion and pre-trained language modeling. In: TREC (2020)","DOI":"10.6028\/NIST.SP.1250.deep-IDST"},{"key":"18_CR29","doi-asserted-by":"crossref","unstructured":"Yue, Y., Patel, R., Roehrig, H.: Beyond position bias: examining result attractiveness as a source of presentation bias in clickthrough data. In: Proceedings of the 19th International Conference on World Wide Web, pp. 1011\u20131018. ACM (2010)","DOI":"10.1145\/1772690.1772793"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99739-7_18","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,21]],"date-time":"2024-09-21T15:43:17Z","timestamp":1726933397000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99739-7_18"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997380","9783030997397"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99739-7_18","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}