{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:15:57Z","timestamp":1784178957473,"version":"3.55.0"},"publisher-location":"Cham","reference-count":32,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030997380","type":"print"},{"value":"9783030997397","type":"electronic"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99739-7_38","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T23:02:47Z","timestamp":1649113367000},"page":"305-310","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":31,"title":["Streamlining Evaluation with\u00a0ir-measures"],"prefix":"10.1007","author":[{"given":"Sean","family":"MacAvaney","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Craig","family":"Macdonald","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Iadh","family":"Ounis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"38_CR1","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., Mackenzie, J., Moffat, A.: ERR is not C\/W\/L: exploring the relationship between expected reciprocal rank and other metrics. In: ICTIR (2021)","DOI":"10.1145\/3471158.3472239"},{"key":"38_CR2","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., Thomas, P., Moffat, A.: Cwl_eval: an evaluation tool for information retrieval. In: SIGIR (2019)","DOI":"10.1145\/3331184.3331398"},{"key":"38_CR3","unstructured":"Bajaj, P., et al.: MS MARCO: a human generated machine reading comprehension dataset. In: CoCo@NIPS (2016)"},{"key":"38_CR4","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: SIGIR (2004)","DOI":"10.1145\/1008992.1009000"},{"key":"38_CR5","volume-title":"Retrieval System Evaluation","author":"C Buckley","year":"2005","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval System Evaluation. MIT Press, Cambridge (2005)"},{"key":"38_CR6","doi-asserted-by":"crossref","unstructured":"Chapelle, O., Metlzer, D., Zhang, Y., Grinspan, P.: Expected reciprocal rank for graded relevance. In: CIKM (2009)","DOI":"10.1145\/1645953.1646033"},{"key":"38_CR7","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., et al.: Novelty and diversity in information retrieval evaluation. In: SIGIR (2008)","DOI":"10.1145\/1390334.1390446"},{"key":"38_CR8","doi-asserted-by":"crossref","unstructured":"Clarke, C.L.A., Kolla, M., Vechtomova, O.: An effectiveness measure for ambiguous and underspecified queries. In: ICTIR (2009)","DOI":"10.1007\/978-3-642-04417-5_17"},{"issue":"3","key":"38_CR9","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3451161","volume":"39","author":"CLA Clarke","year":"2021","unstructured":"Clarke, C.L.A., Vtyurina, A., Smucker, M.D.: Assessing top-k preferences. TOIS 39(3), 1\u201321 (2021)","journal-title":"TOIS"},{"key":"38_CR10","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.: Overview of the TREC 2019 deep learning track. In: TREC (2019)","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"38_CR11","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1145\/3190580.3190586","volume":"51","author":"N Fuhr","year":"2018","unstructured":"Fuhr, N.: Some common mistakes in ir evaluation, and how they can be avoided. SIGIR Forum 51, 32\u201341 (2018)","journal-title":"SIGIR Forum"},{"issue":"4","key":"38_CR12","first-page":"439","volume":"28","author":"D Harman","year":"1992","unstructured":"Harman, D.: Evaluation issues in information retrieval. IPM 28(4), 439\u2013440 (1992)","journal-title":"IPM"},{"issue":"4","key":"38_CR13","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of ir techniques. TOIS 20(4), 422\u2013446 (2002)","journal-title":"TOIS"},{"key":"38_CR14","doi-asserted-by":"crossref","unstructured":"Jose, K.M., Nguyen, T., MacAvaney, S., Dalton, J., Yates, A.: Diffir: exploring differences in ranking models\u2019 behavior. In: SIGIR (2021)","DOI":"10.1145\/3404835.3462784"},{"issue":"2\u20133","key":"38_CR15","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1023\/A:1009902609570","volume":"2","author":"P Kantor","year":"2000","unstructured":"Kantor, P., Voorhees, E.: The TREC-5 confusion track. Inf. Retr. 2(2\u20133), 165\u2013176 (2000)","journal-title":"Inf. Retr."},{"key":"38_CR16","doi-asserted-by":"crossref","unstructured":"Lin, J., et al.: Supporting interoperability between open-source search engines with the common index file format. In: SIGIR (2020)","DOI":"10.1145\/3397271.3401404"},{"key":"38_CR17","doi-asserted-by":"crossref","unstructured":"Lucchese, C., Muntean, C.I., Nardini, F.M., Perego, R., Trani, S.: Rankeval: an evaluation and analysis framework for learning-to-rank solutions. In: SIGIR (2017)","DOI":"10.1145\/3077136.3084140"},{"key":"38_CR18","doi-asserted-by":"crossref","unstructured":"MacAvaney, S.: OpenNIR: a complete neural ad-hoc ranking pipeline. In: WSDM (2020)","DOI":"10.1145\/3336191.3371864"},{"key":"38_CR19","doi-asserted-by":"crossref","unstructured":"MacAvaney, S., Yates, A., Feldman, S., Downey, D., Cohan, A., Goharian, N.: Simplified data wrangling with ir_datasets. In: SIGIR (2021)","DOI":"10.1145\/3404835.3463254"},{"key":"38_CR20","doi-asserted-by":"crossref","unstructured":"Macdonald, C., Tonellotto, N.: Declarative experimentation ininformation retrieval using PyTerrier. In: Proceedings of ICTIR 2020 (2020)","DOI":"10.1145\/3409256.3409829"},{"key":"38_CR21","doi-asserted-by":"crossref","unstructured":"Moffat, A., Bailey, P., Scholer, F., Thomas, P.: Inst: an adaptive metric for information retrieval evaluation. In: Australasian Document Computing Symposium (2015)","DOI":"10.1145\/2838931.2838938"},{"issue":"3","key":"38_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3052768","volume":"35","author":"A Moffat","year":"2017","unstructured":"Moffat, A., Bailey, P., Scholer, F., Thomas, P.: Incorporating user expectations and behavior into the measurement of search effectiveness. TOIS 35(3), 1\u201338 (2017)","journal-title":"TOIS"},{"key":"38_CR23","doi-asserted-by":"crossref","unstructured":"Moffat, A., Scholer, F., Thomas, P.: Models and metrics: IR evaluation as a user process. In: Australasian Document Computing Symposium (2012)","DOI":"10.1145\/2407085.2407092"},{"key":"38_CR24","unstructured":"National Institute of Standards and Technology: trec_eval. https:\/\/github.com\/usnistgov\/trec_eval (1993\u20132021)"},{"key":"38_CR25","doi-asserted-by":"crossref","unstructured":"Palotti, J., Scells, H., Zuccon, G.: TrecTools: an open-source python library for information retrieval practitioners involved in TREC-like campaigns. In: SIGIR (2019)","DOI":"10.1145\/3331184.3331399"},{"key":"38_CR26","doi-asserted-by":"crossref","unstructured":"Piwowarski, B.: Experimaestro and datamaestro: experiment and dataset managers (for IR). In: SIGIR (2020)","DOI":"10.1145\/3397271.3401410"},{"key":"38_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3451964.3451976","volume":"54","author":"T Sakai","year":"2020","unstructured":"Sakai, T.: On Fuhr\u2019s guideline for IR evaluation. SIGIR Forum 54, 1\u20138 (2020)","journal-title":"SIGIR Forum"},{"key":"38_CR28","doi-asserted-by":"crossref","unstructured":"Van Gysel, C., de Rijke, M.: Pytrec_eval: an extremely fast python interface to trec_eval. In: SIGIR (2018)","DOI":"10.1145\/3209978.3210065"},{"key":"38_CR29","unstructured":"Van Rijsbergen, C.J.: Information retrieval (1979)"},{"key":"38_CR30","unstructured":"Voorhees, E., et al.: Trec-covid: constructing a pandemic information retrieval test collection. ArXiv abs\/2005.04474 (2020)"},{"key":"38_CR31","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J.A.: Estimating average precision with incomplete and imperfect judgments. In: CIKM (2006)","DOI":"10.1145\/1183614.1183633"},{"key":"38_CR32","doi-asserted-by":"crossref","unstructured":"Zhang, F., Liu, Y., Li, X., Zhang, M., Xu, Y., Ma, S.: Evaluating web search with a bejeweled player model. In: SIGIR (2017)","DOI":"10.1145\/3077136.3080841"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99739-7_38","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,21]],"date-time":"2024-09-21T15:43:58Z","timestamp":1726933438000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99739-7_38"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997380","9783030997397"],"references-count":32,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99739-7_38","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}