{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T18:16:39Z","timestamp":1742926599941,"version":"3.40.3"},"publisher-location":"Cham","reference-count":26,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031282409"},{"type":"electronic","value":"9783031282416"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-28241-6_2","type":"book-chapter","created":{"date-parts":[[2023,3,16]],"date-time":"2023-03-16T01:02:20Z","timestamp":1678928540000},"page":"20-34","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Stat-Weight: Improving the\u00a0Estimator of\u00a0Interleaved Methods Outcomes with\u00a0Statistical Hypothesis Testing"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8781-8619","authenticated-orcid":false,"given":"Alessandro","family":"Benedetti","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1733-1132","authenticated-orcid":false,"given":"Anna","family":"Ruggero","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,3,16]]},"reference":[{"key":"2_CR1","doi-asserted-by":"crossref","unstructured":"Balog, K., Kelly, L., Schuth, A.: Head first: living labs for ad-hoc search evaluation. In: Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management, pp. 1815\u20131818 (2014)","DOI":"10.1145\/2661829.2661962"},{"issue":"1","key":"2_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2094072.2094078","volume":"30","author":"O Chapelle","year":"2012","unstructured":"Chapelle, O., Joachims, T., Radlinski, F., Yue, Y.: Large-scale validation and analysis of interleaved search evaluation. ACM Trans. Inf. Syst. (TOIS) 30(1), 1\u201341 (2012)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"2_CR3","doi-asserted-by":"crossref","unstructured":"Chuklin, A., Serdyukov, P., De Rijke, M.: Click model-based information retrieval metrics. In: Proceedings of the 36th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 493\u2013502 (2013)","DOI":"10.1145\/2484028.2484071"},{"key":"2_CR4","unstructured":"Cleverdon, C.W., Mills, J., Keen, E.M.: Factors determining the performance of indexing systems, (Volume 1: Design), p. 28. College of Aeronautics, Cranfield (1966)"},{"key":"2_CR5","doi-asserted-by":"crossref","unstructured":"Craswell, N., Zoeter, O., Taylor, M., Ramsey, B.: An experimental comparison of click position-bias models. In: Proceedings of the 2008 International Conference on Web Search and Data Mining, pp. 87\u201394 (2008)","DOI":"10.1145\/1341531.1341545"},{"key":"2_CR6","doi-asserted-by":"publisher","unstructured":"Dalgaard, P.: Power and the computation of sample size. In: Introductory Statistics with R, pp. 155\u2013162. Springer, Cham (2008). https:\/\/doi.org\/10.1007\/0-387-22632-X_8","DOI":"10.1007\/0-387-22632-X_8"},{"key":"2_CR7","doi-asserted-by":"crossref","unstructured":"Guo, F., Li, L., Faloutsos, C.: Tailoring click models to user goals. In: Proceedings of the 2009 workshop on Web Search Click Data, pp. 88\u201392 (2009)","DOI":"10.1145\/1507509.1507523"},{"key":"2_CR8","doi-asserted-by":"crossref","unstructured":"Guo, F., Liu, C., Wang, Y.M.: Efficient multiple-click models in web search. In: Proceedings of the Second ACM International Conference on Web Search and Data Mining, pp. 124\u2013131 (2009)","DOI":"10.1145\/1498759.1498818"},{"key":"2_CR9","doi-asserted-by":"crossref","unstructured":"He, J., Zhai, C., Li, X.: Evaluation of methods for relative comparison of retrieval systems based on ClickThroughs. In: Proceedings of the 18th ACM Conference on Information and Knowledge Management, pp. 2029\u20132032 (2009)","DOI":"10.1145\/1645953.1646293"},{"key":"2_CR10","doi-asserted-by":"crossref","unstructured":"Hofmann, K., Behr, F., Radlinski, F.: On caption bias in interleaving experiments. In: Proceedings of the 21st ACM International Conference on Information And Knowledge Management, pp. 115\u2013124 (2012)","DOI":"10.1145\/2396761.2396780"},{"key":"2_CR11","doi-asserted-by":"crossref","unstructured":"Hofmann, K., Whiteson, S., De Rijke, M.: A probabilistic method for inferring preferences from clicks. In: Proceedings of the 20th ACM International Conference on Information and Knowledge Management, pp. 249\u2013258 (2011)","DOI":"10.1145\/2063576.2063618"},{"issue":"4","key":"2_CR12","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. (TOIS) 20(4), 422\u2013446 (2002)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"2_CR13","doi-asserted-by":"crossref","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: IR evaluation methods for retrieving highly relevant documents. In: ACM SIGIR Forum, vol. 51, pp. 243\u2013250. ACM New York, NY, USA (2017)","DOI":"10.1145\/3130348.3130374"},{"key":"2_CR14","doi-asserted-by":"crossref","unstructured":"Joachims, T.: Optimizing search engines using clickthrough data. In: Proceedings of the Eighth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 133\u2013142 (2002)","DOI":"10.1145\/775047.775067"},{"key":"2_CR15","unstructured":"Joachims, T., et al.: Evaluating retrieval performance using clickthrough data (2003)"},{"key":"2_CR16","doi-asserted-by":"crossref","unstructured":"Kharitonov, E., Macdonald, C., Serdyukov, P., Ounis, I.: Using historical click data to increase interleaving sensitivity. In: Proceedings of the 22nd ACM International Conference on Information & Knowledge Management, pp. 679\u2013688 (2013)","DOI":"10.1145\/2505515.2505687"},{"key":"2_CR17","doi-asserted-by":"crossref","unstructured":"Kharitonov, E., Macdonald, C., Serdyukov, P., Ounis, I.: Generalized team draft interleaving. In: Proceedings of the 24th ACM International on Conference on Information and Knowledge Management, pp. 773\u2013782 (2015)","DOI":"10.1145\/2806416.2806477"},{"key":"2_CR18","unstructured":"Myers, J.L., Well, A.D., Lorch, J.: Developing the fundamentals of hypothesis testing using the binomial distribution. Research Design and Statistical Analysis, pp. 65\u201390 (2010)"},{"key":"2_CR19","unstructured":"Queen, J.P., Quinn, G.P., Keough, M.J.: Experimental Design and Data Analysis for Biologists. Cambridge University Press, Cambridge (2002)"},{"key":"2_CR20","doi-asserted-by":"crossref","unstructured":"Radlinski, F., Craswell, N.: Optimized interleaving for online retrieval evaluation. In: Proceedings of the Sixth ACM International Conference on Web Search and Data Mining, pp. 245\u2013254 (2013)","DOI":"10.1145\/2433396.2433429"},{"key":"2_CR21","doi-asserted-by":"crossref","unstructured":"Radlinski, F., Kurup, M., Joachims, T.: How does clickthrough data reflect retrieval quality? In: Proceedings of the 17th ACM Conference on Information and Knowledge Management, pp. 43\u201352 (2008)","DOI":"10.1145\/1458082.1458092"},{"key":"2_CR22","doi-asserted-by":"crossref","unstructured":"Schuth, A., et al.: Probabilistic multileave for online retrieval evaluation. In: Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 955\u2013958 (2015)","DOI":"10.1145\/2766462.2767838"},{"key":"2_CR23","doi-asserted-by":"crossref","unstructured":"Schuth, A., Hofmann, K., Radlinski, F.: Predicting search satisfaction metrics with interleaved comparisons proceedings of the 38th international ACM SIGIR Conference on Research and Development in Information Retrieval, Santiago, Chile, 9\u201313 August 2015, Ricardo. ACM (2015)","DOI":"10.1145\/2766462.2767695"},{"key":"2_CR24","doi-asserted-by":"crossref","unstructured":"Schuth, A., Sietsma, F., Whiteson, S., Lefortier, D., de Rijke, M.: Multileaved comparisons for fast online evaluation. In: Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management, pp. 71\u201380 (2014)","DOI":"10.1145\/2661829.2661952"},{"key":"2_CR25","doi-asserted-by":"crossref","unstructured":"Sirkin, R.M.: Statistics for the Social Sciences. Sage, London (2006)","DOI":"10.4135\/9781412985987"},{"key":"2_CR26","unstructured":"Sproull, N.L.: Handbook of Research Methods: A Guide for Practitioners and Students in the Social Sciences. Scarecrow Press, Metuchen (2002)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-28241-6_2","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,5]],"date-time":"2024-03-05T13:53:12Z","timestamp":1709646792000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-28241-6_2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031282409","9783031282416"],"references-count":26,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-28241-6_2","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"16 March 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Dublin","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Ireland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 April 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"45","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2023.org\/index.html?v=1.0","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"489","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"77","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"83","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}