{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:05:19Z","timestamp":1755839119102,"version":"3.40.3"},"publisher-location":"Cham","reference-count":28,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030722395"},{"type":"electronic","value":"9783030722401"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-72240-1_3","type":"book-chapter","created":{"date-parts":[[2021,4,1]],"date-time":"2021-04-01T14:49:01Z","timestamp":1617288541000},"page":"33-46","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["System Effect Estimation by Sharding: A Comparison Between ANOVA Approaches to Detect Significant Differences"],"prefix":"10.1007","author":[{"given":"Guglielmo","family":"Faggioli","sequence":"first","affiliation":[]},{"given":"Nicola","family":"Ferro","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,3,30]]},"reference":[{"issue":"1\u20132","key":"3_CR1","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1023\/A:1009984519381","volume":"1","author":"D Banks","year":"1999","unstructured":"Banks, D., Over, P., Zhang, N.F.: Blind men and elephants: six approaches to TREC data. Inf. Retrieval 1(1\u20132), 7\u201334 (1999)","journal-title":"Inf. Retrieval"},{"issue":"1","key":"3_CR2","doi-asserted-by":"crossref","first-page":"289","DOI":"10.1111\/j.2517-6161.1995.tb02031.x","volume":"57","author":"Y Benjamini","year":"1995","unstructured":"Benjamini, Y., Hochberg, Y.: Controlling the false discovery rate: a practical and powerful approach to multiple testing. J. Royal Stat. Soc. 57(1), 289\u2013300 (1995)","journal-title":"J. Royal Stat. Soc."},{"key":"3_CR3","doi-asserted-by":"crossref","unstructured":"Bodoff, D., Li, P.: Test theory for assessing IR test collections. In: Proceedings of SIGIR, pp. 367\u2013374 (2007)","DOI":"10.1145\/1277741.1277805"},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Carterette, B.A.: Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM Trans. Inf. Syst. 30(1), 4:1\u20134:34 (2012)","DOI":"10.1145\/2094072.2094076"},{"key":"3_CR5","doi-asserted-by":"crossref","unstructured":"Clancy, R., Ferro, N., Hauff, C., Sakai, T., Wu, Z.Z.: The SIGIR 2019 open-source IR replicability challenge (OSIRRC 2019). In: Proceedings of SIGIR, pp. 1432\u20131434 (2019)","DOI":"10.1145\/3331184.3331647"},{"key":"3_CR6","doi-asserted-by":"publisher","DOI":"10.1201\/9780429246593","volume-title":"An Introduction to the Bootstrap","author":"B Efron","year":"1994","unstructured":"Efron, B., Tibshirani, R.J.: An Introduction to the Bootstrap. Chapman and Hall\/CRC, Boca Raton (1994)"},{"key":"3_CR7","unstructured":"Ferrari Dacrema, M., Boglio, S., Cremonesi, P., Jannach, D.: A troubling analysis of reproducibility and progress in recommender systems research. User Modeling and User-Adapted Interaction (2019)"},{"key":"3_CR8","doi-asserted-by":"crossref","unstructured":"Ferro, N., Kim, Y., Sanderson, M.: Using collection shards to study retrieval performance effect sizes. ACM Trans. Inf. Syst. 37(3), 30:1\u201330:40 (2019)","DOI":"10.1145\/3310364"},{"key":"3_CR9","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"239","DOI":"10.1007\/978-3-319-98932-7_23","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"N Ferro","year":"2018","unstructured":"Ferro, N., Maistro, M., Sakai, T., Soboroff, I.: Overview of CENTRE@CLEF 2018: a\u00a0first tale in the systematic reproducibility realm. In: Bellot, P., et al. (eds.) CLEF 2018. LNCS, vol. 11018, pp. 239\u2013246. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-98932-7_23"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Ferro, N., Sanderson, M.: Sub-corpora impact on system effectiveness. In: Proceedings of SIGIR, pp. 901\u2013904 (2017)","DOI":"10.1145\/3077136.3080674"},{"key":"3_CR11","doi-asserted-by":"crossref","unstructured":"Ferro, N., Sanderson, M.: Improving the accuracy of system performance estimation by using shards. In: Proceedings of SIGIR, pp. 805\u2013814 (2019)","DOI":"10.1145\/3331184.3338062"},{"issue":"3","key":"3_CR12","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1145\/3190580.3190586","volume":"51","author":"N Fuhr","year":"2017","unstructured":"Fuhr, N.: Some common mistakes in IR evaluation, and how they can be avoided. SIGIR Forum 51(3), 32\u201341 (2017)","journal-title":"SIGIR Forum"},{"key":"3_CR13","doi-asserted-by":"crossref","unstructured":"Harman, D.K.: Overview of the third text REtrieval conference (TREC-3). In: Proceedings of TREC, pp. 1\u201319 (1994)","DOI":"10.6028\/NIST.SP.500-225"},{"key":"3_CR14","doi-asserted-by":"publisher","DOI":"10.1201\/b15074","volume-title":"Multiple Comparisons. Theory and Methods","author":"JC Hsu","year":"1996","unstructured":"Hsu, J.C.: Multiple Comparisons. Theory and Methods. Chapman and Hall\/CRC, Boca Raton (1996)"},{"key":"3_CR15","doi-asserted-by":"crossref","unstructured":"Hull, D.A.: Using statistical testing in the evaluation of retrieval experiments. In: Proceedings of SIGIR, pp. 329\u2013338 (1993)","DOI":"10.1145\/160688.160758"},{"key":"3_CR16","doi-asserted-by":"crossref","unstructured":"Lehmann, E.L., Romano, J.P.: Generalizations of the Familywise Error Rate, pp. 719\u2013735. Boston (2012)","DOI":"10.1007\/978-1-4614-1412-4_57"},{"key":"3_CR17","unstructured":"Marchesin, S., Purpura, A., Silvello, G.: Focal elements of neural information retrieval models. An outlook through a reproducibility study. Inf. Process. Manage. 57, 102\u2013109 (2019)"},{"key":"3_CR18","doi-asserted-by":"crossref","unstructured":"Moffat, A., Scholer, F., Thomas, P.: Models and metrics: IR evaluation as a user process. In: Proceedings of ADCS, pp. 47\u201354 (2012)","DOI":"10.1145\/2407085.2407092"},{"key":"3_CR19","unstructured":"Robertson, S.: On document populations and measures of IR effectiveness. In: Proceedings of the 1st International Conference on the Theory of Information Retrieval (ICTIR 2007), pp. 9\u201322. Foundation for Information Society (2007)"},{"key":"3_CR20","doi-asserted-by":"crossref","unstructured":"Robertson, S.E., Kanoulas, E.: On per-topic variance in IR evaluation. In: Proceedings of SIGIR, pp. 891\u2013900 (2012)","DOI":"10.1145\/2348283.2348402"},{"key":"3_CR21","doi-asserted-by":"crossref","unstructured":"Rutherford, A.: ANOVA and ANCOVA. A GLM Approach, 2nd edn. Wiley, New York (2011)","DOI":"10.1002\/9781118491683"},{"key":"3_CR22","doi-asserted-by":"crossref","unstructured":"Sakai, T.: On Fuhr\u2019s guideline for IR evaluation. SIGIR Forum 54(1), 14:1\u201314:8 (2020)","DOI":"10.1145\/3451964.3451976"},{"issue":"44","key":"3_CR23","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1016\/S0306-4573(97)00027-7","volume":"33","author":"J Savoy","year":"1997","unstructured":"Savoy, J.: Statistical inference in retrieval effectiveness evaluation. Inf. Process. Manage. 33(44), 495\u2013512 (1997)","journal-title":"Inf. Process. Manage."},{"key":"3_CR24","unstructured":"Tague-Sutcliffe, J.M., Blustein, J.: A statistical analysis of the TREC-3 data. In: Proceedings of TREC, pp. 385\u2013398 (1994)"},{"issue":"2","key":"3_CR25","doi-asserted-by":"publisher","first-page":"99","DOI":"10.2307\/3001913","volume":"5","author":"JW Tukey","year":"1949","unstructured":"Tukey, J.W.: Comparing individual means in the analysis of variance. Biometrics 5(2), 99\u2013114 (1949)","journal-title":"Biometrics"},{"key":"3_CR26","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M., Harman, D.K.: Overview of the Eigth Text REtrieval Conference (TREC-8). In: Proceedings of TREC, pp. 1\u201324 (1999)","DOI":"10.6028\/NIST.SP.500-246"},{"key":"3_CR27","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M., Samarov, D., Soboroff, I.: Using replicates in information retrieval evaluation. ACM Trans. Inf. Syst. 36(2), 12:1\u201312:21 (2017)","DOI":"10.1145\/3086701"},{"key":"3_CR28","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-4493-1","volume-title":"Kernel Smoothing","author":"MP Wand","year":"1995","unstructured":"Wand, M.P., Jones, M.C.: Kernel Smoothing. Chapman and Hall\/CRC, Boca Raton (1995)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-72240-1_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,27]],"date-time":"2024-08-27T10:03:03Z","timestamp":1724752983000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-72240-1_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030722395","9783030722401"],"references-count":28,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-72240-1_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"30 March 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 March 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 April 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"43","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.ecir2021.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"436","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"50","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"39","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"11% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}