{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T20:12:41Z","timestamp":1783541561211,"version":"3.55.0"},"publisher-location":"Cham","reference-count":15,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030454418","type":"print"},{"value":"9783030454425","type":"electronic"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-45442-5_6","type":"book-chapter","created":{"date-parts":[[2020,4,10]],"date-time":"2020-04-10T21:03:47Z","timestamp":1586552627000},"page":"43-49","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":9,"title":["Reproducibility is a Process, Not an Achievement: The Replicability of IR Reproducibility Experiments"],"prefix":"10.1007","author":[{"given":"Jimmy","family":"Lin","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qian","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2020,4,8]]},"reference":[{"key":"6_CR1","doi-asserted-by":"crossref","unstructured":"Boldi, P., Vigna, S.: MG4J at TREC 2006. In: TREC (2006)","DOI":"10.6028\/NIST.SP.500-272.terabyte-umilano.vigna"},{"key":"6_CR2","unstructured":"Cartright, M.A., Huston, S., Field, H.: Galago: a modular distributed processing and retrieval system. In: SIGIR 2012 Workshop on Open Source IR (2012)"},{"key":"6_CR3","doi-asserted-by":"crossref","unstructured":"Clancy, R., Ferro, N., Hauff, C., Lin, J., Sakai, T., Wu, Z.Z.: Overview of the 2019 open-source IR replicability challenge (OSIRRC 2019). In: CEUR Workshop Proceedings, Paris, France, vol. 2409, pp. 1\u20137 (2019)","DOI":"10.1145\/3331184.3331647"},{"key":"6_CR4","doi-asserted-by":"crossref","unstructured":"Clarke, C., Craswell, N., Soboroff, I.: Overview of the TREC 2004 terabyte track. In: TREC (2004)","DOI":"10.6028\/NIST.SP.500-261.terabyte-overview"},{"issue":"12","key":"6_CR5","doi-asserted-by":"publisher","first-page":"1972","DOI":"10.14778\/2824032.2824114","volume":"8","author":"J Dittrich","year":"2015","unstructured":"Dittrich, J., Bender, P.: Janiform intra-document analytics for reproducible research. Proc. VLDB Endow. 8(12), 1972\u20131975 (2015)","journal-title":"Proc. VLDB Endow."},{"issue":"1","key":"6_CR6","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1145\/2964797.2964808","volume":"50","author":"N Ferro","year":"2016","unstructured":"Ferro, N., Fuhr, N., J\u00e4rvelin, K., Kando, N., Lippold, M., Zobel, J.: Increasing reproducibility in IR: findings from the Dagstuhl seminar on \u201creproducibility of data-oriented experiments in e-science\u201d. SIGIR Forum 50(1), 68\u201382 (2016)","journal-title":"SIGIR Forum"},{"key":"6_CR7","doi-asserted-by":"publisher","first-page":"589","DOI":"10.1016\/j.procs.2011.04.062","volume":"4","author":"PV Gorp","year":"2011","unstructured":"Gorp, P.V., Mazanek, S.: SHARE: a web portal for creating and sharing executable research papers. Procedia Comput. Sci. 4, 589\u2013597 (2011)","journal-title":"Procedia Comput. Sci."},{"key":"6_CR8","doi-asserted-by":"crossref","unstructured":"Lin, J., et al.: Toward reproducible baselines: the open-source IR reproducibility challenge. In: ECIR, Padua, Italy, pp. 408\u2013420 (2016)","DOI":"10.1007\/978-3-319-30671-1_30"},{"key":"6_CR9","doi-asserted-by":"crossref","unstructured":"Lin, J., Trotman, A.: Anytime ranking for impact-ordered indexes. In: ICTIR, pp. 301\u2013304 (2015)","DOI":"10.1145\/2808194.2809477"},{"key":"6_CR10","doi-asserted-by":"crossref","unstructured":"Lin, J., Yang, P.: The impact of score ties on repeatability in document ranking. In: SIGIR, Paris, France, pp. 1125\u20131128 (2019)","DOI":"10.1145\/3331184.3331339"},{"issue":"1","key":"6_CR11","doi-asserted-by":"publisher","first-page":"91","DOI":"10.2218\/ijdc.v5i1.145","volume":"5","author":"B Matthews","year":"2010","unstructured":"Matthews, B., Shaon, A., Bicarregui, J., Jones, C.: A framework for software preservation. Int. J. Digit. Curation 5(1), 91\u2013105 (2010)","journal-title":"Int. J. Digit. Curation"},{"issue":"5","key":"6_CR12","doi-asserted-by":"publisher","first-page":"735","DOI":"10.1016\/j.ipm.2004.05.001","volume":"40","author":"D Metzler","year":"2004","unstructured":"Metzler, D., Croft, W.B.: Combining the language model and inference network approaches to retrieval. Inf. Process. Manag. 40(5), 735\u2013750 (2004)","journal-title":"Inf. Process. Manag."},{"key":"6_CR13","doi-asserted-by":"crossref","unstructured":"Ounis, I., Amati, G., Plachouras, V., He, B., Macdonald, C., Lioma, C.: Terrier: a high performance and scalable information retrieval platform. In: SIGIR 2006 Workshop on Open Source IR (2006)","DOI":"10.1007\/978-3-540-31865-1_37"},{"key":"6_CR14","volume-title":"The SMART Retrieval System-Experiments in Automatic Document Processing","author":"G Salton","year":"1971","unstructured":"Salton, G.: The SMART Retrieval System-Experiments in Automatic Document Processing. Prentice-Hall, Englewood Cliffs (1971)"},{"key":"6_CR15","unstructured":"Trotman, A., Jia, X.F., Crane, M.: Towards an efficient and effective search engine. In: SIGIR 2012 Workshop on Open Source IR (2012)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-45442-5_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,3]],"date-time":"2024-08-03T20:34:31Z","timestamp":1722717271000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-45442-5_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030454418","9783030454425"],"references-count":15,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-45442-5_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"8 April 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lisbon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17 April 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"42","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2020.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"457","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"55","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"46","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"12% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Also included: 8 reproducibility papers, 10 demonstration papers, 12 CLEF organizers lab track papers, 7 doctoral consortium papers, 4 workshops, 3 tutorials. Due to the COVID-19 pandemic, this conference was held virtually.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}