{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T13:09:42Z","timestamp":1765544982348,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":27,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,2]],"date-time":"2024-08-02T00:00:00Z","timestamp":1722556800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP190101113"],"award-info":[{"award-number":["DP190101113"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,2]]},"DOI":"10.1145\/3664190.3672519","type":"proceedings-article","created":{"date-parts":[[2024,8,5]],"date-time":"2024-08-05T12:39:41Z","timestamp":1722861581000},"page":"35-41","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Query Variability and Experimental Consistency: A Concerning Case Study"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6189-3274","authenticated-orcid":false,"given":"Lida","family":"Rashidi","sequence":"first","affiliation":[{"name":"The University of Melbourne & RMIT Univeristy, Melbourne, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6622-032X","authenticated-orcid":false,"given":"Justin","family":"Zobel","sequence":"additional","affiliation":[{"name":"The University of Melbourne, Melbourne, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6638-0232","authenticated-orcid":false,"given":"Alistair","family":"Moffat","sequence":"additional","affiliation":[{"name":"The University of Melbourne, Melbourne, Australia"}]}],"member":"320","published-online":{"date-parts":[[2024,8,5]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.21236\/ADA460118"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914671"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080839"},{"key":"e_1_3_2_1_4_1","first-page":"339","volume-title":"Proc. SIGIR","author":"Belkin N. J.","year":"1993","unstructured":"N. J. Belkin, C. Cool, W. B. Croft, and J. P. Callan. The effect of multiple query variations on information retrieval system performance. In Proc. SIGIR, pages 339--346, 1993."},{"issue":"4","key":"e_1_3_2_1_5_1","first-page":"41","article-title":"Boosting search performance using query variations","volume":"37","author":"Benham R.","unstructured":"R. Benham, J. Mackenzie, A. Moffat, and J. S. Culpepper. Boosting search performance using query variations. ACM Trans. Inf. Sys., 37(4):41.1--41.25, 2019.","journal-title":"ACM Trans. Inf. Sys."},{"key":"e_1_3_2_1_6_1","first-page":"53","volume-title":"Retrieval system evaluation","author":"Buckley C.","year":"2005","unstructured":"C. Buckley and E. M. Voorhees. Retrieval system evaluation. In E. M. Voorhees and D. K. Harman, editors, TREC: Experiment and Evaluation in Information Retrieval, chapter 3, pages 53--78. MIT Press, 2005."},{"key":"e_1_3_2_1_7_1","volume-title":"Proc. TREC","author":"Buckley C.","year":"1999","unstructured":"C. Buckley and J. Walz. The TREC-8 query track. In Proc. TREC, 1999. NIST Special Publication 500--246."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482438"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835490"},{"key":"e_1_3_2_1_10_1","volume-title":"TREC 2013 web track overview. In Proc. TREC","author":"Collins-Thompson K.","year":"2013","unstructured":"K. Collins-Thompson, P. Bennett, F. Diaz, C. L. A. Clarke, and E. M. Voorhees. TREC 2013 web track overview. In Proc. TREC, 2013. NIST Special Publication 500--302."},{"key":"e_1_3_2_1_11_1","volume-title":"TREC 2014 web track overview. In Proc. TREC, NIST Special Publication 500--308","author":"Collins-Thompson K.","year":"2014","unstructured":"K. Collins-Thompson, C. Macdonald, P. Bennett, F. Diaz, and E. M. Voorhees. TREC 2014 web track overview. In Proc. TREC, NIST Special Publication 500--308, 2014."},{"key":"e_1_3_2_1_12_1","volume-title":"Topic difficulty: Collection and query formulation effects. ACM Trans. Inf. Sys., 40(Article 19):1--36","author":"Culpepper J. S.","year":"2021","unstructured":"J. S. Culpepper, G. Faggioli, N. Ferro, and O. Kurland. Topic difficulty: Collection and query formulation effects. ACM Trans. Inf. Sys., 40(Article 19):1--36, 2021."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076116"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9216-x"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806606"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3052768"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3596511"},{"key":"e_1_3_2_1_20_1","first-page":"109","volume-title":"Proc. TREC","author":"Robertson S. E.","year":"1995","unstructured":"S. E. Robertson, S.Walker, S. Jones, M. Hancock-Beaulieu, and M. Gatford. Okapi at TREC-3. In Proc. TREC, pages 109--126, 1995. NIST Special Publication 500--225."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210078"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3239571"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331253"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130348.3130377"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411998"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983723"}],"event":{"name":"ICTIR '24: The 2024 ACM SIGIR International Conference on the Theory of Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Washington DC USA","acronym":"ICTIR '24"},"container-title":["Proceedings of the 2024 ACM SIGIR International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664190.3672519","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3664190.3672519","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T23:57:58Z","timestamp":1755907078000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664190.3672519"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,2]]},"references-count":27,"alternative-id":["10.1145\/3664190.3672519","10.1145\/3664190"],"URL":"https:\/\/doi.org\/10.1145\/3664190.3672519","relation":{},"subject":[],"published":{"date-parts":[[2024,8,2]]},"assertion":[{"value":"2024-08-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}