{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,25]],"date-time":"2026-02-25T11:44:50Z","timestamp":1772019890395,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":34,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,10,24]],"date-time":"2016-10-24T00:00:00Z","timestamp":1477267200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,10,24]]},"DOI":"10.1145\/2983323.2983723","type":"proceedings-article","created":{"date-parts":[[2016,10,26]],"date-time":"2016-10-26T13:20:08Z","timestamp":1477488008000},"page":"691-700","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["Query Variations and their Effect on Comparing Information Retrieval Systems"],"prefix":"10.1145","author":[{"given":"Guido","family":"Zuccon","sequence":"first","affiliation":[{"name":"Queensland University of Technology, Brisbane, Australia"}]},{"given":"Joao","family":"Palotti","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Vienna, Austria"}]},{"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Vienna, Austria"}]}],"member":"320","published-online":{"date-parts":[[2016,10,24]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/956863.956953"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767728"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914671"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1507509.1507510"},{"key":"e_1_3_2_1_5_1","volume-title":"TREC","author":"Buckley C.","year":"1999","unstructured":"C. Buckley and J. A. Walz . The TREC-8 Query Track . In TREC , 1999 . C. Buckley and J. A. Walz. The TREC-8 Query Track. In TREC, 1999."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2396782"},{"key":"e_1_3_2_1_7_1","volume-title":"TREC 2013 Web Track Overview. In TREC","author":"Collins-Thompson K.","year":"2013","unstructured":"K. Collins-Thompson , P. Bennett , F. Diaz , C. L. Clarke , and E. M. Voorhees . TREC 2013 Web Track Overview. In TREC , 2013 . K. Collins-Thompson, P. Bennett, F. Diaz, C. L. Clarke, and E. M. Voorhees. TREC 2013 Web Track Overview. In TREC, 2013."},{"key":"e_1_3_2_1_8_1","volume-title":"TREC 2014 Web Track Overview. In TREC","author":"Collins-Thompson K.","year":"2014","unstructured":"K. Collins-Thompson , C. Macdonald , P. Bennett , F. Diaz , and E. M. Voorhees . TREC 2014 Web Track Overview. In TREC , 2014 . K. Collins-Thompson, C. Macdonald, P. Bennett, F. Diaz, and E. M. Voorhees. TREC 2014 Web Track Overview. In TREC, 2014."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.5555\/1229015.1229025"},{"key":"e_1_3_2_1_10_1","volume-title":"SIGIR","author":"Din\u00e7er B. T.","year":"2014","unstructured":"B. T. Din\u00e7er , C. Macdonald , and I. Ounis . Hypothesis Testing for the Risk-sensitive Evaluation of Retrieval Systems . In SIGIR , 2014 . B. T. Din\u00e7er, C. Macdonald, and I. Ounis. Hypothesis Testing for the Risk-sensitive Evaluation of Retrieval Systems. In SIGIR, 2014."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914672"},{"key":"e_1_3_2_1_12_1","volume-title":"The journal of finance","author":"Markowitz H.","year":"1952","unstructured":"H. Markowitz . Portfolio Selection . The journal of finance , 1952 . H. Markowitz. Portfolio Selection. The journal of finance, 1952."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806606"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-44564-9_4"},{"key":"e_1_3_2_1_16_1","volume-title":"CLEF","author":"Palotti J.","year":"2015","unstructured":"J. Palotti , G. Zuccon , L. Goeuriot , L. Kelly , CLEF eHealth Evaluation Lab 2015, Task 2: Retrieving information about medical symptoms . In CLEF , 2015 . J. Palotti, G. Zuccon, L. Goeuriot, L. Kelly, et al. CLEF eHealth Evaluation Lab 2015, Task 2: Retrieving information about medical symptoms. In CLEF, 2015."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772859"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183630"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9208-x"},{"key":"e_1_3_2_1_21_1","volume-title":"ADCS","author":"Sanderson M.","year":"2010","unstructured":"M. Sanderson , F. Scholer , and A. Turpin . Relatively relevant: Assessor shift in document judgements . In ADCS , 2010 . M. Sanderson, F. Scholer, and A. Turpin. Relatively relevant: Assessor shift in document judgements. In ADCS, 2010."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383961"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183632"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383963"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-00958-7_4"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1571963"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390435"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661950"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661934"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484127"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1_21"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.5555\/1996889.1996986"},{"key":"e_1_3_2_1_34_1","volume-title":"MedIR","author":"Zuccon G.","year":"2014","unstructured":"G. Zuccon and B. Koopman . Integrating understandability in the evaluation of consumer health search engines . In MedIR , 2014 . G. Zuccon and B. Koopman. Integrating understandability in the evaluation of consumer health search engines. In MedIR, 2014."}],"event":{"name":"CIKM'16: ACM Conference on Information and Knowledge Management","location":"Indianapolis Indiana USA","acronym":"CIKM'16","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 25th ACM International on Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2983323.2983723","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2983323.2983723","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T03:39:54Z","timestamp":1750217994000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2983323.2983723"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,10,24]]},"references-count":34,"alternative-id":["10.1145\/2983323.2983723","10.1145\/2983323"],"URL":"https:\/\/doi.org\/10.1145\/2983323.2983723","relation":{},"subject":[],"published":{"date-parts":[[2016,10,24]]},"assertion":[{"value":"2016-10-24","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}