{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:10:40Z","timestamp":1784178640402,"version":"3.55.0"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030997359","type":"print"},{"value":"9783030997366","type":"electronic"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99736-6_6","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T23:02:47Z","timestamp":1649113367000},"page":"80-94","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["Validating Simulations of User Query Variants"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1765-2449","authenticated-orcid":false,"given":"Timo","family":"Breuer","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0441-6949","authenticated-orcid":false,"given":"Norbert","family":"Fuhr","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8817-4632","authenticated-orcid":false,"given":"Philipp","family":"Schaer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"6_CR1","doi-asserted-by":"crossref","unstructured":"Allan, J., Harman, D., Kanoulas, E., Li, D., Gysel, C.V., Voorhees, E.M.: TREC 2017 common core track overview. In: Proceedings of the TREC (2017)","DOI":"10.6028\/NIST.SP.500-324.core-overview"},{"key":"6_CR2","doi-asserted-by":"crossref","unstructured":"Azzopardi, L.: The economics in interactive information retrieval. In: Proceedings of the SIGIR, pp. 15\u201324 (2011)","DOI":"10.1145\/2009916.2009923"},{"key":"6_CR3","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., de Rijke, M.: Automatic construction of known-item finding test beds. In: Efthimiadis, E.N., Dumais, S.T., Hawking, D., J\u00e4rvelin, K. (eds.) Proceedings of the SIGIR, pp. 603\u2013604 (2006)","DOI":"10.1145\/1148170.1148276"},{"key":"6_CR4","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., de Rijke, M., Balog, K.: Building simulated queries for known-item topics: an analysis using six European languages. In: Proceedings of the SIGIR, pp. 455\u2013462 (2007)","DOI":"10.1145\/1277741.1277820"},{"key":"6_CR5","doi-asserted-by":"crossref","unstructured":"Baskaya, F., Keskustalo, H., J\u00e4rvelin, K.: Time drives interaction: simulating sessions in diverse searching environments. In: Proceedings of the SIGIR, pp. 105\u2013114 (2012)","DOI":"10.1145\/2348283.2348301"},{"key":"6_CR6","doi-asserted-by":"crossref","unstructured":"Benham, R., Culpepper, J.S.: Risk-reward trade-offs in rank fusion. In: Proceedings of the ADCS, pp. 1:1\u20131:8 (2017)","DOI":"10.1145\/3166072.3166084"},{"key":"6_CR7","doi-asserted-by":"crossref","unstructured":"Benham, R., et al.: RMIT at the 2017 TREC CORE track. In: Proceedings of the TREC (2017)","DOI":"10.6028\/NIST.SP.500-324.core-RMIT"},{"issue":"4","key":"6_CR8","doi-asserted-by":"publisher","first-page":"41:1","DOI":"10.1145\/3345001","volume":"37","author":"R Benham","year":"2019","unstructured":"Benham, R., Mackenzie, J.M., Moffat, A., Culpepper, J.S.: Boosting search performance using query variations. ACM Trans. Inf. Syst. 37(4), 41:1-41:25 (2019)","journal-title":"ACM Trans. Inf. Syst."},{"key":"6_CR9","doi-asserted-by":"crossref","unstructured":"Berendsen, R., Tsagkias, M., de Rijke, M., Meij, E.: Generating pseudo test collections for learning to rank scientific articles. In: Proceedings of the CLEF, pp. 42\u201353 (2012)","DOI":"10.1007\/978-3-642-33247-0_6"},{"key":"6_CR10","doi-asserted-by":"crossref","unstructured":"Breuer, T., et al.: How to measure the reproducibility of system-oriented IR experiments. In: Proceedings of the SIGIR, pp. 349\u2013358 (2020)","DOI":"10.1145\/3397271.3401036"},{"key":"6_CR11","doi-asserted-by":"crossref","unstructured":"Breuer, T., Ferro, N., Maistro, M., Schaer, P.: Repro_eval: a python interface to reproducibility measures of system-oriented IR experiments. In: Proceedings of the ECIR, pp. 481\u2013486 (2021)","DOI":"10.1007\/978-3-030-72240-1_51"},{"key":"6_CR12","doi-asserted-by":"crossref","unstructured":"Carterette, B., Bah, A., Zengin, M.: Dynamic test collections for retrieval evaluation. In: Proceedings of the ICTIR, pp. 91\u2013100. ACM (2015)","DOI":"10.1145\/2808194.2809470"},{"key":"6_CR13","doi-asserted-by":"crossref","unstructured":"Chuklin, A., Markov, I., de Rijke, M.: Click models for web search. In: Retrieval, and Services, Morgan & Claypool Publishers, Synthesis Lectures on Information Concepts (2015)","DOI":"10.1007\/978-3-031-02294-4"},{"key":"6_CR14","doi-asserted-by":"crossref","unstructured":"Craswell, N., Campos, D., Mitra, B., Yilmaz, E., Billerbeck, B.: ORCAS: 20 million clicked query-document pairs for analyzing search. In: Proceedings of the CIKM, pp. 2983\u20132989 (2020)","DOI":"10.1145\/3340531.3412779"},{"issue":"4","key":"6_CR15","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1108\/eb026683","volume":"35","author":"WB Croft","year":"1979","unstructured":"Croft, W.B., Harper, D.J.: Using probabilistic models of document retrieval without relevance information. J. Document. 35(4), 285\u2013295 (1979)","journal-title":"J. Document."},{"key":"6_CR16","doi-asserted-by":"crossref","unstructured":"Cronen-Townsend, S., Zhou, Y., Croft, W.B.: Predicting query performance. In: Proceedings of the SIGIR, pp. 299\u2013306 (2002)","DOI":"10.1145\/564376.564429"},{"key":"6_CR17","doi-asserted-by":"crossref","unstructured":"Eickhoff, C., Teevan, J., White, R., Dumais, S.T.: Lessons from the journey: a query log analysis of within-session learning. In: Proceedings of the WSDM, pp. 223\u2013232 (2014)","DOI":"10.1145\/2556195.2556217"},{"key":"6_CR18","doi-asserted-by":"crossref","unstructured":"Faggioli, G., Zendel, O., Culpepper, J.S., Ferro, N., Scholer, F.: An enhanced evaluation framework for query performance prediction. In: Proceedings of the ECIR, pp. 115\u2013129 (2021)","DOI":"10.1007\/978-3-030-72113-8_8"},{"key":"6_CR19","doi-asserted-by":"crossref","unstructured":"Guan, D., Zhang, S., Yang, H.: Utilizing query change for session search. In: Proceedings of the SIGIR, pp. 453\u2013462 (2013)","DOI":"10.1145\/2484028.2484055"},{"key":"6_CR20","unstructured":"G\u00fcnther, S., Hagen, M.: Assessing query suggestions for search session simulation. In: Proceedings of the Sim4IR (2021). http:\/\/ceur-ws.org\/Vol-2911\/paper6.pdf"},{"key":"6_CR21","doi-asserted-by":"crossref","unstructured":"Gysel, C.V., Kanoulas, E., de Rijke, M.: Lexical query modeling in session search. In: Proceedings of the ICTIR, pp. 69\u201372 (2016)","DOI":"10.1145\/2970398.2970422"},{"key":"6_CR22","doi-asserted-by":"crossref","unstructured":"He, Y., Tang, J., Ouyang, H., Kang, C., Yin, D., Chang, Y.: Learning to rewrite queries. In: Proceedings of the CIKM, pp. 1443\u20131452 (2016)","DOI":"10.1145\/2983323.2983835"},{"key":"6_CR23","doi-asserted-by":"crossref","unstructured":"Herdagdelen, A., et al.: Generalized syntactic and semantic models of query reformulation. In: Proceedings of the SIGIR, pp. 283\u2013290 (2010)","DOI":"10.1145\/1835449.1835498"},{"key":"6_CR24","doi-asserted-by":"crossref","unstructured":"Huurnink, B., Hofmann, K., de Rijke, M., Bron, M.: Validating query simulators: an experiment using commercial searches and purchases. In: Proceedings of the CLEF, pp. 40\u201351 (2010)","DOI":"10.1007\/978-3-642-15998-5_6"},{"issue":"7","key":"6_CR25","doi-asserted-by":"publisher","first-page":"1358","DOI":"10.1002\/asi.21071","volume":"60","author":"BJ Jansen","year":"2009","unstructured":"Jansen, B.J., Booth, D.L., Spink, A.: Patterns of query reformulation during web searching. J. Assoc. Inf. Sci. Technol. 60(7), 1358\u20131371 (2009)","journal-title":"J. Assoc. Inf. Sci. Technol."},{"key":"6_CR26","doi-asserted-by":"crossref","unstructured":"J\u00e4rvelin, K., Price, S.L., Delcambre, L.M.L., Nielsen, M.L.: Discounted cumulated gain based evaluation of multiple-query IR sessions. In: Proceedings of the ECIR, pp. 4\u201315 (2008)","DOI":"10.1007\/978-3-540-78646-7_4"},{"key":"6_CR27","doi-asserted-by":"crossref","unstructured":"Jones, R., Rey, B., Madani, O., Greiner, W.: Generating query substitutions. In: Proceedings of the WWW, pp. 387\u2013396 (2006)","DOI":"10.1145\/1135777.1135835"},{"key":"6_CR28","doi-asserted-by":"crossref","unstructured":"Jordan, C., Watters, C.R., Gao, Q.: Using controlled query generation to evaluate blind relevance feedback algorithms. In: Proceedings of the JCDL, pp. 286\u2013295 (2006)","DOI":"10.1145\/1141753.1141818"},{"key":"6_CR29","doi-asserted-by":"crossref","unstructured":"Keskustalo, H., J\u00e4rvelin, K., Pirkola, A., Sharma, T., Lykke, M.: Test collection-based IR evaluation needs extension toward sessions - a case of extremely short queries. In: Proceedings of the AIRS, pp. 63\u201374 (2009)","DOI":"10.1007\/978-3-642-04769-5_6"},{"key":"6_CR30","doi-asserted-by":"crossref","unstructured":"Lin, J., Ma, X., Lin, S., Yang, J., Pradeep, R., Nogueira, R.: Pyserini: a python toolkit for reproducible information retrieval research with sparse and dense representations. In: Proceedings of the SIGIR, pp. 2356\u20132362. ACM (2021)","DOI":"10.1145\/3404835.3463238"},{"key":"6_CR31","doi-asserted-by":"crossref","unstructured":"Liu, B., Craswell, N., Lu, X., Kurland, O., Culpepper, J.S.: A comparative analysis of human and automatic query variants. In: Proceedings of the SIGIR, pp. 47\u201350 (2019)","DOI":"10.1145\/3341981.3344223"},{"key":"6_CR32","doi-asserted-by":"crossref","unstructured":"Liu, J., Sarkar, S., Shah, C.: Identifying and predicting the states of complex search tasks. In: Proceedings of the CHIIR, pp. 193\u2013202 (2020)","DOI":"10.1145\/3343413.3377976"},{"key":"6_CR33","doi-asserted-by":"crossref","unstructured":"Mackenzie, J., Moffat, A.: Modality effects when simulating user querying tasks. In: Proceedings of the ICTIR, pp. 197\u2013201 (2021)","DOI":"10.1145\/3471158.3472244"},{"key":"6_CR34","doi-asserted-by":"crossref","unstructured":"Maxwell, D., Azzopardi, L.: Agents, simulated users and humans: an analysis of performance and behaviour. In: Proceedings of the CIKM, pp. 731\u2013740. ACM (2016)","DOI":"10.1145\/2983323.2983805"},{"key":"6_CR35","doi-asserted-by":"crossref","unstructured":"Maxwell, D., Azzopardi, L.: Simulating interactive information retrieval: simiir: a framework for the simulation of interaction. In: Proceedings of the SIGIR, pp. 1141\u20131144. ACM (2016)","DOI":"10.1145\/2911451.2911469"},{"key":"6_CR36","doi-asserted-by":"crossref","unstructured":"Moffat, A., Scholer, F., Thomas, P., Bailey, P.: Pooled evaluation over query variations: users are as diverse as systems. In: Proceedings of the CIKM, pp. 1759\u20131762 (2015)","DOI":"10.1145\/2806416.2806606"},{"issue":"4","key":"6_CR37","doi-asserted-by":"publisher","first-page":"338","DOI":"10.1007\/s10791-017-9301-2","volume":"20","author":"T P\u00e4\u00e4kk\u00f6nen","year":"2017","unstructured":"P\u00e4\u00e4kk\u00f6nen, T., Kek\u00e4l\u00e4inen, J., Keskustalo, H., Azzopardi, L., Maxwell, D., J\u00e4rvelin, K.: Validating simulated interaction for retrieval evaluation. Inf. Ret. J. 20(4), 338\u2013362 (2017). https:\/\/doi.org\/10.1007\/s10791-017-9301-2","journal-title":"Inf. Ret. J."},{"issue":"2","key":"6_CR38","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1017\/S0269888903000638","volume":"18","author":"I Ruthven","year":"2003","unstructured":"Ruthven, I., Lalmas, M.: A survey on the use of relevance feedback for information access systems. Knowl. Eng. Rev. 18(2), 95\u2013145 (2003)","journal-title":"Knowl. Eng. Rev."},{"issue":"2","key":"6_CR39","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s10791-015-9251-5","volume":"18","author":"M Sloan","year":"2015","unstructured":"Sloan, M., Yang, H., Wang, J.: A term-based methodology for query reformulation understanding. Inf. Retriev. J. 18(2), 145\u2013165 (2015). https:\/\/doi.org\/10.1007\/s10791-015-9251-5","journal-title":"Inf. Retriev. J."},{"key":"6_CR40","doi-asserted-by":"crossref","unstructured":"Tague, J., Nelson, M.J.: Simulation of user judgments in bibliographic retrieval systems. In: Proceedings of the SIGIR, pp. 66\u201371 (1981)","DOI":"10.1145\/1013228.511764"},{"key":"6_CR41","doi-asserted-by":"crossref","unstructured":"Verberne, S., Sappelli, M., J\u00e4rvelin, K., Kraaij, W.: User simulations for interactive search: Evaluating personalized query suggestion. In: Proceedings of the ECIR, pp. 678\u2013690 (2015)","DOI":"10.1007\/978-3-319-16354-3_75"},{"key":"6_CR42","doi-asserted-by":"crossref","unstructured":"Verberne, S., Sappelli, M., Kraaij, W.: Query term suggestion in academic search. In: Proceedings of the ECIR, pp. 560\u2013566 (2014)","DOI":"10.1007\/978-3-319-06028-6_57"},{"key":"6_CR43","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. In: Proceedings of the SIGIR, pp. 315\u2013323 (1998)","DOI":"10.1145\/290941.291017"},{"issue":"4","key":"6_CR44","doi-asserted-by":"publisher","first-page":"20:1","DOI":"10.1145\/2747874","volume":"33","author":"H Yang","year":"2015","unstructured":"Yang, H., Guan, D., Zhang, S.: The query change model: modeling session search as a Markov decision process. ACM Trans. Inf. Syst. 33(4), 20:1-20:33 (2015)","journal-title":"ACM Trans. Inf. Syst."},{"issue":"4","key":"6_CR45","first-page":"16:1","volume":"10","author":"P Yang","year":"2018","unstructured":"Yang, P., Fang, H., Lin, J.: Anserini: reproducible ranking baselines using Lucene. ACM J. Data Inf. Qual. 10(4), 16:1-16:20 (2018)","journal-title":"ACM J. Data Inf. Qual."},{"key":"6_CR46","doi-asserted-by":"crossref","unstructured":"Zhai, C., Lafferty, J.D.: A study of smoothing methods for language models applied to ad hoc information retrieval. In: Proceedings of the SIGIR, pp. 334\u2013342 (2001)","DOI":"10.1145\/383952.384019"},{"key":"6_CR47","doi-asserted-by":"crossref","unstructured":"Zhang, Y., Liu, X., Zhai, C.: Information retrieval evaluation as search simulation: a general formal framework for IR evaluation. In: Proceedings of the ICTIR, pp. 193\u2013200 (2017)","DOI":"10.1145\/3121050.3121070"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99736-6_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,21]],"date-time":"2024-09-21T15:41:11Z","timestamp":1726933271000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99736-6_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997359","9783030997366"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99736-6_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}