{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:15:04Z","timestamp":1775283304319,"version":"3.50.1"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2017,5,6]],"date-time":"2017-05-06T00:00:00Z","timestamp":1494028800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2017,5,6]],"date-time":"2017-05-06T00:00:00Z","timestamp":1494028800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2017,8]]},"DOI":"10.1007\/s10791-017-9301-2","type":"journal-article","created":{"date-parts":[[2017,5,6]],"date-time":"2017-05-06T07:57:33Z","timestamp":1494057453000},"page":"338-362","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":14,"title":["Validating simulated interaction for retrieval evaluation"],"prefix":"10.1007","volume":"20","author":[{"given":"Teemu","family":"P\u00e4\u00e4kk\u00f6nen","sequence":"first","affiliation":[]},{"given":"Jaana","family":"Kek\u00e4l\u00e4inen","sequence":"additional","affiliation":[]},{"given":"Heikki","family":"Keskustalo","sequence":"additional","affiliation":[]},{"given":"Leif","family":"Azzopardi","sequence":"additional","affiliation":[]},{"given":"David","family":"Maxwell","sequence":"additional","affiliation":[]},{"given":"Kalervo","family":"J\u00e4rvelin","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,5,6]]},"reference":[{"issue":"4","key":"9301_CR1","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1145\/582415.582416","volume":"20","author":"G Amati","year":"2002","unstructured":"Amati, G., & van Rijsbergen, C. J. (2002). Probabilistic models of IR based on measuring the divergence from randomness. ACM Transactions on Information Systems, \n                           20(4), 357\u2013389.","journal-title":"ACM Transactions on Information Systems"},{"key":"9301_CR2","doi-asserted-by":"crossref","unstructured":"Azzopardi, L. (2009). Query side evaluation: An empirical analysis of effectiveness and effort. In Proceedings of the 32nd annual international ACM SIGIR conference on research and development in information retrieval (pp. 556\u2013563). New York: ACM.","DOI":"10.1145\/1571941.1572037"},{"key":"9301_CR3","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., de Rijke, M., & Balog, K. (2007). Building simulated queries for known-item topics: An analysis using six European languages. In Proceedings of the 30th annual international ACM SIGIR conference on research and development in information retrieval (pp. 23\u201327). New York: ACM.","DOI":"10.1145\/1277741.1277820"},{"issue":"2","key":"9301_CR4","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1145\/1924475.1924484","volume":"44","author":"L Azzopardi","year":"2010","unstructured":"Azzopardi, L., J\u00e4rvelin, K., Kamps, J., & Smucker, M. (2010). Report on the SIGIR 2010 workshop on the simulation of interaction. SIGIR Forum, \n                           44(2), 35\u201347.","journal-title":"SIGIR Forum"},{"key":"9301_CR5","doi-asserted-by":"crossref","unstructured":"Azzopardi, L., Kelly, D., & Brennan, K. (2013). How query cost affects search behavior. In Proceedings of the 36th Annual International ACM SIGIR conference on research and development in information retrieval (pp. 23\u201332). New York: ACM.","DOI":"10.1145\/2484028.2484049"},{"key":"9301_CR6","doi-asserted-by":"crossref","unstructured":"Baskaya, F., Keskustalo, H., & J\u00e4rvelin, K. (2011). Simulating simple and fallible relevance feedback. In Advances in Information Retrieval: 33rd European conference on IR research proceedings (pp. 593\u2013604). Heidelberg: Springer.","DOI":"10.1007\/978-3-642-20161-5_59"},{"key":"9301_CR7","doi-asserted-by":"crossref","unstructured":"Baskaya, F., Keskustalo, H., & J\u00e4rvelin, K. (2012). Time drives interaction: Simulating sessions in diverse searching environments. In Proceedings of the 35th Annual International ACM SIGIR conference on research and development in information retrieval (pp. 97\u2013106). New York: ACM.","DOI":"10.1145\/2348283.2348301"},{"key":"9301_CR8","doi-asserted-by":"crossref","unstructured":"Baskaya, F., Keskustalo, H., & J\u00e4rvelin, K. (2013). Modeling behavioral factors in interactive information retrieval. In Proceedings of the 22nd ACM international conference on information and knowledge management (pp. 2297\u20132302). New York: ACM.","DOI":"10.1145\/2505515.2505660"},{"issue":"1","key":"9301_CR9","doi-asserted-by":"publisher","first-page":"46","DOI":"10.1145\/1394251.1394261","volume":"42","author":"N Belkin","year":"2008","unstructured":"Belkin, N. (2008). Somewhat grand challenges for IR. SIGIR Forum, \n                           42(1), 46\u201354.","journal-title":"SIGIR Forum"},{"key":"9301_CR10","doi-asserted-by":"crossref","unstructured":"Boytsov, L., Belova, A., & Westfall, P. (2013). Deciding on an adjustment for multiplicity in IR experiments. In Proceedings of the 36th annual international ACM SIGIR conference on research and development in information retrieval (pp. 403\u2013412). New York: ACM.","DOI":"10.1145\/2484028.2484034"},{"key":"9301_CR11","doi-asserted-by":"crossref","unstructured":"Carterette, B., Bah, A., & Zengin, M. (2015). Dynamic test collections for retrieval evaluation. In Proceedings of the 2015 international conference on the theory of information retrieval (pp. 91\u2013100). New York: ACM.","DOI":"10.1145\/2808194.2809470"},{"key":"9301_CR12","doi-asserted-by":"crossref","unstructured":"Carterette, B., Kanoulas, E., & Yilmaz, E. (2011). Simulating simple user behavior for system effectiveness evaluation. In Proceedings of the 20th ACM international conference on information and knowledge management (pp. 611\u2013620). New York: ACM.","DOI":"10.1145\/2063576.2063668"},{"key":"9301_CR13","doi-asserted-by":"crossref","unstructured":"Chen, M-C., Anderson, J.R., & Sohn, M-H. (2001).What can a mouse cursor tell us more? Correlation of eye\/mouse movements on web browsing. In CHI\u201901 extended abstracts on human factors in computing systems (pp. 281\u2013282). New York: ACM.","DOI":"10.1145\/634067.634234"},{"key":"9301_CR14","doi-asserted-by":"crossref","unstructured":"Chuklin, A., Markov, I., & de Rijke, M. (2015). Click models for web search. Synthesis lectures on information concepts, retrieval, and services. San Rafael: Morgan and Claypool.","DOI":"10.1007\/978-3-031-02294-4"},{"key":"9301_CR15","doi-asserted-by":"publisher","first-page":"49","DOI":"10.1016\/j.jda.2012.10.002","volume":"18","author":"G Dupret","year":"2013","unstructured":"Dupret, G., & Piwowarski, B. (2013). Model based comparison of discounted cumulative gain and average precision. Journal of Discrete Algorithms, \n                           18, 49\u201362.","journal-title":"Journal of Discrete Algorithms"},{"key":"9301_CR16","doi-asserted-by":"crossref","unstructured":"Guo, F., Liu, C., Kannan, A., Minka, T., Taylor, M., Wang, Y-M., & Christos Faloutsos, C. (2009a). Click chain model in web search. In Proceedings of the 18th international conference on World Wide Web (pp. 11\u201320). New York: ACM.","DOI":"10.1145\/1526709.1526712"},{"key":"9301_CR17","doi-asserted-by":"crossref","unstructured":"Guo, F., Liu, C., & Wang, Y.M. (2009b). Efficient multiple-click models in web search. In Proceedings of the Second ACM international conference on web search and data mining (pp. 124\u2013131). New York: ACM.","DOI":"10.1145\/1498759.1498818"},{"key":"9301_CR18","doi-asserted-by":"crossref","unstructured":"Harman, D. (1992). Relevance feedback revisited. In Proceedings of the 15th annual international ACM SIGIR conference on research and development in information retrieval (pp. 1\u201310). New York: ACM.","DOI":"10.1145\/133160.133167"},{"key":"9301_CR19","volume-title":"The turn: Integration of information seeking and retrieval in context","author":"P Ingwersen","year":"2005","unstructured":"Ingwersen, P., & J\u00e4rvelin, K. (2005). The turn: Integration of information seeking and retrieval in context. New York: Springer."},{"issue":"4","key":"9301_CR20","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems, \n                           20(4), 422\u2013446.","journal-title":"ACM Transactions on Information Systems"},{"issue":"1\u20132","key":"9301_CR21","first-page":"1","volume":"3","author":"D Kelly","year":"2009","unstructured":"Kelly, D. (2009). Methods for evaluating interactive information retrieval systems with users. Foundations and Trends in Information Retrieval, \n                           3(1\u20132), 1\u2013224.","journal-title":"Foundations and Trends in Information Retrieval"},{"key":"9301_CR22","doi-asserted-by":"crossref","unstructured":"Kelly, D., & Cool, C. (2002). The effects of topic familiarity on information search behavior. In Proceedings of the 2nd ACM\/IEEE-CS joint conference on digital libraries (pp. 74\u201375). New York: ACM.","DOI":"10.1145\/544220.544232"},{"key":"9301_CR23","doi-asserted-by":"crossref","unstructured":"Keskustalo, H., J\u00e4rvelin, K., Pirkola, A., Sharma, T., & Lykke, M. (2009). Test collection-based IR evaluation needs extension toward sessions. In Proceedings of the 5th Asia information retrieval symposium on information retrieval technology (pp. 63\u201374). Berlin: Springer.","DOI":"10.1007\/978-3-642-04769-5_6"},{"key":"9301_CR24","doi-asserted-by":"publisher","unstructured":"Law, A. M. (2008). How to build valid and credible simulation models. In Proceedings of the 2008 winter simulation conference, doi: 10.1109\/WSC.2008.4736054.","DOI":"10.1109\/WSC.2008.4736054"},{"key":"9301_CR25","doi-asserted-by":"crossref","unstructured":"Lin, J., & Smucker, M. D. (2008). How do users find things with PubMed? Towards automatic utility evaluation with user simulations. In Proceedings of the 31st annual international ACM SIGIR conference on research and development in information retrieval (pp. 19\u201326). New York: ACM.","DOI":"10.1145\/1390334.1390340"},{"key":"9301_CR26","doi-asserted-by":"crossref","unstructured":"Maxwell, D., & Azzopardi, L. (2014). Stuck in traffic: How temporal delays affect search behaviour. In Proceedings of the 5th information interaction in context symposium (pp.155\u2013164). New York: ACM.","DOI":"10.1145\/2637002.2637021"},{"key":"9301_CR27","doi-asserted-by":"crossref","unstructured":"Maxwell, D. & Azzopardi, L. (2016), Agents, simulated users and humans: An analysis of performance and behaviour. In Proceedings of the 25th ACM international on conference on information and knowledge management (pp. 731\u2013740). New York: ACM.","DOI":"10.1145\/2983323.2983805"},{"key":"9301_CR28","doi-asserted-by":"crossref","unstructured":"Maxwell, D., Azzopardi, L., J\u00e4rvelin, K., & Keskustalo, H. (2015a). An initial investigation into fixed and adaptive stopping strategies. In Proceedings of the 38th annual international ACM SIGIR conference on research and development in information retrieval (pp. 903\u2013906). New York: ACM.","DOI":"10.1145\/2766462.2767802"},{"key":"9301_CR29","doi-asserted-by":"crossref","unstructured":"Maxwell, D., Azzopardi, L., J\u00e4rvelin, K., & Keskustalo, H. (2015b). Searching and stopping: An analysis of stopping rules and strategies. In Proceedings of the 24th ACM international on conference on information and knowledge management (pp. 313\u2013322). New York: ACM.","DOI":"10.1145\/2806416.2806476"},{"key":"9301_CR30","doi-asserted-by":"crossref","unstructured":"Moffat, A., Scholer, F., Thomas, P. and Bailey, P. (2015). Pooled evaluation over query variations: Users are as diverse as systems. In Proceedings of conference on information and knowlwdge management (pp. 1759\u20131762). New York, ACM.","DOI":"10.1145\/2806416.2806606"},{"key":"9301_CR31","doi-asserted-by":"crossref","unstructured":"P\u00e4\u00e4kk\u00f6nen, T., J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J., Keskustalo, H., Baskaya, F., Maxwell, D., & Azzopardi, L. (2015). Exploring behavioral dimensions in session effectiveness. In Proceedings of 6th international conference of the CLEF association: Experimental IR meets multilinguality, multimodality, and interaction (pp. 178\u2013189). LNCS 9283. Heidelberg: Springer.","DOI":"10.1007\/978-3-319-24027-5_15"},{"key":"9301_CR32","doi-asserted-by":"crossref","first-page":"2997","DOI":"10.1145\/1358628.1358797","volume-title":"CHI \u201808 extended abstracts on human factors in computing systems","author":"K Rodden","year":"2008","unstructured":"Rodden, K., Fu, X., Aula, A., & Spiro, A. (2008). Eye-mouse coordination patterns on web search results pages. CHI \u201808 extended abstracts on human factors in computing systems (pp. 2997\u20133002). New York: ACM."},{"key":"9301_CR33","unstructured":"Saracevic, T. (1996). Modeling interaction in information retrieval: A review and proposal. In Proceedings of the ASIS annual meeting, Vol. 33 (pp. 3\u20139)."},{"key":"9301_CR34","unstructured":"Smucker, M. D. (2011). An analysis of user strategies for examining and processing ranked lists of documents. In Proceedings of the Fifth workshop on HCIR. http:\/\/www.mansci.uwaterloo.ca\/~msmucker\/publications\/smucker-hcir-2011-cluster.pdf. Accessed 3 August 2016."},{"key":"9301_CR35","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., Allan, J., & Carterette, B. (2007). A comparison of statistical significance tests for information retrieval evaluation. In Proceedings of the 16th ACM conference on information and knowledge management (pp. 623\u2013632). New York: ACM.","DOI":"10.1145\/1321440.1321528"},{"key":"9301_CR36","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., & Clarke, C. (2012). Time-based calibration of effectiveness measures. In Proceedings of the 35th annual international ACM SIGIR conference on research and development in information retrieval (pp. 95\u2013104). New York: ACM.","DOI":"10.1145\/2348283.2348300"},{"key":"9301_CR37","doi-asserted-by":"crossref","unstructured":"Thomas, P., Moffat, A., Bailey, P., & Scholer, F. (2014). Modeling decision points in user search behavior. In Proceedings of the 5th information interaction in context symposium (pp. 239\u2013242). New York: ACM.","DOI":"10.1145\/2637002.2637032"},{"key":"9301_CR38","doi-asserted-by":"crossref","unstructured":"Turpin, A., Scholer, F., J\u00e4rvelin, K., Wu, M. F., & Culpepper, S. (2009). Including summaries in system evaluations. In Proceedings of the 32nd annual international ACM SIGIR conference on research and development in information retrieval (pp. 508\u2013515). New York: ACM.","DOI":"10.1145\/1571941.1572029"},{"key":"9301_CR39","doi-asserted-by":"crossref","unstructured":"Verberne, S., Sappelli, M., J\u00e4rvelin, K., & Kraaij, W. (2015). User simulations for interactive search: Evaluating personalized query suggestion. In Advances in information retrieval: 37th European conference on IR research proceedings (pp. 678\u2013690). Cham: Springer International Publishing.","DOI":"10.1007\/978-3-319-16354-3_75"},{"key":"9301_CR40","doi-asserted-by":"crossref","unstructured":"White, R., Dumais, S. T., & Teevan, J. (2009). Characterizing the influence of domain expertise on web search behavior. In Proceedings of the second ACM international conference on web search and data mining (pp. 132\u2013141). New York: ACM.","DOI":"10.1145\/1498759.1498819"},{"key":"9301_CR41","doi-asserted-by":"crossref","unstructured":"White, R. W., Jose, J. M., van Rijsbergen, C. J., & Ruthven, I. (2004). A simulated study of implicit feedback models. In Advances in information retrieval: 26th European conference on IR research proceedings (pp. 311\u2013326). Heidelberg: Springer.","DOI":"10.1007\/978-3-540-24752-4_23"},{"key":"9301_CR42","volume-title":"Theory of modeling and simulation: Integrating discrete event and continuous complex dynamic systems","author":"BP Zeigler","year":"2000","unstructured":"Zeigler, B. P., Praehofer, H., & Kim, T. G. (2000). Theory of modeling and simulation: Integrating discrete event and continuous complex dynamic systems (2nd ed.). Cambridge: Academic Press.","edition":"2"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-017-9301-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-017-9301-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-017-9301-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:33:20Z","timestamp":1704206000000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-017-9301-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,5,6]]},"references-count":42,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2017,8]]}},"alternative-id":["9301"],"URL":"https:\/\/doi.org\/10.1007\/s10791-017-9301-2","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,5,6]]},"assertion":[{"value":"26 August 2016","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 April 2017","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 May 2017","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}