{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T00:51:46Z","timestamp":1740099106257,"version":"3.37.3"},"publisher-location":"Cham","reference-count":21,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319989310"},{"type":"electronic","value":"9783319989327"}],"license":[{"start":{"date-parts":[[2018,1,1]],"date-time":"2018-01-01T00:00:00Z","timestamp":1514764800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018]]},"DOI":"10.1007\/978-3-319-98932-7_31","type":"book-chapter","created":{"date-parts":[[2018,8,14]],"date-time":"2018-08-14T11:51:14Z","timestamp":1534247474000},"page":"362-371","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Overview of the CLEF Dynamic Search Evaluation Lab 2018"],"prefix":"10.1007","author":[{"given":"Evangelos","family":"Kanoulas","sequence":"first","affiliation":[]},{"given":"Leif","family":"Azzopardi","sequence":"additional","affiliation":[]},{"given":"Grace Hui","family":"Yang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,8,15]]},"reference":[{"key":"31_CR1","doi-asserted-by":"crossref","unstructured":"Allan, J.: HARD track overview in TREC 2003 high accuracy retrieval from documents. Technical report, DTIC Document (2005)","DOI":"10.21236\/ADA439446"},{"key":"31_CR2","doi-asserted-by":"publisher","unstructured":"Carterette, B., Clough, P.D., Hall, M.M., Kanoulas, E., Sanderson, M.: Evaluating retrieval over sessions: the TREC session track 2011\u20132014. In: Perego, R., Sebastiani, F., Aslam, J.A., Ruthven, I., Zobel, J. (eds.) Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2016, Pisa, Italy, 17\u201321 July 2016, pp. 685\u2013688. ACM (2016). https:\/\/doi.org\/10.1145\/2911451.2914675","DOI":"10.1145\/2911451.2914675"},{"key":"31_CR3","doi-asserted-by":"crossref","unstructured":"Georgila, K., Henderson, J., Lemon, O.: User simulation for spoken dialogue systems: learning and evaluation. In: Interspeech, pp. 1065\u20131068 (2006)","DOI":"10.21437\/Interspeech.2006-160"},{"key":"31_CR4","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"744","DOI":"10.1007\/978-3-319-56608-5_74","volume-title":"Advances in Information Retrieval","author":"C Gysel Van","year":"2017","unstructured":"Van Gysel, C., Kanoulas, E., de Rijke, M.: Pyndri: a Python interface to the indri search engine. In: Jose, J.M., et al. (eds.) ECIR 2017. LNCS, vol. 10193, pp. 744\u2013748. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-56608-5_74"},{"key":"31_CR5","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1007\/978-3-540-78646-7_4","volume-title":"Advances in Information Retrieval","author":"K J\u00e4rvelin","year":"2008","unstructured":"J\u00e4rvelin, K., Price, S.L., Delcambre, L.M.L., Nielsen, M.L.: Discounted cumulated gain based evaluation of multiple-query IR sessions. In: Macdonald, C., Ounis, I., Plachouras, V., Ruthven, I., White, R.W. (eds.) ECIR 2008. LNCS, vol. 4956, pp. 4\u201315. Springer, Heidelberg (2008). https:\/\/doi.org\/10.1007\/978-3-540-78646-7_4"},{"issue":"4","key":"31_CR6","doi-asserted-by":"publisher","first-page":"479","DOI":"10.1016\/j.csl.2009.03.002","volume":"23","author":"S Jung","year":"2009","unstructured":"Jung, S., Lee, C., Kim, K., Jeong, M., Lee, G.G.: Data-driven user simulation for automated evaluation of spoken dialog systems. Comput. Speech Lang. 23(4), 479\u2013509 (2009). https:\/\/doi.org\/10.1016\/j.csl.2009.03.002","journal-title":"Comput. Speech Lang."},{"key":"31_CR7","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"361","DOI":"10.1007\/978-3-319-65813-1_31","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"E Kanoulas","year":"2017","unstructured":"Kanoulas, E., Azzopardi, L.: CLEF 2017 dynamic search evaluation lab overview. In: Jones, G.J.F., et al. (eds.) CLEF 2017. LNCS, vol. 10456, pp. 361\u2013366. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-65813-1_31"},{"key":"31_CR8","unstructured":"Kanoulas, E., Azzopardi, L.: CLEF 2017 dynamic search lab overview and evaluation. In: Cappellato, L., Ferro, N., Goeuriot, L., Mandl, T. (eds.) Working Notes of CLEF 2017 - Conference and Labs of the Evaluation Forum, Dublin, Ireland, 11\u201314 September 2017. CEUR Workshop Proceedings, vol. 1866. CEUR-WS.org (2017). http:\/\/ceur-ws.org\/Vol-1866\/invited_paper_13.pdf"},{"key":"31_CR9","doi-asserted-by":"publisher","unstructured":"Kanoulas, E., Carterette, B., Clough, P.D., Sanderson, M.: Evaluating multi-query sessions. In: Proceedings of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2011, pp. 1053\u20131062. ACM, New York (2011). https:\/\/doi.org\/10.1145\/2009916.2010056","DOI":"10.1145\/2009916.2010056"},{"key":"31_CR10","doi-asserted-by":"publisher","unstructured":"Luo, J., Wing, C., Yang, H., Hearst, M.A.: The water filling model and the cube test: multi-dimensional evaluation for professional search. In: 22nd ACM International Conference on Information and Knowledge Management, CIKM 2013, San Francisco, CA, USA, 27 October\u20131 November 2013, pp. 709\u2013714 (2013). https:\/\/doi.org\/10.1145\/2505515.2523648","DOI":"10.1145\/2505515.2523648"},{"key":"31_CR11","doi-asserted-by":"crossref","unstructured":"Maxwell, D., Azzopardi, L.: Agents, simulated users and humans: an analysis of performance and behaviour. In: Proceedings of the 25th ACM International on Conference on Information and Knowledge Management, pp. 731\u2013740. CIKM 2016 (2016)","DOI":"10.1145\/2983323.2983805"},{"issue":"3","key":"31_CR12","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1016\/S0306-4573(00)00053-4","volume":"37","author":"P Over","year":"2001","unstructured":"Over, P.: The TREC interactive track: an annotated bibliography. Inf. Process. Manag. 37(3), 369\u2013381 (2001)","journal-title":"Inf. Process. Manag."},{"key":"31_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10791-017-9294-x","volume":"20","author":"T P\u00e4\u00e4kk\u00f6nen","year":"2017","unstructured":"P\u00e4\u00e4kk\u00f6nen, T., Kek\u00e4l\u00e4inen, J., Keskustalo, H., Azzopardi, L., Maxwell, D., J\u00e4rvelin, K.: Validating simulated interaction for retrieval evaluation. Inf. Retr. J. 20, 1\u201325 (2017)","journal-title":"Inf. Retr. J."},{"issue":"01","key":"31_CR14","doi-asserted-by":"publisher","first-page":"59","DOI":"10.1017\/S0269888912000343","volume":"28","author":"O Pietquin","year":"2013","unstructured":"Pietquin, O., Hastie, H.: A survey on metrics for the evaluation of user simulations. Knowl. Eng. Rev. 28(01), 59\u201373 (2013)","journal-title":"Knowl. Eng. Rev."},{"key":"31_CR15","unstructured":"Serban, I.V., Lowe, R., Henderson, P., Charlin, L., Pineau, J.: A survey of available corpora for building data-driven dialogue systems. CoRR abs\/1512.05742 (2015). http:\/\/arxiv.org\/abs\/1512.05742"},{"key":"31_CR16","unstructured":"Verma, M., et al.: Overview of the TREC tasks track 2016. In: Voorhees and Ellis [17] (2016). http:\/\/trec.nist.gov\/pubs\/trec25\/papers\/Overview-T.pdf"},{"key":"31_CR17","unstructured":"Voorhees, E.M., Ellis, A. (eds.): Proceedings of The Twenty-Fifth Text REtrieval Conference, TREC 2016, Gaithersburg, Maryland, USA, 15\u201318 November 2016, vol. Special Publication 500-321. National Institute of Standards and Technology (NIST) (2016). http:\/\/trec.nist.gov\/pubs\/trec25\/trec2016.html"},{"key":"31_CR18","unstructured":"Yang, G.H., Soboroff, I.: TREC 2016 dynamic domain track overview. In: Voorhees and Ellis [17] (2016). http:\/\/trec.nist.gov\/pubs\/trec25\/papers\/Overview-DD.pdf"},{"key":"31_CR19","unstructured":"Yang, G.H., Soboroff, I.: TREC 2017 dynamic domain track overview. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of The Twenty-Sixth Text REtrieval Conference, TREC 2017, Gaithersburg, Maryland, USA, 15\u201318 November 2017. National Institute of Standards and Technology (NIST) (2017)"},{"key":"31_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"164","DOI":"10.1007\/978-3-642-04417-5_15","volume-title":"Advances in Information Retrieval Theory","author":"Y Yang","year":"2009","unstructured":"Yang, Y., Lad, A.: Modeling expected utility of multi-session information distillation. In: Azzopardi, L., et al. (eds.) ICTIR 2009. LNCS, vol. 5766, pp. 164\u2013175. Springer, Heidelberg (2009). https:\/\/doi.org\/10.1007\/978-3-642-04417-5_15"},{"key":"31_CR21","unstructured":"Yilmaz, E., Verma, M., Mehrotra, R., Kanoulas, E., Carterette, B., Craswell, N.: Overview of the TREC 2015 tasks track. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of The Twenty-Fourth Text REtrieval Conference, TREC 2015, Gaithersburg, Maryland, USA, 17\u201320 November 2015, vol. Special Publication 500\u2013319. National Institute of Standards and Technology (NIST) (2015). http:\/\/trec.nist.gov\/pubs\/trec24\/papers\/Overview-T.pdf"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-98932-7_31","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,8,29]],"date-time":"2022-08-29T06:19:14Z","timestamp":1661753954000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-98932-7_31"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018]]},"ISBN":["9783319989310","9783319989327"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-98932-7_31","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2018]]}}}