{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T13:15:18Z","timestamp":1743081318086,"version":"3.40.3"},"publisher-location":"Cham","reference-count":35,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319306704"},{"type":"electronic","value":"9783319306711"}],"license":[{"start":{"date-parts":[[2016,1,1]],"date-time":"2016-01-01T00:00:00Z","timestamp":1451606400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2016]]},"DOI":"10.1007\/978-3-319-30671-1_88","type":"book-chapter","created":{"date-parts":[[2016,3,9]],"date-time":"2016-03-09T16:39:50Z","timestamp":1457541590000},"page":"893-896","source":"Crossref","is-referenced-by-count":0,"title":["Living Labs for Online Evaluation: From Theory to Practice"],"prefix":"10.1007","author":[{"given":"Anne","family":"Schuth","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Krisztian","family":"Balog","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"88_CR1","doi-asserted-by":"crossref","unstructured":"Balog, K., Kelly, L., Schuth, A.: Head first: living labs for ad-hoc search evaluation. In: CIKM 2014, pp. 1815\u20131818. ACM Press, New York, USA, November 2014","DOI":"10.1145\/2661829.2661962"},{"key":"88_CR2","doi-asserted-by":"crossref","unstructured":"Belkin, N.J.: Salton award lecture: people, interacting with information. In: Proceedings of 38th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2015, pp. 1\u20132. ACM (2015)","DOI":"10.1145\/2766462.2767854"},{"key":"88_CR3","series-title":"Synthesis Lectures on Information Concepts, Retrieval, and Services","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-031-02294-4","volume-title":"Click Models for Web Search","author":"A Chuklin","year":"2015","unstructured":"Chuklin, A., Markov, I., de Rijke, M.: Click Models for Web Search. Synthesis Lectures on Information Concepts, Retrieval, and Services. Morgan & Claypool Publishers, San Rafael (2015)"},{"key":"88_CR4","unstructured":"Cleverdon, C.W., Keen, M.: Aslib Cranfield research project-factors determining the performance of indexing systems; Volume 2, Test results, National Science Foundation (1966)"},{"key":"88_CR5","doi-asserted-by":"crossref","unstructured":"Diaz, F., White, R., Buscher, G., Liebling, D.: Robust models of mouse movement on dynamic web search results pages. In: CIKM, pp. 1451\u20131460. ACM Press, October 2013","DOI":"10.1145\/2505515.2505717"},{"key":"88_CR6","unstructured":"Guo, Q., Agichtein, E.: Understanding \u201cabandoned\u201d ads: towards personalized commercial intent inference via mouse movement analysis. In: SIGIR-IRA (2008)"},{"key":"88_CR7","unstructured":"Guo, Q., Agichtein, E.: Towards predicting web searcher gaze position from mouse movements. In: CHI EA, 3601p, April 2010"},{"key":"88_CR8","doi-asserted-by":"crossref","unstructured":"Hassan, A., Shi, X., Craswell, N., Ramsey, B.: Beyond clicks: query reformulation as a predictor of search satisfaction. In: CIKM (2013)","DOI":"10.1145\/2505515.2505682"},{"key":"88_CR9","doi-asserted-by":"crossref","unstructured":"He, J., Zhai, C., Li, X.: Evaluation of methods for relative comparison of retrieval systems based on clickthroughs. In: CIKM 2009, ACM (2009)","DOI":"10.1145\/1645953.1646293"},{"key":"88_CR10","doi-asserted-by":"crossref","unstructured":"He, Y., Wang, K.: Inferring search behaviors using partially observable markov model with duration (POMD). In: WSDM (2011)","DOI":"10.1145\/1935826.1935891"},{"key":"88_CR11","doi-asserted-by":"crossref","unstructured":"Hersh, W., Turpin, A.H., Price, S., Chan, B., Kramer, D., Sacherek, L., Olson, D.: Do batch and user evaluations give the same results? In: SIGIR, pp. 17\u201324 (2000)","DOI":"10.1145\/383952.383992"},{"key":"88_CR12","doi-asserted-by":"crossref","unstructured":"Hofmann, K., Whiteson, S., de Rijke, M.: A probabilistic method for inferring preferences from clicks. In: CIKM 2011, ACM (2011)","DOI":"10.1145\/2063576.2063618"},{"key":"88_CR13","unstructured":"Jeff, H., Thomas, L., Ryen, W.: No search result left behind. In: WSDM, 203p (2012)"},{"issue":"2","key":"88_CR14","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1145\/1229179.1229181","volume":"25","author":"T Joachims","year":"2007","unstructured":"Joachims, T., Granka, L.A., Pan, B., Hembrooke, H., Radlinski, F., Gay, G.: Evaluating the accuracy of implicit feedback from clicks and query reformulations in web search. ACM Trans. Inf. Syst. 25(2), 7 (2007)","journal-title":"ACM Trans. Inf. Syst."},{"key":"88_CR15","doi-asserted-by":"crossref","unstructured":"Kim, Y., Hassan, A., White, R., Zitouni, I.: Modeling dwell time to predict click-level satisfaction. In: WSDM (2014)","DOI":"10.1145\/2556195.2556220"},{"key":"88_CR16","doi-asserted-by":"crossref","unstructured":"Kohavi, R.: Online controlled experiments: introduction, insights, scaling, and humbling statistics. In: Proceedings of UEO 2013 (2013)","DOI":"10.1145\/2512875.2517149"},{"key":"88_CR17","doi-asserted-by":"crossref","unstructured":"Lagun, D., Hsieh, C.H., Webster D., Navalpakkam, V.: Towards better measurement of attention and satisfaction in mobile search. In: SIGIR (2014)","DOI":"10.1145\/2600428.2609631"},{"key":"88_CR18","doi-asserted-by":"crossref","unstructured":"Li, J., Huffman, S., Tokuda, A.: Good abandonment in mobile and pc internet search. In: SIGIR 2009, pp. 43\u201350 (2009)","DOI":"10.1145\/1571941.1571951"},{"key":"88_CR19","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-14267-3","volume-title":"Learning to Rank for Information Retrieval","author":"T-Y Liu","year":"2011","unstructured":"Liu, T.-Y.: Learning to Rank for Information Retrieval. Springer, Heidelberg (2011)"},{"key":"88_CR20","doi-asserted-by":"crossref","unstructured":"Radlinski, F., Craswell, N.: Optimized interleaving for online retrieval evaluation. In: WSDM 2013, ACM (2013)","DOI":"10.1145\/2433396.2433429"},{"key":"88_CR21","doi-asserted-by":"crossref","unstructured":"Radlinski, F., Kurup, M., Joachims, T.: How does clickthrough data reflect retrieval quality? In: CIKM 2008, ACM (2008)","DOI":"10.1145\/1458082.1458092"},{"issue":"4","key":"88_CR22","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1561\/1500000009","volume":"4","author":"M Sanderson","year":"2010","unstructured":"Sanderson, M.: Test collection based evaluation of information retrieval systems. Found. Trends Inf. Retrieval 4(4), 247\u2013375 (2010)","journal-title":"Found. Trends Inf. Retrieval"},{"key":"88_CR23","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1007\/978-3-319-24027-5_47","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"A Schuth","year":"2015","unstructured":"Schuth, A., Balog, K., Kelly, L.: Overview of the living labs for information retrieval evaluation (ll4ir) clef lab. In: Mothe, J., Savoy, J., Kamps, J., Pinel-Sauvagnat, K., Jones, G.J.F., SanJuan, E., Cappellato, L., Ferro, N. (eds.) CLEF 2015. LNCS, pp. 484\u2013496. Springer, Heidelberg (2015)"},{"key":"88_CR24","doi-asserted-by":"crossref","unstructured":"Schuth, A., Bruintjes, R.-J., B\u00fcttner, F., van Doorn, J., Groenland, C., Oosterhuis, H., Tran, C.-N., Veeling, B., van der Velde, J., Wechsler, R., Woudenberg, D., de Rijke, M.: Probabilistic multileave for online retrieval evaluation. In: Proceedings of SIGIR (2015)","DOI":"10.1145\/2766462.2767838"},{"key":"88_CR25","doi-asserted-by":"crossref","unstructured":"Schuth, A., Hofmann, K., Radlinski, F.: Predicting search satisfaction metrics with interleaved comparisons. In: SIGIR 2015 (2015)","DOI":"10.1145\/2766462.2767695"},{"key":"88_CR26","doi-asserted-by":"crossref","unstructured":"Schuth, A., Hofmann, K., Whiteson, S., de Rijke, M.: Lerot: an online learning to rank framework. In: LivingLab 2013, pp. 23\u201326. ACM Press, November 2013","DOI":"10.1145\/2513150.2513162"},{"key":"88_CR27","doi-asserted-by":"crossref","unstructured":"Schuth, A., Sietsma, F., Whiteson, S., Lefortier, D., de Rijke, M.: Multileaved comparisons for fast online evaluation. In: CIKM 2014 (2014)","DOI":"10.1145\/2661829.2661952"},{"key":"88_CR28","doi-asserted-by":"crossref","unstructured":"Song, Y., Shi, X., White, R., Hassan, A.: Context-aware web search abandonment prediction. In: SIGIR (2014)","DOI":"10.1145\/2600428.2609604"},{"key":"88_CR29","doi-asserted-by":"crossref","unstructured":"Teevan, J., Dumais, S., Horvitz, E.: The potential value of personalizing search. In: SIGIR, pp. 756\u2013757 (2007)","DOI":"10.1145\/1277741.1277894"},{"key":"88_CR30","doi-asserted-by":"crossref","unstructured":"Turpin, A., Hersh, W.: Why batch and user evaluations do not give the same results. In: SIGIR, pp. 225\u2013231 (2001)","DOI":"10.1145\/383952.383992"},{"key":"88_CR31","doi-asserted-by":"crossref","unstructured":"Turpin, A., Scholar, F.: User performance versus precision measures for simple search tasks. In: SIGIR, pp. 11\u201318 (2006)","DOI":"10.1145\/1148170.1148176"},{"key":"88_CR32","doi-asserted-by":"crossref","unstructured":"Wang, K., Walker, T., Zheng, Z.: PSkip: estimating relevance ranking quality from web search clickthrough data. In: KDD, pp. 1355\u20131364 (2009)","DOI":"10.1145\/1557019.1557164"},{"key":"88_CR33","doi-asserted-by":"crossref","unstructured":"Wang, K., Gloy, N., Li, X.: Inferring search behaviors using partially observable Markov (POM) model. In: WSDM (2010)","DOI":"10.1145\/1718487.1718514"},{"key":"88_CR34","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Verma, M., Craswell, N., Radlinski, F., Bailey, P.: Relevance and effort: an analysis of document utility. In: CIKM (2014)","DOI":"10.1145\/2661829.2661953"},{"key":"88_CR35","doi-asserted-by":"crossref","unstructured":"Yue, Y., Joachims, T.: Interactively optimizing information retrieval systems as a dueling bandits problem. In: ICML 2009, pp. 1201\u20131208 (2009)","DOI":"10.1145\/1553374.1553527"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-30671-1_88","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,6,14]],"date-time":"2022-06-14T20:32:40Z","timestamp":1655238760000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-30671-1_88"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016]]},"ISBN":["9783319306704","9783319306711"],"references-count":35,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-30671-1_88","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2016]]}}}