{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T06:10:10Z","timestamp":1748758210669,"version":"3.41.0"},"publisher-location":"Cham","reference-count":22,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319289397"},{"type":"electronic","value":"9783319289403"}],"license":[{"start":{"date-parts":[[2015,1,1]],"date-time":"2015-01-01T00:00:00Z","timestamp":1420070400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2015]]},"DOI":"10.1007\/978-3-319-28940-3_26","type":"book-chapter","created":{"date-parts":[[2016,1,21]],"date-time":"2016-01-21T12:04:06Z","timestamp":1453377846000},"page":"332-344","source":"Crossref","is-referenced-by-count":2,"title":["Towards Nuanced System Evaluation Based on Implicit User Expectations"],"prefix":"10.1007","author":[{"given":"Paul","family":"Thomas","sequence":"first","affiliation":[]},{"given":"Peter","family":"Bailey","sequence":"additional","affiliation":[]},{"given":"Alistair","family":"Moffat","sequence":"additional","affiliation":[]},{"given":"Falk","family":"Scholer","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2016,1,22]]},"reference":[{"key":"26_CR1","unstructured":"The roar of the crowd. The Economist (2012)"},{"issue":"6","key":"26_CR2","doi-asserted-by":"publisher","first-page":"716","DOI":"10.1109\/TAC.1974.1100705","volume":"19","author":"H Akaike","year":"1974","unstructured":"Akaike, H.: A new look at the statistical model identification. IEEE Trans. Autom. Control 19(6), 716\u2013723 (1974)","journal-title":"IEEE Trans. Autom. Control"},{"key":"26_CR3","unstructured":"Alonso, O., Mizzaro, S.: Can we get rid of TREC assessors? Using Mechanical Turk for relevance assessment. In: Proceedings of the SIGIR Workshop. Future IR Evaluation, pp. 15\u201316 (2009)"},{"key":"26_CR4","volume-title":"A Taxonomy for Learning, Teaching and Assessing: A Revision of Bloom\u2019s Taxonomy of Educational Objectives","author":"LW Anderson","year":"2001","unstructured":"Anderson, L.W., Krathwohl, D.A.: A Taxonomy for Learning, Teaching and Assessing: A Revision of Bloom\u2019s Taxonomy of Educational Objectives. Longman, New York (2001)"},{"key":"26_CR5","doi-asserted-by":"crossref","unstructured":"Bailey, P., Moffat, A., Scholer, F., Thomas, P.: User variability and IR system evaluation. In: Proceedings of SIGIR, pp. 625\u2013634 (2015)","DOI":"10.1145\/2766462.2767728"},{"key":"26_CR6","doi-asserted-by":"crossref","unstructured":"Bennett, P.N., White, R.W., Chu, W., Dumais, S.T., Bailey, P., Borisyuk, F., Cui, X.: Modeling the impact of short-and long-term behavior on search personalization. In: Proceedings of SIGIR, pp. 185\u2013194 (2012)","DOI":"10.1145\/2348283.2348312"},{"key":"26_CR7","doi-asserted-by":"crossref","unstructured":"Buckley, C., Walz, J.: The TREC-8 query track. In: Proceedings of TREC 1999. NIST Special Publication 500\u2013246 (1999)","DOI":"10.6028\/NIST.SP.500-246.query-overview"},{"key":"26_CR8","doi-asserted-by":"crossref","unstructured":"Burges, C., Shaked, T., Renshaw, E., Lazier, A., Deeds, M., Hamilton, N., Hullender, G.: Learning to rank using gradient descent. In: Proceedings of CIKM, pp. 89\u201396 (2005)","DOI":"10.1145\/1102351.1102363"},{"key":"26_CR9","doi-asserted-by":"crossref","unstructured":"Chapelle, O., Metzler, D., Zhang, Y., Grinspan, P.: Expected reciprocal rank for graded relevance. In: Proceedings of CIKM, pp. 621\u2013630 (2009)","DOI":"10.1145\/1645953.1646033"},{"issue":"4","key":"26_CR10","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. 20(4), 422\u2013446 (2002)","journal-title":"ACM Trans. Inf. Syst."},{"key":"26_CR11","doi-asserted-by":"crossref","unstructured":"Joachims, T., Granka, L., Pan, B., Hembrooke, H., Gay, G.: Accurately interpreting clickthrough data as implicit feedback. In: Proceedings of SIGIR, pp. 154\u2013161 (2005)","DOI":"10.1145\/1076034.1076063"},{"key":"26_CR12","doi-asserted-by":"crossref","unstructured":"Kelly, D., Arguello, J., Edwards, A., Wu, W.C.: Development and evaluation of search tasks for IIR experiments using a cognitive complexity framework. In: Proceeding of ICTIR (2015)","DOI":"10.1145\/2808194.2809465"},{"issue":"4","key":"26_CR13","doi-asserted-by":"publisher","first-page":"393","DOI":"10.1002\/asi.20127","volume":"56","author":"SJ Lin","year":"2005","unstructured":"Lin, S.J., Belkin, N.: Validation of a model of information seeking over multiple search sessions. J. Am. Soc. Inf. Sci. Technol. 56(4), 393\u2013415 (2005)","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"key":"26_CR14","doi-asserted-by":"crossref","unstructured":"Moffat, A., Thomas, P., Scholer, F.: Users versus models: what observation tells us about effectiveness metrics. In: Proceedings of CIKM, pp. 659\u2013668 (2013)","DOI":"10.1145\/2505515.2507665"},{"issue":"1","key":"26_CR15","doi-asserted-by":"publisher","first-page":"2:1","DOI":"10.1145\/1416950.1416952","volume":"27","author":"A Moffat","year":"2008","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Syst. 27(1), 2:1\u20132:27 (2008)","journal-title":"ACM Trans. Inf. Syst."},{"key":"26_CR16","doi-asserted-by":"crossref","unstructured":"Phan, N., Bailey, P., Wilkinson, R.: Understanding the relationship of information need specificity to search query length. In: Proceedings of SIGIR, pp. 709\u2013710 (2007)","DOI":"10.1145\/1277741.1277870"},{"key":"26_CR17","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Clarke, C.L.A.: Time-based calibration of effectiveness measures. In: Proceedings of SIGIR, pp. 95\u2013104 (2012)","DOI":"10.1145\/2348283.2348300"},{"key":"26_CR18","doi-asserted-by":"crossref","unstructured":"Smucker, M., Kazai, G., Lease, M.: The TREC-12 crowdsourcing track. In: Proceedings of TREC 2012. NIST Special Publication 500\u2013298 (2012)","DOI":"10.6028\/NIST.SP.500-298.crowd-overview"},{"key":"26_CR19","doi-asserted-by":"crossref","unstructured":"Sormunen, E.: Liberal relevance criteria of TREC: counting on negligible documents? In: Proceedings of SIGIR, pp. 324\u2013330 (2002)","DOI":"10.1145\/564431.564433"},{"key":"26_CR20","doi-asserted-by":"crossref","unstructured":"Teevan, J., Dumais, S.T., Liebling, D.J.: To personalize or not to personalize: modeling queries with variation in user intent. In: Proceedings of SIGIR, pp. 163\u2013170 (2008)","DOI":"10.1145\/1390334.1390364"},{"key":"26_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"416","DOI":"10.1007\/978-3-642-45068-6_36","volume-title":"Information Retrieval Technology","author":"P Thomas","year":"2013","unstructured":"Thomas, P., Scholer, F., Moffat, A.: What users do: the eyes have it. In: Banchs, R.E., Silvestri, F., Liu, T.-Y., Zhang, M., Gao, S., Lang, J. (eds.) AIRS 2013. LNCS, vol. 8281, pp. 416\u2013427. Springer, Heidelberg (2013)"},{"key":"26_CR22","doi-asserted-by":"crossref","unstructured":"Wu, W.C., Kelly, D., Edwards, A., Arguello, J.: Grannies, tanning beds, tattoos and NASCAR: evaluation of search tasks with varying levels of cognitive complexity. In: Proceedings of IIiX, pp. 254\u2013257 (2012)","DOI":"10.1145\/2362724.2362768"}],"container-title":["Lecture Notes in Computer Science","Information Retrieval Technology"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-28940-3_26","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T05:53:38Z","timestamp":1748757218000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-28940-3_26"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015]]},"ISBN":["9783319289397","9783319289403"],"references-count":22,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-28940-3_26","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2015]]}}}