{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T22:07:41Z","timestamp":1759961261701},"publisher-location":"Cham","reference-count":15,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319445632"},{"type":"electronic","value":"9783319445649"}],"license":[{"start":{"date-parts":[[2016,1,1]],"date-time":"2016-01-01T00:00:00Z","timestamp":1451606400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2016]]},"DOI":"10.1007\/978-3-319-44564-9_4","type":"book-chapter","created":{"date-parts":[[2016,8,21]],"date-time":"2016-08-21T22:00:36Z","timestamp":1471816836000},"page":"40-53","source":"Crossref","is-referenced-by-count":11,"title":["Assessors Agreement: A Case Study Across Assessor Type, Payment Levels, Query Variations and Relevance Dimensions"],"prefix":"10.1007","author":[{"given":"Joao","family":"Palotti","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Guido","family":"Zuccon","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Johannes","family":"Bernhardt","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lorraine","family":"Goeuriot","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2016,8,23]]},"reference":[{"key":"4_CR1","doi-asserted-by":"crossref","unstructured":"Azzopardi, L.: Query side evaluation: an empirical analysis of effectiveness and effort. In: Proceedings of SIGIR, pp. 556\u2013563 (2009)","DOI":"10.1145\/1571941.1572037"},{"key":"4_CR2","doi-asserted-by":"crossref","unstructured":"Bailey, P., Craswell, N., Soboroff, I., Thomas, P., de Vries, A.P., Yilmaz, E.: Relevance assessment: are judges exchangeable and does it matter. In: Proceedings of SIGIR, pp. 667\u2013674 (2008)","DOI":"10.1145\/1390334.1390447"},{"key":"4_CR3","doi-asserted-by":"crossref","unstructured":"Bailey, P., Moffat, A., Scholer, F., Thomas, P.: User variability and IR system evaluation. In: Proceedings of SIGIR, pp. 625\u2013634 (2015)","DOI":"10.1145\/2766462.2767728"},{"key":"4_CR4","doi-asserted-by":"crossref","unstructured":"Carterette, B., Soboroff, I.: The effect of assessor error on IR system evaluation. In: Proceedings of SIGIR, pp. 539\u2013546 (2010)","DOI":"10.1145\/1835449.1835540"},{"key":"4_CR5","doi-asserted-by":"crossref","unstructured":"Koopman, B., Zuccon, G.: Relevation!: an open source system for information retrieval relevance assessment. In: Proceedings of SIGIR, pp. 1243\u20131244. ACM (2014)","DOI":"10.1145\/2600428.2611175"},{"key":"4_CR6","unstructured":"Koopman, B., Zuccon, G.: Why assessing relevance in medical IR is demanding. In: Medical Information Retrieval Workshop at SIGIR 2014 (2014)"},{"issue":"4","key":"4_CR7","doi-asserted-by":"crossref","first-page":"343","DOI":"10.1016\/0020-0271(68)90029-6","volume":"4","author":"ME Lesk","year":"1968","unstructured":"Lesk, M.E., Salton, G.: Relevance assessments and retrieval system evaluation. Inform. Storage Retrieval 4(4), 343\u2013359 (1968)","journal-title":"Inform. Storage Retrieval"},{"issue":"1","key":"4_CR8","first-page":"2","volume":"27","author":"A Moffat","year":"2008","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inform. Syst. (TOIS) 27(1), 2 (2008)","journal-title":"ACM Trans. Inform. Syst. (TOIS)"},{"key":"4_CR9","unstructured":"Palotti, J., Zuccon, G., Goeuriot, L., Kelly, L., Hanbury, A., Jones, G.J., Lupu, M., Pecina, P.: CLEF eHealth evaluation lab: retrieving information about medical symptoms. In: CLEF (2015)"},{"key":"4_CR10","doi-asserted-by":"crossref","unstructured":"Stanton, I., Ieong, S., Mishra, N.: Circumlocution in diagnostic medical queries. In: Proceedings of SIGIR, pp. 133\u2013142. ACM (2014)","DOI":"10.1145\/2600428.2609589"},{"issue":"5","key":"4_CR11","doi-asserted-by":"crossref","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"EM Voorhees","year":"2000","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inform. Process. Manage. 36(5), 697\u2013716 (2000)","journal-title":"Inform. Process. Manage."},{"key":"4_CR12","volume-title":"TREC: Experiment and Evaluation in Information Retrieval","author":"EM Voorhees","year":"2005","unstructured":"Voorhees, E.M., Harman, D.K.: TREC: Experiment and Evaluation in Information Retrieval, vol. 1. MIT Press, Cambridge (2005)"},{"key":"4_CR13","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1007\/978-3-319-30671-1_21","volume-title":"Advances in Information Retrieval","author":"G Zuccon","year":"2016","unstructured":"Zuccon, G.: Understandability biased evaluation for information retrieval. In: Ferro, N., Crestani, F., Moens, M.F., Mothe, J., Silvestri, F., Di Nunzio, G.M., Hauff, C., Silvello, G. (eds.) ECIR 2016. LNCS, vol. 9626, pp. 280\u2013292. Springer, Heielberg (2016)"},{"key":"4_CR14","unstructured":"Zuccon, G., Koopman, B.: Integrating understandability in the evaluation of consumer health search engines. In: Medical Information Retrieval Workshop at SIGIR 2014, p. 32 (2014)"},{"key":"4_CR15","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"crossref","first-page":"562","DOI":"10.1007\/978-3-319-16354-3_62","volume-title":"Advances in Information Retrieval","author":"G Zuccon","year":"2015","unstructured":"Zuccon, G., Koopman, B., Palotti, J.: Diagnose this if you can: on the effectiveness of search engines in finding medical self-diagnosis information. In: Hanbury, A., Kazai, G., Rauber, A., Fuhr, N. (eds.) ECIR 2015. LNCS, vol. 9022, pp. 562\u2013567. Springer, Heidelberg (2015)"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-44564-9_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,6,24]],"date-time":"2017-06-24T17:05:18Z","timestamp":1498323918000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-44564-9_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016]]},"ISBN":["9783319445632","9783319445649"],"references-count":15,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-44564-9_4","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2016]]}}}