{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T10:29:36Z","timestamp":1759228176054},"publisher-location":"Berlin, Heidelberg","reference-count":21,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642332463"},{"type":"electronic","value":"9783642332470"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2012]]},"DOI":"10.1007\/978-3-642-33247-0_14","type":"book-chapter","created":{"date-parts":[[2012,9,11]],"date-time":"2012-09-11T22:41:12Z","timestamp":1347403272000},"page":"124-135","source":"Crossref","is-referenced-by-count":14,"title":["Better than Their Reputation? On the Reliability of Relevance Assessments with Students"],"prefix":"10.1007","author":[{"given":"Philipp","family":"Schaer","sequence":"first","affiliation":[]}],"member":"297","reference":[{"key":"14_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"602","DOI":"10.1007\/978-3-642-12275-0_57","volume-title":"Advances in Information Retrieval","author":"O. Alonso","year":"2010","unstructured":"Alonso, O., Schenkel, R., Theobald, M.: Crowdsourcing Assessments for XML Ranked Retrieval. In: Gurrin, C., He, Y., Kazai, G., Kruschwitz, U., Little, S., Roelleke, T., R\u00fcger, S., van Rijsbergen, K. (eds.) ECIR 2010. LNCS, vol.\u00a05993, pp. 602\u2013606. Springer, Heidelberg (2010)"},{"key":"14_CR2","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1007\/978-3-642-20161-5_15","volume-title":"Advances in Information Retrieval","author":"J. Arguello","year":"2011","unstructured":"Arguello, J., Diaz, F., Callan, J., Carterette, B.: A Methodology for Evaluating Aggregated Search Results. In: Clough, P., Foley, C., Gurrin, C., Jones, G.J.F., Kraaij, W., Lee, H., Mudoch, V. (eds.) ECIR 2011. LNCS, vol.\u00a06611, pp. 141\u2013152. Springer, Heidelberg (2011)"},{"issue":"4","key":"14_CR3","doi-asserted-by":"publisher","first-page":"555","DOI":"10.1162\/coli.07-034-R2","volume":"34","author":"R. Artstein","year":"2008","unstructured":"Artstein, R., Poesio, M.: Inter-coder agreement for computational linguistics. Comput. Linguist.\u00a034(4), 555\u2013596 (2008)","journal-title":"Comput. Linguist."},{"key":"14_CR4","doi-asserted-by":"publisher","first-page":"667","DOI":"10.1145\/1390334.1390447","volume-title":"Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"P. Bailey","year":"2008","unstructured":"Bailey, P., Craswell, N., Soboroff, I., Thomas, P., de Vries, A.P., Yilmaz, E.: Relevance assessment: are judges exchangeable and does it matter. In: Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 667\u2013674. ACM, New York (2008)"},{"key":"14_CR5","doi-asserted-by":"crossref","first-page":"784","DOI":"10.1145\/1571941.1572127","volume-title":"Proceedings of the 32nd International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"A. Bermingham","year":"2009","unstructured":"Bermingham, A., Smeaton, A.F.: A study of inter-annotator agreement for opinion retrieval. In: Proceedings of the 32nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 784\u2013785. ACM, New York (2009)"},{"issue":"10","key":"14_CR6","doi-asserted-by":"publisher","first-page":"913","DOI":"10.1002\/asi.10286","volume":"54","author":"P. Borlund","year":"2003","unstructured":"Borlund, P.: The concept of relevance in IR. Journal of the American Society for Information Science and Technology\u00a054(10), 913\u2013925 (2003)","journal-title":"Journal of the American Society for Information Science and Technology"},{"issue":"5","key":"14_CR7","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"J.L. Fleiss","year":"1971","unstructured":"Fleiss, J.L.: Measuring nominal scale agreement among many raters. Psychological Bulletin\u00a076(5), 378\u2013382 (1971)","journal-title":"Psychological Bulletin"},{"key":"14_CR8","unstructured":"Gamer, M., Lemon, J., Puspendra Singh, I.F.: irr: Various Coefficients of Interrater Reliability and Agreement (2010)"},{"key":"14_CR9","unstructured":"Greve, W., Wentura, D.: Wissenschaftliche Beobachtung: eine Einf\u00fchrung. Beltz, PsychologieVerlagsUnion, Weinheim (1997)"},{"key":"14_CR10","unstructured":"Krippendorff, K.: Computing Krippendorff\u2019s Alpha-Reliability (2011), http:\/\/repository.upenn.edu\/asc_papers\/43"},{"issue":"3","key":"14_CR11","first-page":"411","volume":"30","author":"K. Krippendorff","year":"2004","unstructured":"Krippendorff, K.: Reliability in Content Analysis: Some Common Misconceptions and Recommendations. Human Communication Research\u00a030(3), 411\u2013433 (2004)","journal-title":"Human Communication Research"},{"issue":"1","key":"14_CR12","doi-asserted-by":"publisher","first-page":"159","DOI":"10.2307\/2529310","volume":"33","author":"J.R. Landis","year":"1977","unstructured":"Landis, J.R., Koch, G.G.: The measurement of observer agreement for categorical data. Biometrics\u00a033(1), 159\u2013174 (1977)","journal-title":"Biometrics"},{"key":"14_CR13","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809071","volume-title":"Introduction to Information Retrieval","author":"C.D. Manning","year":"2008","unstructured":"Manning, C.D., Raghavan, P., Sch\u00fctze, H.: Introduction to Information Retrieval. Cambridge University Press, Cambridge (2008)"},{"key":"14_CR14","unstructured":"Mayr, P., Mutschke, P., Petras, V., Schaer, P., Sure, Y.: Applying Science Models for Search. In: 12. Internationales Symposium f\u00fcr Informationswissenschaft (ISI) (2011)"},{"issue":"1","key":"14_CR15","doi-asserted-by":"publisher","first-page":"349","DOI":"10.1007\/s11192-011-0430-x","volume":"89","author":"P. Mutschke","year":"2011","unstructured":"Mutschke, P., Mayr, P., Schaer, P., Sure, Y.: Science models as value-added services for scholarly information systems. Scientometrics\u00a089(1), 349\u2013364 (2011)","journal-title":"Scientometrics"},{"issue":"1","key":"14_CR16","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/1416950.1416951","volume":"27","author":"B. Piwowarski","year":"2008","unstructured":"Piwowarski, B., Trotman, A., Lalmas, M.: Sound and complete relevance assessment for XML retrieval. ACM Trans. Inf. Syst. 27(1), 1:1\u20131:37 (2008)","journal-title":"ACM Trans. Inf. Syst."},{"key":"14_CR17","unstructured":"R Development Core Team: R: A Language and Environment for Statistical Computing, Vienna, Austria (2011)."},{"key":"14_CR18","unstructured":"Schaer, P., Mayr, P., Mutschke, P.: Implications of Inter-Rater Agreement on a Student Information Retrieval Evaluation. In: Atzm\u00fcller, M., Benz, D., Hotho, A., Stumme, G. (eds.) Proceedings of LWA 2010 Workshop-Woche: Lernen, Wissen & Adaptivitaet, Kassel, Germany (2010)"},{"issue":"1","key":"14_CR19","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1016\/j.ipm.2010.02.005","volume":"47","author":"R. Song","year":"2011","unstructured":"Song, R., Guo, Q., Zhang, R., Xin, G., Wen, J.-R., Yu, Y., Hon, H.-W.: Select-the-Best-Ones: A new way to judge relative relevance. Inf. Process. Manage.\u00a047(1), 37\u201352 (2011)","journal-title":"Inf. Process. Manage."},{"key":"14_CR20","doi-asserted-by":"crossref","first-page":"806","DOI":"10.1145\/1571941.1572138","volume-title":"Proceedings of the 32nd International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"E.M. Voorhees","year":"2009","unstructured":"Voorhees, E.M.: Topic set size redux. In: Proceedings of the 32nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 806\u2013807. ACM, New York (2009)"},{"issue":"5","key":"14_CR21","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"E.M. Voorhees","year":"2000","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inf. Process. Manage.\u00a036(5), 697\u2013716 (2000)","journal-title":"Inf. Process. Manage."}],"container-title":["Lecture Notes in Computer Science","Information Access Evaluation. Multilinguality, Multimodality, and Visual Analytics"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-33247-0_14.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,28]],"date-time":"2022-01-28T17:55:22Z","timestamp":1643392522000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-33247-0_14"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012]]},"ISBN":["9783642332463","9783642332470"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-33247-0_14","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2012]]}}}