{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,9]],"date-time":"2025-09-09T22:14:38Z","timestamp":1757456078029,"version":"3.40.3"},"publisher-location":"Berlin, Heidelberg","reference-count":42,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642047923"},{"type":"electronic","value":"9783642047930"}],"license":[{"start":{"date-parts":[[2009,1,1]],"date-time":"2009-01-01T00:00:00Z","timestamp":1230768000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2009]]},"DOI":"10.1007\/978-3-642-04793-0_7","type":"book-chapter","created":{"date-parts":[[2009,9,17]],"date-time":"2009-09-17T10:08:08Z","timestamp":1253182088000},"page":"109-121","source":"Crossref","is-referenced-by-count":6,"title":["Multimodal Corpora Annotation: Validation Methods to Assess Coding Scheme Reliability"],"prefix":"10.1007","author":[{"given":"Federica","family":"Cavicchio","sequence":"first","affiliation":[]},{"given":"Massimo","family":"Poesio","sequence":"additional","affiliation":[]}],"member":"297","reference":[{"key":"7_CR1","unstructured":"Abrilian, S., Buisine, S., Devillers, L., Martin, J.-C.: EmoTV: Annotation of Real-life Emotions for the Specification of Multimodal Affective Interfaces. In: Proceedings of International HCI (2005)"},{"issue":"3-4","key":"7_CR2","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1007\/s10579-007-9061-5","volume":"41","author":"J. Allwood","year":"2007","unstructured":"Allwood, J., Cerrato, L., Jokinen, K., Navarretta, C., Paggio, P.: The MUMIN coding scheme for the annotation of feedback, turn management and sequencing phenomena. Language Resources and Evaluation\u00a041(3-4), 273\u2013287 (2007)","journal-title":"Language Resources and Evaluation"},{"key":"7_CR3","unstructured":"Allwood, J., Cerrato, L., Jokinen, K., Navarretta, C., Paggio, P.: A Coding Scheme for the Annotation of Feedback, Turn Management and Sequencing Phenomena. In: Martin, J.-C., K\u00fchnlein, P., Paggio, P., Stiefelhagen, R., Pianesi, F. (eds.) Multimodal Corpora: From Multimodal Behavior Theories to Usable Models, pp. 38\u201342 (2006)"},{"key":"7_CR4","doi-asserted-by":"publisher","first-page":"555","DOI":"10.1162\/coli.07-034-R2","volume":"34","author":"R. Artstein","year":"2008","unstructured":"Artstein, R., Poesio, M.: Inter-Coder Agreement for Computational Linguistics. Computational Linguistics\u00a034, 555\u2013596 (2008)","journal-title":"Computational Linguistics"},{"key":"7_CR5","unstructured":"Artstein, R., Poesio, M.: Bias decreases in proportion to the number of annotators. In: Proceedings of FG-MoL (2005)"},{"key":"7_CR6","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1007\/s10579-007-9040-x","volume":"41","author":"J. Carletta","year":"2007","unstructured":"Carletta, J.: Unleashing the killer corpus: experiences in creating the multi-everything AMI Meeting Corpus. Language Resources and Evaluation\u00a041, 181\u2013190 (2007)","journal-title":"Language Resources and Evaluation"},{"key":"7_CR7","first-page":"249","volume":"22","author":"J. Carletta","year":"1996","unstructured":"Carletta, J.: Assessing agreement on classification tasks: The kappa statistic. Computational Linguistics\u00a022, 249\u2013254 (1996)","journal-title":"Computational Linguistics"},{"key":"7_CR8","series-title":"Lecture Notes in Artificial Intelligence","doi-asserted-by":"publisher","first-page":"233","DOI":"10.1007\/978-3-540-69369-7_26","volume-title":"Perception in Multimodal Dialogue Systems","author":"F. Cavicchio","year":"2008","unstructured":"Cavicchio, F., Poesio, M.: Annotation of Emotion in Dialogue: The Emotion in Cooperation in Perception. In: Andr\u00e9, E., Dybkj\u00e6r, L., Minker, W., Neumann, H., Pieraccini, R., Weber, M. (eds.) PIT 2008. LNCS (LNAI), vol.\u00a05078, pp. 233\u2013239. Springer, Heidelberg (2008)"},{"key":"7_CR9","unstructured":"Cerrato, L.: A coding scheme for the annotation of feedback phenomena in conversational speech. In: Martin, J.-C., Os, E.D., K\u00fchnlein, P., Boves, L., Paggio, P., Catizone, R. (eds.) Proceedings of Workshop Multimodal Corpora: Models of Human Behavior for the Specification and Evaluation of Multimodal Input and Output Interfaces, pp. 25\u201328 (2004)"},{"key":"7_CR10","unstructured":"Colletta, J.-M., Kunene, R., Venouil, A., Tcherkassof, A.: Double Level Analysis of the Multimodal Expressions of Emotions in Human-machine Interaction. In: Martin, J.-C., Patrizia, P., Kipp, M., Heylen, D. (eds.) Multimodal Corpora: From Models of Natural Interaction to Systems and Applications, pp. 5\u201311 (2008)"},{"key":"7_CR11","series-title":"LNAI","doi-asserted-by":"publisher","first-page":"54","DOI":"10.1007\/978-3-642-04793-0_4","volume-title":"Multimodal Corpora: From Models of Natural Interaction to Systems and Applications","author":"J.-M. Colletta","year":"2009","unstructured":"Colletta, J.-M., Kunene, R.N., Venouil, A., Kaufmann, V., Simon, J.-P.: Multitrack Annotation of Child Language and Gestures. In: Kipp, M., Martin, J.-C., Paggio, P., Heylen, D. (eds.) Multimodal Corpora: From Models of Natural Interaction to Systems and Applications, LNAI 5509. LNCS (LNAI), vol.\u00a05509, pp. 54\u201372. Springer, Heidelberg (2009)"},{"key":"7_CR12","doi-asserted-by":"crossref","unstructured":"Douglas-Cowie, E., Devillers, L., Martin, J.-C., Cowie, R., Savvidou, S., Abrilian, S., Cox, C.: Multimodal Databases of Everyday Emotion: Facing up to Complexity. In: Interspeech 2005, Lisbon, Portugal, September 4-8, pp. 813\u2013816 (2005)","DOI":"10.21437\/Interspeech.2005-381"},{"key":"7_CR13","unstructured":"Dybkj\u00e6r, L., Bernsen, N.O.: Recommendations for Natural Interactivity and Multimodal Annotation Schemes. In: Martin, J.-C., Os, E.D., K\u00fchnlein, P., Boves, L., Paggio, P., Catizone, R. (eds.) Proceedings of Workshop Multimodal Corpora: Models of Human Behavior For The Specification And Evaluation of Multimodal Input And Output Interfaces, pp. 5\u20138 (2004)"},{"key":"7_CR14","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1007\/BF00992253","volume":"10","author":"P. Ekman","year":"1986","unstructured":"Ekman, P., Friesen, W.V.: A new pan-cultural facial expression of emotion. Motivation and emotion\u00a010, 159\u2013168 (1986)","journal-title":"Motivation and emotion"},{"key":"7_CR15","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"J.L. Fleiss","year":"1971","unstructured":"Fleiss, J.L.: Measuring nominal scale agreement among many raters. Psychological Bulletin\u00a076, 378\u2013382 (1971)","journal-title":"Psychological Bulletin"},{"key":"7_CR16","doi-asserted-by":"publisher","first-page":"1094","DOI":"10.1080\/02699930701626582","volume":"22","author":"E. Goeleven","year":"2008","unstructured":"Goeleven, E., De Raedt, R., Leyman, L., Verschuere, B.: The Karolinska Directed Emotional Faces: A validation study. Cognition and Emotion\u00a022, 1094\u20131118 (2008)","journal-title":"Cognition and Emotion"},{"issue":"2","key":"7_CR17","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1080\/08839510601117169","volume":"21","author":"M. Guerini","year":"2007","unstructured":"Guerini, M., Stock, O., Zancanaro, M.: A Taxonomy of Strategies for Multimodal Persuasive Message Generation. Applied Artificial Intelligence\u00a021(2), 99\u2013136 (2007)","journal-title":"Applied Artificial Intelligence"},{"key":"7_CR18","unstructured":"Gut, U., Looks, K., Thies, A., Gibbon, D.: CoGesT\u2013Conversational Gesture Transcription System. Version 1.0. Technical report. Bielefeld University (2003)"},{"key":"7_CR19","doi-asserted-by":"publisher","first-page":"78","DOI":"10.1192\/bjp.137.1.78","volume":"137","author":"J.A. Hall","year":"1980","unstructured":"Hall, J.A., Levin, S.: Affect and verbal-nonverbal discrepancy in schizophrenic and non-schizophrenic family communication. British Journal of Psychiatry\u00a0137, 78\u201392 (1980)","journal-title":"British Journal of Psychiatry"},{"key":"7_CR20","doi-asserted-by":"publisher","first-page":"53","DOI":"10.1023\/B:JONB.0000017867.70191.68","volume":"28","author":"J.K. Hietanen","year":"2004","unstructured":"Hietanen, J.K., Lepp\u00e4nen, J.M., Lehtonen, U.: Perception of Emotions in the Hand Movement Quality of Finnish Sign Language. Journal of Nonverbal Behavior\u00a028, 53\u201364 (2004)","journal-title":"Journal of Nonverbal Behavior"},{"key":"7_CR21","unstructured":"Kipp, M., Neff, M., Albrecht, I.: An Annotation Scheme for Conversational Gestures: How to economically capture timing and form. In: Martin, J.-C., K\u00fchnlein, P., Paggio, P., Stiefelhagen, R., Pianesi, F. (eds.) Multimodal Corpora: From Multimodal Behavior Theories to Usable Models, pp. 24\u201328 (2006)"},{"key":"7_CR22","unstructured":"Kowtko, J.C., Isard, S.D., Doherty, G.M.: Conversational games within dialogue. Technical Report HCRC\/RP-31, Human Communication Research Centre, University of Edinburgh (1992)"},{"key":"7_CR23","first-page":"411","volume":"30","author":"K. Krippendorff","year":"2004","unstructured":"Krippendorff, K.: Reliability in content analysis: Some common misconceptions and recommendations. Human Communication Research\u00a030, 411\u2013433 (2004)","journal-title":"Human Communication Research"},{"key":"7_CR24","volume-title":"Content Analysis: An introduction to its Methodology","author":"K. Krippendorff","year":"1980","unstructured":"Krippendorff, K.: Content Analysis: An introduction to its Methodology. Sage Publications, Thousand Oaks (1980)"},{"key":"7_CR25","doi-asserted-by":"crossref","unstructured":"Litman, D., Hirschberg, J.: Disambiguating cue phrases in text and speech. In: Proceedings of the Thirteenth International Conference on Computational Linguistics, pp. 51\u2013256 (1990)","DOI":"10.3115\/997939.997983"},{"key":"7_CR26","unstructured":"Magno Caldognetto, E., Poggi, I., Cosi, P., Cavicchio, F., Merola, G.: Multimodal Score: an Anvil Based Annotation Scheme for Multimodal Audio-Video Analysis. In: Martin, J.-C., Os, E.D., K\u00fchnlein, P., Boves, L., Paggio, P., Catizone, R. (eds.) Proceedings of Workshop \"Multimodal Corpora: Models of Human Behavior For The Specification And Evaluation of Multimodal Input And Output Interfaces\", pp. 29\u201333 (2004)"},{"key":"7_CR27","unstructured":"Martel, C., Osborn, C., Friedman, J., Howard, P.: The FORM Gesture Annotation System. In: Maybury, M., Martin, J.-C. (eds.) Proceedings of Multimodal Resources and Multimodal Systems Evaluation Workshop, pp. 10\u201315 (2002)"},{"key":"7_CR28","unstructured":"Martell, C., Kroll, J.: Using FORM Gesture Data to Predict Phase Labels. In: Martin, J.-C., K\u00fchnlein, P., Paggio, P., Stiefelhagen, R., Pianesi, F. (eds.) Multimodal Corpora: From Multimodal Behavior Theories to Usable Models, pp. 29\u201332 (2006)"},{"key":"7_CR29","volume-title":"Hand and mind: What the hands reveal about thought","author":"D. McNeill","year":"1992","unstructured":"McNeill, D.: Hand and mind: What the hands reveal about thought. University of Chicago Press, Chicago (1992)"},{"key":"7_CR30","doi-asserted-by":"crossref","unstructured":"Passonneau, R.J., Litman, D.: Intention-based segmentation: human reliability and correlation with linguistic cues. In: Proceedings of the 31st Annual Meeting of the ACL, pp. 148\u2013155 (1993)","DOI":"10.3115\/981574.981594"},{"key":"7_CR31","unstructured":"Pianesi, F., Leonardi, C., Zancanaro, M.: Multimodal Annotated Corpora of Consensus Decision Making Meetings. In: Martin, J.-C., K\u00fchnlein, P., Paggio, P., Stiefelhagen, R., Pianesi, F. (eds.) Multimodal Corpora: From Multimodal Behavior Theories to Usable Models, pp. 6\u20139 (2006)"},{"key":"7_CR32","series-title":"LNAI","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1007\/978-3-642-04793-0_5","volume-title":"Multimodal Corpora: From Models of Natural Interaction to Systems and Applications","author":"I. Poggi","year":"2009","unstructured":"Poggi, I., Vincze, L.: The Persuasive Impact of Gesture and Gaze. In: Kipp, M., Martin, J.-C., Paggio, P., Heylen, D. (eds.) Multimodal Corpora: From Models of Natural Interaction to Systems and Applications, LNAI 5509. LNCS (LNAI), vol.\u00a05509, pp. 73\u201392. Springer, Heidelberg (2009)"},{"key":"7_CR33","volume-title":"Mind, hands, face and body. A goal and belief view of multimodal communication","author":"I. Poggi","year":"2007","unstructured":"Poggi, I.: Mind, hands, face and body. A goal and belief view of multimodal communication. Weidler Buchverlag, Berlin (2007)"},{"issue":"3-4","key":"7_CR34","doi-asserted-by":"publisher","first-page":"215","DOI":"10.1007\/s10579-007-9059-z","volume":"41","author":"I. Poggi","year":"2007","unstructured":"Poggi, I., Cavicchio, F., Magno Caldognetto, E.: Irony in a judicial debate: analyzing the subtleties of irony while testing the subtleties of an annotation scheme. Language Resources and Evaluation\u00a041(3-4), 215\u2013232 (2007)","journal-title":"Language Resources and Evaluation"},{"key":"7_CR35","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1162\/coli.2008.34.3.319","volume":"34","author":"D. Reidsma","year":"2008","unstructured":"Reidsma, D., Carletta, J.: Reliability Measurement without Limits. Computational Linguistics\u00a034, 319\u2013326 (2008)","journal-title":"Computational Linguistics"},{"key":"7_CR36","series-title":"LNAI","doi-asserted-by":"publisher","first-page":"122","DOI":"10.1007\/978-3-642-04793-0_8","volume-title":"Multimodal Corpora: From Models of Natural Interaction to Systems and Applications","author":"D. Reidsma","year":"2009","unstructured":"Reidsma, D., Heylen, D., op den Akker, R.: On the Contextual Analysis of Agreement Scores. In: Kipp, M., Martin, J.-C., Paggio, P., Heylen, D. (eds.) Multimodal Corpora: From Models of Natural Interaction to Systems and Applications. LNCS (LNAI), vol.\u00a05509, pp. 122\u2013137. Springer, Heidelberg (2009)"},{"key":"7_CR37","doi-asserted-by":"publisher","first-page":"321","DOI":"10.1086\/266577","volume":"19","author":"W.A. Scott","year":"1955","unstructured":"Scott, W.A.: Reliability of content analysis: The case of nominal scale coding. Public Opinion Quarterly\u00a019, 321\u2013325 (1955)","journal-title":"Public Opinion Quarterly"},{"key":"7_CR38","volume-title":"Nonparametric Statistics for the Behavioral Sciences","author":"S. Siegel","year":"1988","unstructured":"Siegel, S., Castellan Jr., N.J.: Nonparametric Statistics for the Behavioral Sciences, 2nd edn. McGraw-Hill, New York (1988)","edition":"2"},{"key":"7_CR39","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"193","DOI":"10.1007\/978-3-540-70987-9_22","volume-title":"Adaptive Hypermedia and Adaptive Web-Based Systems","author":"S. Sosnovsky","year":"2008","unstructured":"Sosnovsky, S., Brusilovsky, P., Lee, D.H., Zadorozhny, V., Zhou, X.: Re-assessing the Value of Adaptive Navigation Support in E-Learning Context. In: Nejdl, W., Kay, J., Pu, P., Herder, E. (eds.) AH 2008. LNCS, vol.\u00a05149, pp. 193\u2013203. Springer, Heidelberg (2008)"},{"key":"7_CR40","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/BF00987006","volume":"17","author":"H.L. Wagner","year":"1993","unstructured":"Wagner, H.L.: On measuring performance in category judgment studies on nonverbal behavior. Journal of Non-verbal Behavior\u00a017, 3\u201328 (1993)","journal-title":"Journal of Non-verbal Behavior"},{"key":"7_CR41","doi-asserted-by":"publisher","first-page":"201","DOI":"10.1007\/BF00986922","volume":"15","author":"H.L. Wagner","year":"1991","unstructured":"Wagner, H.L., Smith, J.: Facial Expressions in the Presence of Friends or Strangers. Journal of Nonverbal Behavior\u00a015, 201\u2013214 (1991)","journal-title":"Journal of Nonverbal Behavior"},{"key":"7_CR42","volume-title":"Experimental Psychology","author":"R.S. Woodworth","year":"1938","unstructured":"Woodworth, R.S.: Experimental Psychology, 1st edn. Henry Holt, New York (1938)","edition":"1"}],"container-title":["Lecture Notes in Computer Science","Multimodal Corpora"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-04793-0_7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,10,10]],"date-time":"2021-10-10T23:07:42Z","timestamp":1633907262000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-04793-0_7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009]]},"ISBN":["9783642047923","9783642047930"],"references-count":42,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-04793-0_7","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2009]]}}}