{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,30]],"date-time":"2026-05-30T00:28:41Z","timestamp":1780100921186,"version":"3.54.0"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,10,19]],"date-time":"2020-10-19T00:00:00Z","timestamp":1603065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,10,20]]},"DOI":"10.1145\/3383652.3423892","type":"proceedings-article","created":{"date-parts":[[2020,10,19]],"date-time":"2020-10-19T22:10:31Z","timestamp":1603145431000},"page":"1-8","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Human-in-the-Loop AI for Analysis of Free Response Facial Expression Label Sets"],"prefix":"10.1145","author":[{"given":"Crystal","family":"Butler","sequence":"first","affiliation":[{"name":"Computer Science and Engineering, New York University, New York, NY USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Harriet","family":"Oster","sequence":"additional","affiliation":[{"name":"Psychology, New York University, New York, NY USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Julian","family":"Togelius","sequence":"additional","affiliation":[{"name":"Computer Science and Engineering, New York University, New York, NY USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2020,10,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1037\/a0025827"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cognition.2016.02.004"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1177\/0022022180113003"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3126594.3126640"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10919-019-00294-2"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1037\/emo0000236"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0057410"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1037\/emo0000247"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Jos\u00e9 Miguel Fern\u00e1ndez Dols and James A Russell (James Albert) editors. 2017. The Science of Facial Expression. Oxford University Press.  Jos\u00e9 Miguel Fern\u00e1ndez Dols and James A Russell (James Albert) editors. 2017. The Science of Facial Expression. Oxford University Press.","DOI":"10.1093\/acprof:oso\/9780190613501.003.0024"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1322355111"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-011-9181-9"},{"key":"e_1_3_2_1_12_1","volume-title":"Hager","author":"Ekman Paul","year":"2002","unstructured":"Paul Ekman , Wallace V. Friesen , and Joseph C . Hager . 2002 . A Human Face. In Facial Action Coding System: The manual on CD ROM. Paul Ekman, Wallace V. Friesen, and Joseph C. Hager. 2002. A Human Face. In Facial Action Coding System: The manual on CD ROM."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Paul Ekman and Erika L. Rosenberg (eds.). 2005. What the face reveals: basic and applied studies of spontaneous expression using the facial action coding system (FACS). Oxford University Press Oxford; New York.  Paul Ekman and Erika L. Rosenberg (eds.). 2005. What the face reveals: basic and applied studies of spontaneous expression using the facial action coding system (FACS). Oxford University Press Oxford; New York.","DOI":"10.1093\/acprof:oso\/9780195179644.001.0001"},{"key":"e_1_3_2_1_14_1","volume-title":"Retrieved","author":"Faruqui Manaal","year":"2020","unstructured":"Manaal Faruqui , Jesse Dodge , Sujay K. Jauhar , Chris Dyer , Eduard Hovy , and Noah A. Smith . 2015. Retrofitting Word Vectors to Semantic Lexicons. arXiv:1411.4166 [cs] . Retrieved February 29, 2020 from http:\/\/arxiv.org\/abs\/1411.4166 Manaal Faruqui, Jesse Dodge, Sujay K. Jauhar, Chris Dyer, Eduard Hovy, and Noah A. Smith. 2015. Retrofitting Word Vectors to Semantic Lexicons. arXiv:1411.4166 [cs]. Retrieved February 29, 2020 from http:\/\/arxiv.org\/abs\/1411.4166"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1080\/19312450709336664"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1080\/026999399379267"},{"key":"e_1_3_2_1_17_1","volume-title":"SimLex-999: Evaluating Semantic Models with (Genuine) Similarity Estimation. arXiv:1408.3456 [cs]. Retrieved","author":"Hill Felix","year":"2020","unstructured":"Felix Hill , Roi Reichart , and Anna Korhonen . 2014. SimLex-999: Evaluating Semantic Models with (Genuine) Similarity Estimation. arXiv:1408.3456 [cs]. Retrieved July 5, 2020 from http:\/\/arxiv.org\/abs\/1408.3456 Felix Hill, Roi Reichart, and Anna Korhonen. 2014. SimLex-999: Evaluating Semantic Models with (Genuine) Similarity Estimation. arXiv:1408.3456 [cs]. Retrieved July 5, 2020 from http:\/\/arxiv.org\/abs\/1408.3456"},{"key":"e_1_3_2_1_18_1","volume-title":"The Face of Emotion. Appleton-Century-Crofts","author":"Carroll E.","unstructured":"Carroll E. Izard 1971. Free-Response Labeling of Expressions . In The Face of Emotion. Appleton-Century-Crofts , New York, NY, U.S.A. , 267--281. Carroll E. Izard 1971. Free-Response Labeling of Expressions. In The Face of Emotion. Appleton-Century-Crofts, New York, NY, U.S.A., 267--281."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1200155109"},{"key":"e_1_3_2_1_20_1","volume-title":"Current Issues in Linguistic Theory","author":"Jarmasz Mario","unstructured":"Mario Jarmasz and Stan Szpakowicz . 2004. Roget's thesaurus and semantic similarity . In Current Issues in Linguistic Theory , Nicolas Nicolov, Kalina Bontcheva, Galia Angelova and Ruslan Mitkov (eds.). John Benjamins Publishing Company , Amsterdam , 111. https:\/\/doi.org\/10.1075\/cilt.260.12jar 10.1075\/cilt.260.12jar Mario Jarmasz and Stan Szpakowicz. 2004. Roget's thesaurus and semantic similarity. In Current Issues in Linguistic Theory, Nicolas Nicolov, Kalina Bontcheva, Galia Angelova and Ruslan Mitkov (eds.). John Benjamins Publishing Company, Amsterdam, 111. https:\/\/doi.org\/10.1075\/cilt.260.12jar"},{"key":"e_1_3_2_1_21_1","unstructured":"Thomas K Landauer and Susan T Dutnais. A Solution to Plato's Problem: The Latent Semantic Analysis Theory of Acquisition Induction and Representation of Knowledge. 30.  Thomas K Landauer and Susan T Dutnais. A Solution to Plato's Problem: The Latent Semantic Analysis Theory of Acquisition Induction and Representation of Knowledge. 30."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2010.5543262"},{"key":"e_1_3_2_1_23_1","volume-title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs]. Retrieved","author":"Liu Yinhan","year":"2020","unstructured":"Yinhan Liu , Myle Ott , Naman Goyal , Jingfei Du , Mandar Joshi , Danqi Chen , Omer Levy , Mike Lewis , Luke Zettlemoyer , and Veselin Stoyanov . 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs]. Retrieved August 2, 2020 from http:\/\/arxiv.org\/abs\/1907.11692 Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs]. Retrieved August 2, 2020 from http:\/\/arxiv.org\/abs\/1907.11692"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.87.4.529"},{"key":"e_1_3_2_1_25_1","volume-title":"Retrieved","author":"Lyons Michael","year":"1998","unstructured":"Michael Lyons , Miyuki Kamachi , and Jiro Gyoba . 1998 . Facial Expression Database: Japanese Female Facial Expression (JAFFE) Database . Retrieved October 19, 2016 from http:\/\/www.kasrl.org\/jaffe.html Michael Lyons, Miyuki Kamachi, and Jiro Gyoba. 1998. Facial Expression Database: Japanese Female Facial Expression (JAFFE) Database. Retrieved October 19, 2016 from http:\/\/www.kasrl.org\/jaffe.html"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1080\/01690969108406936"},{"key":"e_1_3_2_1_27_1","volume-title":"Counter-fitting Word Vectors to Linguistic Constraints. arXiv:1603.00892 [cs]. Retrieved","author":"Mrk\u0161i\u0107 Nikola","year":"2020","unstructured":"Nikola Mrk\u0161i\u0107 , Diarmuid \u00d3 S\u00e9aghdha , Blaise Thomson , Milica Ga\u0161i\u0107 , Lina Rojas-Barahona , Pei-Hao Su , David Vandyke , Tsung-Hsien Wen , and Steve Young . 2016. Counter-fitting Word Vectors to Linguistic Constraints. arXiv:1603.00892 [cs]. Retrieved July 5, 2020 from http:\/\/arxiv.org\/abs\/1603.00892 Nikola Mrk\u0161i\u0107, Diarmuid \u00d3 S\u00e9aghdha, Blaise Thomson, Milica Ga\u0161i\u0107, Lina Rojas-Barahona, Pei-Hao Su, David Vandyke, Tsung-Hsien Wen, and Steve Young. 2016. Counter-fitting Word Vectors to Linguistic Constraints. arXiv:1603.00892 [cs]. Retrieved July 5, 2020 from http:\/\/arxiv.org\/abs\/1603.00892"},{"key":"#cr-split#-e_1_3_2_1_28_1.1","doi-asserted-by":"crossref","unstructured":"Shushi Namba Russell S. Kabir Makoto Miyatani and Takashi Nakao. 2017. Spontaneous Facial Actions Map onto Emotional Experiences in a Non-social Context: Toward a Component-Based Approach. Frontiers in Psychology 8. https:\/\/doi.org\/10.3389\/fpsyg.2017.00633 10.3389\/fpsyg.2017.00633","DOI":"10.3389\/fpsyg.2017.00633"},{"key":"#cr-split#-e_1_3_2_1_28_1.2","doi-asserted-by":"crossref","unstructured":"Shushi Namba Russell S. Kabir Makoto Miyatani and Takashi Nakao. 2017. Spontaneous Facial Actions Map onto Emotional Experiences in a Non-social Context: Toward a Component-Based Approach. Frontiers in Psychology 8. https:\/\/doi.org\/10.3389\/fpsyg.2017.00633","DOI":"10.3389\/fpsyg.2017.00633"},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the Twenty-Eighth International Florida Artificial Intelligence Research Society Conference: 6.","author":"Niraula Nobal B","year":"2015","unstructured":"Nobal B Niraula , Dipesh Gautam , Rajendra Banjade , Nabin Maharjan , and Vasile Rus . 2015 . Combining Word Representations for Measuring Word Relatedness and Similarity . Proceedings of the Twenty-Eighth International Florida Artificial Intelligence Research Society Conference: 6. Nobal B Niraula, Dipesh Gautam, Rajendra Banjade, Nabin Maharjan, and Vasile Rus. 2015. Combining Word Representations for Measuring Word Relatedness and Similarity. Proceedings of the Twenty-Eighth International Florida Artificial Intelligence Research Society Conference: 6."},{"key":"e_1_3_2_1_30_1","volume-title":"Ohme","author":"Olszanowski Michal","year":"2015","unstructured":"Michal Olszanowski , Grzegorz Pochwatko , Krzysztof Kuklinski , Michal Scibor-Rylski , Peter Lewinski , and Rafal K . Ohme . 2015 . Warsaw set of emotional facial expression pictures: a validation study of facial display photographs. Frontiers in Psychology 5. https:\/\/doi.org\/10.3389\/fpsyg.2014.01516 10.3389\/fpsyg.2014.01516 Michal Olszanowski, Grzegorz Pochwatko, Krzysztof Kuklinski, Michal Scibor-Rylski, Peter Lewinski, and Rafal K. Ohme. 2015. Warsaw set of emotional facial expression pictures: a validation study of facial display photographs. Frontiers in Psychology 5. https:\/\/doi.org\/10.3389\/fpsyg.2014.01516"},{"key":"e_1_3_2_1_31_1","unstructured":"Harriet Oster and May Kyi. 1999. Free Response Judgments of Spontaneous Facial Expressions.  Harriet Oster and May Kyi. 1999. Free Response Judgments of Spontaneous Facial Expressions."},{"key":"e_1_3_2_1_32_1","volume-title":"Boston: Pearson Addison Wesley.","author":"Michael Pang-Ning","unstructured":"Pang-Ning Tan; Michael Steinbach; Vipin Kumar . 2005. Chapter on Cluster Analysis.pdf. In Introduction to data mining. Boston: Pearson Addison Wesley. Pang-Ning Tan; Michael Steinbach; Vipin Kumar. 2005. Chapter on Cluster Analysis.pdf. In Introduction to data mining. Boston: Pearson Addison Wesley."},{"key":"e_1_3_2_1_33_1","first-page":"14","volume-title":"Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP '14)","author":"Pennington Jeffrey","unstructured":"Jeffrey Pennington , Richard Socher , and Christopher D. Manning . 2014. GloVe: Global Vectors for Word Representation . In Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP '14) , 1532--1543. https:\/\/doi.org\/10.3115\/v1\/D 14 - 1162 10.3115\/v1 Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. GloVe: Global Vectors for Word Representation. In Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP '14), 1532--1543. https:\/\/doi.org\/10.3115\/v1\/D14-1162"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1080\/02643294.2016.1176907"},{"key":"e_1_3_2_1_35_1","volume-title":"WordNet. Retrieved","author":"Princeton University","year":"2015","unstructured":"Princeton University . 2015 . About WordNet . WordNet. Retrieved November 10, 2016 from https:\/\/wordnet.princeton.edu\/ Princeton University. 2015. About WordNet. WordNet. Retrieved November 10, 2016 from https:\/\/wordnet.princeton.edu\/"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02250566"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992324"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.52.6.1061"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1080\/026999398379529"},{"key":"e_1_3_2_1_40_1","volume-title":"Retrieved","author":"Speer Robyn","year":"2016","unstructured":"Robyn Speer , Joshua Chin , and Catherine Havasi . 2016 . ConceptNet 5.5: An Open Multilingual Graph of General Knowledge. arXiv:1612.03975 [cs] . Retrieved January 24, 2019 from http:\/\/arxiv.org\/abs\/1612.03975 Robyn Speer, Joshua Chin, and Catherine Havasi. 2016. ConceptNet 5.5: An Open Multilingual Graph of General Knowledge. arXiv:1612.03975 [cs]. Retrieved January 24, 2019 from http:\/\/arxiv.org\/abs\/1612.03975"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.2934"},{"key":"e_1_3_2_1_42_1","volume-title":"Retrieved","author":"Turney Peter D.","year":"2008","unstructured":"Peter D. Turney . 2008 . A Uniform Approach to Analogies, Synonyms, Antonyms, and Associations. arXiv:0809.0124 [cs] . Retrieved May 10, 2020 from http:\/\/arxiv.org\/abs\/0809.0124 Peter D. Turney. 2008. A Uniform Approach to Analogies, Synonyms, Antonyms, and Associations. arXiv:0809.0124 [cs]. Retrieved May 10, 2020 from http:\/\/arxiv.org\/abs\/0809.0124"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/T-AFFC.2011.27"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00143"},{"key":"e_1_3_2_1_45_1","volume-title":"Retrieved","author":"Database Facial Expression","year":"2016","unstructured":"Facial Expression Database : Japanese Female Facial Expression (JAFFE) Database . Retrieved October 19, 2016 from http:\/\/www.kasrl.org\/jaffe.html Facial Expression Database: Japanese Female Facial Expression (JAFFE) Database. Retrieved October 19, 2016 from http:\/\/www.kasrl.org\/jaffe.html"}],"event":{"name":"IVA '20: ACM International Conference on Intelligent Virtual Agents","location":"Virtual Event Scotland UK","acronym":"IVA '20","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence"]},"container-title":["Proceedings of the 20th ACM International Conference on Intelligent Virtual Agents"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3383652.3423892","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3383652.3423892","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:31:57Z","timestamp":1750195917000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3383652.3423892"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,19]]},"references-count":46,"alternative-id":["10.1145\/3383652.3423892","10.1145\/3383652"],"URL":"https:\/\/doi.org\/10.1145\/3383652.3423892","relation":{},"subject":[],"published":{"date-parts":[[2020,10,19]]},"assertion":[{"value":"2020-10-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}