{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:05:55Z","timestamp":1750309555176,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"EPSRC Centre for Doctoral Training in Digital Health and Care","award":["EP\/S023704\/1"],"award-info":[{"award-number":["EP\/S023704\/1"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706599.3720244","type":"proceedings-article","created":{"date-parts":[[2025,4,23]],"date-time":"2025-04-23T20:15:12Z","timestamp":1745439312000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Can LLMs be used to Quantify the Emotional Salience of Text Statements using an Elo Rating System?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-4040-7196","authenticated-orcid":false,"given":"Kimberley","family":"Beaumont","sequence":"first","affiliation":[{"name":"Digital Health and Care, University of Bristol, Bristol, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-7190-3521","authenticated-orcid":false,"given":"Martin","family":"Oravec","sequence":"additional","affiliation":[{"name":"School of Computer Science, University of Bristol, Bristol, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5829-0261","authenticated-orcid":false,"given":"Harry","family":"Emerson","sequence":"additional","affiliation":[{"name":"School of Engineering Mathematics and Technology, University of Bristol, Bristol, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4232-0953","authenticated-orcid":false,"given":"Ian","family":"Penton-Voak","sequence":"additional","affiliation":[{"name":"School of Psychological Science, University of Bristol, Bristol, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5017-9473","authenticated-orcid":false,"given":"Conor","family":"Houghton","sequence":"additional","affiliation":[{"name":"School of Engineering Mathematics and Technology, University of Bristol, Bristol, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/1520340.1520420"},{"key":"e_1_3_3_2_3_2","unstructured":"Anthropic. 2024. Claude API: Claude-3-Haiku. https:\/\/console.anthropic.com\/ Accessed: 11 December 2024."},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317083"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3659033"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","unstructured":"K. Beaumont I. Penton-Voak and C. Houghton. 2023. Assessment of Momentary Mood and Performance in a Hidden State Foraging Task. 10.17605\/OSF.IO\/MVJ6T Accessed: 27 Feb. 2025.","DOI":"10.17605\/OSF.IO\/MVJ6T"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","unstructured":"K. Beaumont I. Penton-Voak and C. Houghton. 2024. Assessment of Momentary Mood and Performance in a Hidden State Foraging Task \u2013 A Replication Study. 10.17605\/OSF.IO\/XB5PN Accessed: 27 Feb. 2025.","DOI":"10.17605\/OSF.IO\/XB5PN"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","unstructured":"K. Beaumont I. Penton-Voak and C. Houghton. 2024. Effort as a Potential Mediator Between Momentary Mood and Performance in a Hidden State Foraging Task. 10.17605\/OSF.IO\/CY9EW Accessed: 27 Feb. 2025.","DOI":"10.17605\/OSF.IO\/CY9EW"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"crossref","unstructured":"Adam\u00a0J Berinsky Gregory\u00a0A Huber and Gabriel\u00a0S Lenz. 2012. Evaluating online labor markets for experimental research: Amazon. com\u2019s Mechanical Turk. Political Analysis 20 3 (2012) 351\u2013368.","DOI":"10.1093\/pan\/mpr057"},{"key":"e_1_3_3_2_10_2","first-page":"7187","volume-title":"Encyclopedia of Quality of Life and Well-Being Research","author":"Braun Virginia","year":"2024","unstructured":"Virginia Braun and Victoria Clarke. 2024. Thematic analysis. In Encyclopedia of Quality of Life and Well-Being Research. Springer, 7187\u20137193."},{"key":"e_1_3_3_2_11_2","first-page":"103","volume-title":"The Human-Computer Interaction Handbook","author":"Brave Scott","year":"2007","unstructured":"Scott Brave and Cliff Nass. 2007. Emotion in human-computer interaction. In The Human-Computer Interaction Handbook. CRC Press, 103\u2013118."},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"crossref","unstructured":"Andrew\u00a0P Clark Kate\u00a0L Howard Andy\u00a0T Woods Ian\u00a0S Penton-Voak and Christof Neumann. 2018. Why rate when you could compare? Using the \u201cEloChoice\u201d package to assess pairwise comparisons of perceived physical strength. PloS One 13 1 (2018) e0190393.","DOI":"10.1371\/journal.pone.0190393"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642336"},{"key":"e_1_3_3_2_14_2","volume-title":"The method of paired comparisons","author":"David Herbert\u00a0Aron","year":"1963","unstructured":"Herbert\u00a0Aron David. 1963. The method of paired comparisons. Vol.\u00a012. London."},{"key":"e_1_3_3_2_15_2","unstructured":"Arpad\u00a0E Elo and Sam Sloan. 1978. The rating of chessplayers: Past and present. (1978)."},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858535"},{"key":"e_1_3_3_2_17_2","unstructured":"Mark\u00a0E Glickman and Albyn\u00a0C Jones. 1999. Rating the chess rating system. CHANCE-BERLIN THEN NEW YORK- 12 (1999) 21\u201328."},{"key":"e_1_3_3_2_18_2","unstructured":"Google. 2024. Google AI Studio: Gemini-1.5-flash. https:\/\/aistudio.google.com\/app Accessed: 11 December 2024."},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"crossref","unstructured":"Joseph Henrich Steven\u00a0J Heine and Ara Norenzayan. 2010. Most people are not WEIRD. Nature 466 7302 (2010) 29\u201329.","DOI":"10.1038\/466029a"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"crossref","unstructured":"Joseph Henrich Steven\u00a0J Heine and Ara Norenzayan. 2010. The weirdest people in the world? Behavioral and Brain Sciences 33 2-3 (2010) 61\u201383.","DOI":"10.1017\/S0140525X0999152X"},{"key":"e_1_3_3_2_21_2","unstructured":"Jen-tse Huang Man\u00a0Ho Lam Eric\u00a0John Li Shujie Ren Wenxuan Wang Wenxiang Jiao Zhaopeng Tu and Michael\u00a0R Lyu. 2023. Emotionally numb or empathetic? Evaluating how llms feel using emotionbench. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2308.03656 (2023)."},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"crossref","unstructured":"Joseph\u00a0E LeDoux and Stefan\u00a0G Hofmann. 2018. The subjective experience of emotion: A fearful view. Current Opinion in Behavioral Sciences 19 (2018) 67\u201372.","DOI":"10.1016\/j.cobeha.2017.09.011"},{"key":"e_1_3_3_2_23_2","unstructured":"Yan Leng and Yuan Yuan. 2023. Do LLM agents exhibit social behavior? arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2312.15198 (2023)."},{"key":"e_1_3_3_2_24_2","unstructured":"Yinhong Liu Han Zhou Zhijiang Guo Ehsan Shareghi Ivan Vuli\u0107 Anna Korhonen and Nigel Collier. 2024. Aligning with human judgement: The role of pairwise preference in large language model evaluators. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.16950 (2024)."},{"key":"e_1_3_3_2_25_2","unstructured":"Alexandra\u00a0Sasha Luccioni and Joseph\u00a0D Viviano. 2021. What\u2019s in the box? A preliminary analysis of undesirable content in the common crawl corpus. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2105.02732 (2021)."},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124926"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411763.3451799"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"crossref","unstructured":"Hasti Narimanzadeh Arash Badie-Modiri Iuliia\u00a0G Smirnova and Ted Hsuan\u00a0Yun Chen. 2023. Crowdsourcing subjective annotations using pairwise comparisons reduces bias and error compared to the majority-vote method. Proceedings of the ACM on Human-Computer Interaction 7 CSCW2 (2023) 1\u201329.","DOI":"10.1145\/3610183"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512128"},{"key":"e_1_3_3_2_30_2","unstructured":"C. Neumann. 2019. EloChoice: Preference rating for visual stimuli based on Elo ratings (version 0.29). http:\/\/cran.r-project.org\/web\/packages\/EloChoice Accessed: 27 Feb. 2025."},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"crossref","unstructured":"Christof Neumann Julie Duboscq Constance Dubuc Andri Ginting Ade\u00a0Maulana Irwan Muhammad Agil Anja Widdig and Antje Engelhardt. 2011. Assessing dominance hierarchies: Validation and advantages of progressive evaluation with Elo-rating. Animal Behaviour 82 4 (2011) 911\u2013921.","DOI":"10.1016\/j.anbehav.2011.07.016"},{"key":"e_1_3_3_2_32_2","unstructured":"OpenAI. [n. d.]. OpenAI GPT-3 Model Card. https:\/\/github.com\/openai\/gpt-3\/blob\/master\/model-card.md Accessed: 2025-01-17."},{"key":"e_1_3_3_2_33_2","unstructured":"OpenAI. 2024. OpenAI API: ChatGPT-3.5-turbo. https:\/\/platform.openai.com\/ Accessed: 11 December 2024."},{"key":"e_1_3_3_2_34_2","unstructured":"Prolific. 2024. Prolific: Online participant recruitment for surveys and experiments. https:\/\/www.prolific.com\/ Accessed: 2024-11-23."},{"key":"e_1_3_3_2_35_2","unstructured":"Jiaxing Qiu Dongliang Guo Papini Natalie Peace Noelle Levinson Cheri and Teague\u00a0R Henry. 2025. Ensemble of large language models for curated labeling and rating of free-text data. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2501.08413 (2025)."},{"key":"e_1_3_3_2_36_2","unstructured":"Hannah Rashkin. 2018. Towards empathetic open-domain conversation models: A new benchmark and dataset. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1811.00207 (2018)."},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3650798"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP.2011.6116320"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"crossref","unstructured":"Jennifer Routh Sharmini\u00a0Julita Paramasivam Peter Cockcroft Sarah Wood John Remnant Corn\u00e9lie Westermann Alison Reid Patricia Pawson Sheena Warman Vishna\u00a0Devi Nadarajah et\u00a0al. 2023. Rating and ranking preparedness characteristics important for veterinary workplace clinical training: A novel application of pairwise comparisons and the Elo algorithm. Frontiers in Medicine 10 (2023) 1128058.","DOI":"10.3389\/fmed.2023.1128058"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"crossref","unstructured":"Peter Salovey and Daisy Grewal. 2005. The science of emotional intelligence. Current Directions in Psychological Science 14 6 (2005) 281\u2013285.","DOI":"10.1111\/j.0963-7214.2005.00381.x"},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"crossref","unstructured":"Vera Sorin Dana Brin Yiftach Barash Eli Konen Alexander Charney Girish Nadkarni and Eyal Klang. 2024. Large language models and empathy: Systematic review. Journal of Medical Internet Research 26 (2024) e52597.","DOI":"10.2196\/52597"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-60405-8_26"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"crossref","unstructured":"Xuena Wang Xueting Li Zi Yin Yue Wu and Jia Liu. 2023. Emotional intelligence of large language models. Journal of Pacific Rim Psychology 17 (2023) 18344909231213958.","DOI":"10.1177\/18344909231213958"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICME51207.2021.9428220"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"crossref","unstructured":"Xuhai Xu Bingsheng Yao Yuanzhe Dong Saadia Gabriel Hong Yu James Hendler Marzyeh Ghassemi Anind\u00a0K Dey and Dakuo Wang. 2024. Mental-llm: Leveraging large language models for mental health prediction via online text data. Proceedings of the ACM on Interactive Mobile Wearable and Ubiquitous Technologies 8 1 (2024) 1\u201332.","DOI":"10.1145\/3643540"},{"key":"e_1_3_3_2_46_2","first-page":"V006T06A018","volume-title":"International Design Engineering Technical Conferences and Computers and Information in Engineering Conference","volume":"88407","author":"Zhu Qihao","year":"2024","unstructured":"Qihao Zhu, Leah Chong, Maria Yang, and Jianxi Luo. 2024. Reading users\u2019 minds from what they say: An investigation into llm-based empathic mental inference. In International Design Engineering Technical Conferences and Computers and Information in Engineering Conference , Vol.\u00a088407. American Society of Mechanical Engineers, V006T06A018."}],"event":{"name":"CHI EA '25: Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Yokohama Japan","acronym":"CHI EA '25"},"container-title":["Proceedings of the Extended Abstracts of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3720244","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706599.3720244","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:18:51Z","timestamp":1750295931000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3720244"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":45,"alternative-id":["10.1145\/3706599.3720244","10.1145\/3706599"],"URL":"https:\/\/doi.org\/10.1145\/3706599.3720244","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}